From 7f4aa5a58645b95ac5c197babb38b1419e6dd982 Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Thu, 4 Feb 2021 11:17:25 +0100 Subject: [PATCH 01/33] feat(gravsearch): Start implementation of topological sort. --- third_party/dependencies.bzl | 4 ++++ webapi/src/main/scala/org/knora/webapi/messages/BUILD.bazel | 1 + ...plestoreSpecificGravsearchToPrequeryTransformerSpec.scala | 5 +++++ 3 files changed, 10 insertions(+) diff --git a/third_party/dependencies.bzl b/third_party/dependencies.bzl index f783e67940..8bab8aac64 100644 --- a/third_party/dependencies.bzl +++ b/third_party/dependencies.bzl @@ -137,6 +137,9 @@ def dependencies(): # Additional Selenium libraries besides the ones pulled in during init # of io_bazel_rules_webtesting "org.seleniumhq.selenium:selenium-support:3.141.59", + + # Graph for Scala + "org.scala-graph:graph-core_2.12:1.13.1", ], repositories = [ "https://repo.maven.apache.org/maven2", @@ -187,6 +190,7 @@ BASE_TEST_DEPENDENCIES = [ "@maven//:org_scalatest_scalatest_shouldmatchers_2_12", "@maven//:org_scalatest_scalatest_compatible", "@maven//:org_scalactic_scalactic_2_12", + "@maven//:org_scala_graph_graph_core_2_12", ] BASE_TEST_DEPENDENCIES_WITH_JSON = BASE_TEST_DEPENDENCIES + [ diff --git a/webapi/src/main/scala/org/knora/webapi/messages/BUILD.bazel b/webapi/src/main/scala/org/knora/webapi/messages/BUILD.bazel index f6f697eea2..15d4298463 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/BUILD.bazel +++ b/webapi/src/main/scala/org/knora/webapi/messages/BUILD.bazel @@ -44,5 +44,6 @@ scala_library( "@maven//:org_scala_lang_scala_reflect", "@maven//:org_slf4j_slf4j_api", "@maven//:org_springframework_security_spring_security_core", + "@maven//:org_scala_graph_graph_core_2_12", ], ) diff --git a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala index 8805cbdce2..d57c2f818b 100644 --- a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala @@ -21,12 +21,17 @@ import scala.collection.mutable.ArrayBuffer import scala.concurrent.Await import scala.concurrent.duration._ +import scalax.collection.edge.WDiEdge +import scalax.collection.edge.Implicits._ + private object QueryHandler { private val timeout = 10.seconds val anythingUser: UserADM = SharedTestDataADM.anythingAdminUser + protected def reorderPatternsByDependency(patterns: Seq[QueryPattern]): Seq[QueryPattern] = ??? + def transformQuery(query: String, responderData: ResponderData, settings: KnoraSettingsImpl): SelectQuery = { val constructQuery = GravsearchParser.parseQuery(query) From 84938b9a204881597b6a9e9165d9c409c7b37a83 Mon Sep 17 00:00:00 2001 From: Sepideh Alassi Date: Mon, 8 Feb 2021 11:22:56 +0100 Subject: [PATCH 02/33] feat(gravsearch) create a graph from statement patterns --- ...cGravsearchToPrequeryTransformerSpec.scala | 56 +++++++++++++++++-- 1 file changed, 52 insertions(+), 4 deletions(-) diff --git a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala index d57c2f818b..856ac0da30 100644 --- a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala @@ -20,17 +20,59 @@ import org.knora.webapi.util.ApacheLuceneSupport.LuceneQueryString import scala.collection.mutable.ArrayBuffer import scala.concurrent.Await import scala.concurrent.duration._ - -import scalax.collection.edge.WDiEdge +import scalax.collection.edge.LUnDiEdge +import scalax.collection.Graph import scalax.collection.edge.Implicits._ +import scalax.collection.GraphEdge private object QueryHandler { private val timeout = 10.seconds val anythingUser: UserADM = SharedTestDataADM.anythingAdminUser - - protected def reorderPatternsByDependency(patterns: Seq[QueryPattern]): Seq[QueryPattern] = ??? + case class edgeLabel(label: String) + def createAndSortGraph(statementPatterns: Seq[StatementPattern]): Seq[StatementPattern] = { + val graphComponents = statementPatterns.map { statementPattern => + // transform every statementPattern to LUniDiEdge(Subj,Obj)(edgeLabel(pred)) + val node1 = statementPattern.subj.toSparql + val node2 = statementPattern.obj.toSparql + val edge = statementPattern.pred.toSparql + LUnDiEdge(node1, node2)(edgeLabel(edge)) + } + val graph = Graph(graphComponents) + statementPatterns + } + def reorderPatternsByDependency(patterns: Seq[QueryPattern]): Seq[QueryPattern] = { + + val statementPatterns: Seq[StatementPattern] = patterns.collect { case pattern: StatementPattern => pattern } + val sortedStatementPatterns = createAndSortGraph(statementPatterns) + val otherPatterns: Set[QueryPattern] = patterns.filterNot(p => p.isInstanceOf[StatementPattern]).toSet + val sortedOtherPatterns: Set[QueryPattern] = otherPatterns.map { + // sort statements inside each UnionPattern block + case unionPattern: UnionPattern => { + val sortedUnionBlocks = unionPattern.blocks.map(block => reorderPatternsByDependency(block)) + UnionPattern(blocks = sortedUnionBlocks) + } + // sort statements inside OptionalPattern + case optionalPattern: OptionalPattern => { + val sortedOptionalPatterns = reorderPatternsByDependency(optionalPattern.patterns) + OptionalPattern(patterns = sortedOptionalPatterns) + } + // sort statements inside MinusPattern + case minusPattern: MinusPattern => { + val sortedMinusPatterns = reorderPatternsByDependency(minusPattern.patterns) + MinusPattern(patterns = sortedMinusPatterns) + } + // sort statements inside FilterNotExistsPattern + case filterNotExistsPattern: FilterNotExistsPattern => { + val sortedFilterNotExistsPatterns = reorderPatternsByDependency(filterNotExistsPattern.patterns) + FilterNotExistsPattern(patterns = sortedFilterNotExistsPatterns) + } + // return any other query pattern as it is + case pattern: QueryPattern => pattern + } + sortedStatementPatterns ++ sortedOtherPatterns.toSeq + } def transformQuery(query: String, responderData: ResponderData, settings: KnoraSettingsImpl): SelectQuery = { @@ -1932,5 +1974,11 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec assert(transformedQuery === TransformedQueryWithUnionScopes) } + + "reorder query patterns in where clause" in { + val sortedPatterns = QueryHandler.reorderPatternsByDependency( + transformedQueryWithDecimalOptionalSortCriterionAndFilter.whereClause.patterns) + assert(sortedPatterns.head.isInstanceOf[StatementPattern]) + } } } From c01c796120256726b7458c681ee0fe9fd23a5fad Mon Sep 17 00:00:00 2001 From: Sepideh Alassi Date: Mon, 8 Feb 2021 13:36:55 +0100 Subject: [PATCH 03/33] fix (grvsearch): immutable graph --- ...cGravsearchToPrequeryTransformerSpec.scala | 41 +++++++++++++++++-- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala index 856ac0da30..6c27dceaa6 100644 --- a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala @@ -39,7 +39,17 @@ private object QueryHandler { val edge = statementPattern.pred.toSparql LUnDiEdge(node1, node2)(edgeLabel(edge)) } - val graph = Graph(graphComponents) + val createdGraph = graphComponents.foldLeft(Graph.empty[String, LUnDiEdge]) { (graph, edge) => + graph + edge // add nodes and edges to graph + } + // TODO: Is there a cycle in the graph? + if (createdGraph.isCyclic) { + // yes. + println(createdGraph.findCycle) + } else { + // No. sort the graph + createdGraph.topologicalSort + } statementPatterns } def reorderPatternsByDependency(patterns: Seq[QueryPattern]): Seq[QueryPattern] = { @@ -1813,6 +1823,31 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec useDistinct = true ) + val queryToReorder: String = """ + |PREFIX beol: + |PREFIX knora-api: + | + |CONSTRUCT { + | ?letter knora-api:isMainResource true . + | ?letter ?linkingProp1 ?person1 . + | ?letter ?linkingProp2 ?person2 . + | ?letter beol:creationDate ?date . + |} WHERE { + | ?letter beol:creationDate ?date . + | + | ?letter ?linkingProp1 ?person1 . + | FILTER(?linkingProp1 = beol:hasAuthor || ?linkingProp1 = beol:hasRecipient ) + | + | ?letter ?linkingProp2 ?person2 . + | FILTER(?linkingProp2 = beol:hasAuthor || ?linkingProp2 = beol:hasRecipient ) + | + | ?person1 beol:hasIAFIdentifier ?gnd1 . + | ?gnd1 knora-api:valueAsString "(DE-588)118531379" . + | + | ?person2 beol:hasIAFIdentifier ?gnd2 . + | ?gnd2 knora-api:valueAsString "(DE-588)118696149" . + |} ORDER BY ?date""".stripMargin + "The NonTriplestoreSpecificGravsearchToPrequeryGenerator object" should { "transform an input query with an optional property criterion without removing the rdf:type statement" in { @@ -1976,8 +2011,8 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec } "reorder query patterns in where clause" in { - val sortedPatterns = QueryHandler.reorderPatternsByDependency( - transformedQueryWithDecimalOptionalSortCriterionAndFilter.whereClause.patterns) + val transformedQuery = QueryHandler.transformQuery(queryToReorder, responderData, settings) + val sortedPatterns = QueryHandler.reorderPatternsByDependency(transformedQuery.whereClause.patterns) assert(sortedPatterns.head.isInstanceOf[StatementPattern]) } } From 2d2afd7df8e139351002ea1043dd5b89a21672b4 Mon Sep 17 00:00:00 2001 From: Sepideh Alassi Date: Mon, 8 Feb 2021 14:09:59 +0100 Subject: [PATCH 04/33] fix (gravsearch) is not cyclic, sort the graph --- ...riplestoreSpecificGravsearchToPrequeryTransformerSpec.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala index 6c27dceaa6..ee58abbe0e 100644 --- a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala @@ -48,7 +48,8 @@ private object QueryHandler { println(createdGraph.findCycle) } else { // No. sort the graph - createdGraph.topologicalSort + val sortedGraph = createdGraph.topologicalSort + println(sortedGraph) } statementPatterns } From 41c8be2eb1fff68320b44f5e55334059049ce28a Mon Sep 17 00:00:00 2001 From: Sepideh Alassi Date: Mon, 8 Feb 2021 14:23:39 +0100 Subject: [PATCH 05/33] fix (gravsearch): use input query for sorting --- ...estoreSpecificGravsearchToPrequeryTransformerSpec.scala | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala index ee58abbe0e..c5345e5d99 100644 --- a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala @@ -45,7 +45,8 @@ private object QueryHandler { // TODO: Is there a cycle in the graph? if (createdGraph.isCyclic) { // yes. - println(createdGraph.findCycle) + val cycle = createdGraph.findCycle + println(cycle) } else { // No. sort the graph val sortedGraph = createdGraph.topologicalSort @@ -2012,8 +2013,8 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec } "reorder query patterns in where clause" in { - val transformedQuery = QueryHandler.transformQuery(queryToReorder, responderData, settings) - val sortedPatterns = QueryHandler.reorderPatternsByDependency(transformedQuery.whereClause.patterns) + val constructQuery = GravsearchParser.parseQuery(queryToReorder) + val sortedPatterns = QueryHandler.reorderPatternsByDependency(constructQuery.whereClause.patterns) assert(sortedPatterns.head.isInstanceOf[StatementPattern]) } } From e5140502bc7ec41d6a4b83e42ccd6deb91ff582e Mon Sep 17 00:00:00 2001 From: Sepideh Alassi Date: Mon, 8 Feb 2021 15:16:40 +0100 Subject: [PATCH 06/33] feat(gravsearch) use directed hyper edge --- ...icGravsearchToPrequeryTransformerSpec.scala | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala index c5345e5d99..d11d9770a3 100644 --- a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala @@ -20,10 +20,8 @@ import org.knora.webapi.util.ApacheLuceneSupport.LuceneQueryString import scala.collection.mutable.ArrayBuffer import scala.concurrent.Await import scala.concurrent.duration._ -import scalax.collection.edge.LUnDiEdge -import scalax.collection.Graph -import scalax.collection.edge.Implicits._ -import scalax.collection.GraphEdge +import scalax.collection.{Graph} +import scalax.collection.GraphEdge.DiHyperEdge private object QueryHandler { @@ -37,11 +35,12 @@ private object QueryHandler { val node1 = statementPattern.subj.toSparql val node2 = statementPattern.obj.toSparql val edge = statementPattern.pred.toSparql - LUnDiEdge(node1, node2)(edgeLabel(edge)) + DiHyperEdge(node1, node2) } - val createdGraph = graphComponents.foldLeft(Graph.empty[String, LUnDiEdge]) { (graph, edge) => + val createdGraph = graphComponents.foldLeft(Graph.empty[String, DiHyperEdge]) { (graph, edge) => graph + edge // add nodes and edges to graph } + // TODO: Is there a cycle in the graph? if (createdGraph.isCyclic) { // yes. @@ -49,8 +48,11 @@ private object QueryHandler { println(cycle) } else { // No. sort the graph - val sortedGraph = createdGraph.topologicalSort - println(sortedGraph) + createdGraph.topologicalSort match { + case Right(topOrder) => println(topOrder) + case Left(cycleNode) => + throw new Error(s"Graph contains a cycle at node: ${cycleNode}.") + } } statementPatterns } From 2778e5717ff9d817139626c51118387db0c3bb67 Mon Sep 17 00:00:00 2001 From: Sepideh Alassi Date: Tue, 9 Feb 2021 08:28:39 +0100 Subject: [PATCH 07/33] feat(gravearch) change to DiHyperedge --- ...cGravsearchToPrequeryTransformerSpec.scala | 50 +++++++++++-------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala index d11d9770a3..6726b2d285 100644 --- a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala @@ -30,31 +30,39 @@ private object QueryHandler { val anythingUser: UserADM = SharedTestDataADM.anythingAdminUser case class edgeLabel(label: String) def createAndSortGraph(statementPatterns: Seq[StatementPattern]): Seq[StatementPattern] = { - val graphComponents = statementPatterns.map { statementPattern => - // transform every statementPattern to LUniDiEdge(Subj,Obj)(edgeLabel(pred)) - val node1 = statementPattern.subj.toSparql - val node2 = statementPattern.obj.toSparql - val edge = statementPattern.pred.toSparql - DiHyperEdge(node1, node2) - } - val createdGraph = graphComponents.foldLeft(Graph.empty[String, DiHyperEdge]) { (graph, edge) => - graph + edge // add nodes and edges to graph + def createGraph: Graph[String, DiHyperEdge] = { + val graphComponents = statementPatterns.map { statementPattern => + // transform every statementPattern to LUniDiEdge(Subj,Obj)(edgeLabel(pred)) + val node1 = statementPattern.subj.toSparql + val node2 = statementPattern.obj.toSparql +// val edge = statementPattern.pred.toSparql + DiHyperEdge(node1, node2) + } + val graph = graphComponents.foldLeft(Graph.empty[String, DiHyperEdge]) { (graph, edge) => + graph + edge // add nodes and edges to graph + } + graph } - - // TODO: Is there a cycle in the graph? - if (createdGraph.isCyclic) { - // yes. - val cycle = createdGraph.findCycle - println(cycle) - } else { - // No. sort the graph - createdGraph.topologicalSort match { - case Right(topOrder) => println(topOrder) + def sortStatementPatterns(createdGraph: Graph[String, DiHyperEdge], + statementPatterns: Seq[StatementPattern]): Seq[StatementPattern] = { + // Try topological sorting of graph + val topologicalOrder: createdGraph.TopologicalOrder[createdGraph.NodeT] = createdGraph.topologicalSort match { + // Is there a cycle in the graph? case Left(cycleNode) => - throw new Error(s"Graph contains a cycle at node: ${cycleNode}.") + // TODO: yes. break the cycle + throw new Error(s"Graph contains a cycle at node: ${cycleNode}, entire cycle is ${createdGraph.findCycle}.") + case Right(topOrder) => { + // No. return the topological order + topOrder + } } + val leafToRootOrder: Seq[createdGraph.NodeT] = topologicalOrder.iterator.toSeq.reverse + statementPatterns } - statementPatterns + + val createdGraph = createGraph + sortStatementPatterns(createdGraph, statementPatterns) + } def reorderPatternsByDependency(patterns: Seq[QueryPattern]): Seq[QueryPattern] = { From 41147b5176d670a9b579479c5fad08c2842dbd0b Mon Sep 17 00:00:00 2001 From: Sepideh Alassi Date: Tue, 9 Feb 2021 18:04:40 +0100 Subject: [PATCH 08/33] feat(gravsearch): sort statements --- ...cGravsearchToPrequeryTransformerSpec.scala | 44 ++++++++++++------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala index 6726b2d285..49a77666aa 100644 --- a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala @@ -20,16 +20,18 @@ import org.knora.webapi.util.ApacheLuceneSupport.LuceneQueryString import scala.collection.mutable.ArrayBuffer import scala.concurrent.Await import scala.concurrent.duration._ -import scalax.collection.{Graph} +import scalax.collection.Graph import scalax.collection.GraphEdge.DiHyperEdge +import scala.collection.immutable.TreeSet + private object QueryHandler { private val timeout = 10.seconds val anythingUser: UserADM = SharedTestDataADM.anythingAdminUser case class edgeLabel(label: String) - def createAndSortGraph(statementPatterns: Seq[StatementPattern]): Seq[StatementPattern] = { + def createAndSortGraph(statementPatterns: Seq[StatementPattern]): Set[QueryPattern] = { def createGraph: Graph[String, DiHyperEdge] = { val graphComponents = statementPatterns.map { statementPattern => // transform every statementPattern to LUniDiEdge(Subj,Obj)(edgeLabel(pred)) @@ -44,7 +46,7 @@ private object QueryHandler { graph } def sortStatementPatterns(createdGraph: Graph[String, DiHyperEdge], - statementPatterns: Seq[StatementPattern]): Seq[StatementPattern] = { + statementPatterns: Seq[StatementPattern]): Set[QueryPattern] = { // Try topological sorting of graph val topologicalOrder: createdGraph.TopologicalOrder[createdGraph.NodeT] = createdGraph.topologicalSort match { // Is there a cycle in the graph? @@ -53,18 +55,28 @@ private object QueryHandler { throw new Error(s"Graph contains a cycle at node: ${cycleNode}, entire cycle is ${createdGraph.findCycle}.") case Right(topOrder) => { // No. return the topological order + println(topOrder) topOrder } } - val leafToRootOrder: Seq[createdGraph.NodeT] = topologicalOrder.iterator.toSeq.reverse - statementPatterns + // take the main resource out. + val topologicalOrderWithoutRoot: Iterable[createdGraph.NodeT] = topologicalOrder.tail + val sortedStatements: Set[QueryPattern] = + topologicalOrderWithoutRoot.foldRight(Set.empty[QueryPattern]) { (node, sortedStatements) => + val statementsOfNode: Set[QueryPattern] = statementPatterns + .filter(p => p.obj.toSparql.equals(node.value)) + .toSet[QueryPattern] + println(sortedStatements ++ statementsOfNode) + sortedStatements ++ statementsOfNode + } + sortedStatements } val createdGraph = createGraph sortStatementPatterns(createdGraph, statementPatterns) } - def reorderPatternsByDependency(patterns: Seq[QueryPattern]): Seq[QueryPattern] = { + def reorderPatternsByDependency(patterns: Seq[QueryPattern]): Set[QueryPattern] = { val statementPatterns: Seq[StatementPattern] = patterns.collect { case pattern: StatementPattern => pattern } val sortedStatementPatterns = createAndSortGraph(statementPatterns) @@ -72,28 +84,30 @@ private object QueryHandler { val sortedOtherPatterns: Set[QueryPattern] = otherPatterns.map { // sort statements inside each UnionPattern block case unionPattern: UnionPattern => { - val sortedUnionBlocks = unionPattern.blocks.map(block => reorderPatternsByDependency(block)) + val sortedUnionBlocks: Seq[Seq[QueryPattern]] = + unionPattern.blocks.map(block => reorderPatternsByDependency(block).toSeq) UnionPattern(blocks = sortedUnionBlocks) } // sort statements inside OptionalPattern case optionalPattern: OptionalPattern => { - val sortedOptionalPatterns = reorderPatternsByDependency(optionalPattern.patterns) - OptionalPattern(patterns = sortedOptionalPatterns) + val sortedOptionalPatterns: Set[QueryPattern] = reorderPatternsByDependency(optionalPattern.patterns) + OptionalPattern(patterns = sortedOptionalPatterns.toSeq) } // sort statements inside MinusPattern case minusPattern: MinusPattern => { - val sortedMinusPatterns = reorderPatternsByDependency(minusPattern.patterns) - MinusPattern(patterns = sortedMinusPatterns) + val sortedMinusPatterns: Set[QueryPattern] = reorderPatternsByDependency(minusPattern.patterns) + MinusPattern(patterns = sortedMinusPatterns.toSeq) } // sort statements inside FilterNotExistsPattern case filterNotExistsPattern: FilterNotExistsPattern => { - val sortedFilterNotExistsPatterns = reorderPatternsByDependency(filterNotExistsPattern.patterns) - FilterNotExistsPattern(patterns = sortedFilterNotExistsPatterns) + val sortedFilterNotExistsPatterns: Set[QueryPattern] = + reorderPatternsByDependency(filterNotExistsPattern.patterns) + FilterNotExistsPattern(patterns = sortedFilterNotExistsPatterns.toSeq) } // return any other query pattern as it is case pattern: QueryPattern => pattern } - sortedStatementPatterns ++ sortedOtherPatterns.toSeq + sortedStatementPatterns ++ sortedOtherPatterns } def transformQuery(query: String, responderData: ResponderData, settings: KnoraSettingsImpl): SelectQuery = { @@ -2025,7 +2039,7 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec "reorder query patterns in where clause" in { val constructQuery = GravsearchParser.parseQuery(queryToReorder) val sortedPatterns = QueryHandler.reorderPatternsByDependency(constructQuery.whereClause.patterns) - assert(sortedPatterns.head.isInstanceOf[StatementPattern]) + assert(sortedPatterns.head.isInstanceOf[QueryPattern]) } } } From 106f33595ef10f599723ce8cd3057ce345279912 Mon Sep 17 00:00:00 2001 From: Sepideh Alassi Date: Tue, 9 Feb 2021 19:07:32 +0100 Subject: [PATCH 09/33] fix (gravsearch): correctly preserve the order of statements as indicated by topologicalOrder --- ...iplestoreSpecificGravsearchToPrequeryTransformerSpec.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala index 49a77666aa..7d29bfb9e7 100644 --- a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala @@ -23,7 +23,7 @@ import scala.concurrent.duration._ import scalax.collection.Graph import scalax.collection.GraphEdge.DiHyperEdge -import scala.collection.immutable.TreeSet +import scala.collection.immutable.ListSet private object QueryHandler { @@ -62,7 +62,7 @@ private object QueryHandler { // take the main resource out. val topologicalOrderWithoutRoot: Iterable[createdGraph.NodeT] = topologicalOrder.tail val sortedStatements: Set[QueryPattern] = - topologicalOrderWithoutRoot.foldRight(Set.empty[QueryPattern]) { (node, sortedStatements) => + topologicalOrderWithoutRoot.foldRight(ListSet.empty[QueryPattern]) { (node, sortedStatements) => val statementsOfNode: Set[QueryPattern] = statementPatterns .filter(p => p.obj.toSparql.equals(node.value)) .toSet[QueryPattern] From 6d7168a7e68931c75f3c1b26c6a478062bc3c1fd Mon Sep 17 00:00:00 2001 From: Sepideh Alassi Date: Wed, 10 Feb 2021 13:10:02 +0100 Subject: [PATCH 10/33] test (gravsearch) test the recursive function for sorting statements inside scopes --- ...cGravsearchToPrequeryTransformerSpec.scala | 112 +++++++++++++++++- 1 file changed, 110 insertions(+), 2 deletions(-) diff --git a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala index 7d29bfb9e7..385486acde 100644 --- a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala @@ -107,7 +107,8 @@ private object QueryHandler { // return any other query pattern as it is case pattern: QueryPattern => pattern } - sortedStatementPatterns ++ sortedOtherPatterns + val sorted = sortedStatementPatterns ++ sortedOtherPatterns + sorted } def transformQuery(query: String, responderData: ResponderData, settings: KnoraSettingsImpl): SelectQuery = { @@ -1874,6 +1875,53 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec | ?gnd2 knora-api:valueAsString "(DE-588)118696149" . |} ORDER BY ?date""".stripMargin + val queryToReorderWithMinus = + """PREFIX knora-api: + |PREFIX anything: + | + |CONSTRUCT { + | ?thing knora-api:isMainResource true . + |} WHERE { + | ?thing a knora-api:Resource . + | ?thing a anything:Thing . + | MINUS { + | ?thing anything:hasInteger ?intVal . + | anything:hasInteger knora-api:objectType xsd:integer . + | ?intVal a xsd:integer . + | FILTER(?intVal = 123454321 || ?intVal = 999999999) + | } + |}""".stripMargin + + val queryToReorderWithUnion: String = + s"""PREFIX knora-api: + |PREFIX anything: + |CONSTRUCT { + | ?thing knora-api:isMainResource true . + | ?thing anything:hasInteger ?int . + | ?thing anything:hasRichtext ?richtext . + | ?thing anything:hasText ?text . + |} WHERE { + | ?thing a knora-api:Resource . + | ?thing a anything:Thing . + | + | { + | ?thing anything:hasRichtext ?richtext . + | FILTER knora-api:matchText(?richtext, "test") + | + | ?thing anything:hasInteger ?int . + | ?int knora-api:intValueAsInt 1 . + | } + | UNION + | { + | ?thing anything:hasText ?text . + | FILTER knora-api:matchText(?text, "test") + | + | ?thing anything:hasInteger ?int . + | ?int knora-api:intValueAsInt 1 . + | } + |} + |ORDER BY (?int)""".stripMargin + "The NonTriplestoreSpecificGravsearchToPrequeryGenerator object" should { "transform an input query with an optional property criterion without removing the rdf:type statement" in { @@ -2039,7 +2087,67 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec "reorder query patterns in where clause" in { val constructQuery = GravsearchParser.parseQuery(queryToReorder) val sortedPatterns = QueryHandler.reorderPatternsByDependency(constructQuery.whereClause.patterns) - assert(sortedPatterns.head.isInstanceOf[QueryPattern]) + val statements = sortedPatterns.filter(p => p.isInstanceOf[StatementPattern]) + // check that statements are brought to top + val topElements = sortedPatterns.slice(0, statements.size) + assert(topElements.equals(statements)) + // check the order of statements + val mainResourceStatements = statements.slice(statements.size - 3, statements.size) + assert(!mainResourceStatements.exists(p => p.asInstanceOf[StatementPattern].subj.toSparql !== "?letter")) + } + "reorder query patterns in where clause with union" in { + val constructQuery = GravsearchParser.parseQuery(queryToReorderWithUnion) + val sortedPatterns = QueryHandler.reorderPatternsByDependency(constructQuery.whereClause.patterns) + val statements = sortedPatterns.filter(p => p.isInstanceOf[StatementPattern]) + // check that statements are brought to top + val topElements = sortedPatterns.slice(0, statements.size) + assert(topElements.equals(statements)) + //check order of statements in each block + val unionPattern: Set[UnionPattern] = sortedPatterns.collect { case pattern: UnionPattern => pattern } + val firstBlock = unionPattern.head.blocks.head + val firstBlockStatements = firstBlock.collect { + case pattern: StatementPattern => pattern + } + assert(firstBlockStatements.head.subj.toSparql == "?int") + assert(firstBlockStatements.last.subj.toSparql == "?thing") + assert(firstBlockStatements.last.obj.toSparql == "?richtext") + val secondBlock = unionPattern.head.blocks.last + val secondBlockStatements = secondBlock.collect { + case pattern: StatementPattern => pattern + } + assert(secondBlockStatements.head.subj.toSparql == "?int") + assert(secondBlockStatements.last.subj.toSparql == "?thing") + assert(secondBlockStatements.last.obj.toSparql == "?text") + } + + "reorder query patterns in where clause with optional" in { + val constructQuery = GravsearchParser.parseQuery(queryWithOptional) + val sortedPatterns = QueryHandler.reorderPatternsByDependency(constructQuery.whereClause.patterns) + val statements = sortedPatterns.filter(p => p.isInstanceOf[StatementPattern]) + // check that statements are brought to top + val topElements = sortedPatterns.slice(0, statements.size) + assert(topElements.equals(statements)) + // check statements inside optional pattern + val optionalPattern: Set[OptionalPattern] = sortedPatterns.collect { case pattern: OptionalPattern => pattern } + val optionalPatternStatements = optionalPattern.head.patterns.collect { + case pattern: StatementPattern => pattern + } + assert(optionalPatternStatements.last.subj.toSparql == "?document") + assert(optionalPatternStatements.last.obj.toSparql == "?recipient") + } + + "reorder query patterns with minus scope" in { + val constructQuery = GravsearchParser.parseQuery(queryToReorderWithMinus) + val sortedPatterns = QueryHandler.reorderPatternsByDependency(constructQuery.whereClause.patterns) + val statements = sortedPatterns.filter(p => p.isInstanceOf[StatementPattern]) + // check that statements are brought to top + val topElements = sortedPatterns.slice(0, statements.size) + assert(topElements.equals(statements)) + // check statements inside minus pattern + val minusPattern: Set[MinusPattern] = sortedPatterns.collect { case pattern: MinusPattern => pattern } + val minusPatternStatements = minusPattern.head.patterns.collect { case pattern: StatementPattern => pattern } + assert(minusPatternStatements.last.subj.toSparql == "?thing") + assert(minusPatternStatements.last.obj.toSparql == "?intVal") } } } From 68a3bbdd5083f97c4c936315deb73eb3f4c4005c Mon Sep 17 00:00:00 2001 From: Sepideh Alassi Date: Wed, 10 Feb 2021 13:45:04 +0100 Subject: [PATCH 11/33] fix the failing test --- ...cGravsearchToPrequeryTransformerSpec.scala | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala index 385486acde..1f9d5362a8 100644 --- a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala @@ -24,6 +24,7 @@ import scalax.collection.Graph import scalax.collection.GraphEdge.DiHyperEdge import scala.collection.immutable.ListSet +import scala.util.Failure private object QueryHandler { @@ -1875,6 +1876,22 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec | ?gnd2 knora-api:valueAsString "(DE-588)118696149" . |} ORDER BY ?date""".stripMargin + val queryToReorderWithCycle: String = """ + |PREFIX beol: + |PREFIX knora-api: + | + |CONSTRUCT { + | ?letter knora-api:isMainResource true . + | ?letter ?linkingProp1 ?person1 . + |} WHERE { + | ?letter beol:hasAuthor ?person1 . + | ?letter beol:letterHasTranslation ?letter2. + | ?letter2 beol:hasAuthor ?person1 . + | ?person1 beol:hasIAFIdentifier ?gnd1 . + | ?gnd1 knora-api:valueAsString "(DE-588)118531379" . + | + |} ORDER BY ?date""".stripMargin + val queryToReorderWithMinus = """PREFIX knora-api: |PREFIX anything: @@ -1917,7 +1934,7 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec | FILTER knora-api:matchText(?text, "test") | | ?thing anything:hasInteger ?int . - | ?int knora-api:intValueAsInt 1 . + | ?int knora-api:intValueAsInt 3 . | } |} |ORDER BY (?int)""".stripMargin @@ -2109,15 +2126,15 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec case pattern: StatementPattern => pattern } assert(firstBlockStatements.head.subj.toSparql == "?int") + assert(firstBlockStatements.head.obj.asInstanceOf[XsdLiteral].value == "1") assert(firstBlockStatements.last.subj.toSparql == "?thing") - assert(firstBlockStatements.last.obj.toSparql == "?richtext") val secondBlock = unionPattern.head.blocks.last val secondBlockStatements = secondBlock.collect { case pattern: StatementPattern => pattern } assert(secondBlockStatements.head.subj.toSparql == "?int") + assert(secondBlockStatements.head.obj.asInstanceOf[XsdLiteral].value == "3") assert(secondBlockStatements.last.subj.toSparql == "?thing") - assert(secondBlockStatements.last.obj.toSparql == "?text") } "reorder query patterns in where clause with optional" in { @@ -2149,5 +2166,6 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec assert(minusPatternStatements.last.subj.toSparql == "?thing") assert(minusPatternStatements.last.obj.toSparql == "?intVal") } + } } From ce490e1ea47f1d4f8cf67557a9255abfed489fe2 Mon Sep 17 00:00:00 2001 From: Sepideh Alassi Date: Wed, 10 Feb 2021 18:07:46 +0100 Subject: [PATCH 12/33] feat(gravsearch): break cycles in graph --- ...cGravsearchToPrequeryTransformerSpec.scala | 105 +++++++++++------- 1 file changed, 66 insertions(+), 39 deletions(-) diff --git a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala index 1f9d5362a8..98c818d218 100644 --- a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala @@ -33,44 +33,69 @@ private object QueryHandler { val anythingUser: UserADM = SharedTestDataADM.anythingAdminUser case class edgeLabel(label: String) def createAndSortGraph(statementPatterns: Seq[StatementPattern]): Set[QueryPattern] = { + def makeGraphWithoutCycles(graphComponents: Seq[(String, String)]): Graph[String, DiHyperEdge] = { + val graph = graphComponents.foldLeft(Graph.empty[String, DiHyperEdge]) { (graph, edgeDef) => + val edge = DiHyperEdge(edgeDef._1, edgeDef._2) + graph + edge // add nodes and edges to graph + } + val acyclicGraph = if (graph.isCyclic) { + // get the cycle + val cycle: graph.Cycle = graph.findCycle.get + // the cyclic node is the one that cycle starts and ends with + val cyclicNode: graph.NodeT = cycle.endNode + val cyclicEdge: graph.EdgeT = cyclicNode.edges.last + val originNodeOfCyclicEdge: String = cyclicEdge._1.value + val TargetNodeOfCyclicEdge: String = cyclicEdge._2.value + val graphComponenetsWithOutCycle = + graphComponents.filterNot(edgeDef => edgeDef.equals((originNodeOfCyclicEdge, TargetNodeOfCyclicEdge))) + + makeGraphWithoutCycles(graphComponenetsWithOutCycle) + } else { + graph + } + acyclicGraph + } def createGraph: Graph[String, DiHyperEdge] = { - val graphComponents = statementPatterns.map { statementPattern => - // transform every statementPattern to LUniDiEdge(Subj,Obj)(edgeLabel(pred)) + val graphComponents: Seq[(String, String)] = statementPatterns.map { statementPattern => + // transform every statementPattern to pair of nodes that will consist an edge. val node1 = statementPattern.subj.toSparql val node2 = statementPattern.obj.toSparql -// val edge = statementPattern.pred.toSparql - DiHyperEdge(node1, node2) - } - val graph = graphComponents.foldLeft(Graph.empty[String, DiHyperEdge]) { (graph, edge) => - graph + edge // add nodes and edges to graph + (node1, node2) } - graph + makeGraphWithoutCycles(graphComponents) } def sortStatementPatterns(createdGraph: Graph[String, DiHyperEdge], statementPatterns: Seq[StatementPattern]): Set[QueryPattern] = { // Try topological sorting of graph - val topologicalOrder: createdGraph.TopologicalOrder[createdGraph.NodeT] = createdGraph.topologicalSort match { - // Is there a cycle in the graph? - case Left(cycleNode) => - // TODO: yes. break the cycle - throw new Error(s"Graph contains a cycle at node: ${cycleNode}, entire cycle is ${createdGraph.findCycle}.") - case Right(topOrder) => { - // No. return the topological order - println(topOrder) - topOrder + val topologicalOrderSeq: Seq[createdGraph.TopologicalOrder[createdGraph.NodeT]] = + createdGraph.topologicalSort match { + // Is there still a cycle in the graph? + case Left(cycleNode) => + // Don't try sorting, return statements as they are given. + Seq.empty[createdGraph.TopologicalOrder[createdGraph.NodeT]] + case Right(topOrder) => + // No. return the topological order + Seq(topOrder) } + + // Is there a topological order found? + val sortedPatterns = if (topologicalOrderSeq.nonEmpty) { + // Topological sort algorithm return only one perturbation; i.e. one topologicalOrder. + val topologicalOrder: Iterable[createdGraph.NodeT] = topologicalOrderSeq.head + // Start from the end of the ordered list (the nodes with lowest degree); + // for each node, find statements which have the node as object and bring them to top. + val sortedStatements: Set[QueryPattern] = + topologicalOrder.foldRight(ListSet.empty[QueryPattern]) { (node, sortedStatements) => + val statementsOfNode: Set[QueryPattern] = statementPatterns + .filter(p => p.obj.toSparql.equals(node.value)) + .toSet[QueryPattern] + sortedStatements ++ statementsOfNode + } + sortedStatements + } else { + statementPatterns.toSet[QueryPattern] } - // take the main resource out. - val topologicalOrderWithoutRoot: Iterable[createdGraph.NodeT] = topologicalOrder.tail - val sortedStatements: Set[QueryPattern] = - topologicalOrderWithoutRoot.foldRight(ListSet.empty[QueryPattern]) { (node, sortedStatements) => - val statementsOfNode: Set[QueryPattern] = statementPatterns - .filter(p => p.obj.toSparql.equals(node.value)) - .toSet[QueryPattern] - println(sortedStatements ++ statementsOfNode) - sortedStatements ++ statementsOfNode - } - sortedStatements + sortedPatterns } val createdGraph = createGraph @@ -1877,20 +1902,16 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec |} ORDER BY ?date""".stripMargin val queryToReorderWithCycle: String = """ - |PREFIX beol: + |PREFIX anything: |PREFIX knora-api: | |CONSTRUCT { - | ?letter knora-api:isMainResource true . - | ?letter ?linkingProp1 ?person1 . + | ?thing knora-api:isMainResource true . |} WHERE { - | ?letter beol:hasAuthor ?person1 . - | ?letter beol:letterHasTranslation ?letter2. - | ?letter2 beol:hasAuthor ?person1 . - | ?person1 beol:hasIAFIdentifier ?gnd1 . - | ?gnd1 knora-api:valueAsString "(DE-588)118531379" . - | - |} ORDER BY ?date""".stripMargin + | ?thing anything:hasOtherThing ?thing1 . + | ?thing1 anything:hasOtherThing ?thing2 . + | ?thing2 anything:hasOtherThing ?thing . + |} """.stripMargin val queryToReorderWithMinus = """PREFIX knora-api: @@ -2166,6 +2187,12 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec assert(minusPatternStatements.last.subj.toSparql == "?thing") assert(minusPatternStatements.last.obj.toSparql == "?intVal") } - + "reorder a query with a cycle" in { + val constructQuery = GravsearchParser.parseQuery(queryToReorderWithCycle) + val sortedPatterns = QueryHandler.reorderPatternsByDependency(constructQuery.whereClause.patterns) + // checks that all statement patterns which created a cycle are returned + val statements = sortedPatterns.filter(p => p.isInstanceOf[StatementPattern]) + assert(statements.size === 3) + } } } From 29aba9d59a70cc14e8278684ab1a5b8d4d3297c9 Mon Sep 17 00:00:00 2001 From: Sepideh Alassi Date: Thu, 11 Feb 2021 15:02:53 +0100 Subject: [PATCH 13/33] refactor (gravsearch) clean up --- ...cGravsearchToPrequeryTransformerSpec.scala | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala index 98c818d218..c0a8629d28 100644 --- a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala @@ -32,7 +32,7 @@ private object QueryHandler { val anythingUser: UserADM = SharedTestDataADM.anythingAdminUser case class edgeLabel(label: String) - def createAndSortGraph(statementPatterns: Seq[StatementPattern]): Set[QueryPattern] = { + def createAndSortGraph(statementPatterns: Seq[StatementPattern]): Seq[QueryPattern] = { def makeGraphWithoutCycles(graphComponents: Seq[(String, String)]): Graph[String, DiHyperEdge] = { val graph = graphComponents.foldLeft(Graph.empty[String, DiHyperEdge]) { (graph, edgeDef) => val edge = DiHyperEdge(edgeDef._1, edgeDef._2) @@ -65,7 +65,7 @@ private object QueryHandler { makeGraphWithoutCycles(graphComponents) } def sortStatementPatterns(createdGraph: Graph[String, DiHyperEdge], - statementPatterns: Seq[StatementPattern]): Set[QueryPattern] = { + statementPatterns: Seq[StatementPattern]): Seq[QueryPattern] = { // Try topological sorting of graph val topologicalOrderSeq: Seq[createdGraph.TopologicalOrder[createdGraph.NodeT]] = createdGraph.topologicalSort match { @@ -84,16 +84,16 @@ private object QueryHandler { val topologicalOrder: Iterable[createdGraph.NodeT] = topologicalOrderSeq.head // Start from the end of the ordered list (the nodes with lowest degree); // for each node, find statements which have the node as object and bring them to top. - val sortedStatements: Set[QueryPattern] = - topologicalOrder.foldRight(ListSet.empty[QueryPattern]) { (node, sortedStatements) => + val sortedStatements: Seq[QueryPattern] = + topologicalOrder.foldRight(Vector.empty[QueryPattern]) { (node, sortedStatements) => val statementsOfNode: Set[QueryPattern] = statementPatterns .filter(p => p.obj.toSparql.equals(node.value)) .toSet[QueryPattern] - sortedStatements ++ statementsOfNode + sortedStatements ++ statementsOfNode.toVector } sortedStatements } else { - statementPatterns.toSet[QueryPattern] + statementPatterns } sortedPatterns } @@ -102,12 +102,12 @@ private object QueryHandler { sortStatementPatterns(createdGraph, statementPatterns) } - def reorderPatternsByDependency(patterns: Seq[QueryPattern]): Set[QueryPattern] = { + def reorderPatternsByDependency(patterns: Seq[QueryPattern]): Seq[QueryPattern] = { val statementPatterns: Seq[StatementPattern] = patterns.collect { case pattern: StatementPattern => pattern } - val sortedStatementPatterns = createAndSortGraph(statementPatterns) - val otherPatterns: Set[QueryPattern] = patterns.filterNot(p => p.isInstanceOf[StatementPattern]).toSet - val sortedOtherPatterns: Set[QueryPattern] = otherPatterns.map { + val sortedStatementPatterns: Seq[QueryPattern] = createAndSortGraph(statementPatterns) + val otherPatterns: Seq[QueryPattern] = patterns.filterNot(p => p.isInstanceOf[StatementPattern]) + val sortedOtherPatterns: Seq[QueryPattern] = otherPatterns.map { // sort statements inside each UnionPattern block case unionPattern: UnionPattern => { val sortedUnionBlocks: Seq[Seq[QueryPattern]] = @@ -116,19 +116,19 @@ private object QueryHandler { } // sort statements inside OptionalPattern case optionalPattern: OptionalPattern => { - val sortedOptionalPatterns: Set[QueryPattern] = reorderPatternsByDependency(optionalPattern.patterns) + val sortedOptionalPatterns: Seq[QueryPattern] = reorderPatternsByDependency(optionalPattern.patterns) OptionalPattern(patterns = sortedOptionalPatterns.toSeq) } // sort statements inside MinusPattern case minusPattern: MinusPattern => { - val sortedMinusPatterns: Set[QueryPattern] = reorderPatternsByDependency(minusPattern.patterns) + val sortedMinusPatterns: Seq[QueryPattern] = reorderPatternsByDependency(minusPattern.patterns) MinusPattern(patterns = sortedMinusPatterns.toSeq) } // sort statements inside FilterNotExistsPattern case filterNotExistsPattern: FilterNotExistsPattern => { - val sortedFilterNotExistsPatterns: Set[QueryPattern] = + val sortedFilterNotExistsPatterns: Seq[QueryPattern] = reorderPatternsByDependency(filterNotExistsPattern.patterns) - FilterNotExistsPattern(patterns = sortedFilterNotExistsPatterns.toSeq) + FilterNotExistsPattern(patterns = sortedFilterNotExistsPatterns) } // return any other query pattern as it is case pattern: QueryPattern => pattern @@ -2141,7 +2141,7 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec val topElements = sortedPatterns.slice(0, statements.size) assert(topElements.equals(statements)) //check order of statements in each block - val unionPattern: Set[UnionPattern] = sortedPatterns.collect { case pattern: UnionPattern => pattern } + val unionPattern: Seq[UnionPattern] = sortedPatterns.collect { case pattern: UnionPattern => pattern } val firstBlock = unionPattern.head.blocks.head val firstBlockStatements = firstBlock.collect { case pattern: StatementPattern => pattern @@ -2166,7 +2166,7 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec val topElements = sortedPatterns.slice(0, statements.size) assert(topElements.equals(statements)) // check statements inside optional pattern - val optionalPattern: Set[OptionalPattern] = sortedPatterns.collect { case pattern: OptionalPattern => pattern } + val optionalPattern: Seq[OptionalPattern] = sortedPatterns.collect { case pattern: OptionalPattern => pattern } val optionalPatternStatements = optionalPattern.head.patterns.collect { case pattern: StatementPattern => pattern } @@ -2182,7 +2182,7 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec val topElements = sortedPatterns.slice(0, statements.size) assert(topElements.equals(statements)) // check statements inside minus pattern - val minusPattern: Set[MinusPattern] = sortedPatterns.collect { case pattern: MinusPattern => pattern } + val minusPattern: Seq[MinusPattern] = sortedPatterns.collect { case pattern: MinusPattern => pattern } val minusPatternStatements = minusPattern.head.patterns.collect { case pattern: StatementPattern => pattern } assert(minusPatternStatements.last.subj.toSparql == "?thing") assert(minusPatternStatements.last.obj.toSparql == "?intVal") From d0290a019770eebc32855f5bb71da01a706fbd29 Mon Sep 17 00:00:00 2001 From: Sepideh Alassi Date: Mon, 15 Feb 2021 10:53:36 +0100 Subject: [PATCH 14/33] move the topological sort to prequery generator --- .../prequery/AbstractPrequeryGenerator.scala | 108 ++++++++ ...GravsearchToCountPrequeryTransformer.scala | 3 +- ...cificGravsearchToPrequeryTransformer.scala | 3 +- ...cGravsearchToPrequeryTransformerSpec.scala | 256 +++++------------- 4 files changed, 185 insertions(+), 185 deletions(-) diff --git a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/AbstractPrequeryGenerator.scala b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/AbstractPrequeryGenerator.scala index 71f4c2f5df..bd229892f3 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/AbstractPrequeryGenerator.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/AbstractPrequeryGenerator.scala @@ -28,6 +28,8 @@ import org.knora.webapi.messages.util.search.gravsearch.types._ import org.knora.webapi.messages.v2.responder.valuemessages.DateValueContentV2 import org.knora.webapi.messages.{OntologyConstants, SmartIri, StringFormatter} import org.knora.webapi.util.ApacheLuceneSupport.LuceneQueryString +import scalax.collection.Graph +import scalax.collection.GraphEdge.DiHyperEdge import scala.collection.mutable @@ -2091,4 +2093,110 @@ abstract class AbstractPrequeryGenerator(constructClause: ConstructClause, } optimisedPatterns } + + private def createAndSortGraph(statementPatterns: Seq[StatementPattern]): Seq[QueryPattern] = { + def makeGraphWithoutCycles(graphComponents: Seq[(String, String)]): Graph[String, DiHyperEdge] = { + val graph = graphComponents.foldLeft(Graph.empty[String, DiHyperEdge]) { (graph, edgeDef) => + val edge = DiHyperEdge(edgeDef._1, edgeDef._2) + graph + edge // add nodes and edges to graph + } + val acyclicGraph = if (graph.isCyclic) { + // get the cycle + val cycle: graph.Cycle = graph.findCycle.get + // the cyclic node is the one that cycle starts and ends with + val cyclicNode: graph.NodeT = cycle.endNode + val cyclicEdge: graph.EdgeT = cyclicNode.edges.last + val originNodeOfCyclicEdge: String = cyclicEdge._1.value + val TargetNodeOfCyclicEdge: String = cyclicEdge._2.value + val graphComponenetsWithOutCycle = + graphComponents.filterNot(edgeDef => edgeDef.equals((originNodeOfCyclicEdge, TargetNodeOfCyclicEdge))) + + makeGraphWithoutCycles(graphComponenetsWithOutCycle) + } else { + graph + } + acyclicGraph + } + def createGraph: Graph[String, DiHyperEdge] = { + val graphComponents: Seq[(String, String)] = statementPatterns.map { statementPattern => + // transform every statementPattern to pair of nodes that will consist an edge. + val node1 = statementPattern.subj.toSparql + val node2 = statementPattern.obj.toSparql + (node1, node2) + } + makeGraphWithoutCycles(graphComponents) + } + def sortStatementPatterns(createdGraph: Graph[String, DiHyperEdge], + statementPatterns: Seq[StatementPattern]): Seq[QueryPattern] = { + // Try topological sorting of graph + val topologicalOrderSeq: Seq[createdGraph.TopologicalOrder[createdGraph.NodeT]] = + createdGraph.topologicalSort match { + // Is there still a cycle in the graph? + case Left(cycleNode) => + // Don't try sorting, return statements as they are given. + Seq.empty[createdGraph.TopologicalOrder[createdGraph.NodeT]] + case Right(topOrder) => + // No. return the topological order + Seq(topOrder) + } + + // Is there a topological order found? + val sortedPatterns = if (topologicalOrderSeq.nonEmpty) { + // Topological sort algorithm return only one perturbation; i.e. one topologicalOrder. + val topologicalOrder: Iterable[createdGraph.NodeT] = topologicalOrderSeq.head + // Start from the end of the ordered list (the nodes with lowest degree); + // for each node, find statements which have the node as object and bring them to top. + val sortedStatements: Seq[QueryPattern] = + topologicalOrder.foldRight(Vector.empty[QueryPattern]) { (node, sortedStatements) => + val statementsOfNode: Set[QueryPattern] = statementPatterns + .filter(p => p.obj.toSparql.equals(node.value)) + .toSet[QueryPattern] + sortedStatements ++ statementsOfNode.toVector + } + sortedStatements + } else { + statementPatterns + } + sortedPatterns + } + + val createdGraph = createGraph + sortStatementPatterns(createdGraph, statementPatterns) + + } + + protected def reorderPatternsByDependency(patterns: Seq[QueryPattern]): Seq[QueryPattern] = { + + val statementPatterns: Seq[StatementPattern] = patterns.collect { case pattern: StatementPattern => pattern } + val sortedStatementPatterns: Seq[QueryPattern] = createAndSortGraph(statementPatterns) + val otherPatterns: Seq[QueryPattern] = patterns.filterNot(p => p.isInstanceOf[StatementPattern]) + val sortedOtherPatterns: Seq[QueryPattern] = otherPatterns.map { + // sort statements inside each UnionPattern block + case unionPattern: UnionPattern => { + val sortedUnionBlocks: Seq[Seq[QueryPattern]] = + unionPattern.blocks.map(block => reorderPatternsByDependency(block).toSeq) + UnionPattern(blocks = sortedUnionBlocks) + } + // sort statements inside OptionalPattern + case optionalPattern: OptionalPattern => { + val sortedOptionalPatterns: Seq[QueryPattern] = reorderPatternsByDependency(optionalPattern.patterns) + OptionalPattern(patterns = sortedOptionalPatterns.toSeq) + } + // sort statements inside MinusPattern + case minusPattern: MinusPattern => { + val sortedMinusPatterns: Seq[QueryPattern] = reorderPatternsByDependency(minusPattern.patterns) + MinusPattern(patterns = sortedMinusPatterns.toSeq) + } + // sort statements inside FilterNotExistsPattern + case filterNotExistsPattern: FilterNotExistsPattern => { + val sortedFilterNotExistsPatterns: Seq[QueryPattern] = + reorderPatternsByDependency(filterNotExistsPattern.patterns) + FilterNotExistsPattern(patterns = sortedFilterNotExistsPatterns) + } + // return any other query pattern as it is + case pattern: QueryPattern => pattern + } + val sorted = sortedStatementPatterns ++ sortedOtherPatterns + sorted + } } diff --git a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToCountPrequeryTransformer.scala b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToCountPrequeryTransformer.scala index 202a3e6921..521a0c8fc9 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToCountPrequeryTransformer.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToCountPrequeryTransformer.scala @@ -87,7 +87,8 @@ class NonTriplestoreSpecificGravsearchToCountPrequeryTransformer(constructClause } override def optimiseQueryPatterns(patterns: Seq[QueryPattern]): Seq[QueryPattern] = { - removeEntitiesInferredFromProperty(patterns) + val patternsWithoutInferredEntities = removeEntitiesInferredFromProperty(patterns) + reorderPatternsByDependency(patternsWithoutInferredEntities) } override def transformLuceneQueryPattern(luceneQueryPattern: LuceneQueryPattern): Seq[QueryPattern] = diff --git a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformer.scala b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformer.scala index e1861c65ef..7fe440a624 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformer.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformer.scala @@ -408,6 +408,7 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformer(constructClause: Con * @return the optimised query patterns. */ override def optimiseQueryPatterns(patterns: Seq[QueryPattern]): Seq[QueryPattern] = { - removeEntitiesInferredFromProperty(patterns) + val patternsWithoutInferredEntities = removeEntitiesInferredFromProperty(patterns) + reorderPatternsByDependency(patternsWithoutInferredEntities) } } diff --git a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala index c0a8629d28..440bf8ed95 100644 --- a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala @@ -20,122 +20,12 @@ import org.knora.webapi.util.ApacheLuceneSupport.LuceneQueryString import scala.collection.mutable.ArrayBuffer import scala.concurrent.Await import scala.concurrent.duration._ -import scalax.collection.Graph -import scalax.collection.GraphEdge.DiHyperEdge - -import scala.collection.immutable.ListSet -import scala.util.Failure private object QueryHandler { private val timeout = 10.seconds val anythingUser: UserADM = SharedTestDataADM.anythingAdminUser - case class edgeLabel(label: String) - def createAndSortGraph(statementPatterns: Seq[StatementPattern]): Seq[QueryPattern] = { - def makeGraphWithoutCycles(graphComponents: Seq[(String, String)]): Graph[String, DiHyperEdge] = { - val graph = graphComponents.foldLeft(Graph.empty[String, DiHyperEdge]) { (graph, edgeDef) => - val edge = DiHyperEdge(edgeDef._1, edgeDef._2) - graph + edge // add nodes and edges to graph - } - val acyclicGraph = if (graph.isCyclic) { - // get the cycle - val cycle: graph.Cycle = graph.findCycle.get - // the cyclic node is the one that cycle starts and ends with - val cyclicNode: graph.NodeT = cycle.endNode - val cyclicEdge: graph.EdgeT = cyclicNode.edges.last - val originNodeOfCyclicEdge: String = cyclicEdge._1.value - val TargetNodeOfCyclicEdge: String = cyclicEdge._2.value - val graphComponenetsWithOutCycle = - graphComponents.filterNot(edgeDef => edgeDef.equals((originNodeOfCyclicEdge, TargetNodeOfCyclicEdge))) - - makeGraphWithoutCycles(graphComponenetsWithOutCycle) - } else { - graph - } - acyclicGraph - } - def createGraph: Graph[String, DiHyperEdge] = { - val graphComponents: Seq[(String, String)] = statementPatterns.map { statementPattern => - // transform every statementPattern to pair of nodes that will consist an edge. - val node1 = statementPattern.subj.toSparql - val node2 = statementPattern.obj.toSparql - (node1, node2) - } - makeGraphWithoutCycles(graphComponents) - } - def sortStatementPatterns(createdGraph: Graph[String, DiHyperEdge], - statementPatterns: Seq[StatementPattern]): Seq[QueryPattern] = { - // Try topological sorting of graph - val topologicalOrderSeq: Seq[createdGraph.TopologicalOrder[createdGraph.NodeT]] = - createdGraph.topologicalSort match { - // Is there still a cycle in the graph? - case Left(cycleNode) => - // Don't try sorting, return statements as they are given. - Seq.empty[createdGraph.TopologicalOrder[createdGraph.NodeT]] - case Right(topOrder) => - // No. return the topological order - Seq(topOrder) - } - - // Is there a topological order found? - val sortedPatterns = if (topologicalOrderSeq.nonEmpty) { - // Topological sort algorithm return only one perturbation; i.e. one topologicalOrder. - val topologicalOrder: Iterable[createdGraph.NodeT] = topologicalOrderSeq.head - // Start from the end of the ordered list (the nodes with lowest degree); - // for each node, find statements which have the node as object and bring them to top. - val sortedStatements: Seq[QueryPattern] = - topologicalOrder.foldRight(Vector.empty[QueryPattern]) { (node, sortedStatements) => - val statementsOfNode: Set[QueryPattern] = statementPatterns - .filter(p => p.obj.toSparql.equals(node.value)) - .toSet[QueryPattern] - sortedStatements ++ statementsOfNode.toVector - } - sortedStatements - } else { - statementPatterns - } - sortedPatterns - } - - val createdGraph = createGraph - sortStatementPatterns(createdGraph, statementPatterns) - - } - def reorderPatternsByDependency(patterns: Seq[QueryPattern]): Seq[QueryPattern] = { - - val statementPatterns: Seq[StatementPattern] = patterns.collect { case pattern: StatementPattern => pattern } - val sortedStatementPatterns: Seq[QueryPattern] = createAndSortGraph(statementPatterns) - val otherPatterns: Seq[QueryPattern] = patterns.filterNot(p => p.isInstanceOf[StatementPattern]) - val sortedOtherPatterns: Seq[QueryPattern] = otherPatterns.map { - // sort statements inside each UnionPattern block - case unionPattern: UnionPattern => { - val sortedUnionBlocks: Seq[Seq[QueryPattern]] = - unionPattern.blocks.map(block => reorderPatternsByDependency(block).toSeq) - UnionPattern(blocks = sortedUnionBlocks) - } - // sort statements inside OptionalPattern - case optionalPattern: OptionalPattern => { - val sortedOptionalPatterns: Seq[QueryPattern] = reorderPatternsByDependency(optionalPattern.patterns) - OptionalPattern(patterns = sortedOptionalPatterns.toSeq) - } - // sort statements inside MinusPattern - case minusPattern: MinusPattern => { - val sortedMinusPatterns: Seq[QueryPattern] = reorderPatternsByDependency(minusPattern.patterns) - MinusPattern(patterns = sortedMinusPatterns.toSeq) - } - // sort statements inside FilterNotExistsPattern - case filterNotExistsPattern: FilterNotExistsPattern => { - val sortedFilterNotExistsPatterns: Seq[QueryPattern] = - reorderPatternsByDependency(filterNotExistsPattern.patterns) - FilterNotExistsPattern(patterns = sortedFilterNotExistsPatterns) - } - // return any other query pattern as it is - case pattern: QueryPattern => pattern - } - val sorted = sortedStatementPatterns ++ sortedOtherPatterns - sorted - } def transformQuery(query: String, responderData: ResponderData, settings: KnoraSettingsImpl): SelectQuery = { @@ -2121,78 +2011,78 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec assert(transformedQuery === TransformedQueryWithUnionScopes) } - - "reorder query patterns in where clause" in { - val constructQuery = GravsearchParser.parseQuery(queryToReorder) - val sortedPatterns = QueryHandler.reorderPatternsByDependency(constructQuery.whereClause.patterns) - val statements = sortedPatterns.filter(p => p.isInstanceOf[StatementPattern]) - // check that statements are brought to top - val topElements = sortedPatterns.slice(0, statements.size) - assert(topElements.equals(statements)) - // check the order of statements - val mainResourceStatements = statements.slice(statements.size - 3, statements.size) - assert(!mainResourceStatements.exists(p => p.asInstanceOf[StatementPattern].subj.toSparql !== "?letter")) - } - "reorder query patterns in where clause with union" in { - val constructQuery = GravsearchParser.parseQuery(queryToReorderWithUnion) - val sortedPatterns = QueryHandler.reorderPatternsByDependency(constructQuery.whereClause.patterns) - val statements = sortedPatterns.filter(p => p.isInstanceOf[StatementPattern]) - // check that statements are brought to top - val topElements = sortedPatterns.slice(0, statements.size) - assert(topElements.equals(statements)) - //check order of statements in each block - val unionPattern: Seq[UnionPattern] = sortedPatterns.collect { case pattern: UnionPattern => pattern } - val firstBlock = unionPattern.head.blocks.head - val firstBlockStatements = firstBlock.collect { - case pattern: StatementPattern => pattern - } - assert(firstBlockStatements.head.subj.toSparql == "?int") - assert(firstBlockStatements.head.obj.asInstanceOf[XsdLiteral].value == "1") - assert(firstBlockStatements.last.subj.toSparql == "?thing") - val secondBlock = unionPattern.head.blocks.last - val secondBlockStatements = secondBlock.collect { - case pattern: StatementPattern => pattern - } - assert(secondBlockStatements.head.subj.toSparql == "?int") - assert(secondBlockStatements.head.obj.asInstanceOf[XsdLiteral].value == "3") - assert(secondBlockStatements.last.subj.toSparql == "?thing") - } - - "reorder query patterns in where clause with optional" in { - val constructQuery = GravsearchParser.parseQuery(queryWithOptional) - val sortedPatterns = QueryHandler.reorderPatternsByDependency(constructQuery.whereClause.patterns) - val statements = sortedPatterns.filter(p => p.isInstanceOf[StatementPattern]) - // check that statements are brought to top - val topElements = sortedPatterns.slice(0, statements.size) - assert(topElements.equals(statements)) - // check statements inside optional pattern - val optionalPattern: Seq[OptionalPattern] = sortedPatterns.collect { case pattern: OptionalPattern => pattern } - val optionalPatternStatements = optionalPattern.head.patterns.collect { - case pattern: StatementPattern => pattern - } - assert(optionalPatternStatements.last.subj.toSparql == "?document") - assert(optionalPatternStatements.last.obj.toSparql == "?recipient") - } - - "reorder query patterns with minus scope" in { - val constructQuery = GravsearchParser.parseQuery(queryToReorderWithMinus) - val sortedPatterns = QueryHandler.reorderPatternsByDependency(constructQuery.whereClause.patterns) - val statements = sortedPatterns.filter(p => p.isInstanceOf[StatementPattern]) - // check that statements are brought to top - val topElements = sortedPatterns.slice(0, statements.size) - assert(topElements.equals(statements)) - // check statements inside minus pattern - val minusPattern: Seq[MinusPattern] = sortedPatterns.collect { case pattern: MinusPattern => pattern } - val minusPatternStatements = minusPattern.head.patterns.collect { case pattern: StatementPattern => pattern } - assert(minusPatternStatements.last.subj.toSparql == "?thing") - assert(minusPatternStatements.last.obj.toSparql == "?intVal") - } - "reorder a query with a cycle" in { - val constructQuery = GravsearchParser.parseQuery(queryToReorderWithCycle) - val sortedPatterns = QueryHandler.reorderPatternsByDependency(constructQuery.whereClause.patterns) - // checks that all statement patterns which created a cycle are returned - val statements = sortedPatterns.filter(p => p.isInstanceOf[StatementPattern]) - assert(statements.size === 3) - } +// +// "reorder query patterns in where clause" in { +// val constructQuery = GravsearchParser.parseQuery(queryToReorder) +// val sortedPatterns = QueryHandler.reorderPatternsByDependency(constructQuery.whereClause.patterns) +// val statements = sortedPatterns.filter(p => p.isInstanceOf[StatementPattern]) +// // check that statements are brought to top +// val topElements = sortedPatterns.slice(0, statements.size) +// assert(topElements.equals(statements)) +// // check the order of statements +// val mainResourceStatements = statements.slice(statements.size - 3, statements.size) +// assert(!mainResourceStatements.exists(p => p.asInstanceOf[StatementPattern].subj.toSparql !== "?letter")) +// } +// "reorder query patterns in where clause with union" in { +// val constructQuery = GravsearchParser.parseQuery(queryToReorderWithUnion) +// val sortedPatterns = QueryHandler.reorderPatternsByDependency(constructQuery.whereClause.patterns) +// val statements = sortedPatterns.filter(p => p.isInstanceOf[StatementPattern]) +// // check that statements are brought to top +// val topElements = sortedPatterns.slice(0, statements.size) +// assert(topElements.equals(statements)) +// //check order of statements in each block +// val unionPattern: Seq[UnionPattern] = sortedPatterns.collect { case pattern: UnionPattern => pattern } +// val firstBlock = unionPattern.head.blocks.head +// val firstBlockStatements = firstBlock.collect { +// case pattern: StatementPattern => pattern +// } +// assert(firstBlockStatements.head.subj.toSparql == "?int") +// assert(firstBlockStatements.head.obj.asInstanceOf[XsdLiteral].value == "1") +// assert(firstBlockStatements.last.subj.toSparql == "?thing") +// val secondBlock = unionPattern.head.blocks.last +// val secondBlockStatements = secondBlock.collect { +// case pattern: StatementPattern => pattern +// } +// assert(secondBlockStatements.head.subj.toSparql == "?int") +// assert(secondBlockStatements.head.obj.asInstanceOf[XsdLiteral].value == "3") +// assert(secondBlockStatements.last.subj.toSparql == "?thing") +// } +// +// "reorder query patterns in where clause with optional" in { +// val constructQuery = GravsearchParser.parseQuery(queryWithOptional) +// val sortedPatterns = QueryHandler.reorderPatternsByDependency(constructQuery.whereClause.patterns) +// val statements = sortedPatterns.filter(p => p.isInstanceOf[StatementPattern]) +// // check that statements are brought to top +// val topElements = sortedPatterns.slice(0, statements.size) +// assert(topElements.equals(statements)) +// // check statements inside optional pattern +// val optionalPattern: Seq[OptionalPattern] = sortedPatterns.collect { case pattern: OptionalPattern => pattern } +// val optionalPatternStatements = optionalPattern.head.patterns.collect { +// case pattern: StatementPattern => pattern +// } +// assert(optionalPatternStatements.last.subj.toSparql == "?document") +// assert(optionalPatternStatements.last.obj.toSparql == "?recipient") +// } +// +// "reorder query patterns with minus scope" in { +// val constructQuery = GravsearchParser.parseQuery(queryToReorderWithMinus) +// val sortedPatterns = QueryHandler.reorderPatternsByDependency(constructQuery.whereClause.patterns) +// val statements = sortedPatterns.filter(p => p.isInstanceOf[StatementPattern]) +// // check that statements are brought to top +// val topElements = sortedPatterns.slice(0, statements.size) +// assert(topElements.equals(statements)) +// // check statements inside minus pattern +// val minusPattern: Seq[MinusPattern] = sortedPatterns.collect { case pattern: MinusPattern => pattern } +// val minusPatternStatements = minusPattern.head.patterns.collect { case pattern: StatementPattern => pattern } +// assert(minusPatternStatements.last.subj.toSparql == "?thing") +// assert(minusPatternStatements.last.obj.toSparql == "?intVal") +// } +// "reorder a query with a cycle" in { +// val constructQuery = GravsearchParser.parseQuery(queryToReorderWithCycle) +// val sortedPatterns = QueryHandler.reorderPatternsByDependency(constructQuery.whereClause.patterns) +// // checks that all statement patterns which created a cycle are returned +// val statements = sortedPatterns.filter(p => p.isInstanceOf[StatementPattern]) +// assert(statements.size === 3) +// } } } From 4eca3b577593ac22e08f3d94b13b3ef390cc5605 Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Mon, 15 Feb 2021 16:04:53 +0100 Subject: [PATCH 15/33] style(gravsearch): Clean up a few things. --- .../prequery/AbstractPrequeryGenerator.scala | 94 ++++---- ...cGravsearchToPrequeryTransformerSpec.scala | 204 ++++++++++++------ 2 files changed, 196 insertions(+), 102 deletions(-) diff --git a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/AbstractPrequeryGenerator.scala b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/AbstractPrequeryGenerator.scala index bd229892f3..031908962f 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/AbstractPrequeryGenerator.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/AbstractPrequeryGenerator.scala @@ -2057,10 +2057,9 @@ abstract class AbstractPrequeryGenerator(constructClause: ConstructClause, protected def removeEntitiesInferredFromProperty(patterns: Seq[QueryPattern]): Seq[QueryPattern] = { // Collect all entities which are used as subject or object of an OptionalPattern. - val optionalEntities = patterns - .filter { - case OptionalPattern(_) => true - case _ => false + val optionalEntities: Seq[TypeableEntity] = patterns + .collect { + case optionalPattern: OptionalPattern => optionalPattern } .flatMap { case optionalPattern: OptionalPattern => @@ -2068,13 +2067,15 @@ abstract class AbstractPrequeryGenerator(constructClause: ConstructClause, case pattern: StatementPattern => GravsearchTypeInspectionUtil.maybeTypeableEntity(pattern.subj) ++ GravsearchTypeInspectionUtil .maybeTypeableEntity(pattern.obj) + case _ => None } + case _ => None } // remove statements whose predicate is rdf:type, type of subject is inferred from a property, and the subject is not in optionalEntities. - val optimisedPatterns = patterns.filter { + patterns.filter { case statementPattern: StatementPattern => statementPattern.pred match { case iriRef: IriRef => @@ -2091,32 +2092,34 @@ abstract class AbstractPrequeryGenerator(constructClause: ConstructClause, } case _ => true } - optimisedPatterns } private def createAndSortGraph(statementPatterns: Seq[StatementPattern]): Seq[QueryPattern] = { + @scala.annotation.tailrec def makeGraphWithoutCycles(graphComponents: Seq[(String, String)]): Graph[String, DiHyperEdge] = { val graph = graphComponents.foldLeft(Graph.empty[String, DiHyperEdge]) { (graph, edgeDef) => val edge = DiHyperEdge(edgeDef._1, edgeDef._2) graph + edge // add nodes and edges to graph } - val acyclicGraph = if (graph.isCyclic) { + + if (graph.isCyclic) { // get the cycle val cycle: graph.Cycle = graph.findCycle.get + // the cyclic node is the one that cycle starts and ends with val cyclicNode: graph.NodeT = cycle.endNode val cyclicEdge: graph.EdgeT = cyclicNode.edges.last val originNodeOfCyclicEdge: String = cyclicEdge._1.value val TargetNodeOfCyclicEdge: String = cyclicEdge._2.value - val graphComponenetsWithOutCycle = + val graphComponentsWithOutCycle = graphComponents.filterNot(edgeDef => edgeDef.equals((originNodeOfCyclicEdge, TargetNodeOfCyclicEdge))) - makeGraphWithoutCycles(graphComponenetsWithOutCycle) + makeGraphWithoutCycles(graphComponentsWithOutCycle) } else { graph } - acyclicGraph } + def createGraph: Graph[String, DiHyperEdge] = { val graphComponents: Seq[(String, String)] = statementPatterns.map { statementPattern => // transform every statementPattern to pair of nodes that will consist an edge. @@ -2124,8 +2127,10 @@ abstract class AbstractPrequeryGenerator(constructClause: ConstructClause, val node2 = statementPattern.obj.toSparql (node1, node2) } + makeGraphWithoutCycles(graphComponents) } + def sortStatementPatterns(createdGraph: Graph[String, DiHyperEdge], statementPatterns: Seq[StatementPattern]): Seq[QueryPattern] = { // Try topological sorting of graph @@ -2141,62 +2146,73 @@ abstract class AbstractPrequeryGenerator(constructClause: ConstructClause, } // Is there a topological order found? - val sortedPatterns = if (topologicalOrderSeq.nonEmpty) { - // Topological sort algorithm return only one perturbation; i.e. one topologicalOrder. + if (topologicalOrderSeq.nonEmpty) { + // Yes. Topological sort algorithm return only one perturbation; i.e. one topologicalOrder. val topologicalOrder: Iterable[createdGraph.NodeT] = topologicalOrderSeq.head + // Start from the end of the ordered list (the nodes with lowest degree); // for each node, find statements which have the node as object and bring them to top. - val sortedStatements: Seq[QueryPattern] = - topologicalOrder.foldRight(Vector.empty[QueryPattern]) { (node, sortedStatements) => - val statementsOfNode: Set[QueryPattern] = statementPatterns - .filter(p => p.obj.toSparql.equals(node.value)) - .toSet[QueryPattern] - sortedStatements ++ statementsOfNode.toVector - } - sortedStatements + topologicalOrder.foldRight(Vector.empty[QueryPattern]) { (node, sortedStatements) => + val statementsOfNode: Set[QueryPattern] = statementPatterns + .filter(p => p.obj.toSparql.equals(node.value)) + .toSet[QueryPattern] + sortedStatements ++ statementsOfNode.toVector + } } else { + // No topological order found. statementPatterns } - sortedPatterns } - val createdGraph = createGraph - sortStatementPatterns(createdGraph, statementPatterns) - + sortStatementPatterns(createGraph, statementPatterns) } + /** + * Optimises query patterns by reordering them on the basis of dependencies between subjects and objects. + * + * @param patterns the patterns to be optimised. + * @return the optimised patterns. + */ protected def reorderPatternsByDependency(patterns: Seq[QueryPattern]): Seq[QueryPattern] = { + // Separate the statement patterns from the other patterns. + val (statementPatterns: Seq[StatementPattern], otherPatterns: Seq[QueryPattern]) = + patterns.foldLeft((Vector.empty[StatementPattern], Vector.empty[QueryPattern])) { + case ((statementPatternAcc, otherPatternAcc), pattern: QueryPattern) => + pattern match { + case statementPattern: StatementPattern => (statementPatternAcc :+ statementPattern, otherPatternAcc) + case _ => (statementPatternAcc, otherPatternAcc :+ pattern) + } + } - val statementPatterns: Seq[StatementPattern] = patterns.collect { case pattern: StatementPattern => pattern } val sortedStatementPatterns: Seq[QueryPattern] = createAndSortGraph(statementPatterns) - val otherPatterns: Seq[QueryPattern] = patterns.filterNot(p => p.isInstanceOf[StatementPattern]) + val sortedOtherPatterns: Seq[QueryPattern] = otherPatterns.map { // sort statements inside each UnionPattern block - case unionPattern: UnionPattern => { + case unionPattern: UnionPattern => val sortedUnionBlocks: Seq[Seq[QueryPattern]] = - unionPattern.blocks.map(block => reorderPatternsByDependency(block).toSeq) + unionPattern.blocks.map(block => reorderPatternsByDependency(block)) UnionPattern(blocks = sortedUnionBlocks) - } + // sort statements inside OptionalPattern - case optionalPattern: OptionalPattern => { + case optionalPattern: OptionalPattern => val sortedOptionalPatterns: Seq[QueryPattern] = reorderPatternsByDependency(optionalPattern.patterns) - OptionalPattern(patterns = sortedOptionalPatterns.toSeq) - } + OptionalPattern(patterns = sortedOptionalPatterns) + // sort statements inside MinusPattern - case minusPattern: MinusPattern => { + case minusPattern: MinusPattern => val sortedMinusPatterns: Seq[QueryPattern] = reorderPatternsByDependency(minusPattern.patterns) - MinusPattern(patterns = sortedMinusPatterns.toSeq) - } + MinusPattern(patterns = sortedMinusPatterns) + // sort statements inside FilterNotExistsPattern - case filterNotExistsPattern: FilterNotExistsPattern => { + case filterNotExistsPattern: FilterNotExistsPattern => val sortedFilterNotExistsPatterns: Seq[QueryPattern] = reorderPatternsByDependency(filterNotExistsPattern.patterns) FilterNotExistsPattern(patterns = sortedFilterNotExistsPatterns) - } + // return any other query pattern as it is case pattern: QueryPattern => pattern } - val sorted = sortedStatementPatterns ++ sortedOtherPatterns - sorted + + sortedStatementPatterns ++ sortedOtherPatterns } } diff --git a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala index 440bf8ed95..4afcc27a84 100644 --- a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala @@ -5,7 +5,7 @@ import org.knora.webapi.exceptions.AssertionException import org.knora.webapi.messages.IriConversions._ import org.knora.webapi.messages.StringFormatter import org.knora.webapi.messages.admin.responder.usersmessages.UserADM -import org.knora.webapi.messages.util.ResponderData +import org.knora.webapi.messages.util.{MessageUtil, ResponderData} import org.knora.webapi.messages.util.search._ import org.knora.webapi.messages.util.search.gravsearch.prequery.NonTriplestoreSpecificGravsearchToPrequeryTransformer import org.knora.webapi.messages.util.search.gravsearch.types.{ @@ -967,12 +967,13 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec val transformedQueryWithDecimalOptionalSortCriterionAndFilterComplex: SelectQuery = SelectQuery( + fromClause = None, variables = Vector( QueryVariable(variableName = "thing"), GroupConcat( inputVariable = QueryVariable(variableName = "decimal"), separator = StringFormatter.INFORMATION_SEPARATOR_ONE, - outputVariableName = "decimal__Concat", + outputVariableName = "decimal__Concat" ) ), offset = 0, @@ -991,7 +992,7 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec ) ), whereClause = WhereClause( - patterns = ArrayBuffer( + patterns = Vector( StatementPattern( subj = QueryVariable(variableName = "thing"), pred = IriRef( @@ -1022,6 +1023,15 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec ), OptionalPattern( patterns = Vector( + StatementPattern( + subj = QueryVariable(variableName = "decimal"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#valueHasDecimal".toSmartIri, + propertyPathOperator = None + ), + obj = QueryVariable(variableName = "decimalVal"), + namedGraph = None + ), StatementPattern( subj = QueryVariable(variableName = "thing"), pred = IriRef( @@ -1060,15 +1070,6 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec propertyPathOperator = None )) ), - StatementPattern( - subj = QueryVariable(variableName = "decimal"), - pred = IriRef( - iri = "http://www.knora.org/ontology/knora-base#valueHasDecimal".toSmartIri, - propertyPathOperator = None - ), - obj = QueryVariable(variableName = "decimalVal"), - namedGraph = None - ), FilterPattern(expression = CompareExpression( leftArg = QueryVariable(variableName = "decimalVal"), operator = CompareExpressionOperator.GREATER_THAN, @@ -1114,7 +1115,8 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec |} """.stripMargin - val TransformedQueryWithRdfsLabelAndLiteral: SelectQuery = SelectQuery( + val TransformedQueryWithRdfsLabelAndLiteralVersion1: SelectQuery = SelectQuery( + fromClause = None, variables = Vector(QueryVariable(variableName = "book")), offset = 0, groupBy = Vector(QueryVariable(variableName = "book")), @@ -1124,7 +1126,7 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec isAscending = true )), whereClause = WhereClause( - patterns = ArrayBuffer( + patterns = Vector( StatementPattern( subj = QueryVariable(variableName = "book"), pred = IriRef( @@ -1173,6 +1175,66 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec useDistinct = true ) + val TransformedQueryWithRdfsLabelAndLiteralVersion2: SelectQuery = SelectQuery( + fromClause = None, + variables = Vector(QueryVariable(variableName = "book")), + offset = 0, + groupBy = Vector(QueryVariable(variableName = "book")), + orderBy = Vector( + OrderCriterion( + queryVariable = QueryVariable(variableName = "book"), + isAscending = true + )), + whereClause = WhereClause( + patterns = Vector( + StatementPattern( + subj = QueryVariable(variableName = "book"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "false", + datatype = "http://www.w3.org/2001/XMLSchema#boolean".toSmartIri + ), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = QueryVariable(variableName = "book"), + pred = IriRef( + iri = "http://www.w3.org/2000/01/rdf-schema#label".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "Zeitgl\u00F6cklein des Lebens und Leidens Christi", + datatype = "http://www.w3.org/2001/XMLSchema#string".toSmartIri + ), + namedGraph = None + ), + StatementPattern( + subj = QueryVariable(variableName = "book"), + pred = IriRef( + iri = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type".toSmartIri, + propertyPathOperator = None + ), + obj = IriRef( + iri = "http://www.knora.org/ontology/0803/incunabula#book".toSmartIri, + propertyPathOperator = None + ), + namedGraph = None + ) + ), + positiveEntities = Set(), + querySchema = None + ), + limit = Some(25), + useDistinct = true + ) + val InputQueryWithRdfsLabelAndVariableInSimpleSchema: String = """ |PREFIX incunabula: @@ -1232,6 +1294,7 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec |}""".stripMargin val TransformedQueryWithRdfsLabelAndVariable: SelectQuery = SelectQuery( + fromClause = None, variables = Vector(QueryVariable(variableName = "book")), offset = 0, groupBy = Vector(QueryVariable(variableName = "book")), @@ -1241,7 +1304,7 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec isAscending = true )), whereClause = WhereClause( - patterns = ArrayBuffer( + patterns = Vector( StatementPattern( subj = QueryVariable(variableName = "book"), pred = IriRef( @@ -1297,6 +1360,7 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec ) val TransformedQueryWithRdfsLabelAndRegex: SelectQuery = SelectQuery( + fromClause = None, variables = Vector(QueryVariable(variableName = "book")), offset = 0, groupBy = Vector(QueryVariable(variableName = "book")), @@ -1306,7 +1370,7 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec isAscending = true )), whereClause = WhereClause( - patterns = ArrayBuffer( + patterns = Vector( StatementPattern( subj = QueryVariable(variableName = "book"), pred = IriRef( @@ -1424,6 +1488,47 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec ), OptionalPattern( patterns = Vector( + StatementPattern( + subj = QueryVariable(variableName = "recipient"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "false", + datatype = "http://www.w3.org/2001/XMLSchema#boolean".toSmartIri + ), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = QueryVariable(variableName = "recipient"), + pred = IriRef( + iri = "http://www.knora.org/ontology/0801/beol#hasFamilyName".toSmartIri, + propertyPathOperator = None + ), + obj = QueryVariable(variableName = "familyName"), + namedGraph = None + ), + StatementPattern( + subj = QueryVariable(variableName = "familyName"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "false", + datatype = "http://www.w3.org/2001/XMLSchema#boolean".toSmartIri + ), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), StatementPattern( subj = QueryVariable(variableName = "document"), pred = IriRef( @@ -1491,47 +1596,6 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec propertyPathOperator = None )) ), - StatementPattern( - subj = QueryVariable(variableName = "recipient"), - pred = IriRef( - iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, - propertyPathOperator = None - ), - obj = XsdLiteral( - value = "false", - datatype = "http://www.w3.org/2001/XMLSchema#boolean".toSmartIri - ), - namedGraph = Some( - IriRef( - iri = "http://www.knora.org/explicit".toSmartIri, - propertyPathOperator = None - )) - ), - StatementPattern( - subj = QueryVariable(variableName = "recipient"), - pred = IriRef( - iri = "http://www.knora.org/ontology/0801/beol#hasFamilyName".toSmartIri, - propertyPathOperator = None - ), - obj = QueryVariable(variableName = "familyName"), - namedGraph = None - ), - StatementPattern( - subj = QueryVariable(variableName = "familyName"), - pred = IriRef( - iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, - propertyPathOperator = None - ), - obj = XsdLiteral( - value = "false", - datatype = "http://www.w3.org/2001/XMLSchema#boolean".toSmartIri - ), - namedGraph = Some( - IriRef( - iri = "http://www.knora.org/explicit".toSmartIri, - propertyPathOperator = None - )) - ), LuceneQueryPattern( subj = QueryVariable(variableName = "familyName"), obj = QueryVariable(variableName = "familyName__valueHasString"), @@ -1803,7 +1867,7 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec | ?thing2 anything:hasOtherThing ?thing . |} """.stripMargin - val queryToReorderWithMinus = + val queryToReorderWithMinus: String = """PREFIX knora-api: |PREFIX anything: | @@ -1964,12 +2028,26 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec } "transform an input query using rdfs:label and a literal in the simple schema" in { - val transformedQuery = - QueryHandler.transformQuery(InputQueryWithRdfsLabelAndLiteralInSimpleSchema, responderData, settings) - - assert(transformedQuery === TransformedQueryWithRdfsLabelAndLiteral) + val result = new ArrayBuffer[String] + + for (i <- 1 to 20) { + val transformedQuery = + QueryHandler.transformQuery(InputQueryWithRdfsLabelAndLiteralInSimpleSchema, responderData, settings) + + if (transformedQuery === TransformedQueryWithRdfsLabelAndLiteralVersion1) { + result += s"$i. transformedQuery == TransformedQueryWithRdfsLabelAndLiteralVersion1" + } else if (transformedQuery === TransformedQueryWithRdfsLabelAndLiteralVersion2) { + result += s"$i. transformedQuery == TransformedQueryWithRdfsLabelAndLiteralVersion2" + } else { + throw new Exception( + "transformedQuery is different from both TransformedQueryWithRdfsLabelAndLiteralVersion1 and TransformedQueryWithRdfsLabelAndLiteralVersion2") + } + } + + throw new Exception(result.mkString("\n")) } + /* "transform an input query using rdfs:label and a literal in the complex schema" in { val transformedQuery = QueryHandler.transformQuery(InputQueryWithRdfsLabelAndLiteralInComplexSchema, responderData, settings) @@ -2010,7 +2088,7 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec QueryHandler.transformQuery(InputQueryWithUnionScopes, responderData, settings) assert(transformedQuery === TransformedQueryWithUnionScopes) - } + } */ // // "reorder query patterns in where clause" in { // val constructQuery = GravsearchParser.parseQuery(queryToReorder) From cfbc7508ae4ae090dd179efc9d43540dea0e5971 Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Mon, 15 Feb 2021 16:11:31 +0100 Subject: [PATCH 16/33] test(gravsearch): Improve test. --- ...toreSpecificGravsearchToPrequeryTransformerSpec.scala | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala index 4afcc27a84..a7019c17e8 100644 --- a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala @@ -2030,13 +2030,18 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec "transform an input query using rdfs:label and a literal in the simple schema" in { val result = new ArrayBuffer[String] + println("=========== Version 1 ============") + println(TransformedQueryWithRdfsLabelAndLiteralVersion1.toSparql) + println("=========== Version 2 ============") + println(TransformedQueryWithRdfsLabelAndLiteralVersion2.toSparql) + for (i <- 1 to 20) { val transformedQuery = QueryHandler.transformQuery(InputQueryWithRdfsLabelAndLiteralInSimpleSchema, responderData, settings) - if (transformedQuery === TransformedQueryWithRdfsLabelAndLiteralVersion1) { + if (transformedQuery == TransformedQueryWithRdfsLabelAndLiteralVersion1) { result += s"$i. transformedQuery == TransformedQueryWithRdfsLabelAndLiteralVersion1" - } else if (transformedQuery === TransformedQueryWithRdfsLabelAndLiteralVersion2) { + } else if (transformedQuery == TransformedQueryWithRdfsLabelAndLiteralVersion2) { result += s"$i. transformedQuery == TransformedQueryWithRdfsLabelAndLiteralVersion2" } else { throw new Exception( From cde90b6a5c8b5d16e8f08c2e4412e599451f5f6b Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Tue, 16 Feb 2021 10:57:24 +0100 Subject: [PATCH 17/33] feat(gravsearch): Add utility for finding all topological orders of a graph. --- .../prequery/AbstractPrequeryGenerator.scala | 4 + .../prequery/TopologicalSortUtil.scala | 98 +++++++++++++++++++ .../webapi/messages/util/search/BUILD.bazel | 18 ++++ .../prequery/TopologicalSortUtilSpec.scala | 30 ++++++ 4 files changed, 150 insertions(+) create mode 100644 webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtil.scala create mode 100644 webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtilSpec.scala diff --git a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/AbstractPrequeryGenerator.scala b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/AbstractPrequeryGenerator.scala index 031908962f..9ec6948822 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/AbstractPrequeryGenerator.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/AbstractPrequeryGenerator.scala @@ -2134,6 +2134,10 @@ abstract class AbstractPrequeryGenerator(constructClause: ConstructClause, def sortStatementPatterns(createdGraph: Graph[String, DiHyperEdge], statementPatterns: Seq[StatementPattern]): Seq[QueryPattern] = { // Try topological sorting of graph + + // TODO: get all valid orders using TopologicalSortUtil, get the lowest ordered node. + // From statements, find the statements whose object is this node. + // check the predicate of these statements, choose the order with the node that is object of a statement whose predicate is not rdf:type val topologicalOrderSeq: Seq[createdGraph.TopologicalOrder[createdGraph.NodeT]] = createdGraph.topologicalSort match { // Is there still a cycle in the graph? diff --git a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtil.scala b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtil.scala new file mode 100644 index 0000000000..5a0435764a --- /dev/null +++ b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtil.scala @@ -0,0 +1,98 @@ +/* + * Copyright © 2015-2018 the contributors (see Contributors.md). + * + * This file is part of Knora. + * + * Knora is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Knora is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public + * License along with Knora. If not, see . + */ + +package org.knora.webapi.messages.util.search.gravsearch.prequery + +import org.knora.webapi.exceptions.AssertionException +import scalax.collection.Graph +import scalax.collection.GraphEdge.DiHyperEdge + +import scala.collection.mutable + +/** + * A utility for finding all topological orders of a graph. + * Based on [[https://github.com/scala-graph/scala-graph/issues/129#issuecomment-485398400]]. + */ +object TopologicalSortUtil { + + /** + * Finds all possible topological orders of a graph. + * + * @param graph the graph to be sorted. + * @tparam T the type of the nodes in the graph. + */ + def allTopologicalOrders[T](graph: Graph[T, DiHyperEdge]): Set[List[graph.NodeT]] = { + + /** + * Represents arguments to be put on the simulated call stack. + */ + case class StackItem(sources: Set[graph.NodeT], + inDegrees: Map[graph.NodeT, Int], + topOrder: List[graph.NodeT], + count: Int) + + val inDegree: Map[graph.NodeT, Int] = graph.nodes.map(node => (node, node.inDegree)).toMap + + def isSource(node: graph.NodeT): Boolean = inDegree(node) == 0 + def getSources: Set[graph.NodeT] = graph.nodes.filter(node => isSource(node)).toSet + + // Replaces the program stack by our own. + val stack: mutable.ArrayStack[StackItem] = new mutable.ArrayStack() + + // Accumulates topological orders. + var allOrders = Set[List[graph.NodeT]]() + + // Push arguments onto the stack. + stack.push(StackItem(getSources, inDegree, List[graph.NodeT](), 0)) + + while (stack.nonEmpty) { + // Fetch arguments + val stackItem = stack.pop() + + if (stackItem.sources.nonEmpty) { + // `sources` contains all the nodes we can pick. Generate all possibilities. + for (source <- stackItem.sources) { + val newTopOrder = source :: stackItem.topOrder + var newSources = stackItem.sources - source + + // Decrease the in-degree of all adjacent nodes. + var newInDegrees = stackItem.inDegrees + + for (adjacent <- source.diSuccessors) { + val newInDegree = newInDegrees(adjacent) - 1 + newInDegrees = newInDegrees.updated(adjacent, newInDegree) + + // If in-degree becomes zero, add to sources. + if (newInDegree == 0) { + newSources = newSources + adjacent + } + } + + stack.push(StackItem(newSources, newInDegrees, newTopOrder, stackItem.count + 1)) + } + } else if (stackItem.count != graph.nodes.size) { + throw AssertionException("Expected an an acyclic graph, but there is a cycle") + } else { + allOrders += stackItem.topOrder.reverse + } + } + + allOrders + } +} diff --git a/webapi/src/test/scala/org/knora/webapi/messages/util/search/BUILD.bazel b/webapi/src/test/scala/org/knora/webapi/messages/util/search/BUILD.bazel index 9bedb5f558..f0d968721f 100644 --- a/webapi/src/test/scala/org/knora/webapi/messages/util/search/BUILD.bazel +++ b/webapi/src/test/scala/org/knora/webapi/messages/util/search/BUILD.bazel @@ -32,6 +32,24 @@ scala_test( ] + BASE_TEST_DEPENDENCIES, ) +scala_test( + name = "TopologicalSortUtilSpec", + size = "small", # 60s + srcs = [ + "gravsearch/prequery/TopologicalSortUtilSpec.scala", + ], + data = [ + "//knora-ontologies", + "//test_data", + ], + jvm_flags = ["-Dconfig.resource=fuseki.conf"], + # unused_dependency_checker_mode = "warn", + deps = ALL_WEBAPI_MAIN_DEPENDENCIES + [ + "//webapi:main_library", + "//webapi:test_library", + ] + BASE_TEST_DEPENDENCIES, +) + scala_test( name = "NonTriplestoreSpecificGravsearchToCountPrequeryTransformerSpec", size = "small", # 60s diff --git a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtilSpec.scala b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtilSpec.scala new file mode 100644 index 0000000000..8b8e01dd86 --- /dev/null +++ b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtilSpec.scala @@ -0,0 +1,30 @@ +package org.knora.webapi.util.search.gravsearch.prequery + +import org.knora.webapi.CoreSpec +import org.knora.webapi.messages.util.search.gravsearch.prequery.TopologicalSortUtil +import scalax.collection.Graph +import scalax.collection.GraphEdge._ + +class TopologicalSortUtilSpec extends CoreSpec() { + "TopologicalSortUtilSpec" should { + + "return all topological orders of a graph" in { + val graph: Graph[Int, DiHyperEdge] = + Graph[Int, DiHyperEdge](DiHyperEdge[Int](2, 4), DiHyperEdge[Int](2, 7), DiHyperEdge[Int](4, 5)) + + val allOrders: Set[List[Int]] = TopologicalSortUtil + .allTopologicalOrders(graph) + .map { order: List[Graph[Int, DiHyperEdge]#NodeT] => + order.map(_.value) + } + + val expectedOrders = Set( + List(2, 4, 5, 7), + List(2, 4, 7, 5), + List(2, 7, 4, 5) + ) + + assert(allOrders == expectedOrders) + } + } +} From f092bf3068d251263d7b6acfe38af45f21b0c3df Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Tue, 16 Feb 2021 17:48:44 +0100 Subject: [PATCH 18/33] feat(gravsearch): Fix topological sort bugs, add tests. --- .../prequery/AbstractPrequeryGenerator.scala | 102 +++++++++++---- .../prequery/TopologicalSortUtil.scala | 28 ++-- ...cGravsearchToPrequeryTransformerSpec.scala | 121 +++--------------- .../prequery/TopologicalSortUtilSpec.scala | 43 +++++-- 4 files changed, 148 insertions(+), 146 deletions(-) diff --git a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/AbstractPrequeryGenerator.scala b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/AbstractPrequeryGenerator.scala index 9ec6948822..c7074014c6 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/AbstractPrequeryGenerator.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/AbstractPrequeryGenerator.scala @@ -771,7 +771,7 @@ abstract class AbstractPrequeryGenerator(constructClause: ConstructClause, case xsdLiteral: XsdLiteral if xsdLiteral.datatype.toString == OntologyConstants.KnoraApiV2Simple.ListNode => xsdLiteral.value - case other => + case _ => throw GravsearchException(s"Invalid type for literal ${OntologyConstants.KnoraApiV2Simple.ListNode}") } @@ -1261,7 +1261,7 @@ abstract class AbstractPrequeryGenerator(constructClause: ConstructClause, val langLiteral: XsdLiteral = compareExpression.rightArg match { case strLiteral: XsdLiteral if strLiteral.datatype == OntologyConstants.Xsd.String.toSmartIri => strLiteral - case other => + case _ => throw GravsearchException( s"Right argument of comparison statement must be a string literal for use with 'lang' function call") } @@ -2131,31 +2131,87 @@ abstract class AbstractPrequeryGenerator(constructClause: ConstructClause, makeGraphWithoutCycles(graphComponents) } + /** + * Tries to find the best topological order for the graph, by finding all possible topological orders, + * and eliminating those whose first node is the object of rdf:type. + * + * @param graph the graph to be ordered. + * @param statementPatterns the statement patterns that were used to create the graph. + * @return a topological order. + */ + def findBestTopologicalOrder(graph: Graph[String, DiHyperEdge], + statementPatterns: Seq[StatementPattern]): Vector[Graph[String, DiHyperEdge]#NodeT] = { + type NodeT = Graph[String, DiHyperEdge]#NodeT + + /** + * An ordering for sorting topological orders. + */ + object TopologicalOrderOrdering extends Ordering[Vector[NodeT]] { + private def orderToString(order: Vector[NodeT]) = order.map(_.value).mkString("|") + + override def compare(left: Vector[NodeT], right: Vector[NodeT]): Int = + orderToString(left).compare(orderToString(right)) + } + + // Get all the possible topological orders for the graph. + val allTopologicalOrders: Set[Vector[NodeT]] = TopologicalSortUtil.findAllTopologicalOrders(graph) + + if (allTopologicalOrders.isEmpty) { + Vector.empty + } else { + // Is there only one order? + val preferredOrders: Set[Vector[NodeT]] = if (allTopologicalOrders.size == 1) { + // Yes. Don't bother filtering. + allTopologicalOrders + } else { + // There's more than one order. Find the nodes that are objects of rdf:type in the statement patterns. + val nodesThatAreObjectsOfRdfType: Set[String] = statementPatterns + .filter { statementPattern => + statementPattern.pred match { + case iriRef: IriRef => iriRef.iri.toString == OntologyConstants.Rdf.Type + case _ => false + } + } + .map { statementPattern => + statementPattern.obj.toSparql + } + .toSet + + // Filter out the topological orders that end with any of those nodes. + allTopologicalOrders.filterNot { order: Vector[NodeT] => + order.lastOption match { + case Some(node) => nodesThatAreObjectsOfRdfType.contains(node.value) + case None => true + } + } + } + + // Are there any preferred orders? + val ordersToSort = if (preferredOrders.nonEmpty) { + // Yes. Use one of those. + preferredOrders + } else { + // No. Use any order. + allTopologicalOrders + } + + // Sort the orders into a deterministic order, and return the first one. + ordersToSort.toArray.min(TopologicalOrderOrdering) + } + } + def sortStatementPatterns(createdGraph: Graph[String, DiHyperEdge], statementPatterns: Seq[StatementPattern]): Seq[QueryPattern] = { - // Try topological sorting of graph - - // TODO: get all valid orders using TopologicalSortUtil, get the lowest ordered node. - // From statements, find the statements whose object is this node. - // check the predicate of these statements, choose the order with the node that is object of a statement whose predicate is not rdf:type - val topologicalOrderSeq: Seq[createdGraph.TopologicalOrder[createdGraph.NodeT]] = - createdGraph.topologicalSort match { - // Is there still a cycle in the graph? - case Left(cycleNode) => - // Don't try sorting, return statements as they are given. - Seq.empty[createdGraph.TopologicalOrder[createdGraph.NodeT]] - case Right(topOrder) => - // No. return the topological order - Seq(topOrder) - } + type NodeT = Graph[String, DiHyperEdge]#NodeT - // Is there a topological order found? - if (topologicalOrderSeq.nonEmpty) { - // Yes. Topological sort algorithm return only one perturbation; i.e. one topologicalOrder. - val topologicalOrder: Iterable[createdGraph.NodeT] = topologicalOrderSeq.head + // Try to find the best topological order for the graph. + val topologicalOrder: Vector[NodeT] = + findBestTopologicalOrder(graph = createdGraph, statementPatterns = statementPatterns) - // Start from the end of the ordered list (the nodes with lowest degree); - // for each node, find statements which have the node as object and bring them to top. + // Was a topological order found? + if (topologicalOrder.nonEmpty) { + // Start from the end of the ordered list (the nodes with lowest degree). + // For each node, find statements which have the node as object and bring them to top. topologicalOrder.foldRight(Vector.empty[QueryPattern]) { (node, sortedStatements) => val statementsOfNode: Set[QueryPattern] = statementPatterns .filter(p => p.obj.toSparql.equals(node.value)) diff --git a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtil.scala b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtil.scala index 5a0435764a..ebedaf47ef 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtil.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtil.scala @@ -19,7 +19,6 @@ package org.knora.webapi.messages.util.search.gravsearch.prequery -import org.knora.webapi.exceptions.AssertionException import scalax.collection.Graph import scalax.collection.GraphEdge.DiHyperEdge @@ -37,29 +36,27 @@ object TopologicalSortUtil { * @param graph the graph to be sorted. * @tparam T the type of the nodes in the graph. */ - def allTopologicalOrders[T](graph: Graph[T, DiHyperEdge]): Set[List[graph.NodeT]] = { + def findAllTopologicalOrders[T](graph: Graph[T, DiHyperEdge]): Set[Vector[Graph[T, DiHyperEdge]#NodeT]] = { + type NodeT = Graph[T, DiHyperEdge]#NodeT /** * Represents arguments to be put on the simulated call stack. */ - case class StackItem(sources: Set[graph.NodeT], - inDegrees: Map[graph.NodeT, Int], - topOrder: List[graph.NodeT], - count: Int) + case class StackItem(sources: Set[NodeT], inDegrees: Map[NodeT, Int], topOrder: Vector[NodeT], count: Int) - val inDegree: Map[graph.NodeT, Int] = graph.nodes.map(node => (node, node.inDegree)).toMap + val inDegrees: Map[NodeT, Int] = graph.nodes.map(node => (node, node.inDegree)).toMap - def isSource(node: graph.NodeT): Boolean = inDegree(node) == 0 - def getSources: Set[graph.NodeT] = graph.nodes.filter(node => isSource(node)).toSet + def isSource(node: NodeT): Boolean = inDegrees(node) == 0 + def getSources: Set[NodeT] = graph.nodes.filter(node => isSource(node)).toSet // Replaces the program stack by our own. val stack: mutable.ArrayStack[StackItem] = new mutable.ArrayStack() // Accumulates topological orders. - var allOrders = Set[List[graph.NodeT]]() + var allTopologicalOrders = Set[Vector[NodeT]]() // Push arguments onto the stack. - stack.push(StackItem(getSources, inDegree, List[graph.NodeT](), 0)) + stack.push(StackItem(sources = getSources, inDegrees = inDegrees, topOrder = Vector[NodeT](), count = 0)) while (stack.nonEmpty) { // Fetch arguments @@ -68,7 +65,7 @@ object TopologicalSortUtil { if (stackItem.sources.nonEmpty) { // `sources` contains all the nodes we can pick. Generate all possibilities. for (source <- stackItem.sources) { - val newTopOrder = source :: stackItem.topOrder + val newTopOrder = source +: stackItem.topOrder var newSources = stackItem.sources - source // Decrease the in-degree of all adjacent nodes. @@ -87,12 +84,13 @@ object TopologicalSortUtil { stack.push(StackItem(newSources, newInDegrees, newTopOrder, stackItem.count + 1)) } } else if (stackItem.count != graph.nodes.size) { - throw AssertionException("Expected an an acyclic graph, but there is a cycle") + // The graph has a cycle, so don't try to sort it. + () } else { - allOrders += stackItem.topOrder.reverse + allTopologicalOrders += stackItem.topOrder.reverse } } - allOrders + allTopologicalOrders.filter(_.nonEmpty) } } diff --git a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala index a7019c17e8..b83f2727ed 100644 --- a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala @@ -5,7 +5,7 @@ import org.knora.webapi.exceptions.AssertionException import org.knora.webapi.messages.IriConversions._ import org.knora.webapi.messages.StringFormatter import org.knora.webapi.messages.admin.responder.usersmessages.UserADM -import org.knora.webapi.messages.util.{MessageUtil, ResponderData} +import org.knora.webapi.messages.util.ResponderData import org.knora.webapi.messages.util.search._ import org.knora.webapi.messages.util.search.gravsearch.prequery.NonTriplestoreSpecificGravsearchToPrequeryTransformer import org.knora.webapi.messages.util.search.gravsearch.types.{ @@ -1115,67 +1115,7 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec |} """.stripMargin - val TransformedQueryWithRdfsLabelAndLiteralVersion1: SelectQuery = SelectQuery( - fromClause = None, - variables = Vector(QueryVariable(variableName = "book")), - offset = 0, - groupBy = Vector(QueryVariable(variableName = "book")), - orderBy = Vector( - OrderCriterion( - queryVariable = QueryVariable(variableName = "book"), - isAscending = true - )), - whereClause = WhereClause( - patterns = Vector( - StatementPattern( - subj = QueryVariable(variableName = "book"), - pred = IriRef( - iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, - propertyPathOperator = None - ), - obj = XsdLiteral( - value = "false", - datatype = "http://www.w3.org/2001/XMLSchema#boolean".toSmartIri - ), - namedGraph = Some( - IriRef( - iri = "http://www.knora.org/explicit".toSmartIri, - propertyPathOperator = None - )) - ), - StatementPattern( - subj = QueryVariable(variableName = "book"), - pred = IriRef( - iri = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type".toSmartIri, - propertyPathOperator = None - ), - obj = IriRef( - iri = "http://www.knora.org/ontology/0803/incunabula#book".toSmartIri, - propertyPathOperator = None - ), - namedGraph = None - ), - StatementPattern( - subj = QueryVariable(variableName = "book"), - pred = IriRef( - iri = "http://www.w3.org/2000/01/rdf-schema#label".toSmartIri, - propertyPathOperator = None - ), - obj = XsdLiteral( - value = "Zeitgl\u00F6cklein des Lebens und Leidens Christi", - datatype = "http://www.w3.org/2001/XMLSchema#string".toSmartIri - ), - namedGraph = None - ) - ), - positiveEntities = Set(), - querySchema = None - ), - limit = Some(25), - useDistinct = true - ) - - val TransformedQueryWithRdfsLabelAndLiteralVersion2: SelectQuery = SelectQuery( + val TransformedQueryWithRdfsLabelAndLiteral: SelectQuery = SelectQuery( fromClause = None, variables = Vector(QueryVariable(variableName = "book")), offset = 0, @@ -1324,22 +1264,22 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec StatementPattern( subj = QueryVariable(variableName = "book"), pred = IriRef( - iri = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type".toSmartIri, - propertyPathOperator = None - ), - obj = IriRef( - iri = "http://www.knora.org/ontology/0803/incunabula#book".toSmartIri, + iri = "http://www.w3.org/2000/01/rdf-schema#label".toSmartIri, propertyPathOperator = None ), + obj = QueryVariable(variableName = "label"), namedGraph = None ), StatementPattern( subj = QueryVariable(variableName = "book"), pred = IriRef( - iri = "http://www.w3.org/2000/01/rdf-schema#label".toSmartIri, + iri = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type".toSmartIri, + propertyPathOperator = None + ), + obj = IriRef( + iri = "http://www.knora.org/ontology/0803/incunabula#book".toSmartIri, propertyPathOperator = None ), - obj = QueryVariable(variableName = "label"), namedGraph = None ), FilterPattern( @@ -1390,22 +1330,22 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec StatementPattern( subj = QueryVariable(variableName = "book"), pred = IriRef( - iri = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type".toSmartIri, - propertyPathOperator = None - ), - obj = IriRef( - iri = "http://www.knora.org/ontology/0803/incunabula#book".toSmartIri, + iri = "http://www.w3.org/2000/01/rdf-schema#label".toSmartIri, propertyPathOperator = None ), + obj = QueryVariable(variableName = "bookLabel"), namedGraph = None ), StatementPattern( subj = QueryVariable(variableName = "book"), pred = IriRef( - iri = "http://www.w3.org/2000/01/rdf-schema#label".toSmartIri, + iri = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type".toSmartIri, + propertyPathOperator = None + ), + obj = IriRef( + iri = "http://www.knora.org/ontology/0803/incunabula#book".toSmartIri, propertyPathOperator = None ), - obj = QueryVariable(variableName = "bookLabel"), namedGraph = None ), FilterPattern( @@ -2028,31 +1968,12 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec } "transform an input query using rdfs:label and a literal in the simple schema" in { - val result = new ArrayBuffer[String] - - println("=========== Version 1 ============") - println(TransformedQueryWithRdfsLabelAndLiteralVersion1.toSparql) - println("=========== Version 2 ============") - println(TransformedQueryWithRdfsLabelAndLiteralVersion2.toSparql) - - for (i <- 1 to 20) { - val transformedQuery = - QueryHandler.transformQuery(InputQueryWithRdfsLabelAndLiteralInSimpleSchema, responderData, settings) - - if (transformedQuery == TransformedQueryWithRdfsLabelAndLiteralVersion1) { - result += s"$i. transformedQuery == TransformedQueryWithRdfsLabelAndLiteralVersion1" - } else if (transformedQuery == TransformedQueryWithRdfsLabelAndLiteralVersion2) { - result += s"$i. transformedQuery == TransformedQueryWithRdfsLabelAndLiteralVersion2" - } else { - throw new Exception( - "transformedQuery is different from both TransformedQueryWithRdfsLabelAndLiteralVersion1 and TransformedQueryWithRdfsLabelAndLiteralVersion2") - } - } - - throw new Exception(result.mkString("\n")) + val transformedQuery = + QueryHandler.transformQuery(InputQueryWithRdfsLabelAndLiteralInSimpleSchema, responderData, settings) + + assert(transformedQuery == TransformedQueryWithRdfsLabelAndLiteral) } - /* "transform an input query using rdfs:label and a literal in the complex schema" in { val transformedQuery = QueryHandler.transformQuery(InputQueryWithRdfsLabelAndLiteralInComplexSchema, responderData, settings) @@ -2093,7 +2014,7 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec QueryHandler.transformQuery(InputQueryWithUnionScopes, responderData, settings) assert(transformedQuery === TransformedQueryWithUnionScopes) - } */ + } // // "reorder query patterns in where clause" in { // val constructQuery = GravsearchParser.parseQuery(queryToReorder) diff --git a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtilSpec.scala b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtilSpec.scala index 8b8e01dd86..3afd47f402 100644 --- a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtilSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtilSpec.scala @@ -6,25 +6,52 @@ import scalax.collection.Graph import scalax.collection.GraphEdge._ class TopologicalSortUtilSpec extends CoreSpec() { + type NodeT = Graph[Int, DiHyperEdge]#NodeT + + private def nodesToValues(orders: Set[Vector[NodeT]]): Set[Vector[Int]] = { + orders.map { order: Vector[NodeT] => + order.map(_.value) + } + } + "TopologicalSortUtilSpec" should { "return all topological orders of a graph" in { val graph: Graph[Int, DiHyperEdge] = Graph[Int, DiHyperEdge](DiHyperEdge[Int](2, 4), DiHyperEdge[Int](2, 7), DiHyperEdge[Int](4, 5)) - val allOrders: Set[List[Int]] = TopologicalSortUtil - .allTopologicalOrders(graph) - .map { order: List[Graph[Int, DiHyperEdge]#NodeT] => - order.map(_.value) - } + val allOrders: Set[Vector[Int]] = nodesToValues( + TopologicalSortUtil + .findAllTopologicalOrders(graph)) val expectedOrders = Set( - List(2, 4, 5, 7), - List(2, 4, 7, 5), - List(2, 7, 4, 5) + Vector(2, 4, 5, 7), + Vector(2, 4, 7, 5), + Vector(2, 7, 4, 5) ) assert(allOrders == expectedOrders) } + + "return an empty set of orders for an empty graph" in { + val graph: Graph[Int, DiHyperEdge] = Graph[Int, DiHyperEdge]() + + val allOrders: Set[Vector[Int]] = nodesToValues( + TopologicalSortUtil + .findAllTopologicalOrders(graph)) + + assert(allOrders.isEmpty) + } + + "return an empty set of orders for a cyclic graph" in { + val graph: Graph[Int, DiHyperEdge] = + Graph[Int, DiHyperEdge](DiHyperEdge[Int](2, 4), DiHyperEdge[Int](4, 7), DiHyperEdge[Int](7, 2)) + + val allOrders: Set[Vector[Int]] = nodesToValues( + TopologicalSortUtil + .findAllTopologicalOrders(graph)) + + assert(allOrders.isEmpty) + } } } From 431bd75a9f85391121f136f176876292cc168507 Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Wed, 17 Feb 2021 11:24:56 +0100 Subject: [PATCH 19/33] test(gravsearch): Fix test. --- ...searchToCountPrequeryTransformerSpec.scala | 72 +++++++++---------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToCountPrequeryTransformerSpec.scala b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToCountPrequeryTransformerSpec.scala index 70794bb12a..0fd8f8a3d4 100644 --- a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToCountPrequeryTransformerSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToCountPrequeryTransformerSpec.scala @@ -71,30 +71,6 @@ class NonTriplestoreSpecificGravsearchToCountPrequeryTransformerSpec extends Cor implicit val stringFormatter: StringFormatter = StringFormatter.getGeneralInstance - "The NonTriplestoreSpecificGravsearchToCountPrequeryGenerator object" should { - - "transform an input query with a decimal as an optional sort criterion and a filter" in { - - val transformedQuery = - CountQueryHandler.transformQuery(inputQueryWithDecimalOptionalSortCriterionAndFilter, responderData, settings) - - assert(transformedQuery === transformedQueryWithDecimalOptionalSortCriterionAndFilter) - - } - - "transform an input query with a decimal as an optional sort criterion and a filter (submitted in complex schema)" in { - - val transformedQuery = - CountQueryHandler.transformQuery(inputQueryWithDecimalOptionalSortCriterionAndFilterComplex, - responderData, - settings) - - assert(transformedQuery === transformedQueryWithDecimalOptionalSortCriterionAndFilterComplex) - - } - - } - val inputQueryWithDecimalOptionalSortCriterionAndFilter: String = """ |PREFIX anything: @@ -244,17 +220,18 @@ class NonTriplestoreSpecificGravsearchToCountPrequeryTransformerSpec extends Cor val transformedQueryWithDecimalOptionalSortCriterionAndFilterComplex: SelectQuery = SelectQuery( + fromClause = None, variables = Vector( Count( - inputVariable = QueryVariable(variableName = "thing"), + outputVariableName = "count", distinct = true, - outputVariableName = "count" + inputVariable = QueryVariable(variableName = "thing") )), offset = 0, groupBy = Nil, orderBy = Nil, whereClause = WhereClause( - patterns = ArrayBuffer( + patterns = Vector( StatementPattern( subj = QueryVariable(variableName = "thing"), pred = IriRef( @@ -285,6 +262,15 @@ class NonTriplestoreSpecificGravsearchToCountPrequeryTransformerSpec extends Cor ), OptionalPattern( patterns = Vector( + StatementPattern( + subj = QueryVariable(variableName = "decimal"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#valueHasDecimal".toSmartIri, + propertyPathOperator = None + ), + obj = QueryVariable(variableName = "decimalVal"), + namedGraph = None + ), StatementPattern( subj = QueryVariable(variableName = "thing"), pred = IriRef( @@ -310,15 +296,6 @@ class NonTriplestoreSpecificGravsearchToCountPrequeryTransformerSpec extends Cor propertyPathOperator = None )) ), - StatementPattern( - subj = QueryVariable(variableName = "decimal"), - pred = IriRef( - iri = "http://www.knora.org/ontology/knora-base#valueHasDecimal".toSmartIri, - propertyPathOperator = None - ), - obj = QueryVariable(variableName = "decimalVal"), - namedGraph = None - ), FilterPattern(expression = CompareExpression( leftArg = QueryVariable(variableName = "decimalVal"), operator = CompareExpressionOperator.GREATER_THAN, @@ -336,4 +313,27 @@ class NonTriplestoreSpecificGravsearchToCountPrequeryTransformerSpec extends Cor useDistinct = true ) + "The NonTriplestoreSpecificGravsearchToCountPrequeryGenerator object" should { + + "transform an input query with a decimal as an optional sort criterion and a filter" in { + + val transformedQuery = + CountQueryHandler.transformQuery(inputQueryWithDecimalOptionalSortCriterionAndFilter, responderData, settings) + + assert(transformedQuery === transformedQueryWithDecimalOptionalSortCriterionAndFilter) + + } + + "transform an input query with a decimal as an optional sort criterion and a filter (submitted in complex schema)" in { + + val transformedQuery = + CountQueryHandler.transformQuery(inputQueryWithDecimalOptionalSortCriterionAndFilterComplex, + responderData, + settings) + + assert(transformedQuery === transformedQueryWithDecimalOptionalSortCriterionAndFilterComplex) + + } + + } } From bb5e48b08c070999f27a0eebde42aab565ecdf73 Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Wed, 17 Feb 2021 17:27:53 +0100 Subject: [PATCH 20/33] feat(gravsearch): Prefer topological orders that don't put rdf:type statements first. - Fix DSP-1361. --- .../prequery/AbstractPrequeryGenerator.scala | 165 +++++++++++----- ...cGravsearchToPrequeryTransformerSpec.scala | 182 ++++++++++++++++++ 2 files changed, 300 insertions(+), 47 deletions(-) diff --git a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/AbstractPrequeryGenerator.scala b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/AbstractPrequeryGenerator.scala index c7074014c6..d4b559d66a 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/AbstractPrequeryGenerator.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/AbstractPrequeryGenerator.scala @@ -2074,23 +2074,78 @@ abstract class AbstractPrequeryGenerator(constructClause: ConstructClause, case _ => None } - // remove statements whose predicate is rdf:type, type of subject is inferred from a property, and the subject is not in optionalEntities. - patterns.filter { + // Remove statements whose predicate is rdf:type, type of subject is inferred from a property, + // and the subject is not in optionalEntities. If the subject is a standoff tag, remove the statement only + // if the object is the IRI knora-api:StandoffTag, because type inspection doesn't return subtypes of StandoffTag. + patterns.filterNot { case statementPattern: StatementPattern => + // Is the predicate an IRI? statementPattern.pred match { - case iriRef: IriRef => - val subject = GravsearchTypeInspectionUtil.maybeTypeableEntity(statementPattern.subj) - subject match { - case Some(typeableEntity) => - !(iriRef.iri.toString == OntologyConstants.Rdf.Type && typeInspectionResult.entitiesInferredFromProperties.keySet - .contains(typeableEntity) - && !optionalEntities.contains(typeableEntity)) - case _ => true + case predicateIriRef: IriRef => + // Yes. Is this an rdf:type statement? + if (predicateIriRef.iri.toString == OntologyConstants.Rdf.Type) { + // Yes. Is the subject a typeable entity? + val subjectAsTypeableEntity = GravsearchTypeInspectionUtil.maybeTypeableEntity(statementPattern.subj) + + subjectAsTypeableEntity match { + case Some(typeableEntity) => + // Yes. Determine whether it represents a standoff tag. + val subjectIsStandoffTag: Boolean = + typeInspectionResult.getTypeOfEntity(statementPattern.subj) match { + case Some(definedSubjectType) => + definedSubjectType match { + case nonPropertyTypeInfo: NonPropertyTypeInfo + if nonPropertyTypeInfo.typeIri.toString == OntologyConstants.KnoraApiV2Complex.StandoffTag => + true + case _ => false + } + + case None => + throw GravsearchException(s"No type information found for ${statementPattern.subj.toSparql}") + } + + // Determine whether the object is the IRI knora-api:StandoffTag. + val objectIsStandoffTagType: Boolean = statementPattern.obj match { + case iriRef: IriRef => iriRef.iri.toString == OntologyConstants.KnoraApiV2Complex.StandoffTag + case _ => false + } + + // Was the type of the subject inferred from another predicate? + if (typeInspectionResult.entitiesInferredFromProperties.keySet.contains(typeableEntity)) { + // Yes. Is the subject in optional entities? + if (optionalEntities.contains(typeableEntity)) { + // Yes. Keep the statement. + false + } else if (subjectIsStandoffTag && !objectIsStandoffTagType) { + // No. The subject is a standoff tag, and the object is not knora-api:StandoffTag. Keep + // the statement. + false + } else { + // Remove the statement. + true + } + } else { + // The type of the subject was not inferred from another predicate. Keep the statement. + false + } + + case _ => + // The subject isn't a typeable entity. Keep the statement. + false + } + } else { + // This isn't an rdf:type statement. Keep it. + false } - case _ => true + case _ => + // The predicate isn't an IRI. Keep the statement. + false } - case _ => true + + case _ => + // This isn't a statement pattern. Keep it. + false } } @@ -2132,8 +2187,39 @@ abstract class AbstractPrequeryGenerator(constructClause: ConstructClause, } /** - * Tries to find the best topological order for the graph, by finding all possible topological orders, - * and eliminating those whose first node is the object of rdf:type. + * Finds topological orders that don't end with an object of rdf:type. + * + * @param orders the orders to be filtered. + * @param statementPatterns the statement patterns that the orders are based on. + * @return the filtered topological orders. + */ + def findOrdersNotEndingWithObjectOfRdfType( + orders: Set[Vector[Graph[String, DiHyperEdge]#NodeT]], + statementPatterns: Seq[StatementPattern]): Set[Vector[Graph[String, DiHyperEdge]#NodeT]] = { + type NodeT = Graph[String, DiHyperEdge]#NodeT + + // Find the nodes that are objects of rdf:type in the statement patterns. + val nodesThatAreObjectsOfRdfType: Set[String] = statementPatterns + .filter { statementPattern => + statementPattern.pred match { + case iriRef: IriRef => iriRef.iri.toString == OntologyConstants.Rdf.Type + case _ => false + } + } + .map { statementPattern => + statementPattern.obj.toSparql + } + .toSet + + // Filter out the topological orders that end with any of those nodes. + orders.filterNot { order: Vector[NodeT] => + nodesThatAreObjectsOfRdfType.contains(order.last.value) + } + } + + /** + * Tries to find the best topological order for the graph, by finding all possible topological orders + * and eliminating those whose last node is the object of rdf:type. * * @param graph the graph to be ordered. * @param statementPatterns the statement patterns that were used to create the graph. @@ -2156,47 +2242,32 @@ abstract class AbstractPrequeryGenerator(constructClause: ConstructClause, // Get all the possible topological orders for the graph. val allTopologicalOrders: Set[Vector[NodeT]] = TopologicalSortUtil.findAllTopologicalOrders(graph) + // Did we find any topological orders? if (allTopologicalOrders.isEmpty) { + // No, the graph is cyclical. Vector.empty } else { - // Is there only one order? - val preferredOrders: Set[Vector[NodeT]] = if (allTopologicalOrders.size == 1) { + // Yes. Is there only one possible order? + if (allTopologicalOrders.size == 1) { // Yes. Don't bother filtering. - allTopologicalOrders + allTopologicalOrders.head } else { - // There's more than one order. Find the nodes that are objects of rdf:type in the statement patterns. - val nodesThatAreObjectsOfRdfType: Set[String] = statementPatterns - .filter { statementPattern => - statementPattern.pred match { - case iriRef: IriRef => iriRef.iri.toString == OntologyConstants.Rdf.Type - case _ => false - } - } - .map { statementPattern => - statementPattern.obj.toSparql - } - .toSet - - // Filter out the topological orders that end with any of those nodes. - allTopologicalOrders.filterNot { order: Vector[NodeT] => - order.lastOption match { - case Some(node) => nodesThatAreObjectsOfRdfType.contains(node.value) - case None => true - } + // There's more than one possible order. Find orders that don't end with an object of rdf:type. + val ordersNotEndingWithObjectOfRdfType: Set[Vector[NodeT]] = + findOrdersNotEndingWithObjectOfRdfType(allTopologicalOrders, statementPatterns) + + // Are there any? + val preferredOrders = if (ordersNotEndingWithObjectOfRdfType.nonEmpty) { + // Yes. Use one of those. + ordersNotEndingWithObjectOfRdfType + } else { + // No. Use any order. + allTopologicalOrders } - } - // Are there any preferred orders? - val ordersToSort = if (preferredOrders.nonEmpty) { - // Yes. Use one of those. - preferredOrders - } else { - // No. Use any order. - allTopologicalOrders + // Sort the preferred orders to produce a deterministic result, and return one of them. + preferredOrders.min(TopologicalOrderOrdering) } - - // Sort the orders into a deterministic order, and return the first one. - ordersToSort.toArray.min(TopologicalOrderOrdering) } } diff --git a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala index b83f2727ed..c46ef41327 100644 --- a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala @@ -1854,6 +1854,181 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec |} |ORDER BY (?int)""".stripMargin + val queryWithStandoffTagHasStartAncestor: String = + """ + |PREFIX knora-api: + |PREFIX standoff: + |PREFIX anything: + |PREFIX knora-api-simple: + | + |CONSTRUCT { + | ?thing knora-api:isMainResource true . + | ?thing anything:hasText ?text . + |} WHERE { + | ?thing a anything:Thing . + | ?thing anything:hasText ?text . + | ?text knora-api:textValueHasStandoff ?standoffDateTag . + | ?standoffDateTag a knora-api:StandoffDateTag . + | FILTER(knora-api:toSimpleDate(?standoffDateTag) = "GREGORIAN:2016-12-24 CE"^^knora-api-simple:Date) + | ?standoffDateTag knora-api:standoffTagHasStartAncestor ?standoffParagraphTag . + | ?standoffParagraphTag a standoff:StandoffParagraphTag . + |}""".stripMargin + + val transformedQueryWithStandoffTagHasStartAncestor: SelectQuery = SelectQuery( + fromClause = None, + variables = Vector( + QueryVariable(variableName = "thing"), + GroupConcat( + inputVariable = QueryVariable(variableName = "text"), + separator = StringFormatter.INFORMATION_SEPARATOR_ONE, + outputVariableName = "text__Concat" + ) + ), + offset = 0, + groupBy = Vector(QueryVariable(variableName = "thing")), + orderBy = Vector( + OrderCriterion( + queryVariable = QueryVariable(variableName = "thing"), + isAscending = true + )), + whereClause = WhereClause( + patterns = Vector( + StatementPattern( + subj = QueryVariable(variableName = "standoffParagraphTag"), + pred = IriRef( + iri = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type".toSmartIri, + propertyPathOperator = None + ), + obj = IriRef( + iri = "http://www.knora.org/ontology/standoff#StandoffParagraphTag".toSmartIri, + propertyPathOperator = None + ), + namedGraph = None + ), + StatementPattern( + subj = QueryVariable(variableName = "standoffDateTag"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#standoffTagHasStartAncestor".toSmartIri, + propertyPathOperator = None + ), + obj = QueryVariable(variableName = "standoffParagraphTag"), + namedGraph = None + ), + StatementPattern( + subj = QueryVariable(variableName = "standoffDateTag"), + pred = IriRef( + iri = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type".toSmartIri, + propertyPathOperator = None + ), + obj = IriRef( + iri = "http://www.knora.org/ontology/knora-base#StandoffDateTag".toSmartIri, + propertyPathOperator = None + ), + namedGraph = None + ), + StatementPattern( + subj = QueryVariable(variableName = "text"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#valueHasStandoff".toSmartIri, + propertyPathOperator = None + ), + obj = QueryVariable(variableName = "standoffDateTag"), + namedGraph = None + ), + StatementPattern( + subj = QueryVariable(variableName = "thing"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "false", + datatype = "http://www.w3.org/2001/XMLSchema#boolean".toSmartIri + ), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = QueryVariable(variableName = "thing"), + pred = IriRef( + iri = "http://www.knora.org/ontology/0001/anything#hasText".toSmartIri, + propertyPathOperator = None + ), + obj = QueryVariable(variableName = "text"), + namedGraph = None + ), + StatementPattern( + subj = QueryVariable(variableName = "text"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "false", + datatype = "http://www.w3.org/2001/XMLSchema#boolean".toSmartIri + ), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = QueryVariable(variableName = "standoffDateTag"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#valueHasStartJDN".toSmartIri, + propertyPathOperator = None + ), + obj = QueryVariable(variableName = "standoffDateTag__valueHasStartJDN"), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = QueryVariable(variableName = "standoffDateTag"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#valueHasEndJDN".toSmartIri, + propertyPathOperator = None + ), + obj = QueryVariable(variableName = "standoffDateTag__valueHasEndJDN"), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + FilterPattern( + expression = AndExpression( + leftArg = CompareExpression( + leftArg = XsdLiteral( + value = "2457747", + datatype = "http://www.w3.org/2001/XMLSchema#integer".toSmartIri + ), + operator = CompareExpressionOperator.LESS_THAN_OR_EQUAL_TO, + rightArg = QueryVariable(variableName = "standoffDateTag__valueHasEndJDN") + ), + rightArg = CompareExpression( + leftArg = XsdLiteral( + value = "2457747", + datatype = "http://www.w3.org/2001/XMLSchema#integer".toSmartIri + ), + operator = CompareExpressionOperator.GREATER_THAN_OR_EQUAL_TO, + rightArg = QueryVariable(variableName = "standoffDateTag__valueHasStartJDN") + ) + )) + ), + positiveEntities = Set(), + querySchema = None + ), + limit = Some(25), + useDistinct = true + ) + "The NonTriplestoreSpecificGravsearchToPrequeryGenerator object" should { "transform an input query with an optional property criterion without removing the rdf:type statement" in { @@ -2015,6 +2190,13 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec assert(transformedQuery === TransformedQueryWithUnionScopes) } + + "transform an input query with knora-api:standoffTagHasStartAncestor" in { + val transformedQuery = + QueryHandler.transformQuery(queryWithStandoffTagHasStartAncestor, responderData, settings) + + assert(transformedQuery === transformedQueryWithStandoffTagHasStartAncestor) + } // // "reorder query patterns in where clause" in { // val constructQuery = GravsearchParser.parseQuery(queryToReorder) From f33a90d1a3e2239ab70e56ea67b6db3f577d918e Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Thu, 18 Feb 2021 14:51:08 +0100 Subject: [PATCH 21/33] fix(gravsearch): Correctly handle standoff classes in optimisations. --- .../prequery/AbstractPrequeryGenerator.scala | 51 +--- .../GravsearchTypeInspectionResult.scala | 30 +- .../InferringGravsearchTypeInspector.scala | 260 ++++++++++-------- 3 files changed, 176 insertions(+), 165 deletions(-) diff --git a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/AbstractPrequeryGenerator.scala b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/AbstractPrequeryGenerator.scala index d4b559d66a..5d4b2aa000 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/AbstractPrequeryGenerator.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/AbstractPrequeryGenerator.scala @@ -57,10 +57,6 @@ abstract class AbstractPrequeryGenerator(constructClause: ConstructClause, // suffix appended to variables that are returned by a SPARQL aggregation function. protected val groupConcatVariableSuffix = "__Concat" - // A set of types that can be treated as dates by the knora-api:toSimpleDate function. - private val dateTypes: Set[IRI] = - Set(OntologyConstants.KnoraApiV2Complex.DateValue, OntologyConstants.KnoraApiV2Complex.StandoffTag) - /** * A container for a generated variable representing a value literal. * @@ -354,14 +350,14 @@ abstract class AbstractPrequeryGenerator(constructClause: ConstructClause, } - val (maybeSubjectTypeIri: Option[SmartIri], subjectIsResource: Boolean) = + val maybeSubjectType: Option[NonPropertyTypeInfo] = typeInspectionResult.getTypeOfEntity(statementPattern.subj) match { - case Some(NonPropertyTypeInfo(subjectTypeIri, isResourceType, _)) => (Some(subjectTypeIri), isResourceType) - case _ => (None, false) + case Some(nonPropertyTypeInfo: NonPropertyTypeInfo) => Some(nonPropertyTypeInfo) + case _ => None } // Is the subject of the statement a resource? - if (subjectIsResource) { + if (maybeSubjectType.exists(_.isResourceType)) { // Yes. Is the object of the statement also a resource? if (propertyTypeInfo.objectIsResourceType) { // Yes. This is a link property. Make sure that the object is either an IRI or a variable (cannot be a literal). @@ -492,7 +488,7 @@ abstract class AbstractPrequeryGenerator(constructClause: ConstructClause, if (querySchema == ApiV2Complex) { // Yes. If the subject is a standoff tag and the object is a resource, that's an error, because the client // has to use the knora-api:standoffLink function instead. - if (maybeSubjectTypeIri.contains(OntologyConstants.KnoraApiV2Complex.StandoffTag.toSmartIri) && propertyTypeInfo.objectIsResourceType) { + if (maybeSubjectType.exists(_.isStandoffTagType) && propertyTypeInfo.objectIsResourceType) { throw GravsearchException( s"Invalid statement pattern (use the knora-api:standoffLink function instead): ${statementPattern.toSparql.trim}") } else { @@ -1823,9 +1819,8 @@ abstract class AbstractPrequeryGenerator(constructClause: ConstructClause, val standoffTagVar = functionCallExpression.getArgAsQueryVar(pos = 1) typeInspectionResult.getTypeOfEntity(standoffTagVar) match { - case Some(nonPropertyTypeInfo: NonPropertyTypeInfo) - if nonPropertyTypeInfo.typeIri.toString == OntologyConstants.KnoraApiV2Complex.StandoffTag => - () + case Some(nonPropertyTypeInfo: NonPropertyTypeInfo) if nonPropertyTypeInfo.isStandoffTagType => () + case _ => throw GravsearchException( s"The second argument of ${functionIri.toSparql} must represent a knora-api:StandoffTag") @@ -1932,7 +1927,7 @@ abstract class AbstractPrequeryGenerator(constructClause: ConstructClause, typeInspectionResult.getTypeOfEntity(dateBaseVar) match { case Some(nonPropInfo: NonPropertyTypeInfo) => - if (!dateTypes.contains(nonPropInfo.typeIri.toString)) { + if (!(nonPropInfo.isStandoffTagType || nonPropInfo.typeIri.toString == OntologyConstants.KnoraApiV2Complex.DateValue)) { throw GravsearchException( s"${dateBaseVar.toSparql} must represent a knora-api:DateValue or a knora-api:StandoffDateTag") } @@ -2075,8 +2070,7 @@ abstract class AbstractPrequeryGenerator(constructClause: ConstructClause, } // Remove statements whose predicate is rdf:type, type of subject is inferred from a property, - // and the subject is not in optionalEntities. If the subject is a standoff tag, remove the statement only - // if the object is the IRI knora-api:StandoffTag, because type inspection doesn't return subtypes of StandoffTag. + // and the subject is not in optionalEntities. patterns.filterNot { case statementPattern: StatementPattern => // Is the predicate an IRI? @@ -2089,37 +2083,12 @@ abstract class AbstractPrequeryGenerator(constructClause: ConstructClause, subjectAsTypeableEntity match { case Some(typeableEntity) => - // Yes. Determine whether it represents a standoff tag. - val subjectIsStandoffTag: Boolean = - typeInspectionResult.getTypeOfEntity(statementPattern.subj) match { - case Some(definedSubjectType) => - definedSubjectType match { - case nonPropertyTypeInfo: NonPropertyTypeInfo - if nonPropertyTypeInfo.typeIri.toString == OntologyConstants.KnoraApiV2Complex.StandoffTag => - true - case _ => false - } - - case None => - throw GravsearchException(s"No type information found for ${statementPattern.subj.toSparql}") - } - - // Determine whether the object is the IRI knora-api:StandoffTag. - val objectIsStandoffTagType: Boolean = statementPattern.obj match { - case iriRef: IriRef => iriRef.iri.toString == OntologyConstants.KnoraApiV2Complex.StandoffTag - case _ => false - } - - // Was the type of the subject inferred from another predicate? + // Yes. Was the type of the subject inferred from another predicate? if (typeInspectionResult.entitiesInferredFromProperties.keySet.contains(typeableEntity)) { // Yes. Is the subject in optional entities? if (optionalEntities.contains(typeableEntity)) { // Yes. Keep the statement. false - } else if (subjectIsStandoffTag && !objectIsStandoffTagType) { - // No. The subject is a standoff tag, and the object is not knora-api:StandoffTag. Keep - // the statement. - false } else { // Remove the statement. true diff --git a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/types/GravsearchTypeInspectionResult.scala b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/types/GravsearchTypeInspectionResult.scala index 1bbb0d126b..6a7875f306 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/types/GravsearchTypeInspectionResult.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/types/GravsearchTypeInspectionResult.scala @@ -33,12 +33,24 @@ sealed trait GravsearchEntityTypeInfo * @param objectTypeIri an IRI representing the type of the objects of the property. * @param objectIsResourceType `true` if the property's object type is a resource type. Property is a link. * @param objectIsValueType `true` if the property's object type is a value type. Property is not a link. + * @param objectIsStandoffTagType `true` if the property's object type is a standoff tag type. Property is not a link. */ case class PropertyTypeInfo(objectTypeIri: SmartIri, objectIsResourceType: Boolean = false, - objectIsValueType: Boolean = false) + objectIsValueType: Boolean = false, + objectIsStandoffTagType: Boolean = false) extends GravsearchEntityTypeInfo { override def toString: String = s"knora-api:objectType ${IriRef(objectTypeIri).toSparql}" + + /** + * Converts this [[PropertyTypeInfo]] to a [[NonPropertyTypeInfo]]. + */ + def toNonPropertyTypeInfo: NonPropertyTypeInfo = NonPropertyTypeInfo( + typeIri = objectTypeIri, + isResourceType = objectIsResourceType, + isValueType = objectIsValueType, + isStandoffTagType = objectIsStandoffTagType + ) } /** @@ -48,10 +60,24 @@ case class PropertyTypeInfo(objectTypeIri: SmartIri, * @param typeIri an IRI representing the entity's type. * @param isResourceType `true` if this is a resource type. * @param isValueType `true` if this is a value type. + * @param isStandoffTagType `true` if this is a standoff tag type. */ -case class NonPropertyTypeInfo(typeIri: SmartIri, isResourceType: Boolean = false, isValueType: Boolean = false) +case class NonPropertyTypeInfo(typeIri: SmartIri, + isResourceType: Boolean = false, + isValueType: Boolean = false, + isStandoffTagType: Boolean = false) extends GravsearchEntityTypeInfo { override def toString: String = s"rdf:type ${IriRef(typeIri).toSparql}" + + /** + * Converts this [[NonPropertyTypeInfo]] to a [[PropertyTypeInfo]]. + */ + def toPropertyTypeInfo: PropertyTypeInfo = PropertyTypeInfo( + objectTypeIri = typeIri, + objectIsResourceType = isResourceType, + objectIsValueType = isValueType, + objectIsStandoffTagType = isStandoffTagType + ) } /** diff --git a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/types/InferringGravsearchTypeInspector.scala b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/types/InferringGravsearchTypeInspector.scala index 44b3f4fa72..06c97d95b1 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/types/InferringGravsearchTypeInspector.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/types/InferringGravsearchTypeInspector.scala @@ -138,15 +138,13 @@ class InferringGravsearchTypeInspector(nextInspector: Option[GravsearchTypeInspe case Some(classDef) => // Yes. Is it a resource class? if (classDef.isResourceClass) { - // Yes. Infer rdf:type knora-api:Resource. - val inferredType = NonPropertyTypeInfo(classDef.entityInfoContent.classIri, - isResourceType = classDef.isResourceClass, - isValueType = classDef.isValueClass) + // Yes. Use that class as the inferred type. + val inferredType = NonPropertyTypeInfo(classDef.entityInfoContent.classIri, isResourceType = true) log.debug("InferTypeOfSubjectOfRdfTypePredicate: {} {} .", entityToType, inferredType) Some(inferredType) } else if (classDef.isStandoffClass) { - // It's not a resource class, it's a standoff class. Infer rdf:type knora-api:StandoffTag. - val inferredType = NonPropertyTypeInfo(OntologyConstants.KnoraApiV2Complex.StandoffTag.toSmartIri) + val inferredType = + NonPropertyTypeInfo(classDef.entityInfoContent.classIri, isStandoffTagType = true) log.debug("InferTypeOfSubjectOfRdfTypePredicate: {} {} .", entityToType, inferredType) Some(inferredType) } else { @@ -212,19 +210,11 @@ class InferringGravsearchTypeInspector(nextInspector: Option[GravsearchTypeInspe entityInfo.propertyInfoMap.get(iri) match { case Some(readPropertyInfo: ReadPropertyInfoV2) => // Yes. Try to infer its knora-api:objectType from the provided information. - InferenceRuleUtil.readPropertyInfoToObjectType(readPropertyInfo, entityInfo, usageIndex.querySchema) match { - case Some(objectTypeIri: SmartIri) => - val isValue = GravsearchTypeInspectionUtil.GravsearchValueTypeIris.contains(objectTypeIri.toString) - val inferredType = PropertyTypeInfo(objectTypeIri = objectTypeIri, - objectIsResourceType = readPropertyInfo.isLinkProp, - objectIsValueType = isValue) - log.debug("InferTypeOfPropertyFromItsIri: {} {} .", entityToType, inferredType) - Set(inferredType) - - case None => - // Its knora-api:objectType couldn't be inferred. - Set.empty[GravsearchEntityTypeInfo] - } + val inferredObjectTypes: Set[GravsearchEntityTypeInfo] = InferenceRuleUtil + .readPropertyInfoToObjectType(readPropertyInfo, entityInfo, usageIndex.querySchema) + .toSet + log.debug("InferTypeOfPropertyFromItsIri: {} {} .", entityToType, inferredObjectTypes.mkString(", ")) + inferredObjectTypes case None => // The ontology responder hasn't provided a definition of this property. This should have caused @@ -280,10 +270,7 @@ class InferringGravsearchTypeInspector(nextInspector: Option[GravsearchTypeInspe // Yes. Use the knora-api:objectType of each PropertyTypeInfo. entityTypes.flatMap { case propertyTypeInfo: PropertyTypeInfo => - val inferredType: GravsearchEntityTypeInfo = - NonPropertyTypeInfo(propertyTypeInfo.objectTypeIri, - isResourceType = propertyTypeInfo.objectIsResourceType, - isValueType = propertyTypeInfo.objectIsValueType) + val inferredType: GravsearchEntityTypeInfo = propertyTypeInfo.toNonPropertyTypeInfo log.debug("InferTypeOfObjectFromPredicate: {} {} .", entityToType, inferredType) Some(inferredType) case _ => @@ -384,24 +371,11 @@ class InferringGravsearchTypeInspector(nextInspector: Option[GravsearchTypeInspe // Yes. Has the ontology responder provided a property definition for it? entityInfo.propertyInfoMap.get(predIri) match { case Some(readPropertyInfo: ReadPropertyInfoV2) => - // Yes. Can we infer the property's knora-api:subjectType from that definition? - InferenceRuleUtil.readPropertyInfoToSubjectType(readPropertyInfo, - entityInfo, - usageIndex.querySchema) match { - case Some(subjectTypeIri: SmartIri) => - // Yes. Use that type. - val isValue = - GravsearchTypeInspectionUtil.GravsearchValueTypeIris.contains(subjectTypeIri.toString) - val inferredType = NonPropertyTypeInfo(subjectTypeIri, - isResourceType = readPropertyInfo.isResourceProp, - isValueType = isValue) - log.debug("InferTypeOfSubjectFromPredicateIri: {} {} .", entityToType, inferredType) - Some(inferredType) - - case None => - // No. This rule can't infer the entity's type. - None - } + // Yes. Try to get the property's knora-api:subjectType from that definition, + // and infer that type as the type of the entity. + InferenceRuleUtil + .readPropertyInfoToSubjectType(readPropertyInfo, entityInfo, usageIndex.querySchema) + .map(_.toNonPropertyTypeInfo) case None => // The ontology responder hasn't provided a definition of this property. This should have caused @@ -455,14 +429,36 @@ class InferringGravsearchTypeInspector(nextInspector: Option[GravsearchTypeInspe updatedIntermediateResult.entities.get(typeableObj) match { case Some(entityTypes: Set[GravsearchEntityTypeInfo]) => // Yes. Use those types. + + val alreadyInferredPropertyTypes: Set[PropertyTypeInfo] = + updatedIntermediateResult.entities.getOrElse(entityToType, Set.empty).collect { + case propertyTypeInfo: PropertyTypeInfo => propertyTypeInfo + } + entityTypes.flatMap { case nonPropertyTypeInfo: NonPropertyTypeInfo => - val inferredType: GravsearchEntityTypeInfo = - PropertyTypeInfo(objectTypeIri = nonPropertyTypeInfo.typeIri, - objectIsResourceType = nonPropertyTypeInfo.isResourceType, - nonPropertyTypeInfo.isValueType) - log.debug("InferTypeOfPredicateFromObject: {} {} .", entityToType, inferredType) - Some(inferredType) + // Is this type a subclass of an object type that we already have for this property, + // which we may have got from the property's definition in an ontology? + val baseClassesOfInferredType: Set[SmartIri] = + entityInfo.classInfoMap.get(nonPropertyTypeInfo.typeIri) match { + case Some(classDef) => classDef.allBaseClasses.toSet + case None => Set.empty + } + + val isSubclassOfAlreadyInferredType: Boolean = alreadyInferredPropertyTypes.exists { + alreadyInferredType: PropertyTypeInfo => + baseClassesOfInferredType.contains(alreadyInferredType.objectTypeIri) + } + + if (!isSubclassOfAlreadyInferredType) { + // No. Use the inferred type. + val inferredType: GravsearchEntityTypeInfo = nonPropertyTypeInfo.toPropertyTypeInfo + log.debug("InferTypeOfPredicateFromObject: {} {} .", entityToType, inferredType) + Some(inferredType) + } else { + // Yes. Don't infer the more specific type for the property. + None + } case _ => None @@ -506,9 +502,14 @@ class InferringGravsearchTypeInspector(nextInspector: Option[GravsearchTypeInspe val typesFromFilters: Set[GravsearchEntityTypeInfo] = usageIndex.typedEntitiesInFilters.get(entityToType) match { case Some(typesFromFilters: Set[SmartIri]) => // Yes. Return those types. - typesFromFilters.map { typeFromFilter => - val isValue = GravsearchTypeInspectionUtil.GravsearchValueTypeIris.contains(typeFromFilter.toString) - val inferredType = NonPropertyTypeInfo(typeFromFilter, isResourceType = !isValue, isValueType = isValue) + typesFromFilters.map { typeFromFilter: SmartIri => + val isValueType = GravsearchTypeInspectionUtil.GravsearchValueTypeIris.contains(typeFromFilter.toString) + val isStandoffTagType = typeFromFilter.toString == OntologyConstants.KnoraApiV2Complex.StandoffTag + val isResourceType = !(isValueType || isStandoffTagType) + val inferredType = NonPropertyTypeInfo(typeFromFilter, + isResourceType = isResourceType, + isValueType = isValueType, + isStandoffTagType = isStandoffTagType) log.debug("InferTypeOfEntityFromKnownTypeInFilter: {} {} .", entityToType, inferredType) inferredType } @@ -551,24 +552,10 @@ class InferringGravsearchTypeInspector(nextInspector: Option[GravsearchTypeInspe // Has the ontology responder provided a definition of this property? entityInfo.propertyInfoMap.get(propertyIri) match { case Some(readPropertyInfo: ReadPropertyInfoV2) => - // Yes. Can we determine the property's knora-api:objectType from that definition? - InferenceRuleUtil.readPropertyInfoToObjectType(readPropertyInfo, - entityInfo, - usageIndex.querySchema) match { - case Some(objectTypeIri: SmartIri) => - // Yes. Use that type. - val isValue = - GravsearchTypeInspectionUtil.GravsearchValueTypeIris.contains(objectTypeIri.toString) - val inferredType = PropertyTypeInfo(objectTypeIri = objectTypeIri, - objectIsResourceType = readPropertyInfo.isLinkProp, - objectIsValueType = isValue) - log.debug("InferTypeOfEntityFromKnownTypeInFilter: {} {} .", variableToType, inferredType) - Some(inferredType) - - case None => - // No knora-api:objectType could be determined for the property IRI. - None - } + // Yes. Try to determine the property's knora-api:objectType from that definition. + InferenceRuleUtil + .readPropertyInfoToObjectType(readPropertyInfo, entityInfo, usageIndex.querySchema) + .toSet case None => // The ontology responder hasn't provided a definition of this property. This should have caused @@ -675,7 +662,7 @@ class InferringGravsearchTypeInspector(nextInspector: Option[GravsearchTypeInspe */ def readPropertyInfoToSubjectType(readPropertyInfo: ReadPropertyInfoV2, entityInfo: EntityInfoGetResponseV2, - querySchema: ApiV2Schema): Option[SmartIri] = { + querySchema: ApiV2Schema): Option[PropertyTypeInfo] = { // Get the knora-api:subjectType that the ontology responder provided. readPropertyInfo.entityInfoContent .getPredicateIriObject(OntologyConstants.KnoraApiV2Simple.SubjectType.toSmartIri) @@ -687,14 +674,14 @@ class InferringGravsearchTypeInspector(nextInspector: Option[GravsearchTypeInspe // Is it a resource class? if (readPropertyInfo.isResourceProp) { // Yes. Use it. - Some(subjectType) + Some(PropertyTypeInfo(subjectType, objectIsResourceType = true)) } else if (subjectTypeStr == OntologyConstants.KnoraApiV2Complex.Value || OntologyConstants.KnoraApiV2Complex.ValueBaseClasses .contains(subjectTypeStr)) { // If it's knora-api:Value or one of the knora-api:ValueBase classes, don't use it. None } else if (OntologyConstants.KnoraApiV2Complex.FileValueClasses.contains(subjectTypeStr)) { // If it's a file value class, return the representation of file values in the specified schema. - Some(getFileTypeForSchema(querySchema)) + Some(PropertyTypeInfo(getFileTypeForSchema(querySchema), objectIsValueType = true)) } else { // It's not any of those types. Is it a standoff class? val isStandoffClass: Boolean = entityInfo.classInfoMap.get(subjectType) match { @@ -703,11 +690,11 @@ class InferringGravsearchTypeInspector(nextInspector: Option[GravsearchTypeInspe } if (isStandoffClass) { - // Yes. Infer knora-api:subjectType knora-api:StandoffTag. - Some(OntologyConstants.KnoraApiV2Complex.StandoffTag.toSmartIri) + // Yes. Return it as a standoff tag type. + Some(PropertyTypeInfo(subjectType, objectIsStandoffTagType = true)) } else if (GravsearchTypeInspectionUtil.GravsearchValueTypeIris.contains(subjectTypeStr)) { // It's not any of those. If it's a value type, return it. - Some(subjectType) + Some(PropertyTypeInfo(subjectType, objectIsValueType = true)) } else { // It's not valid in a type inspection result. This must mean it's not allowed in Gravsearch queries. throw GravsearchException( @@ -719,7 +706,7 @@ class InferringGravsearchTypeInspector(nextInspector: Option[GravsearchTypeInspe // Subject type of the predicate is not known but this is a resource property? if (readPropertyInfo.isResourceProp) { // Yes. Infer knora-api:subjectType knora-api:Resource. - Some(getResourceTypeIriForSchema(querySchema)) + Some(PropertyTypeInfo(getResourceTypeIriForSchema(querySchema), objectIsResourceType = true)) } else None } } @@ -733,11 +720,11 @@ class InferringGravsearchTypeInspector(nextInspector: Option[GravsearchTypeInspe */ def readPropertyInfoToObjectType(readPropertyInfo: ReadPropertyInfoV2, entityInfo: EntityInfoGetResponseV2, - querySchema: ApiV2Schema): Option[SmartIri] = { + querySchema: ApiV2Schema): Option[PropertyTypeInfo] = { // Is this a file value property? if (readPropertyInfo.isFileValueProp) { // Yes, return the representation of file values in the specified schema. - Some(getFileTypeForSchema(querySchema)) + Some(PropertyTypeInfo(getFileTypeForSchema(querySchema), objectIsValueType = true)) } else { // It's not a link property. Get the knora-api:objectType that the ontology responder provided. readPropertyInfo.entityInfoContent @@ -759,14 +746,14 @@ class InferringGravsearchTypeInspector(nextInspector: Option[GravsearchTypeInspe } if (isStandoffClass) { - // Yes. Infer knora-api:objectType knora-api:StandoffTag. - Some(OntologyConstants.KnoraApiV2Complex.StandoffTag.toSmartIri) + // Yes. Return it as a standoff tag type. + Some(PropertyTypeInfo(objectType, objectIsStandoffTagType = true)) } else if (readPropertyInfo.isLinkProp) { // No. Is this a link property? - // Yes. return the object type resource class. - Some(objectType) + // Yes. Return it as a resource type. + Some(PropertyTypeInfo(objectType, objectIsResourceType = true)) } else if (GravsearchTypeInspectionUtil.GravsearchValueTypeIris.contains(objectTypeStr)) { // It's not any of those. If it's a value type, return it. - Some(objectType) + Some(PropertyTypeInfo(objectType, objectIsValueType = true)) } else { // No. This must mean it's not allowed in Gravsearch queries. throw GravsearchException( @@ -781,7 +768,9 @@ class InferringGravsearchTypeInspector(nextInspector: Option[GravsearchTypeInspe } // The inference rule pipeline for the first iteration. Includes rules that cannot return additional - // information if they are run more than once. + // information if they are run more than once. It's important that InferTypeOfPropertyFromItsIri + // is run before InferTypeOfPredicateFromObject, so that the latter doesn't add a subtype of a type + // already added by the former. private val firstIterationRulePipeline = new InferTypeOfSubjectOfRdfTypePredicate( Some(new InferTypeOfPropertyFromItsIri(Some(new InferTypeOfSubjectFromPredicateIri( Some(new InferTypeOfEntityFromKnownTypeInFilter(Some(new InferTypeOfVariableFromComparisonWithPropertyIriInFilter( @@ -1009,23 +998,14 @@ class InferringGravsearchTypeInspector(nextInspector: Option[GravsearchTypeInspe entityInfo: EntityInfoGetResponseV2): IntermediateTypeInspectionResult = { /** - * Returns `true` if the specified [[GravsearchEntityTypeInfo]] refers to a resource type. - */ - def getIsResourceFlags(typeInfo: GravsearchEntityTypeInfo): Boolean = { - typeInfo match { - case propertyTypeInfo: PropertyTypeInfo => propertyTypeInfo.objectIsResourceType - case nonPropertyTypeInfo: NonPropertyTypeInfo => nonPropertyTypeInfo.isResourceType - case _ => throw GravsearchException(s"There is an invalid type") - } - } - - /** - * Given a set of resource classes, this method finds a common base class. + * Given a set of classes, this method finds a common base class. * - * @param typesToBeChecked a set of resource classes. + * @param typesToBeChecked a set of classes. + * @param defaultBaseClassIri the default base class IRI if none is found. * @return the IRI of a common base class. */ - def findCommonBaseResourceClass(typesToBeChecked: Set[GravsearchEntityTypeInfo]): SmartIri = { + def findCommonBaseClass(typesToBeChecked: Set[GravsearchEntityTypeInfo], + defaultBaseClassIri: SmartIri): SmartIri = { val baseClassesOfFirstType: Seq[SmartIri] = entityInfo.classInfoMap.get(iriOfGravsearchTypeInfo(typesToBeChecked.head)) match { case Some(classDef: ReadClassInfoV2) => classDef.allBaseClasses @@ -1048,26 +1028,26 @@ class InferringGravsearchTypeInspector(nextInspector: Option[GravsearchTypeInspe // returns the most specific common base class. commonBaseClasses.head } else { - InferenceRuleUtil.getResourceTypeIriForSchema(querySchema) + defaultBaseClassIri } } else { - InferenceRuleUtil.getResourceTypeIriForSchema(querySchema) + defaultBaseClassIri } } /** - * Replaces inconsistent resource types with a common base class. + * Replaces inconsistent types with a common base class. */ - def replaceInconsistentResourceTypes(acc: IntermediateTypeInspectionResult, - typedEntity: TypeableEntity, - typesToBeChecked: Set[GravsearchEntityTypeInfo], - newType: GravsearchEntityTypeInfo): IntermediateTypeInspectionResult = { + def replaceInconsistentTypes(acc: IntermediateTypeInspectionResult, + typedEntity: TypeableEntity, + typesToBeChecked: Set[GravsearchEntityTypeInfo], + newType: GravsearchEntityTypeInfo): IntermediateTypeInspectionResult = { val withoutInconsistentTypes: IntermediateTypeInspectionResult = typesToBeChecked.foldLeft(acc) { - (sanitizeResults, currType) => - sanitizeResults.removeType(typedEntity, currType) + (sanitizeResults: IntermediateTypeInspectionResult, currType: GravsearchEntityTypeInfo) => + sanitizeResults.removeType(entity = typedEntity, typeToRemove = currType) } - withoutInconsistentTypes.addTypes(typedEntity, Set(newType)) + withoutInconsistentTypes.addTypes(entity = typedEntity, entityTypes = Set(newType)) } // get inconsistent types @@ -1078,26 +1058,62 @@ class InferringGravsearchTypeInspector(nextInspector: Option[GravsearchTypeInspe inconsistentEntities.keySet.foldLeft(lastResults) { (acc, typedEntity) => // all inconsistent types val typesToBeChecked: Set[GravsearchEntityTypeInfo] = inconsistentEntities.getOrElse(typedEntity, Set.empty) - val commonBaseClassIri: SmartIri = findCommonBaseResourceClass(typesToBeChecked) - - // Are all inconsistent types NonPropertyTypeInfo and resourceType? - if (typesToBeChecked.count(elem => elem.isInstanceOf[NonPropertyTypeInfo]) == typesToBeChecked.size && - typesToBeChecked.count(elem => getIsResourceFlags(elem)) == typesToBeChecked.size) { + // Are all inconsistent types NonPropertyTypeInfo representing resource classes? + if (typesToBeChecked.forall { + case nonPropertyTypeInfo: NonPropertyTypeInfo => nonPropertyTypeInfo.isResourceType + case _ => false + }) { // Yes. Remove inconsistent types and replace with a common base class. + val commonBaseClassIri: SmartIri = + findCommonBaseClass(typesToBeChecked, InferenceRuleUtil.getResourceTypeIriForSchema(querySchema)) val newResourceType = NonPropertyTypeInfo(commonBaseClassIri, isResourceType = true) - replaceInconsistentResourceTypes(acc, typedEntity, typesToBeChecked, newResourceType) - - // No. Are they PropertyTypeInfo types with a object of a resource type? - } else if (typesToBeChecked.count(elem => elem.isInstanceOf[PropertyTypeInfo]) == typesToBeChecked.size && - typesToBeChecked.count(elem => getIsResourceFlags(elem)) == typesToBeChecked.size) { - - // Yes. Remove inconsistent types and replace with a common base class + replaceInconsistentTypes(acc = acc, + typedEntity = typedEntity, + typesToBeChecked = typesToBeChecked, + newType = newResourceType) + } else if (typesToBeChecked.forall { + case nonPropertyTypeInfo: NonPropertyTypeInfo => nonPropertyTypeInfo.isStandoffTagType + case _ => false + }) { + // No, they're NonPropertyTypeInfo representing standoff tag classes. + // Yes. Remove inconsistent types and replace with a common base class. + val commonBaseClassIri: SmartIri = + findCommonBaseClass(typesToBeChecked, OntologyConstants.KnoraApiV2Complex.StandoffTag.toSmartIri) + val newStandoffTagType = NonPropertyTypeInfo(commonBaseClassIri, isStandoffTagType = true) + replaceInconsistentTypes(acc = acc, + typedEntity = typedEntity, + typesToBeChecked = typesToBeChecked, + newType = newStandoffTagType) + } else if (typesToBeChecked.forall { + case nonPropertyTypeInfo: PropertyTypeInfo => nonPropertyTypeInfo.objectIsResourceType + case _ => false + }) { + // No, they're PropertyTypeInfo types with object types representing resource classes. + // Remove inconsistent types and replace with a common base class. + val commonBaseClassIri: SmartIri = + findCommonBaseClass(typesToBeChecked, InferenceRuleUtil.getResourceTypeIriForSchema(querySchema)) val newObjectType = PropertyTypeInfo(commonBaseClassIri, objectIsResourceType = true) - replaceInconsistentResourceTypes(acc, typedEntity, typesToBeChecked, newObjectType) - - // No. Don't touch the determined inconsistent types, later an error is returned for this. + replaceInconsistentTypes(acc = acc, + typedEntity = typedEntity, + typesToBeChecked = typesToBeChecked, + newType = newObjectType) + + } else if (typesToBeChecked.forall { + case nonPropertyTypeInfo: PropertyTypeInfo => nonPropertyTypeInfo.objectIsStandoffTagType + case _ => false + }) { + // No, they're PropertyTypeInfo types with object types representing standoff tag classes. + // Remove inconsistent types and replace with a common base class. + val commonBaseClassIri: SmartIri = + findCommonBaseClass(typesToBeChecked, OntologyConstants.KnoraApiV2Complex.StandoffTag.toSmartIri) + val newObjectType = PropertyTypeInfo(commonBaseClassIri, objectIsStandoffTagType = true) + replaceInconsistentTypes(acc = acc, + typedEntity = typedEntity, + typesToBeChecked = typesToBeChecked, + newType = newObjectType) } else { + // None of the above. Don't touch the determined inconsistent types, later an error is returned for this. acc } } From 1339b0c261cca1ddc968a543f7bc0b76136d1c7e Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Thu, 18 Feb 2021 17:10:46 +0100 Subject: [PATCH 22/33] test(gravsearch): Update toplogical reordering tests. --- ...cGravsearchToPrequeryTransformerSpec.scala | 1127 +++++++++++++++-- 1 file changed, 1050 insertions(+), 77 deletions(-) diff --git a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala index c46ef41327..a03be77e41 100644 --- a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala @@ -1795,9 +1795,376 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec | ?gnd2 knora-api:valueAsString "(DE-588)118696149" . |} ORDER BY ?date""".stripMargin + val transformedQueryToReorder: SelectQuery = SelectQuery( + fromClause = None, + variables = Vector( + QueryVariable(variableName = "letter"), + GroupConcat( + inputVariable = QueryVariable(variableName = "person1"), + separator = StringFormatter.INFORMATION_SEPARATOR_ONE, + outputVariableName = "person1__Concat" + ), + GroupConcat( + inputVariable = QueryVariable(variableName = "person2"), + separator = StringFormatter.INFORMATION_SEPARATOR_ONE, + outputVariableName = "person2__Concat" + ), + GroupConcat( + inputVariable = QueryVariable(variableName = "date"), + separator = StringFormatter.INFORMATION_SEPARATOR_ONE, + outputVariableName = "date__Concat" + ), + GroupConcat( + inputVariable = QueryVariable(variableName = "letter__linkingProp1__person1__LinkValue"), + separator = StringFormatter.INFORMATION_SEPARATOR_ONE, + outputVariableName = "letter__linkingProp1__person1__LinkValue__Concat" + ), + GroupConcat( + inputVariable = QueryVariable(variableName = "letter__linkingProp2__person2__LinkValue"), + separator = StringFormatter.INFORMATION_SEPARATOR_ONE, + outputVariableName = "letter__linkingProp2__person2__LinkValue__Concat" + ) + ), + offset = 0, + groupBy = Vector( + QueryVariable(variableName = "letter"), + QueryVariable(variableName = "date__valueHasStartJDN") + ), + orderBy = Vector( + OrderCriterion( + queryVariable = QueryVariable(variableName = "date__valueHasStartJDN"), + isAscending = true + ), + OrderCriterion( + queryVariable = QueryVariable(variableName = "letter"), + isAscending = true + ) + ), + whereClause = WhereClause( + patterns = Vector( + StatementPattern( + subj = QueryVariable(variableName = "gnd2"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#valueHasString".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "(DE-588)118696149", + datatype = "http://www.w3.org/2001/XMLSchema#string".toSmartIri + ), + namedGraph = None + ), + StatementPattern( + subj = QueryVariable(variableName = "person2"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "false", + datatype = "http://www.w3.org/2001/XMLSchema#boolean".toSmartIri + ), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = QueryVariable(variableName = "person2"), + pred = IriRef( + iri = "http://www.knora.org/ontology/0801/beol#hasIAFIdentifier".toSmartIri, + propertyPathOperator = None + ), + obj = QueryVariable(variableName = "gnd2"), + namedGraph = None + ), + StatementPattern( + subj = QueryVariable(variableName = "gnd2"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "false", + datatype = "http://www.w3.org/2001/XMLSchema#boolean".toSmartIri + ), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = QueryVariable(variableName = "letter"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "false", + datatype = "http://www.w3.org/2001/XMLSchema#boolean".toSmartIri + ), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = QueryVariable(variableName = "letter"), + pred = QueryVariable(variableName = "linkingProp2"), + obj = QueryVariable(variableName = "person2"), + namedGraph = None + ), + StatementPattern( + subj = QueryVariable(variableName = "letter"), + pred = QueryVariable(variableName = "linkingProp2__hasLinkToValue"), + obj = QueryVariable(variableName = "letter__linkingProp2__person2__LinkValue"), + namedGraph = None + ), + StatementPattern( + subj = QueryVariable(variableName = "letter__linkingProp2__person2__LinkValue"), + pred = IriRef( + iri = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type".toSmartIri, + propertyPathOperator = None + ), + obj = IriRef( + iri = "http://www.knora.org/ontology/knora-base#LinkValue".toSmartIri, + propertyPathOperator = None + ), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = QueryVariable(variableName = "letter__linkingProp2__person2__LinkValue"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "false", + datatype = "http://www.w3.org/2001/XMLSchema#boolean".toSmartIri + ), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = QueryVariable(variableName = "letter__linkingProp2__person2__LinkValue"), + pred = IriRef( + iri = "http://www.w3.org/1999/02/22-rdf-syntax-ns#object".toSmartIri, + propertyPathOperator = None + ), + obj = QueryVariable(variableName = "person2"), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = QueryVariable(variableName = "gnd1"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#valueHasString".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "(DE-588)118531379", + datatype = "http://www.w3.org/2001/XMLSchema#string".toSmartIri + ), + namedGraph = None + ), + StatementPattern( + subj = QueryVariable(variableName = "person1"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "false", + datatype = "http://www.w3.org/2001/XMLSchema#boolean".toSmartIri + ), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = QueryVariable(variableName = "person1"), + pred = IriRef( + iri = "http://www.knora.org/ontology/0801/beol#hasIAFIdentifier".toSmartIri, + propertyPathOperator = None + ), + obj = QueryVariable(variableName = "gnd1"), + namedGraph = None + ), + StatementPattern( + subj = QueryVariable(variableName = "gnd1"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "false", + datatype = "http://www.w3.org/2001/XMLSchema#boolean".toSmartIri + ), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = QueryVariable(variableName = "letter"), + pred = QueryVariable(variableName = "linkingProp1"), + obj = QueryVariable(variableName = "person1"), + namedGraph = None + ), + StatementPattern( + subj = QueryVariable(variableName = "letter"), + pred = QueryVariable(variableName = "linkingProp1__hasLinkToValue"), + obj = QueryVariable(variableName = "letter__linkingProp1__person1__LinkValue"), + namedGraph = None + ), + StatementPattern( + subj = QueryVariable(variableName = "letter__linkingProp1__person1__LinkValue"), + pred = IriRef( + iri = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type".toSmartIri, + propertyPathOperator = None + ), + obj = IriRef( + iri = "http://www.knora.org/ontology/knora-base#LinkValue".toSmartIri, + propertyPathOperator = None + ), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = QueryVariable(variableName = "letter__linkingProp1__person1__LinkValue"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "false", + datatype = "http://www.w3.org/2001/XMLSchema#boolean".toSmartIri + ), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = QueryVariable(variableName = "letter__linkingProp1__person1__LinkValue"), + pred = IriRef( + iri = "http://www.w3.org/1999/02/22-rdf-syntax-ns#object".toSmartIri, + propertyPathOperator = None + ), + obj = QueryVariable(variableName = "person1"), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = QueryVariable(variableName = "letter"), + pred = IriRef( + iri = "http://www.knora.org/ontology/0801/beol#creationDate".toSmartIri, + propertyPathOperator = None + ), + obj = QueryVariable(variableName = "date"), + namedGraph = None + ), + StatementPattern( + subj = QueryVariable(variableName = "date"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "false", + datatype = "http://www.w3.org/2001/XMLSchema#boolean".toSmartIri + ), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = QueryVariable(variableName = "date"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#valueHasStartJDN".toSmartIri, + propertyPathOperator = None + ), + obj = QueryVariable(variableName = "date__valueHasStartJDN"), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + FilterPattern( + expression = OrExpression( + leftArg = CompareExpression( + leftArg = QueryVariable(variableName = "linkingProp1"), + operator = CompareExpressionOperator.EQUALS, + rightArg = IriRef( + iri = "http://www.knora.org/ontology/0801/beol#hasAuthor".toSmartIri, + propertyPathOperator = None + ) + ), + rightArg = CompareExpression( + leftArg = QueryVariable(variableName = "linkingProp1"), + operator = CompareExpressionOperator.EQUALS, + rightArg = IriRef( + iri = "http://www.knora.org/ontology/0801/beol#hasRecipient".toSmartIri, + propertyPathOperator = None + ) + ) + )), + FilterPattern( + expression = OrExpression( + leftArg = CompareExpression( + leftArg = QueryVariable(variableName = "linkingProp2"), + operator = CompareExpressionOperator.EQUALS, + rightArg = IriRef( + iri = "http://www.knora.org/ontology/0801/beol#hasAuthor".toSmartIri, + propertyPathOperator = None + ) + ), + rightArg = CompareExpression( + leftArg = QueryVariable(variableName = "linkingProp2"), + operator = CompareExpressionOperator.EQUALS, + rightArg = IriRef( + iri = "http://www.knora.org/ontology/0801/beol#hasRecipient".toSmartIri, + propertyPathOperator = None + ) + ) + )) + ), + positiveEntities = Set(), + querySchema = None + ), + limit = Some(25), + useDistinct = true + ) + val queryToReorderWithCycle: String = """ |PREFIX anything: - |PREFIX knora-api: + |PREFIX knora-api: | |CONSTRUCT { | ?thing knora-api:isMainResource true . @@ -1807,6 +2174,275 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec | ?thing2 anything:hasOtherThing ?thing . |} """.stripMargin + val transformedQueryToReorderWithCycle: SelectQuery = SelectQuery( + fromClause = None, + variables = Vector(QueryVariable(variableName = "thing")), + offset = 0, + groupBy = Vector(QueryVariable(variableName = "thing")), + orderBy = Vector( + OrderCriterion( + queryVariable = QueryVariable(variableName = "thing"), + isAscending = true + )), + whereClause = WhereClause( + patterns = Vector( + StatementPattern( + subj = QueryVariable(variableName = "thing2"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "false", + datatype = "http://www.w3.org/2001/XMLSchema#boolean".toSmartIri + ), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = QueryVariable(variableName = "thing2"), + pred = IriRef( + iri = "http://www.knora.org/ontology/0001/anything#hasOtherThing".toSmartIri, + propertyPathOperator = None + ), + obj = QueryVariable(variableName = "thing"), + namedGraph = None + ), + StatementPattern( + subj = QueryVariable(variableName = "thing2"), + pred = IriRef( + iri = "http://www.knora.org/ontology/0001/anything#hasOtherThingValue".toSmartIri, + propertyPathOperator = None + ), + obj = + QueryVariable(variableName = "thing2__httpwwwknoraorgontology0001anythinghasOtherThing__thing__LinkValue"), + namedGraph = None + ), + StatementPattern( + subj = + QueryVariable(variableName = "thing2__httpwwwknoraorgontology0001anythinghasOtherThing__thing__LinkValue"), + pred = IriRef( + iri = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type".toSmartIri, + propertyPathOperator = None + ), + obj = IriRef( + iri = "http://www.knora.org/ontology/knora-base#LinkValue".toSmartIri, + propertyPathOperator = None + ), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = + QueryVariable(variableName = "thing2__httpwwwknoraorgontology0001anythinghasOtherThing__thing__LinkValue"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "false", + datatype = "http://www.w3.org/2001/XMLSchema#boolean".toSmartIri + ), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = + QueryVariable(variableName = "thing2__httpwwwknoraorgontology0001anythinghasOtherThing__thing__LinkValue"), + pred = IriRef( + iri = "http://www.w3.org/1999/02/22-rdf-syntax-ns#object".toSmartIri, + propertyPathOperator = None + ), + obj = QueryVariable(variableName = "thing"), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = QueryVariable(variableName = "thing"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "false", + datatype = "http://www.w3.org/2001/XMLSchema#boolean".toSmartIri + ), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = QueryVariable(variableName = "thing1"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "false", + datatype = "http://www.w3.org/2001/XMLSchema#boolean".toSmartIri + ), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = QueryVariable(variableName = "thing1"), + pred = IriRef( + iri = "http://www.knora.org/ontology/0001/anything#hasOtherThing".toSmartIri, + propertyPathOperator = None + ), + obj = QueryVariable(variableName = "thing2"), + namedGraph = None + ), + StatementPattern( + subj = QueryVariable(variableName = "thing1"), + pred = IriRef( + iri = "http://www.knora.org/ontology/0001/anything#hasOtherThingValue".toSmartIri, + propertyPathOperator = None + ), + obj = + QueryVariable(variableName = "thing1__httpwwwknoraorgontology0001anythinghasOtherThing__thing2__LinkValue"), + namedGraph = None + ), + StatementPattern( + subj = + QueryVariable(variableName = "thing1__httpwwwknoraorgontology0001anythinghasOtherThing__thing2__LinkValue"), + pred = IriRef( + iri = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type".toSmartIri, + propertyPathOperator = None + ), + obj = IriRef( + iri = "http://www.knora.org/ontology/knora-base#LinkValue".toSmartIri, + propertyPathOperator = None + ), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = + QueryVariable(variableName = "thing1__httpwwwknoraorgontology0001anythinghasOtherThing__thing2__LinkValue"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "false", + datatype = "http://www.w3.org/2001/XMLSchema#boolean".toSmartIri + ), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = + QueryVariable(variableName = "thing1__httpwwwknoraorgontology0001anythinghasOtherThing__thing2__LinkValue"), + pred = IriRef( + iri = "http://www.w3.org/1999/02/22-rdf-syntax-ns#object".toSmartIri, + propertyPathOperator = None + ), + obj = QueryVariable(variableName = "thing2"), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = QueryVariable(variableName = "thing"), + pred = IriRef( + iri = "http://www.knora.org/ontology/0001/anything#hasOtherThing".toSmartIri, + propertyPathOperator = None + ), + obj = QueryVariable(variableName = "thing1"), + namedGraph = None + ), + StatementPattern( + subj = QueryVariable(variableName = "thing"), + pred = IriRef( + iri = "http://www.knora.org/ontology/0001/anything#hasOtherThingValue".toSmartIri, + propertyPathOperator = None + ), + obj = + QueryVariable(variableName = "thing__httpwwwknoraorgontology0001anythinghasOtherThing__thing1__LinkValue"), + namedGraph = None + ), + StatementPattern( + subj = + QueryVariable(variableName = "thing__httpwwwknoraorgontology0001anythinghasOtherThing__thing1__LinkValue"), + pred = IriRef( + iri = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type".toSmartIri, + propertyPathOperator = None + ), + obj = IriRef( + iri = "http://www.knora.org/ontology/knora-base#LinkValue".toSmartIri, + propertyPathOperator = None + ), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = + QueryVariable(variableName = "thing__httpwwwknoraorgontology0001anythinghasOtherThing__thing1__LinkValue"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "false", + datatype = "http://www.w3.org/2001/XMLSchema#boolean".toSmartIri + ), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = + QueryVariable(variableName = "thing__httpwwwknoraorgontology0001anythinghasOtherThing__thing1__LinkValue"), + pred = IriRef( + iri = "http://www.w3.org/1999/02/22-rdf-syntax-ns#object".toSmartIri, + propertyPathOperator = None + ), + obj = QueryVariable(variableName = "thing1"), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ) + ), + positiveEntities = Set(), + querySchema = None + ), + limit = Some(25), + useDistinct = true + ) + val queryToReorderWithMinus: String = """PREFIX knora-api: |PREFIX anything: @@ -1818,12 +2454,101 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec | ?thing a anything:Thing . | MINUS { | ?thing anything:hasInteger ?intVal . - | anything:hasInteger knora-api:objectType xsd:integer . | ?intVal a xsd:integer . | FILTER(?intVal = 123454321 || ?intVal = 999999999) | } |}""".stripMargin + val transformedQueryToReorderWithMinus: SelectQuery = SelectQuery( + fromClause = None, + variables = Vector(QueryVariable(variableName = "thing")), + offset = 0, + groupBy = Vector(QueryVariable(variableName = "thing")), + orderBy = Vector( + OrderCriterion( + queryVariable = QueryVariable(variableName = "thing"), + isAscending = true + )), + whereClause = WhereClause( + patterns = Vector( + MinusPattern(patterns = Vector( + StatementPattern( + subj = QueryVariable(variableName = "thing"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "false", + datatype = "http://www.w3.org/2001/XMLSchema#boolean".toSmartIri + ), + namedGraph = Some(IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = QueryVariable(variableName = "thing"), + pred = IriRef( + iri = "http://www.knora.org/ontology/0001/anything#hasInteger".toSmartIri, + propertyPathOperator = None + ), + obj = QueryVariable(variableName = "intVal"), + namedGraph = None + ), + StatementPattern( + subj = QueryVariable(variableName = "intVal"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "false", + datatype = "http://www.w3.org/2001/XMLSchema#boolean".toSmartIri + ), + namedGraph = Some(IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = QueryVariable(variableName = "intVal"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#valueHasInteger".toSmartIri, + propertyPathOperator = None + ), + obj = QueryVariable(variableName = "intVal__valueHasInteger"), + namedGraph = Some(IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + FilterPattern(expression = OrExpression( + leftArg = CompareExpression( + leftArg = QueryVariable(variableName = "intVal__valueHasInteger"), + operator = CompareExpressionOperator.EQUALS, + rightArg = XsdLiteral( + value = "123454321", + datatype = "http://www.w3.org/2001/XMLSchema#integer".toSmartIri + ) + ), + rightArg = CompareExpression( + leftArg = QueryVariable(variableName = "intVal__valueHasInteger"), + operator = CompareExpressionOperator.EQUALS, + rightArg = XsdLiteral( + value = "999999999", + datatype = "http://www.w3.org/2001/XMLSchema#integer".toSmartIri + ) + ) + )) + ))), + positiveEntities = Set(), + querySchema = None + ), + limit = Some(25), + useDistinct = true + ) + val queryToReorderWithUnion: String = s"""PREFIX knora-api: |PREFIX anything: @@ -1835,6 +2560,7 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec |} WHERE { | ?thing a knora-api:Resource . | ?thing a anything:Thing . + | ?thing anything:hasInteger ?int . | | { | ?thing anything:hasRichtext ?richtext . @@ -1848,12 +2574,302 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec | ?thing anything:hasText ?text . | FILTER knora-api:matchText(?text, "test") | - | ?thing anything:hasInteger ?int . - | ?int knora-api:intValueAsInt 3 . + | ?thing anything:hasInteger ?int . + | ?int knora-api:intValueAsInt 3 . | } |} |ORDER BY (?int)""".stripMargin + val transformedQueryToReorderWithUnion: SelectQuery = SelectQuery( + fromClause = None, + variables = Vector( + QueryVariable(variableName = "thing"), + GroupConcat( + inputVariable = QueryVariable(variableName = "int"), + separator = StringFormatter.INFORMATION_SEPARATOR_ONE, + outputVariableName = "int__Concat" + ), + GroupConcat( + inputVariable = QueryVariable(variableName = "richtext"), + separator = StringFormatter.INFORMATION_SEPARATOR_ONE, + outputVariableName = "richtext__Concat" + ), + GroupConcat( + inputVariable = QueryVariable(variableName = "text"), + separator = StringFormatter.INFORMATION_SEPARATOR_ONE, + outputVariableName = "text__Concat" + ) + ), + offset = 0, + groupBy = Vector( + QueryVariable(variableName = "thing"), + QueryVariable(variableName = "int__valueHasInteger") + ), + orderBy = Vector( + OrderCriterion( + queryVariable = QueryVariable(variableName = "int__valueHasInteger"), + isAscending = true + ), + OrderCriterion( + queryVariable = QueryVariable(variableName = "thing"), + isAscending = true + ) + ), + whereClause = WhereClause( + patterns = Vector( + StatementPattern( + subj = QueryVariable(variableName = "thing"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "false", + datatype = "http://www.w3.org/2001/XMLSchema#boolean".toSmartIri + ), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = QueryVariable(variableName = "thing"), + pred = IriRef( + iri = "http://www.knora.org/ontology/0001/anything#hasInteger".toSmartIri, + propertyPathOperator = None + ), + obj = QueryVariable(variableName = "int"), + namedGraph = None + ), + StatementPattern( + subj = QueryVariable(variableName = "int"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "false", + datatype = "http://www.w3.org/2001/XMLSchema#boolean".toSmartIri + ), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = QueryVariable(variableName = "int"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#valueHasInteger".toSmartIri, + propertyPathOperator = None + ), + obj = QueryVariable(variableName = "int__valueHasInteger"), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + UnionPattern( + blocks = Vector( + Vector( + StatementPattern( + subj = QueryVariable(variableName = "thing"), + pred = IriRef( + iri = "http://www.knora.org/ontology/0001/anything#hasRichtext".toSmartIri, + propertyPathOperator = None + ), + obj = QueryVariable(variableName = "richtext"), + namedGraph = None + ), + StatementPattern( + subj = QueryVariable(variableName = "richtext"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "false", + datatype = "http://www.w3.org/2001/XMLSchema#boolean".toSmartIri + ), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = QueryVariable(variableName = "int"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#valueHasInteger".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "1", + datatype = "http://www.w3.org/2001/XMLSchema#integer".toSmartIri + ), + namedGraph = None + ), + StatementPattern( + subj = QueryVariable(variableName = "thing"), + pred = IriRef( + iri = "http://www.knora.org/ontology/0001/anything#hasInteger".toSmartIri, + propertyPathOperator = None + ), + obj = QueryVariable(variableName = "int"), + namedGraph = None + ), + StatementPattern( + subj = QueryVariable(variableName = "int"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "false", + datatype = "http://www.w3.org/2001/XMLSchema#boolean".toSmartIri + ), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = QueryVariable(variableName = "int"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#valueHasInteger".toSmartIri, + propertyPathOperator = None + ), + obj = QueryVariable(variableName = "int__valueHasInteger"), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + LuceneQueryPattern( + subj = QueryVariable(variableName = "richtext"), + obj = QueryVariable(variableName = "richtext__valueHasString"), + queryString = LuceneQueryString(queryString = "test"), + literalStatement = Some(StatementPattern( + subj = QueryVariable(variableName = "richtext"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#valueHasString".toSmartIri, + propertyPathOperator = None + ), + obj = QueryVariable(variableName = "richtext__valueHasString"), + namedGraph = Some(IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + )) + ) + ), + Vector( + StatementPattern( + subj = QueryVariable(variableName = "thing"), + pred = IriRef( + iri = "http://www.knora.org/ontology/0001/anything#hasText".toSmartIri, + propertyPathOperator = None + ), + obj = QueryVariable(variableName = "text"), + namedGraph = None + ), + StatementPattern( + subj = QueryVariable(variableName = "text"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "false", + datatype = "http://www.w3.org/2001/XMLSchema#boolean".toSmartIri + ), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = QueryVariable(variableName = "int"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#valueHasInteger".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "3", + datatype = "http://www.w3.org/2001/XMLSchema#integer".toSmartIri + ), + namedGraph = None + ), + StatementPattern( + subj = QueryVariable(variableName = "thing"), + pred = IriRef( + iri = "http://www.knora.org/ontology/0001/anything#hasInteger".toSmartIri, + propertyPathOperator = None + ), + obj = QueryVariable(variableName = "int"), + namedGraph = None + ), + StatementPattern( + subj = QueryVariable(variableName = "int"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "false", + datatype = "http://www.w3.org/2001/XMLSchema#boolean".toSmartIri + ), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + StatementPattern( + subj = QueryVariable(variableName = "int"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#valueHasInteger".toSmartIri, + propertyPathOperator = None + ), + obj = QueryVariable(variableName = "int__valueHasInteger"), + namedGraph = Some( + IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + ), + LuceneQueryPattern( + subj = QueryVariable(variableName = "text"), + obj = QueryVariable(variableName = "text__valueHasString"), + queryString = LuceneQueryString(queryString = "test"), + literalStatement = Some(StatementPattern( + subj = QueryVariable(variableName = "text"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#valueHasString".toSmartIri, + propertyPathOperator = None + ), + obj = QueryVariable(variableName = "text__valueHasString"), + namedGraph = Some(IriRef( + iri = "http://www.knora.org/explicit".toSmartIri, + propertyPathOperator = None + )) + )) + ) + ) + )) + ), + positiveEntities = Set(), + querySchema = None + ), + limit = Some(25), + useDistinct = true + ) + val queryWithStandoffTagHasStartAncestor: String = """ |PREFIX knora-api: @@ -2197,78 +3213,35 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec assert(transformedQuery === transformedQueryWithStandoffTagHasStartAncestor) } -// -// "reorder query patterns in where clause" in { -// val constructQuery = GravsearchParser.parseQuery(queryToReorder) -// val sortedPatterns = QueryHandler.reorderPatternsByDependency(constructQuery.whereClause.patterns) -// val statements = sortedPatterns.filter(p => p.isInstanceOf[StatementPattern]) -// // check that statements are brought to top -// val topElements = sortedPatterns.slice(0, statements.size) -// assert(topElements.equals(statements)) -// // check the order of statements -// val mainResourceStatements = statements.slice(statements.size - 3, statements.size) -// assert(!mainResourceStatements.exists(p => p.asInstanceOf[StatementPattern].subj.toSparql !== "?letter")) -// } -// "reorder query patterns in where clause with union" in { -// val constructQuery = GravsearchParser.parseQuery(queryToReorderWithUnion) -// val sortedPatterns = QueryHandler.reorderPatternsByDependency(constructQuery.whereClause.patterns) -// val statements = sortedPatterns.filter(p => p.isInstanceOf[StatementPattern]) -// // check that statements are brought to top -// val topElements = sortedPatterns.slice(0, statements.size) -// assert(topElements.equals(statements)) -// //check order of statements in each block -// val unionPattern: Seq[UnionPattern] = sortedPatterns.collect { case pattern: UnionPattern => pattern } -// val firstBlock = unionPattern.head.blocks.head -// val firstBlockStatements = firstBlock.collect { -// case pattern: StatementPattern => pattern -// } -// assert(firstBlockStatements.head.subj.toSparql == "?int") -// assert(firstBlockStatements.head.obj.asInstanceOf[XsdLiteral].value == "1") -// assert(firstBlockStatements.last.subj.toSparql == "?thing") -// val secondBlock = unionPattern.head.blocks.last -// val secondBlockStatements = secondBlock.collect { -// case pattern: StatementPattern => pattern -// } -// assert(secondBlockStatements.head.subj.toSparql == "?int") -// assert(secondBlockStatements.head.obj.asInstanceOf[XsdLiteral].value == "3") -// assert(secondBlockStatements.last.subj.toSparql == "?thing") -// } -// -// "reorder query patterns in where clause with optional" in { -// val constructQuery = GravsearchParser.parseQuery(queryWithOptional) -// val sortedPatterns = QueryHandler.reorderPatternsByDependency(constructQuery.whereClause.patterns) -// val statements = sortedPatterns.filter(p => p.isInstanceOf[StatementPattern]) -// // check that statements are brought to top -// val topElements = sortedPatterns.slice(0, statements.size) -// assert(topElements.equals(statements)) -// // check statements inside optional pattern -// val optionalPattern: Seq[OptionalPattern] = sortedPatterns.collect { case pattern: OptionalPattern => pattern } -// val optionalPatternStatements = optionalPattern.head.patterns.collect { -// case pattern: StatementPattern => pattern -// } -// assert(optionalPatternStatements.last.subj.toSparql == "?document") -// assert(optionalPatternStatements.last.obj.toSparql == "?recipient") -// } -// -// "reorder query patterns with minus scope" in { -// val constructQuery = GravsearchParser.parseQuery(queryToReorderWithMinus) -// val sortedPatterns = QueryHandler.reorderPatternsByDependency(constructQuery.whereClause.patterns) -// val statements = sortedPatterns.filter(p => p.isInstanceOf[StatementPattern]) -// // check that statements are brought to top -// val topElements = sortedPatterns.slice(0, statements.size) -// assert(topElements.equals(statements)) -// // check statements inside minus pattern -// val minusPattern: Seq[MinusPattern] = sortedPatterns.collect { case pattern: MinusPattern => pattern } -// val minusPatternStatements = minusPattern.head.patterns.collect { case pattern: StatementPattern => pattern } -// assert(minusPatternStatements.last.subj.toSparql == "?thing") -// assert(minusPatternStatements.last.obj.toSparql == "?intVal") -// } -// "reorder a query with a cycle" in { -// val constructQuery = GravsearchParser.parseQuery(queryToReorderWithCycle) -// val sortedPatterns = QueryHandler.reorderPatternsByDependency(constructQuery.whereClause.patterns) -// // checks that all statement patterns which created a cycle are returned -// val statements = sortedPatterns.filter(p => p.isInstanceOf[StatementPattern]) -// assert(statements.size === 3) -// } + + "reorder query patterns in where clause" in { + val transformedQuery = QueryHandler.transformQuery(queryToReorder, responderData, settings) + + assert(transformedQuery === transformedQueryToReorder) + } + + "reorder query patterns in where clause with union" in { + val transformedQuery = QueryHandler.transformQuery(queryToReorderWithUnion, responderData, settings) + + assert(transformedQuery === transformedQueryToReorderWithUnion) + } + + "reorder query patterns in where clause with optional" in { + val transformedQuery = QueryHandler.transformQuery(queryWithOptional, responderData, settings) + + assert(transformedQuery === TransformedQueryWithOptional) + } + + "reorder query patterns with minus scope" in { + val transformedQuery = QueryHandler.transformQuery(queryToReorderWithMinus, responderData, settings) + + assert(transformedQuery == transformedQueryToReorderWithMinus) + } + + "reorder a query with a cycle" in { + val transformedQuery = QueryHandler.transformQuery(queryToReorderWithCycle, responderData, settings) + + assert(transformedQuery == transformedQueryToReorderWithCycle) + } } } From 47e74d4de0878e4aa9411f866ed15e53db2888d5 Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Fri, 19 Feb 2021 13:34:28 +0100 Subject: [PATCH 23/33] test(gravsearch): Fix test. --- .../types/GravsearchTypeInspectorSpec.scala | 217 +++++++++++++++--- 1 file changed, 179 insertions(+), 38 deletions(-) diff --git a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/types/GravsearchTypeInspectorSpec.scala b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/types/GravsearchTypeInspectorSpec.scala index 9acdec81f7..4a0af7ab57 100644 --- a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/types/GravsearchTypeInspectorSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/types/GravsearchTypeInspectorSpec.scala @@ -767,86 +767,198 @@ class GravsearchTypeInspectorSpec extends CoreSpec() with ImplicitSender { entities = Map( TypeableVariable(variableName = "book4") -> NonPropertyTypeInfo( typeIri = "http://0.0.0.0:3333/ontology/0803/incunabula/simple/v2#book".toSmartIri, - isResourceType = true), + isResourceType = true, + isValueType = false, + isStandoffTagType = false + ), TypeableVariable(variableName = "titleProp1") -> PropertyTypeInfo( objectTypeIri = "http://www.w3.org/2001/XMLSchema#string".toSmartIri, - objectIsValueType = true), + objectIsResourceType = false, + objectIsValueType = true, + objectIsStandoffTagType = false + ), TypeableVariable(variableName = "page1") -> NonPropertyTypeInfo( typeIri = "http://0.0.0.0:3333/ontology/0803/incunabula/simple/v2#page".toSmartIri, - isResourceType = true), + isResourceType = true, + isValueType = false, + isStandoffTagType = false + ), TypeableVariable(variableName = "book1") -> NonPropertyTypeInfo( typeIri = "http://0.0.0.0:3333/ontology/0803/incunabula/simple/v2#book".toSmartIri, - isResourceType = true), + isResourceType = true, + isValueType = false, + isStandoffTagType = false + ), TypeableVariable(variableName = "titleProp2") -> PropertyTypeInfo( objectTypeIri = "http://www.w3.org/2001/XMLSchema#string".toSmartIri, - objectIsValueType = true), + objectIsResourceType = false, + objectIsValueType = true, + objectIsStandoffTagType = false + ), TypeableVariable(variableName = "page3") -> NonPropertyTypeInfo( typeIri = "http://0.0.0.0:3333/ontology/0803/incunabula/simple/v2#page".toSmartIri, - isResourceType = true), + isResourceType = true, + isValueType = false, + isStandoffTagType = false + ), TypeableIri(iri = "http://0.0.0.0:3333/ontology/0803/incunabula/simple/v2#partOf".toSmartIri) -> PropertyTypeInfo( objectTypeIri = "http://0.0.0.0:3333/ontology/0803/incunabula/simple/v2#book".toSmartIri, - objectIsResourceType = true), + objectIsResourceType = true, + objectIsValueType = false, + objectIsStandoffTagType = false + ), TypeableIri(iri = "http://rdfh.ch/1749ad09ac06".toSmartIri) -> NonPropertyTypeInfo( typeIri = "http://api.knora.org/ontology/knora-api/simple/v2#Region".toSmartIri, - isResourceType = true), + isResourceType = true, + isValueType = false, + isStandoffTagType = false + ), TypeableVariable(variableName = "linkObj") -> NonPropertyTypeInfo( typeIri = "http://api.knora.org/ontology/knora-api/simple/v2#Resource".toSmartIri, - isResourceType = true), + isResourceType = true, + isValueType = false, + isStandoffTagType = false + ), TypeableVariable(variableName = "title2") -> NonPropertyTypeInfo( typeIri = "http://www.w3.org/2001/XMLSchema#string".toSmartIri, - isValueType = true), + isResourceType = false, + isValueType = true, + isStandoffTagType = false + ), TypeableIri(iri = "http://rdfh.ch/52431ecfab06".toSmartIri) -> NonPropertyTypeInfo( typeIri = "http://api.knora.org/ontology/knora-api/simple/v2#Region".toSmartIri, - isResourceType = true), + isResourceType = true, + isValueType = false, + isStandoffTagType = false + ), TypeableVariable(variableName = "title3") -> NonPropertyTypeInfo( typeIri = "http://www.w3.org/2001/XMLSchema#string".toSmartIri, - isValueType = true), + isResourceType = false, + isValueType = true, + isStandoffTagType = false + ), TypeableIri(iri = "http://rdfh.ch/dc4e3c44ac06".toSmartIri) -> NonPropertyTypeInfo( typeIri = "http://api.knora.org/ontology/knora-api/simple/v2#Region".toSmartIri, - isResourceType = true), + isResourceType = true, + isValueType = false, + isStandoffTagType = false + ), TypeableIri(iri = "http://api.knora.org/ontology/knora-api/simple/v2#isRegionOf".toSmartIri) -> PropertyTypeInfo( - objectTypeIri = "http://0.0.0.0:3333/ontology/0803/incunabula/simple/v2#page".toSmartIri, - objectIsResourceType = true), + objectTypeIri = "http://api.knora.org/ontology/knora-api/simple/v2#Representation".toSmartIri, + objectIsResourceType = true, + objectIsValueType = false, + objectIsStandoffTagType = false + ), TypeableVariable(variableName = "page2") -> NonPropertyTypeInfo( typeIri = "http://0.0.0.0:3333/ontology/0803/incunabula/simple/v2#page".toSmartIri, - isResourceType = true), + isResourceType = true, + isValueType = false, + isStandoffTagType = false + ), TypeableVariable(variableName = "page4") -> NonPropertyTypeInfo( typeIri = "http://0.0.0.0:3333/ontology/0803/incunabula/simple/v2#page".toSmartIri, - isResourceType = true), + isResourceType = true, + isValueType = false, + isStandoffTagType = false + ), TypeableIri(iri = "http://api.knora.org/ontology/knora-api/simple/v2#hasLinkTo".toSmartIri) -> PropertyTypeInfo( - objectTypeIri = "http://api.knora.org/ontology/knora-api/simple/v2#Region".toSmartIri, - objectIsResourceType = true), + objectTypeIri = "http://api.knora.org/ontology/knora-api/simple/v2#Resource".toSmartIri, + objectIsResourceType = true, + objectIsValueType = false, + objectIsStandoffTagType = false + ), TypeableVariable(variableName = "titleProp4") -> PropertyTypeInfo( objectTypeIri = "http://www.w3.org/2001/XMLSchema#string".toSmartIri, - objectIsValueType = true), + objectIsResourceType = false, + objectIsValueType = true, + objectIsStandoffTagType = false + ), TypeableIri(iri = "http://rdfh.ch/8d3d8f94ab06".toSmartIri) -> NonPropertyTypeInfo( typeIri = "http://api.knora.org/ontology/knora-api/simple/v2#Region".toSmartIri, - isResourceType = true), + isResourceType = true, + isValueType = false, + isStandoffTagType = false + ), TypeableVariable(variableName = "title1") -> NonPropertyTypeInfo( typeIri = "http://www.w3.org/2001/XMLSchema#string".toSmartIri, - isValueType = true), + isResourceType = false, + isValueType = true, + isStandoffTagType = false + ), TypeableVariable(variableName = "titleProp3") -> PropertyTypeInfo( objectTypeIri = "http://www.w3.org/2001/XMLSchema#string".toSmartIri, - objectIsValueType = true), + objectIsResourceType = false, + objectIsValueType = true, + objectIsStandoffTagType = false + ), TypeableVariable(variableName = "linkProp2") -> PropertyTypeInfo( objectTypeIri = "http://0.0.0.0:3333/ontology/0803/incunabula/simple/v2#page".toSmartIri, - objectIsResourceType = true), + objectIsResourceType = true, + objectIsValueType = false, + objectIsStandoffTagType = false + ), TypeableVariable(variableName = "partOfProp") -> PropertyTypeInfo( objectTypeIri = "http://0.0.0.0:3333/ontology/0803/incunabula/simple/v2#book".toSmartIri, - objectIsResourceType = true), + objectIsResourceType = true, + objectIsValueType = false, + objectIsStandoffTagType = false + ), TypeableVariable(variableName = "title4") -> NonPropertyTypeInfo( typeIri = "http://www.w3.org/2001/XMLSchema#string".toSmartIri, - isValueType = true), + isResourceType = false, + isValueType = true, + isStandoffTagType = false + ), TypeableVariable(variableName = "book3") -> NonPropertyTypeInfo( typeIri = "http://0.0.0.0:3333/ontology/0803/incunabula/simple/v2#book".toSmartIri, - isResourceType = true), + isResourceType = true, + isValueType = false, + isStandoffTagType = false + ), TypeableVariable(variableName = "linkProp1") -> PropertyTypeInfo( objectTypeIri = "http://api.knora.org/ontology/knora-api/simple/v2#Region".toSmartIri, - objectIsResourceType = true), + objectIsResourceType = true, + objectIsValueType = false, + objectIsStandoffTagType = false + ), TypeableVariable(variableName = "book2") -> NonPropertyTypeInfo( typeIri = "http://0.0.0.0:3333/ontology/0803/incunabula/simple/v2#book".toSmartIri, - isResourceType = true) - )) + isResourceType = true, + isValueType = false, + isStandoffTagType = false + ) + ), + entitiesInferredFromProperties = Map( + TypeableVariable(variableName = "page1") -> Set( + NonPropertyTypeInfo( + typeIri = "http://0.0.0.0:3333/ontology/0803/incunabula/simple/v2#page".toSmartIri, + isResourceType = true, + isValueType = false, + isStandoffTagType = false + )), + TypeableVariable(variableName = "linkObj") -> Set( + NonPropertyTypeInfo( + typeIri = "http://api.knora.org/ontology/knora-api/simple/v2#Resource".toSmartIri, + isResourceType = true, + isValueType = false, + isStandoffTagType = false + )), + TypeableIri(iri = "http://rdfh.ch/8d3d8f94ab06".toSmartIri) -> Set( + NonPropertyTypeInfo( + typeIri = "http://api.knora.org/ontology/knora-api/simple/v2#Region".toSmartIri, + isResourceType = true, + isValueType = false, + isStandoffTagType = false + )), + TypeableVariable(variableName = "book1") -> Set( + NonPropertyTypeInfo( + typeIri = "http://0.0.0.0:3333/ontology/0803/incunabula/simple/v2#book".toSmartIri, + isResourceType = true, + isValueType = false, + isStandoffTagType = false + )) + ) + ) val TypeInferenceResult1: GravsearchTypeInspectionResult = GravsearchTypeInspectionResult( entities = Map( @@ -984,21 +1096,50 @@ class GravsearchTypeInspectorSpec extends CoreSpec() with ImplicitSender { entities = Map( TypeableIri(iri = "http://0.0.0.0:3333/ontology/0801/beol/v2#hasText".toSmartIri) -> PropertyTypeInfo( objectTypeIri = "http://api.knora.org/ontology/knora-api/v2#TextValue".toSmartIri, - objectIsValueType = true), + objectIsResourceType = false, + objectIsValueType = true, + objectIsStandoffTagType = false + ), TypeableVariable(variableName = "text") -> NonPropertyTypeInfo( typeIri = "http://api.knora.org/ontology/knora-api/v2#TextValue".toSmartIri, - isValueType = true), + isResourceType = false, + isValueType = true, + isStandoffTagType = false + ), TypeableVariable(variableName = "letter") -> NonPropertyTypeInfo( typeIri = "http://0.0.0.0:3333/ontology/0801/beol/v2#letter".toSmartIri, - isResourceType = true), + isResourceType = true, + isValueType = false, + isStandoffTagType = false + ), TypeableIri(iri = "http://rdfh.ch/biblio/up0Q0ZzPSLaULC2tlTs1sA".toSmartIri) -> NonPropertyTypeInfo( typeIri = "http://api.knora.org/ontology/knora-api/v2#Resource".toSmartIri, - isResourceType = true), + isResourceType = true, + isValueType = false, + isStandoffTagType = false + ), TypeableIri(iri = "http://api.knora.org/ontology/knora-api/v2#textValueHasStandoff".toSmartIri) -> PropertyTypeInfo( - objectTypeIri = "http://api.knora.org/ontology/knora-api/v2#StandoffTag".toSmartIri), + objectTypeIri = "http://api.knora.org/ontology/knora-api/v2#StandoffTag".toSmartIri, + objectIsResourceType = false, + objectIsValueType = false, + objectIsStandoffTagType = true + ), TypeableVariable(variableName = "standoffLinkTag") -> NonPropertyTypeInfo( - typeIri = "http://api.knora.org/ontology/knora-api/v2#StandoffTag".toSmartIri) - )) + typeIri = "http://api.knora.org/ontology/knora-api/v2#StandoffLinkTag".toSmartIri, + isResourceType = false, + isValueType = false, + isStandoffTagType = true + ) + ), + entitiesInferredFromProperties = Map( + TypeableVariable(variableName = "text") -> Set( + NonPropertyTypeInfo( + typeIri = "http://api.knora.org/ontology/knora-api/v2#TextValue".toSmartIri, + isResourceType = false, + isValueType = true, + isStandoffTagType = false + ))) + ) val QueryWithRdfsLabelAndLiteral: String = """ @@ -1508,7 +1649,7 @@ class GravsearchTypeInspectorSpec extends CoreSpec() with ImplicitSender { assert(result.entities == TypeInferenceResult6.entities) } - "infer the types in a query that requires 6 iterations" in { + "infer the types in a query that requires 6 iterations test1" in { val typeInspectionRunner = new GravsearchTypeInspectionRunner(responderData = responderData, inferTypes = true) val parsedQuery = GravsearchParser.parseQuery(PathologicalQuery) val resultFuture: Future[GravsearchTypeInspectionResult] = From 2bf4b0aa29be9478d818ec0d84e010b79897d8a8 Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Fri, 19 Feb 2021 16:28:21 +0100 Subject: [PATCH 24/33] feat(gravsearch): Add feature toggle for topological sort optimisation. --- webapi/src/main/resources/application.conf | 15 + .../prequery/AbstractPrequeryGenerator.scala | 278 ------------ .../GravsearchQueryOptimisationFactory.scala | 395 ++++++++++++++++++ ...GravsearchToCountPrequeryTransformer.scala | 12 +- ...cificGravsearchToPrequeryTransformer.scala | 12 +- .../responders/v2/SearchResponderV2.scala | 6 +- ...searchToCountPrequeryTransformerSpec.scala | 15 +- ...cGravsearchToPrequeryTransformerSpec.scala | 124 ++++-- 8 files changed, 538 insertions(+), 319 deletions(-) create mode 100644 webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/GravsearchQueryOptimisationFactory.scala diff --git a/webapi/src/main/resources/application.conf b/webapi/src/main/resources/application.conf index 749d1dfb0d..ac560f2b6b 100644 --- a/webapi/src/main/resources/application.conf +++ b/webapi/src/main/resources/application.conf @@ -292,6 +292,21 @@ app { "Benjamin Geer " ] } + + gravsearch-dependency-optimisation { + description = "Optimise Gravsearch queries by reordering query patterns according to their dependencies." + + available-versions = [ 1 ] + default-version = 1 + enabled-by-default = yes + override-allowed = yes + expiration-date = "2021-12-01T00:00:00Z" + + developer-emails = [ + "Sepideh Alassi " + "Benjamin Geer " + ] + } } shacl { diff --git a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/AbstractPrequeryGenerator.scala b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/AbstractPrequeryGenerator.scala index 5d4b2aa000..60bd9718a0 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/AbstractPrequeryGenerator.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/AbstractPrequeryGenerator.scala @@ -28,8 +28,6 @@ import org.knora.webapi.messages.util.search.gravsearch.types._ import org.knora.webapi.messages.v2.responder.valuemessages.DateValueContentV2 import org.knora.webapi.messages.{OntologyConstants, SmartIri, StringFormatter} import org.knora.webapi.util.ApacheLuceneSupport.LuceneQueryString -import scalax.collection.Graph -import scalax.collection.GraphEdge.DiHyperEdge import scala.collection.mutable @@ -2039,280 +2037,4 @@ abstract class AbstractPrequeryGenerator(constructClause: ConstructClause, } } - - /** - * Optimises a query by removing `rdf:type` statements that are known to be redundant. A redundant - * `rdf:type` statement gives the type of a variable whose type is already restricted by its - * use with a property that can only be used with that type (unless the property - * statement is in an `OPTIONAL` block). - * - * @param patterns the query patterns. - * @return the optimised query patterns. - */ - protected def removeEntitiesInferredFromProperty(patterns: Seq[QueryPattern]): Seq[QueryPattern] = { - - // Collect all entities which are used as subject or object of an OptionalPattern. - val optionalEntities: Seq[TypeableEntity] = patterns - .collect { - case optionalPattern: OptionalPattern => optionalPattern - } - .flatMap { - case optionalPattern: OptionalPattern => - optionalPattern.patterns.flatMap { - case pattern: StatementPattern => - GravsearchTypeInspectionUtil.maybeTypeableEntity(pattern.subj) ++ GravsearchTypeInspectionUtil - .maybeTypeableEntity(pattern.obj) - - case _ => None - } - - case _ => None - } - - // Remove statements whose predicate is rdf:type, type of subject is inferred from a property, - // and the subject is not in optionalEntities. - patterns.filterNot { - case statementPattern: StatementPattern => - // Is the predicate an IRI? - statementPattern.pred match { - case predicateIriRef: IriRef => - // Yes. Is this an rdf:type statement? - if (predicateIriRef.iri.toString == OntologyConstants.Rdf.Type) { - // Yes. Is the subject a typeable entity? - val subjectAsTypeableEntity = GravsearchTypeInspectionUtil.maybeTypeableEntity(statementPattern.subj) - - subjectAsTypeableEntity match { - case Some(typeableEntity) => - // Yes. Was the type of the subject inferred from another predicate? - if (typeInspectionResult.entitiesInferredFromProperties.keySet.contains(typeableEntity)) { - // Yes. Is the subject in optional entities? - if (optionalEntities.contains(typeableEntity)) { - // Yes. Keep the statement. - false - } else { - // Remove the statement. - true - } - } else { - // The type of the subject was not inferred from another predicate. Keep the statement. - false - } - - case _ => - // The subject isn't a typeable entity. Keep the statement. - false - } - } else { - // This isn't an rdf:type statement. Keep it. - false - } - - case _ => - // The predicate isn't an IRI. Keep the statement. - false - } - - case _ => - // This isn't a statement pattern. Keep it. - false - } - } - - private def createAndSortGraph(statementPatterns: Seq[StatementPattern]): Seq[QueryPattern] = { - @scala.annotation.tailrec - def makeGraphWithoutCycles(graphComponents: Seq[(String, String)]): Graph[String, DiHyperEdge] = { - val graph = graphComponents.foldLeft(Graph.empty[String, DiHyperEdge]) { (graph, edgeDef) => - val edge = DiHyperEdge(edgeDef._1, edgeDef._2) - graph + edge // add nodes and edges to graph - } - - if (graph.isCyclic) { - // get the cycle - val cycle: graph.Cycle = graph.findCycle.get - - // the cyclic node is the one that cycle starts and ends with - val cyclicNode: graph.NodeT = cycle.endNode - val cyclicEdge: graph.EdgeT = cyclicNode.edges.last - val originNodeOfCyclicEdge: String = cyclicEdge._1.value - val TargetNodeOfCyclicEdge: String = cyclicEdge._2.value - val graphComponentsWithOutCycle = - graphComponents.filterNot(edgeDef => edgeDef.equals((originNodeOfCyclicEdge, TargetNodeOfCyclicEdge))) - - makeGraphWithoutCycles(graphComponentsWithOutCycle) - } else { - graph - } - } - - def createGraph: Graph[String, DiHyperEdge] = { - val graphComponents: Seq[(String, String)] = statementPatterns.map { statementPattern => - // transform every statementPattern to pair of nodes that will consist an edge. - val node1 = statementPattern.subj.toSparql - val node2 = statementPattern.obj.toSparql - (node1, node2) - } - - makeGraphWithoutCycles(graphComponents) - } - - /** - * Finds topological orders that don't end with an object of rdf:type. - * - * @param orders the orders to be filtered. - * @param statementPatterns the statement patterns that the orders are based on. - * @return the filtered topological orders. - */ - def findOrdersNotEndingWithObjectOfRdfType( - orders: Set[Vector[Graph[String, DiHyperEdge]#NodeT]], - statementPatterns: Seq[StatementPattern]): Set[Vector[Graph[String, DiHyperEdge]#NodeT]] = { - type NodeT = Graph[String, DiHyperEdge]#NodeT - - // Find the nodes that are objects of rdf:type in the statement patterns. - val nodesThatAreObjectsOfRdfType: Set[String] = statementPatterns - .filter { statementPattern => - statementPattern.pred match { - case iriRef: IriRef => iriRef.iri.toString == OntologyConstants.Rdf.Type - case _ => false - } - } - .map { statementPattern => - statementPattern.obj.toSparql - } - .toSet - - // Filter out the topological orders that end with any of those nodes. - orders.filterNot { order: Vector[NodeT] => - nodesThatAreObjectsOfRdfType.contains(order.last.value) - } - } - - /** - * Tries to find the best topological order for the graph, by finding all possible topological orders - * and eliminating those whose last node is the object of rdf:type. - * - * @param graph the graph to be ordered. - * @param statementPatterns the statement patterns that were used to create the graph. - * @return a topological order. - */ - def findBestTopologicalOrder(graph: Graph[String, DiHyperEdge], - statementPatterns: Seq[StatementPattern]): Vector[Graph[String, DiHyperEdge]#NodeT] = { - type NodeT = Graph[String, DiHyperEdge]#NodeT - - /** - * An ordering for sorting topological orders. - */ - object TopologicalOrderOrdering extends Ordering[Vector[NodeT]] { - private def orderToString(order: Vector[NodeT]) = order.map(_.value).mkString("|") - - override def compare(left: Vector[NodeT], right: Vector[NodeT]): Int = - orderToString(left).compare(orderToString(right)) - } - - // Get all the possible topological orders for the graph. - val allTopologicalOrders: Set[Vector[NodeT]] = TopologicalSortUtil.findAllTopologicalOrders(graph) - - // Did we find any topological orders? - if (allTopologicalOrders.isEmpty) { - // No, the graph is cyclical. - Vector.empty - } else { - // Yes. Is there only one possible order? - if (allTopologicalOrders.size == 1) { - // Yes. Don't bother filtering. - allTopologicalOrders.head - } else { - // There's more than one possible order. Find orders that don't end with an object of rdf:type. - val ordersNotEndingWithObjectOfRdfType: Set[Vector[NodeT]] = - findOrdersNotEndingWithObjectOfRdfType(allTopologicalOrders, statementPatterns) - - // Are there any? - val preferredOrders = if (ordersNotEndingWithObjectOfRdfType.nonEmpty) { - // Yes. Use one of those. - ordersNotEndingWithObjectOfRdfType - } else { - // No. Use any order. - allTopologicalOrders - } - - // Sort the preferred orders to produce a deterministic result, and return one of them. - preferredOrders.min(TopologicalOrderOrdering) - } - } - } - - def sortStatementPatterns(createdGraph: Graph[String, DiHyperEdge], - statementPatterns: Seq[StatementPattern]): Seq[QueryPattern] = { - type NodeT = Graph[String, DiHyperEdge]#NodeT - - // Try to find the best topological order for the graph. - val topologicalOrder: Vector[NodeT] = - findBestTopologicalOrder(graph = createdGraph, statementPatterns = statementPatterns) - - // Was a topological order found? - if (topologicalOrder.nonEmpty) { - // Start from the end of the ordered list (the nodes with lowest degree). - // For each node, find statements which have the node as object and bring them to top. - topologicalOrder.foldRight(Vector.empty[QueryPattern]) { (node, sortedStatements) => - val statementsOfNode: Set[QueryPattern] = statementPatterns - .filter(p => p.obj.toSparql.equals(node.value)) - .toSet[QueryPattern] - sortedStatements ++ statementsOfNode.toVector - } - } else { - // No topological order found. - statementPatterns - } - } - - sortStatementPatterns(createGraph, statementPatterns) - } - - /** - * Optimises query patterns by reordering them on the basis of dependencies between subjects and objects. - * - * @param patterns the patterns to be optimised. - * @return the optimised patterns. - */ - protected def reorderPatternsByDependency(patterns: Seq[QueryPattern]): Seq[QueryPattern] = { - // Separate the statement patterns from the other patterns. - val (statementPatterns: Seq[StatementPattern], otherPatterns: Seq[QueryPattern]) = - patterns.foldLeft((Vector.empty[StatementPattern], Vector.empty[QueryPattern])) { - case ((statementPatternAcc, otherPatternAcc), pattern: QueryPattern) => - pattern match { - case statementPattern: StatementPattern => (statementPatternAcc :+ statementPattern, otherPatternAcc) - case _ => (statementPatternAcc, otherPatternAcc :+ pattern) - } - } - - val sortedStatementPatterns: Seq[QueryPattern] = createAndSortGraph(statementPatterns) - - val sortedOtherPatterns: Seq[QueryPattern] = otherPatterns.map { - // sort statements inside each UnionPattern block - case unionPattern: UnionPattern => - val sortedUnionBlocks: Seq[Seq[QueryPattern]] = - unionPattern.blocks.map(block => reorderPatternsByDependency(block)) - UnionPattern(blocks = sortedUnionBlocks) - - // sort statements inside OptionalPattern - case optionalPattern: OptionalPattern => - val sortedOptionalPatterns: Seq[QueryPattern] = reorderPatternsByDependency(optionalPattern.patterns) - OptionalPattern(patterns = sortedOptionalPatterns) - - // sort statements inside MinusPattern - case minusPattern: MinusPattern => - val sortedMinusPatterns: Seq[QueryPattern] = reorderPatternsByDependency(minusPattern.patterns) - MinusPattern(patterns = sortedMinusPatterns) - - // sort statements inside FilterNotExistsPattern - case filterNotExistsPattern: FilterNotExistsPattern => - val sortedFilterNotExistsPatterns: Seq[QueryPattern] = - reorderPatternsByDependency(filterNotExistsPattern.patterns) - FilterNotExistsPattern(patterns = sortedFilterNotExistsPatterns) - - // return any other query pattern as it is - case pattern: QueryPattern => pattern - } - - sortedStatementPatterns ++ sortedOtherPatterns - } } diff --git a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/GravsearchQueryOptimisationFactory.scala b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/GravsearchQueryOptimisationFactory.scala new file mode 100644 index 0000000000..d76ba1b88e --- /dev/null +++ b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/GravsearchQueryOptimisationFactory.scala @@ -0,0 +1,395 @@ +/* + * Copyright © 2015-2018 the contributors (see Contributors.md). + * + * This file is part of Knora. + * + * Knora is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Knora is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public + * License along with Knora. If not, see . + */ + +package org.knora.webapi.messages.util.search.gravsearch.prequery + +import org.knora.webapi.ApiV2Schema +import org.knora.webapi.feature.{Feature, FeatureFactory, FeatureFactoryConfig} +import org.knora.webapi.messages.OntologyConstants +import org.knora.webapi.messages.util.search.gravsearch.types.{ + GravsearchTypeInspectionResult, + GravsearchTypeInspectionUtil, + TypeableEntity +} +import org.knora.webapi.messages.util.search.{ + FilterNotExistsPattern, + IriRef, + MinusPattern, + OptionalPattern, + QueryPattern, + StatementPattern, + UnionPattern +} +import scalax.collection.Graph +import scalax.collection.GraphEdge.DiHyperEdge + +/** + * Represents optimisation algorithms that transform Gravsearch input queries. + * + * @param typeInspectionResult the type inspection result. + * @param querySchema the query schema. + */ +abstract class GravsearchQueryOptimisationFeature(protected val typeInspectionResult: GravsearchTypeInspectionResult, + protected val querySchema: ApiV2Schema) { + + /** + * Performs the optimisation. + * + * @param patterns the query patterns. + * @return the optimised query patterns. + */ + def optimiseQueryPatterns(patterns: Seq[QueryPattern]): Seq[QueryPattern] +} + +/** + * A feature factory that constructs Gravsearch query optimisation algorithms. + */ +object GravsearchQueryOptimisationFactory extends FeatureFactory { + + /** + * Returns a [[GravsearchQueryOptimisationFeature]] implementing one or more optimisations, depending + * on the feature factory configuration. + * + * @param typeInspectionResult the type inspection result. + * @param querySchema the query schema. + * @param featureFactoryConfig the feature factory configuration. + * @return a [[GravsearchQueryOptimisationFeature]] implementing one or more optimisations. + */ + def getGravsearchQueryOptimisationFeature( + typeInspectionResult: GravsearchTypeInspectionResult, + querySchema: ApiV2Schema, + featureFactoryConfig: FeatureFactoryConfig): GravsearchQueryOptimisationFeature = { + new GravsearchQueryOptimisationFeature(typeInspectionResult: GravsearchTypeInspectionResult, + querySchema: ApiV2Schema) { + override def optimiseQueryPatterns(patterns: Seq[QueryPattern]): Seq[QueryPattern] = { + if (featureFactoryConfig.getToggle("gravsearch-dependency-optimisation").isEnabled) { + new ReorderPatternsByDependencyOptimisationFeature(typeInspectionResult, querySchema).optimiseQueryPatterns( + new RemoveEntitiesInferredFromPropertyOptimisationFeature(typeInspectionResult, querySchema) + .optimiseQueryPatterns(patterns) + ) + } else { + new RemoveEntitiesInferredFromPropertyOptimisationFeature(typeInspectionResult, querySchema) + .optimiseQueryPatterns(patterns) + } + } + } + } +} + +/** + * Optimises a query by removing `rdf:type` statements that are known to be redundant. A redundant + * `rdf:type` statement gives the type of a variable whose type is already restricted by its + * use with a property that can only be used with that type (unless the property + * statement is in an `OPTIONAL` block). + */ +class RemoveEntitiesInferredFromPropertyOptimisationFeature(typeInspectionResult: GravsearchTypeInspectionResult, + querySchema: ApiV2Schema) + extends GravsearchQueryOptimisationFeature(typeInspectionResult, querySchema) + with Feature { + + /** + * Performs the optimisation. + * + * @param patterns the query patterns. + * @return the optimised query patterns. + */ + override def optimiseQueryPatterns(patterns: Seq[QueryPattern]): Seq[QueryPattern] = { + + // Collect all entities which are used as subject or object of an OptionalPattern. + val optionalEntities: Seq[TypeableEntity] = patterns + .collect { + case optionalPattern: OptionalPattern => optionalPattern + } + .flatMap { + case optionalPattern: OptionalPattern => + optionalPattern.patterns.flatMap { + case pattern: StatementPattern => + GravsearchTypeInspectionUtil.maybeTypeableEntity(pattern.subj) ++ GravsearchTypeInspectionUtil + .maybeTypeableEntity(pattern.obj) + + case _ => None + } + + case _ => None + } + + // Remove statements whose predicate is rdf:type, type of subject is inferred from a property, + // and the subject is not in optionalEntities. + patterns.filterNot { + case statementPattern: StatementPattern => + // Is the predicate an IRI? + statementPattern.pred match { + case predicateIriRef: IriRef => + // Yes. Is this an rdf:type statement? + if (predicateIriRef.iri.toString == OntologyConstants.Rdf.Type) { + // Yes. Is the subject a typeable entity? + val subjectAsTypeableEntity = GravsearchTypeInspectionUtil.maybeTypeableEntity(statementPattern.subj) + + subjectAsTypeableEntity match { + case Some(typeableEntity) => + // Yes. Was the type of the subject inferred from another predicate? + if (typeInspectionResult.entitiesInferredFromProperties.keySet.contains(typeableEntity)) { + // Yes. Is the subject in optional entities? + if (optionalEntities.contains(typeableEntity)) { + // Yes. Keep the statement. + false + } else { + // Remove the statement. + true + } + } else { + // The type of the subject was not inferred from another predicate. Keep the statement. + false + } + + case _ => + // The subject isn't a typeable entity. Keep the statement. + false + } + } else { + // This isn't an rdf:type statement. Keep it. + false + } + + case _ => + // The predicate isn't an IRI. Keep the statement. + false + } + + case _ => + // This isn't a statement pattern. Keep it. + false + } + } +} + +/** + * Optimises query patterns by reordering them on the basis of dependencies between subjects and objects. + */ +class ReorderPatternsByDependencyOptimisationFeature(typeInspectionResult: GravsearchTypeInspectionResult, + querySchema: ApiV2Schema) + extends GravsearchQueryOptimisationFeature(typeInspectionResult, querySchema) + with Feature { + + /** + * Converts a sequence of query patterns into DAG representing dependencies between + * the subjects and objects used, performs a topological sort of the graph, and reorders + * the query patterns according to the topological order. + * + * @param statementPatterns the query patterns to be reordered. + * @return the reordered query patterns. + */ + private def createAndSortGraph(statementPatterns: Seq[StatementPattern]): Seq[QueryPattern] = { + @scala.annotation.tailrec + def makeGraphWithoutCycles(graphComponents: Seq[(String, String)]): Graph[String, DiHyperEdge] = { + val graph = graphComponents.foldLeft(Graph.empty[String, DiHyperEdge]) { (graph, edgeDef) => + val edge = DiHyperEdge(edgeDef._1, edgeDef._2) + graph + edge // add nodes and edges to graph + } + + if (graph.isCyclic) { + // get the cycle + val cycle: graph.Cycle = graph.findCycle.get + + // the cyclic node is the one that cycle starts and ends with + val cyclicNode: graph.NodeT = cycle.endNode + val cyclicEdge: graph.EdgeT = cyclicNode.edges.last + val originNodeOfCyclicEdge: String = cyclicEdge._1.value + val TargetNodeOfCyclicEdge: String = cyclicEdge._2.value + val graphComponentsWithOutCycle = + graphComponents.filterNot(edgeDef => edgeDef.equals((originNodeOfCyclicEdge, TargetNodeOfCyclicEdge))) + + makeGraphWithoutCycles(graphComponentsWithOutCycle) + } else { + graph + } + } + + def createGraph: Graph[String, DiHyperEdge] = { + val graphComponents: Seq[(String, String)] = statementPatterns.map { statementPattern => + // transform every statementPattern to pair of nodes that will consist an edge. + val node1 = statementPattern.subj.toSparql + val node2 = statementPattern.obj.toSparql + (node1, node2) + } + + makeGraphWithoutCycles(graphComponents) + } + + /** + * Finds topological orders that don't end with an object of rdf:type. + * + * @param orders the orders to be filtered. + * @param statementPatterns the statement patterns that the orders are based on. + * @return the filtered topological orders. + */ + def findOrdersNotEndingWithObjectOfRdfType( + orders: Set[Vector[Graph[String, DiHyperEdge]#NodeT]], + statementPatterns: Seq[StatementPattern]): Set[Vector[Graph[String, DiHyperEdge]#NodeT]] = { + type NodeT = Graph[String, DiHyperEdge]#NodeT + + // Find the nodes that are objects of rdf:type in the statement patterns. + val nodesThatAreObjectsOfRdfType: Set[String] = statementPatterns + .filter { statementPattern => + statementPattern.pred match { + case iriRef: IriRef => iriRef.iri.toString == OntologyConstants.Rdf.Type + case _ => false + } + } + .map { statementPattern => + statementPattern.obj.toSparql + } + .toSet + + // Filter out the topological orders that end with any of those nodes. + orders.filterNot { order: Vector[NodeT] => + nodesThatAreObjectsOfRdfType.contains(order.last.value) + } + } + + /** + * Tries to find the best topological order for the graph, by finding all possible topological orders + * and eliminating those whose last node is the object of rdf:type. + * + * @param graph the graph to be ordered. + * @param statementPatterns the statement patterns that were used to create the graph. + * @return a topological order. + */ + def findBestTopologicalOrder(graph: Graph[String, DiHyperEdge], + statementPatterns: Seq[StatementPattern]): Vector[Graph[String, DiHyperEdge]#NodeT] = { + type NodeT = Graph[String, DiHyperEdge]#NodeT + + /** + * An ordering for sorting topological orders. + */ + object TopologicalOrderOrdering extends Ordering[Vector[NodeT]] { + private def orderToString(order: Vector[NodeT]) = order.map(_.value).mkString("|") + + override def compare(left: Vector[NodeT], right: Vector[NodeT]): Int = + orderToString(left).compare(orderToString(right)) + } + + // Get all the possible topological orders for the graph. + val allTopologicalOrders: Set[Vector[NodeT]] = TopologicalSortUtil.findAllTopologicalOrders(graph) + + // Did we find any topological orders? + if (allTopologicalOrders.isEmpty) { + // No, the graph is cyclical. + Vector.empty + } else { + // Yes. Is there only one possible order? + if (allTopologicalOrders.size == 1) { + // Yes. Don't bother filtering. + allTopologicalOrders.head + } else { + // There's more than one possible order. Find orders that don't end with an object of rdf:type. + val ordersNotEndingWithObjectOfRdfType: Set[Vector[NodeT]] = + findOrdersNotEndingWithObjectOfRdfType(allTopologicalOrders, statementPatterns) + + // Are there any? + val preferredOrders = if (ordersNotEndingWithObjectOfRdfType.nonEmpty) { + // Yes. Use one of those. + ordersNotEndingWithObjectOfRdfType + } else { + // No. Use any order. + allTopologicalOrders + } + + // Sort the preferred orders to produce a deterministic result, and return one of them. + preferredOrders.min(TopologicalOrderOrdering) + } + } + } + + def sortStatementPatterns(createdGraph: Graph[String, DiHyperEdge], + statementPatterns: Seq[StatementPattern]): Seq[QueryPattern] = { + type NodeT = Graph[String, DiHyperEdge]#NodeT + + // Try to find the best topological order for the graph. + val topologicalOrder: Vector[NodeT] = + findBestTopologicalOrder(graph = createdGraph, statementPatterns = statementPatterns) + + // Was a topological order found? + if (topologicalOrder.nonEmpty) { + // Start from the end of the ordered list (the nodes with lowest degree). + // For each node, find statements which have the node as object and bring them to top. + topologicalOrder.foldRight(Vector.empty[QueryPattern]) { (node, sortedStatements) => + val statementsOfNode: Set[QueryPattern] = statementPatterns + .filter(p => p.obj.toSparql.equals(node.value)) + .toSet[QueryPattern] + sortedStatements ++ statementsOfNode.toVector + } + } else { + // No topological order found. + statementPatterns + } + } + + sortStatementPatterns(createGraph, statementPatterns) + } + + /** + * Performs the optimisation. + * + * @param patterns the query patterns. + * @return the optimised query patterns. + */ + override def optimiseQueryPatterns(patterns: Seq[QueryPattern]): Seq[QueryPattern] = { + // Separate the statement patterns from the other patterns. + val (statementPatterns: Seq[StatementPattern], otherPatterns: Seq[QueryPattern]) = + patterns.foldLeft((Vector.empty[StatementPattern], Vector.empty[QueryPattern])) { + case ((statementPatternAcc, otherPatternAcc), pattern: QueryPattern) => + pattern match { + case statementPattern: StatementPattern => (statementPatternAcc :+ statementPattern, otherPatternAcc) + case _ => (statementPatternAcc, otherPatternAcc :+ pattern) + } + } + + val sortedStatementPatterns: Seq[QueryPattern] = createAndSortGraph(statementPatterns) + + val sortedOtherPatterns: Seq[QueryPattern] = otherPatterns.map { + // sort statements inside each UnionPattern block + case unionPattern: UnionPattern => + val sortedUnionBlocks: Seq[Seq[QueryPattern]] = + unionPattern.blocks.map(block => optimiseQueryPatterns(block)) + UnionPattern(blocks = sortedUnionBlocks) + + // sort statements inside OptionalPattern + case optionalPattern: OptionalPattern => + val sortedOptionalPatterns: Seq[QueryPattern] = optimiseQueryPatterns(optionalPattern.patterns) + OptionalPattern(patterns = sortedOptionalPatterns) + + // sort statements inside MinusPattern + case minusPattern: MinusPattern => + val sortedMinusPatterns: Seq[QueryPattern] = optimiseQueryPatterns(minusPattern.patterns) + MinusPattern(patterns = sortedMinusPatterns) + + // sort statements inside FilterNotExistsPattern + case filterNotExistsPattern: FilterNotExistsPattern => + val sortedFilterNotExistsPatterns: Seq[QueryPattern] = + optimiseQueryPatterns(filterNotExistsPattern.patterns) + FilterNotExistsPattern(patterns = sortedFilterNotExistsPatterns) + + // return any other query pattern as it is + case pattern: QueryPattern => pattern + } + + sortedStatementPatterns ++ sortedOtherPatterns + } +} diff --git a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToCountPrequeryTransformer.scala b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToCountPrequeryTransformer.scala index 521a0c8fc9..084aff64a9 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToCountPrequeryTransformer.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToCountPrequeryTransformer.scala @@ -20,6 +20,7 @@ package org.knora.webapi.messages.util.search.gravsearch.prequery import org.knora.webapi.ApiV2Schema +import org.knora.webapi.feature.FeatureFactoryConfig import org.knora.webapi.messages.util.search._ import org.knora.webapi.messages.util.search.gravsearch.types.GravsearchTypeInspectionResult @@ -31,10 +32,12 @@ import org.knora.webapi.messages.util.search.gravsearch.types.GravsearchTypeInsp * @param constructClause the CONSTRUCT clause from the input query. * @param typeInspectionResult the result of type inspection of the input query. * @param querySchema the ontology schema used in the input query. + * @param featureFactoryConfig the feature factory configuration. */ class NonTriplestoreSpecificGravsearchToCountPrequeryTransformer(constructClause: ConstructClause, typeInspectionResult: GravsearchTypeInspectionResult, - querySchema: ApiV2Schema) + querySchema: ApiV2Schema, + featureFactoryConfig: FeatureFactoryConfig) extends AbstractPrequeryGenerator(constructClause = constructClause, typeInspectionResult = typeInspectionResult, querySchema = querySchema) @@ -87,8 +90,11 @@ class NonTriplestoreSpecificGravsearchToCountPrequeryTransformer(constructClause } override def optimiseQueryPatterns(patterns: Seq[QueryPattern]): Seq[QueryPattern] = { - val patternsWithoutInferredEntities = removeEntitiesInferredFromProperty(patterns) - reorderPatternsByDependency(patternsWithoutInferredEntities) + GravsearchQueryOptimisationFactory + .getGravsearchQueryOptimisationFeature(typeInspectionResult = typeInspectionResult, + querySchema = querySchema, + featureFactoryConfig = featureFactoryConfig) + .optimiseQueryPatterns(patterns) } override def transformLuceneQueryPattern(luceneQueryPattern: LuceneQueryPattern): Seq[QueryPattern] = diff --git a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformer.scala b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformer.scala index 7fe440a624..1ef31a96ef 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformer.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformer.scala @@ -21,6 +21,7 @@ package org.knora.webapi.messages.util.search.gravsearch.prequery import org.knora.webapi._ import org.knora.webapi.exceptions.{AssertionException, GravsearchException} +import org.knora.webapi.feature.FeatureFactoryConfig import org.knora.webapi.messages.util.search._ import org.knora.webapi.messages.util.search.gravsearch.types.{ GravsearchTypeInspectionResult, @@ -39,11 +40,13 @@ import org.knora.webapi.settings.KnoraSettingsImpl * @param typeInspectionResult the result of type inspection of the input query. * @param querySchema the ontology schema used in the input query. * @param settings application settings. + * @param featureFactoryConfig the feature factory configuration. */ class NonTriplestoreSpecificGravsearchToPrequeryTransformer(constructClause: ConstructClause, typeInspectionResult: GravsearchTypeInspectionResult, querySchema: ApiV2Schema, - settings: KnoraSettingsImpl) + settings: KnoraSettingsImpl, + featureFactoryConfig: FeatureFactoryConfig) extends AbstractPrequeryGenerator( constructClause = constructClause, typeInspectionResult = typeInspectionResult, @@ -408,7 +411,10 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformer(constructClause: Con * @return the optimised query patterns. */ override def optimiseQueryPatterns(patterns: Seq[QueryPattern]): Seq[QueryPattern] = { - val patternsWithoutInferredEntities = removeEntitiesInferredFromProperty(patterns) - reorderPatternsByDependency(patternsWithoutInferredEntities) + GravsearchQueryOptimisationFactory + .getGravsearchQueryOptimisationFeature(typeInspectionResult = typeInspectionResult, + querySchema = querySchema, + featureFactoryConfig = featureFactoryConfig) + .optimiseQueryPatterns(patterns) } } diff --git a/webapi/src/main/scala/org/knora/webapi/responders/v2/SearchResponderV2.scala b/webapi/src/main/scala/org/knora/webapi/responders/v2/SearchResponderV2.scala index 2eacb892f4..7159124955 100644 --- a/webapi/src/main/scala/org/knora/webapi/responders/v2/SearchResponderV2.scala +++ b/webapi/src/main/scala/org/knora/webapi/responders/v2/SearchResponderV2.scala @@ -407,7 +407,8 @@ class SearchResponderV2(responderData: ResponderData) extends ResponderWithStand nonTriplestoreSpecificConstructToSelectTransformer: NonTriplestoreSpecificGravsearchToCountPrequeryTransformer = new NonTriplestoreSpecificGravsearchToCountPrequeryTransformer( constructClause = inputQuery.constructClause, typeInspectionResult = typeInspectionResult, - querySchema = inputQuery.querySchema.getOrElse(throw AssertionException(s"WhereClause has no querySchema")) + querySchema = inputQuery.querySchema.getOrElse(throw AssertionException(s"WhereClause has no querySchema")), + featureFactoryConfig = featureFactoryConfig ) nonTriplestoreSpecificPrequery: SelectQuery = QueryTraverser.transformConstructToSelect( @@ -493,7 +494,8 @@ class SearchResponderV2(responderData: ResponderData) extends ResponderWithStand constructClause = inputQuery.constructClause, typeInspectionResult = typeInspectionResult, querySchema = inputQuery.querySchema.getOrElse(throw AssertionException(s"WhereClause has no querySchema")), - settings = settings + settings = settings, + featureFactoryConfig = featureFactoryConfig ) // TODO: if the ORDER BY criterion is a property whose occurrence is not 1, then the logic does not work correctly diff --git a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToCountPrequeryTransformerSpec.scala b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToCountPrequeryTransformerSpec.scala index 0fd8f8a3d4..1f27f2f9a0 100644 --- a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToCountPrequeryTransformerSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToCountPrequeryTransformerSpec.scala @@ -2,6 +2,7 @@ package org.knora.webapi.util.search.gravsearch.prequery import org.knora.webapi.CoreSpec import org.knora.webapi.exceptions.AssertionException +import org.knora.webapi.feature.FeatureFactoryConfig import org.knora.webapi.messages.IriConversions._ import org.knora.webapi.messages.StringFormatter import org.knora.webapi.messages.admin.responder.usersmessages.UserADM @@ -13,7 +14,6 @@ import org.knora.webapi.messages.util.search.gravsearch.types.{ GravsearchTypeInspectionUtil } import org.knora.webapi.messages.util.search.gravsearch.{GravsearchParser, GravsearchQueryChecker} -import org.knora.webapi.settings.KnoraSettingsImpl import org.knora.webapi.sharedtestdata.SharedTestDataADM import scala.collection.mutable.ArrayBuffer @@ -26,7 +26,9 @@ private object CountQueryHandler { val anythingUser: UserADM = SharedTestDataADM.anythingAdminUser - def transformQuery(query: String, responderData: ResponderData, settings: KnoraSettingsImpl): SelectQuery = { + def transformQuery(query: String, + responderData: ResponderData, + featureFactoryConfig: FeatureFactoryConfig): SelectQuery = { val constructQuery = GravsearchParser.parseQuery(query) @@ -51,7 +53,8 @@ private object CountQueryHandler { new NonTriplestoreSpecificGravsearchToCountPrequeryTransformer( constructClause = constructQuery.constructClause, typeInspectionResult = typeInspectionResult, - querySchema = constructQuery.querySchema.getOrElse(throw AssertionException(s"WhereClause has no querySchema")) + querySchema = constructQuery.querySchema.getOrElse(throw AssertionException(s"WhereClause has no querySchema")), + featureFactoryConfig = featureFactoryConfig ) val nonTriplestoreSpecficPrequery: SelectQuery = QueryTraverser.transformConstructToSelect( @@ -318,7 +321,9 @@ class NonTriplestoreSpecificGravsearchToCountPrequeryTransformerSpec extends Cor "transform an input query with a decimal as an optional sort criterion and a filter" in { val transformedQuery = - CountQueryHandler.transformQuery(inputQueryWithDecimalOptionalSortCriterionAndFilter, responderData, settings) + CountQueryHandler.transformQuery(inputQueryWithDecimalOptionalSortCriterionAndFilter, + responderData, + defaultFeatureFactoryConfig) assert(transformedQuery === transformedQueryWithDecimalOptionalSortCriterionAndFilter) @@ -329,7 +334,7 @@ class NonTriplestoreSpecificGravsearchToCountPrequeryTransformerSpec extends Cor val transformedQuery = CountQueryHandler.transformQuery(inputQueryWithDecimalOptionalSortCriterionAndFilterComplex, responderData, - settings) + defaultFeatureFactoryConfig) assert(transformedQuery === transformedQueryWithDecimalOptionalSortCriterionAndFilterComplex) diff --git a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala index a03be77e41..af41717c4e 100644 --- a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala @@ -2,6 +2,7 @@ package org.knora.webapi.util.search.gravsearch.prequery import org.knora.webapi.CoreSpec import org.knora.webapi.exceptions.AssertionException +import org.knora.webapi.feature.FeatureFactoryConfig import org.knora.webapi.messages.IriConversions._ import org.knora.webapi.messages.StringFormatter import org.knora.webapi.messages.admin.responder.usersmessages.UserADM @@ -27,7 +28,10 @@ private object QueryHandler { val anythingUser: UserADM = SharedTestDataADM.anythingAdminUser - def transformQuery(query: String, responderData: ResponderData, settings: KnoraSettingsImpl): SelectQuery = { + def transformQuery(query: String, + responderData: ResponderData, + settings: KnoraSettingsImpl, + featureFactoryConfig: FeatureFactoryConfig): SelectQuery = { val constructQuery = GravsearchParser.parseQuery(query) @@ -53,7 +57,8 @@ private object QueryHandler { constructClause = constructQuery.constructClause, typeInspectionResult = typeInspectionResult, querySchema = constructQuery.querySchema.getOrElse(throw AssertionException(s"WhereClause has no querySchema")), - settings = settings + settings = settings, + featureFactoryConfig = featureFactoryConfig ) val nonTriplestoreSpecificPrequery: SelectQuery = QueryTraverser.transformConstructToSelect( @@ -3049,14 +3054,18 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec "transform an input query with an optional property criterion without removing the rdf:type statement" in { - val transformedQuery = QueryHandler.transformQuery(queryWithOptional, responderData, settings) + val transformedQuery = + QueryHandler.transformQuery(queryWithOptional, responderData, settings, defaultFeatureFactoryConfig) assert(transformedQuery === TransformedQueryWithOptional) } "transform an input query with a date as a non optional sort criterion" in { val transformedQuery = - QueryHandler.transformQuery(inputQueryWithDateNonOptionalSortCriterion, responderData, settings) + QueryHandler.transformQuery(inputQueryWithDateNonOptionalSortCriterion, + responderData, + settings, + defaultFeatureFactoryConfig) assert(transformedQuery === transformedQueryWithDateNonOptionalSortCriterion) @@ -3065,7 +3074,10 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec "transform an input query with a date as a non optional sort criterion (submitted in complex schema)" in { val transformedQuery = - QueryHandler.transformQuery(inputQueryWithDateNonOptionalSortCriterionComplex, responderData, settings) + QueryHandler.transformQuery(inputQueryWithDateNonOptionalSortCriterionComplex, + responderData, + settings, + defaultFeatureFactoryConfig) assert(transformedQuery === transformedQueryWithDateNonOptionalSortCriterion) @@ -3074,7 +3086,10 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec "transform an input query with a date as non optional sort criterion and a filter" in { val transformedQuery = - QueryHandler.transformQuery(inputQueryWithDateNonOptionalSortCriterionAndFilter, responderData, settings) + QueryHandler.transformQuery(inputQueryWithDateNonOptionalSortCriterionAndFilter, + responderData, + settings, + defaultFeatureFactoryConfig) assert(transformedQuery === transformedQueryWithDateNonOptionalSortCriterionAndFilter) @@ -3083,7 +3098,10 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec "transform an input query with a date as non optional sort criterion and a filter (submitted in complex schema)" in { val transformedQuery = - QueryHandler.transformQuery(inputQueryWithDateNonOptionalSortCriterionAndFilterComplex, responderData, settings) + QueryHandler.transformQuery(inputQueryWithDateNonOptionalSortCriterionAndFilterComplex, + responderData, + settings, + defaultFeatureFactoryConfig) assert(transformedQuery === transformedQueryWithDateNonOptionalSortCriterionAndFilter) @@ -3092,7 +3110,10 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec "transform an input query with a date as an optional sort criterion" in { val transformedQuery = - QueryHandler.transformQuery(inputQueryWithDateOptionalSortCriterion, responderData, settings) + QueryHandler.transformQuery(inputQueryWithDateOptionalSortCriterion, + responderData, + settings, + defaultFeatureFactoryConfig) assert(transformedQuery === transformedQueryWithDateOptionalSortCriterion) @@ -3101,7 +3122,10 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec "transform an input query with a date as an optional sort criterion (submitted in complex schema)" in { val transformedQuery = - QueryHandler.transformQuery(inputQueryWithDateOptionalSortCriterionComplex, responderData, settings) + QueryHandler.transformQuery(inputQueryWithDateOptionalSortCriterionComplex, + responderData, + settings, + defaultFeatureFactoryConfig) assert(transformedQuery === transformedQueryWithDateOptionalSortCriterion) @@ -3110,7 +3134,10 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec "transform an input query with a date as an optional sort criterion and a filter" in { val transformedQuery = - QueryHandler.transformQuery(inputQueryWithDateOptionalSortCriterionAndFilter, responderData, settings) + QueryHandler.transformQuery(inputQueryWithDateOptionalSortCriterionAndFilter, + responderData, + settings, + defaultFeatureFactoryConfig) assert(transformedQuery === transformedQueryWithDateOptionalSortCriterionAndFilter) @@ -3119,7 +3146,10 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec "transform an input query with a date as an optional sort criterion and a filter (submitted in complex schema)" in { val transformedQuery = - QueryHandler.transformQuery(inputQueryWithDateOptionalSortCriterionAndFilterComplex, responderData, settings) + QueryHandler.transformQuery(inputQueryWithDateOptionalSortCriterionAndFilterComplex, + responderData, + settings, + defaultFeatureFactoryConfig) assert(transformedQuery === transformedQueryWithDateOptionalSortCriterionAndFilter) @@ -3128,7 +3158,10 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec "transform an input query with a decimal as an optional sort criterion" in { val transformedQuery = - QueryHandler.transformQuery(inputQueryWithDecimalOptionalSortCriterion, responderData, settings) + QueryHandler.transformQuery(inputQueryWithDecimalOptionalSortCriterion, + responderData, + settings, + defaultFeatureFactoryConfig) assert(transformedQuery === transformedQueryWithDecimalOptionalSortCriterion) } @@ -3136,7 +3169,10 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec "transform an input query with a decimal as an optional sort criterion (submitted in complex schema)" in { val transformedQuery = - QueryHandler.transformQuery(inputQueryWithDecimalOptionalSortCriterionComplex, responderData, settings) + QueryHandler.transformQuery(inputQueryWithDecimalOptionalSortCriterionComplex, + responderData, + settings, + defaultFeatureFactoryConfig) assert(transformedQuery === transformedQueryWithDecimalOptionalSortCriterion) } @@ -3144,7 +3180,10 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec "transform an input query with a decimal as an optional sort criterion and a filter" in { val transformedQuery = - QueryHandler.transformQuery(inputQueryWithDecimalOptionalSortCriterionAndFilter, responderData, settings) + QueryHandler.transformQuery(inputQueryWithDecimalOptionalSortCriterionAndFilter, + responderData, + settings, + defaultFeatureFactoryConfig) assert(transformedQuery === transformedQueryWithDecimalOptionalSortCriterionAndFilter) } @@ -3152,7 +3191,10 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec "transform an input query with a decimal as an optional sort criterion and a filter (submitted in complex schema)" in { val transformedQuery = - QueryHandler.transformQuery(inputQueryWithDecimalOptionalSortCriterionAndFilterComplex, responderData, settings) + QueryHandler.transformQuery(inputQueryWithDecimalOptionalSortCriterionAndFilterComplex, + responderData, + settings, + defaultFeatureFactoryConfig) // TODO: user provided statements and statement generated for sorting should be unified (https://github.com/dhlab-basel/Knora/issues/1195) assert(transformedQuery === transformedQueryWithDecimalOptionalSortCriterionAndFilterComplex) @@ -3160,86 +3202,112 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec "transform an input query using rdfs:label and a literal in the simple schema" in { val transformedQuery = - QueryHandler.transformQuery(InputQueryWithRdfsLabelAndLiteralInSimpleSchema, responderData, settings) + QueryHandler.transformQuery(InputQueryWithRdfsLabelAndLiteralInSimpleSchema, + responderData, + settings, + defaultFeatureFactoryConfig) assert(transformedQuery == TransformedQueryWithRdfsLabelAndLiteral) } "transform an input query using rdfs:label and a literal in the complex schema" in { val transformedQuery = - QueryHandler.transformQuery(InputQueryWithRdfsLabelAndLiteralInComplexSchema, responderData, settings) + QueryHandler.transformQuery(InputQueryWithRdfsLabelAndLiteralInComplexSchema, + responderData, + settings, + defaultFeatureFactoryConfig) assert(transformedQuery === TransformedQueryWithRdfsLabelAndLiteral) } "transform an input query using rdfs:label and a variable in the simple schema" in { val transformedQuery = - QueryHandler.transformQuery(InputQueryWithRdfsLabelAndVariableInSimpleSchema, responderData, settings) + QueryHandler.transformQuery(InputQueryWithRdfsLabelAndVariableInSimpleSchema, + responderData, + settings, + defaultFeatureFactoryConfig) assert(transformedQuery === TransformedQueryWithRdfsLabelAndVariable) } "transform an input query using rdfs:label and a variable in the complex schema" in { val transformedQuery = - QueryHandler.transformQuery(InputQueryWithRdfsLabelAndVariableInComplexSchema, responderData, settings) + QueryHandler.transformQuery(InputQueryWithRdfsLabelAndVariableInComplexSchema, + responderData, + settings, + defaultFeatureFactoryConfig) assert(transformedQuery === TransformedQueryWithRdfsLabelAndVariable) } "transform an input query using rdfs:label and a regex in the simple schema" in { val transformedQuery = - QueryHandler.transformQuery(InputQueryWithRdfsLabelAndRegexInSimpleSchema, responderData, settings) + QueryHandler.transformQuery(InputQueryWithRdfsLabelAndRegexInSimpleSchema, + responderData, + settings, + defaultFeatureFactoryConfig) assert(transformedQuery === TransformedQueryWithRdfsLabelAndRegex) } "transform an input query using rdfs:label and a regex in the complex schema" in { val transformedQuery = - QueryHandler.transformQuery(InputQueryWithRdfsLabelAndRegexInComplexSchema, responderData, settings) + QueryHandler.transformQuery(InputQueryWithRdfsLabelAndRegexInComplexSchema, + responderData, + settings, + defaultFeatureFactoryConfig) assert(transformedQuery === TransformedQueryWithRdfsLabelAndRegex) } "transform an input query with UNION scopes in the simple schema" in { val transformedQuery = - QueryHandler.transformQuery(InputQueryWithUnionScopes, responderData, settings) + QueryHandler.transformQuery(InputQueryWithUnionScopes, responderData, settings, defaultFeatureFactoryConfig) assert(transformedQuery === TransformedQueryWithUnionScopes) } "transform an input query with knora-api:standoffTagHasStartAncestor" in { val transformedQuery = - QueryHandler.transformQuery(queryWithStandoffTagHasStartAncestor, responderData, settings) + QueryHandler.transformQuery(queryWithStandoffTagHasStartAncestor, + responderData, + settings, + defaultFeatureFactoryConfig) assert(transformedQuery === transformedQueryWithStandoffTagHasStartAncestor) } "reorder query patterns in where clause" in { - val transformedQuery = QueryHandler.transformQuery(queryToReorder, responderData, settings) + val transformedQuery = + QueryHandler.transformQuery(queryToReorder, responderData, settings, defaultFeatureFactoryConfig) assert(transformedQuery === transformedQueryToReorder) } "reorder query patterns in where clause with union" in { - val transformedQuery = QueryHandler.transformQuery(queryToReorderWithUnion, responderData, settings) + val transformedQuery = + QueryHandler.transformQuery(queryToReorderWithUnion, responderData, settings, defaultFeatureFactoryConfig) assert(transformedQuery === transformedQueryToReorderWithUnion) } "reorder query patterns in where clause with optional" in { - val transformedQuery = QueryHandler.transformQuery(queryWithOptional, responderData, settings) + val transformedQuery = + QueryHandler.transformQuery(queryWithOptional, responderData, settings, defaultFeatureFactoryConfig) assert(transformedQuery === TransformedQueryWithOptional) } "reorder query patterns with minus scope" in { - val transformedQuery = QueryHandler.transformQuery(queryToReorderWithMinus, responderData, settings) + val transformedQuery = + QueryHandler.transformQuery(queryToReorderWithMinus, responderData, settings, defaultFeatureFactoryConfig) assert(transformedQuery == transformedQueryToReorderWithMinus) } "reorder a query with a cycle" in { - val transformedQuery = QueryHandler.transformQuery(queryToReorderWithCycle, responderData, settings) + val transformedQuery = + QueryHandler.transformQuery(queryToReorderWithCycle, responderData, settings, defaultFeatureFactoryConfig) assert(transformedQuery == transformedQueryToReorderWithCycle) } From 018dd4707a3984b4883f0997492114e15f5766e4 Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Fri, 19 Feb 2021 17:50:08 +0100 Subject: [PATCH 25/33] test(gravsearch): Clean up test. --- .../search/gravsearch/types/GravsearchTypeInspectorSpec.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/types/GravsearchTypeInspectorSpec.scala b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/types/GravsearchTypeInspectorSpec.scala index 4a0af7ab57..f6573b6747 100644 --- a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/types/GravsearchTypeInspectorSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/types/GravsearchTypeInspectorSpec.scala @@ -1649,7 +1649,7 @@ class GravsearchTypeInspectorSpec extends CoreSpec() with ImplicitSender { assert(result.entities == TypeInferenceResult6.entities) } - "infer the types in a query that requires 6 iterations test1" in { + "infer the types in a query that requires 6 iterations" in { val typeInspectionRunner = new GravsearchTypeInspectionRunner(responderData = responderData, inferTypes = true) val parsedQuery = GravsearchParser.parseQuery(PathologicalQuery) val resultFuture: Future[GravsearchTypeInspectionResult] = From 4eb807eac849e8a10e7f267fab156644b2692795 Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Fri, 19 Feb 2021 18:50:15 +0100 Subject: [PATCH 26/33] style(gravsearch): Use import wildcard. --- .../prequery/GravsearchQueryOptimisationFactory.scala | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/GravsearchQueryOptimisationFactory.scala b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/GravsearchQueryOptimisationFactory.scala index d76ba1b88e..d577ec58a3 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/GravsearchQueryOptimisationFactory.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/GravsearchQueryOptimisationFactory.scala @@ -22,20 +22,12 @@ package org.knora.webapi.messages.util.search.gravsearch.prequery import org.knora.webapi.ApiV2Schema import org.knora.webapi.feature.{Feature, FeatureFactory, FeatureFactoryConfig} import org.knora.webapi.messages.OntologyConstants +import org.knora.webapi.messages.util.search._ import org.knora.webapi.messages.util.search.gravsearch.types.{ GravsearchTypeInspectionResult, GravsearchTypeInspectionUtil, TypeableEntity } -import org.knora.webapi.messages.util.search.{ - FilterNotExistsPattern, - IriRef, - MinusPattern, - OptionalPattern, - QueryPattern, - StatementPattern, - UnionPattern -} import scalax.collection.Graph import scalax.collection.GraphEdge.DiHyperEdge From 35587e5e33854d93db6294c702ac7ed0ee4d4829 Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Tue, 23 Feb 2021 12:00:03 +0100 Subject: [PATCH 27/33] style(test): Add copyright. --- .../prequery/TopologicalSortUtilSpec.scala | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtilSpec.scala b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtilSpec.scala index 3afd47f402..a8f7929b28 100644 --- a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtilSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtilSpec.scala @@ -1,3 +1,22 @@ +/* + * Copyright © 2015-2018 the contributors (see Contributors.md). + * + * This file is part of Knora. + * + * Knora is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Knora is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public + * License along with Knora. If not, see . + */ + package org.knora.webapi.util.search.gravsearch.prequery import org.knora.webapi.CoreSpec @@ -5,6 +24,9 @@ import org.knora.webapi.messages.util.search.gravsearch.prequery.TopologicalSort import scalax.collection.Graph import scalax.collection.GraphEdge._ +/** + * Tests [[TopologicalSortUtil]]. + */ class TopologicalSortUtilSpec extends CoreSpec() { type NodeT = Graph[Int, DiHyperEdge]#NodeT From 420f873082099f1f5d753d17bad58ae59763f319 Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Tue, 23 Feb 2021 12:01:37 +0100 Subject: [PATCH 28/33] style(gravsearch): Add comment. --- .../util/search/gravsearch/prequery/TopologicalSortUtil.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtil.scala b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtil.scala index ebedaf47ef..b22bf946f7 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtil.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtil.scala @@ -31,7 +31,7 @@ import scala.collection.mutable object TopologicalSortUtil { /** - * Finds all possible topological orders of a graph. + * Finds all possible topological orders of a graph. If the graph is cyclical, returns an empty set. * * @param graph the graph to be sorted. * @tparam T the type of the nodes in the graph. From 75e9023c3c434ab3f0459f5c1233a8b67b4e86e2 Mon Sep 17 00:00:00 2001 From: Sepideh Alassi Date: Mon, 1 Mar 2021 13:05:05 +0100 Subject: [PATCH 29/33] feat (gravsearch) get all permutations of topological order according to Kahn's algorithm --- .../GravsearchQueryOptimisationFactory.scala | 2 +- .../prequery/TopologicalSortUtil.scala | 92 +++++------- ...cGravsearchToPrequeryTransformerSpec.scala | 140 +++++++++--------- 3 files changed, 109 insertions(+), 125 deletions(-) diff --git a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/GravsearchQueryOptimisationFactory.scala b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/GravsearchQueryOptimisationFactory.scala index d577ec58a3..23c38cd519 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/GravsearchQueryOptimisationFactory.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/GravsearchQueryOptimisationFactory.scala @@ -278,7 +278,7 @@ class ReorderPatternsByDependencyOptimisationFeature(typeInspectionResult: Gravs } // Get all the possible topological orders for the graph. - val allTopologicalOrders: Set[Vector[NodeT]] = TopologicalSortUtil.findAllTopologicalOrders(graph) + val allTopologicalOrders: Set[Vector[NodeT]] = TopologicalSortUtil.findAllTopologicalOrderPermutations(graph) // Did we find any topological orders? if (allTopologicalOrders.isEmpty) { diff --git a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtil.scala b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtil.scala index b22bf946f7..feebe27f0c 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtil.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtil.scala @@ -31,66 +31,50 @@ import scala.collection.mutable object TopologicalSortUtil { /** - * Finds all possible topological orders of a graph. If the graph is cyclical, returns an empty set. - * - * @param graph the graph to be sorted. - * @tparam T the type of the nodes in the graph. - */ - def findAllTopologicalOrders[T](graph: Graph[T, DiHyperEdge]): Set[Vector[Graph[T, DiHyperEdge]#NodeT]] = { + * Finds all possible topological order permutations of a graph. If the graph is cyclical, returns an empty set. + * + * @param graph the graph to be sorted. + * @tparam T the type of the nodes in the graph. + */ + def findAllTopologicalOrderPermutations[T](graph: Graph[T, DiHyperEdge]): Set[Vector[Graph[T, DiHyperEdge]#NodeT]] = { type NodeT = Graph[T, DiHyperEdge]#NodeT + def findPermutations(listOfLists: List[Vector[NodeT]]): List[Vector[NodeT]] = { + def makePermutations(next: Vector[NodeT], acc: List[Vector[NodeT]]): List[Vector[NodeT]] = { + next.permutations.toList.flatMap(i => acc.map(j => j ++ i)) + } - /** - * Represents arguments to be put on the simulated call stack. - */ - case class StackItem(sources: Set[NodeT], inDegrees: Map[NodeT, Int], topOrder: Vector[NodeT], count: Int) - - val inDegrees: Map[NodeT, Int] = graph.nodes.map(node => (node, node.inDegree)).toMap - - def isSource(node: NodeT): Boolean = inDegrees(node) == 0 - def getSources: Set[NodeT] = graph.nodes.filter(node => isSource(node)).toSet + def makePermutationsRec(next: Vector[NodeT], + rest: List[Vector[NodeT]], + acc: List[Vector[NodeT]]): List[Vector[NodeT]] = { + if (rest.isEmpty) { + makePermutations(next, acc) + } else { + makePermutationsRec(rest.head, rest.tail, makePermutations(next, acc)) + } + } - // Replaces the program stack by our own. - val stack: mutable.ArrayStack[StackItem] = new mutable.ArrayStack() + listOfLists match { + case Nil => Nil + case one :: Nil => one.permutations.toList + case one :: two :: tail => makePermutationsRec(two, tail, one.permutations.toList) + } + } // Accumulates topological orders. - var allTopologicalOrders = Set[Vector[NodeT]]() - - // Push arguments onto the stack. - stack.push(StackItem(sources = getSources, inDegrees = inDegrees, topOrder = Vector[NodeT](), count = 0)) - - while (stack.nonEmpty) { - // Fetch arguments - val stackItem = stack.pop() - - if (stackItem.sources.nonEmpty) { - // `sources` contains all the nodes we can pick. Generate all possibilities. - for (source <- stackItem.sources) { - val newTopOrder = source +: stackItem.topOrder - var newSources = stackItem.sources - source - - // Decrease the in-degree of all adjacent nodes. - var newInDegrees = stackItem.inDegrees - - for (adjacent <- source.diSuccessors) { - val newInDegree = newInDegrees(adjacent) - 1 - newInDegrees = newInDegrees.updated(adjacent, newInDegree) - - // If in-degree becomes zero, add to sources. - if (newInDegree == 0) { - newSources = newSources + adjacent - } + val allOrders = graph.topologicalSort match { + // Is there any topological order? + case Right(topOrder) => + // Yes, find all valid permutations + val nodesOfLayers: List[Vector[NodeT]] = + topOrder.toLayered.iterator.foldRight(List.empty[Vector[NodeT]]) { (layer, acc) => + val layerNodes: Vector[NodeT] = layer._2.toVector + layerNodes +: acc } - - stack.push(StackItem(newSources, newInDegrees, newTopOrder, stackItem.count + 1)) - } - } else if (stackItem.count != graph.nodes.size) { - // The graph has a cycle, so don't try to sort it. - () - } else { - allTopologicalOrders += stackItem.topOrder.reverse - } + findPermutations(nodesOfLayers).toSet + case Left(_) => + // No, The graph has a cycle, so don't try to sort it. + Set.empty[Vector[NodeT]] } - - allTopologicalOrders.filter(_.nonEmpty) + allOrders.filter(_.nonEmpty) } } diff --git a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala index af41717c4e..2893741de9 100644 --- a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec.scala @@ -1859,6 +1859,18 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec ), namedGraph = None ), + StatementPattern( + subj = QueryVariable(variableName = "gnd1"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#valueHasString".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "(DE-588)118531379", + datatype = "http://www.w3.org/2001/XMLSchema#string".toSmartIri + ), + namedGraph = None + ), StatementPattern( subj = QueryVariable(variableName = "person2"), pred = IriRef( @@ -1901,7 +1913,7 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec )) ), StatementPattern( - subj = QueryVariable(variableName = "letter"), + subj = QueryVariable(variableName = "person1"), pred = IriRef( iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, propertyPathOperator = None @@ -1917,26 +1929,23 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec )) ), StatementPattern( - subj = QueryVariable(variableName = "letter"), - pred = QueryVariable(variableName = "linkingProp2"), - obj = QueryVariable(variableName = "person2"), - namedGraph = None - ), - StatementPattern( - subj = QueryVariable(variableName = "letter"), - pred = QueryVariable(variableName = "linkingProp2__hasLinkToValue"), - obj = QueryVariable(variableName = "letter__linkingProp2__person2__LinkValue"), + subj = QueryVariable(variableName = "person1"), + pred = IriRef( + iri = "http://www.knora.org/ontology/0801/beol#hasIAFIdentifier".toSmartIri, + propertyPathOperator = None + ), + obj = QueryVariable(variableName = "gnd1"), namedGraph = None ), StatementPattern( - subj = QueryVariable(variableName = "letter__linkingProp2__person2__LinkValue"), + subj = QueryVariable(variableName = "gnd1"), pred = IriRef( - iri = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type".toSmartIri, + iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, propertyPathOperator = None ), - obj = IriRef( - iri = "http://www.knora.org/ontology/knora-base#LinkValue".toSmartIri, - propertyPathOperator = None + obj = XsdLiteral( + value = "false", + datatype = "http://www.w3.org/2001/XMLSchema#boolean".toSmartIri ), namedGraph = Some( IriRef( @@ -1945,7 +1954,7 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec )) ), StatementPattern( - subj = QueryVariable(variableName = "letter__linkingProp2__person2__LinkValue"), + subj = QueryVariable(variableName = "letter"), pred = IriRef( iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, propertyPathOperator = None @@ -1960,13 +1969,28 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec propertyPathOperator = None )) ), + StatementPattern( + subj = QueryVariable(variableName = "letter"), + pred = QueryVariable(variableName = "linkingProp2"), + obj = QueryVariable(variableName = "person2"), + namedGraph = None + ), + StatementPattern( + subj = QueryVariable(variableName = "letter"), + pred = QueryVariable(variableName = "linkingProp2__hasLinkToValue"), + obj = QueryVariable(variableName = "letter__linkingProp2__person2__LinkValue"), + namedGraph = None + ), StatementPattern( subj = QueryVariable(variableName = "letter__linkingProp2__person2__LinkValue"), pred = IriRef( - iri = "http://www.w3.org/1999/02/22-rdf-syntax-ns#object".toSmartIri, + iri = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type".toSmartIri, + propertyPathOperator = None + ), + obj = IriRef( + iri = "http://www.knora.org/ontology/knora-base#LinkValue".toSmartIri, propertyPathOperator = None ), - obj = QueryVariable(variableName = "person2"), namedGraph = Some( IriRef( iri = "http://www.knora.org/explicit".toSmartIri, @@ -1974,19 +1998,7 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec )) ), StatementPattern( - subj = QueryVariable(variableName = "gnd1"), - pred = IriRef( - iri = "http://www.knora.org/ontology/knora-base#valueHasString".toSmartIri, - propertyPathOperator = None - ), - obj = XsdLiteral( - value = "(DE-588)118531379", - datatype = "http://www.w3.org/2001/XMLSchema#string".toSmartIri - ), - namedGraph = None - ), - StatementPattern( - subj = QueryVariable(variableName = "person1"), + subj = QueryVariable(variableName = "letter__linkingProp2__person2__LinkValue"), pred = IriRef( iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, propertyPathOperator = None @@ -2002,24 +2014,12 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec )) ), StatementPattern( - subj = QueryVariable(variableName = "person1"), - pred = IriRef( - iri = "http://www.knora.org/ontology/0801/beol#hasIAFIdentifier".toSmartIri, - propertyPathOperator = None - ), - obj = QueryVariable(variableName = "gnd1"), - namedGraph = None - ), - StatementPattern( - subj = QueryVariable(variableName = "gnd1"), + subj = QueryVariable(variableName = "letter__linkingProp2__person2__LinkValue"), pred = IriRef( - iri = "http://www.knora.org/ontology/knora-base#isDeleted".toSmartIri, + iri = "http://www.w3.org/1999/02/22-rdf-syntax-ns#object".toSmartIri, propertyPathOperator = None ), - obj = XsdLiteral( - value = "false", - datatype = "http://www.w3.org/2001/XMLSchema#boolean".toSmartIri - ), + obj = QueryVariable(variableName = "person2"), namedGraph = Some( IriRef( iri = "http://www.knora.org/explicit".toSmartIri, @@ -2679,6 +2679,18 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec UnionPattern( blocks = Vector( Vector( + StatementPattern( + subj = QueryVariable(variableName = "int"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#valueHasInteger".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "1", + datatype = "http://www.w3.org/2001/XMLSchema#integer".toSmartIri + ), + namedGraph = None + ), StatementPattern( subj = QueryVariable(variableName = "thing"), pred = IriRef( @@ -2704,18 +2716,6 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec propertyPathOperator = None )) ), - StatementPattern( - subj = QueryVariable(variableName = "int"), - pred = IriRef( - iri = "http://www.knora.org/ontology/knora-base#valueHasInteger".toSmartIri, - propertyPathOperator = None - ), - obj = XsdLiteral( - value = "1", - datatype = "http://www.w3.org/2001/XMLSchema#integer".toSmartIri - ), - namedGraph = None - ), StatementPattern( subj = QueryVariable(variableName = "thing"), pred = IriRef( @@ -2773,6 +2773,18 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec ) ), Vector( + StatementPattern( + subj = QueryVariable(variableName = "int"), + pred = IriRef( + iri = "http://www.knora.org/ontology/knora-base#valueHasInteger".toSmartIri, + propertyPathOperator = None + ), + obj = XsdLiteral( + value = "3", + datatype = "http://www.w3.org/2001/XMLSchema#integer".toSmartIri + ), + namedGraph = None + ), StatementPattern( subj = QueryVariable(variableName = "thing"), pred = IriRef( @@ -2798,18 +2810,6 @@ class NonTriplestoreSpecificGravsearchToPrequeryTransformerSpec extends CoreSpec propertyPathOperator = None )) ), - StatementPattern( - subj = QueryVariable(variableName = "int"), - pred = IriRef( - iri = "http://www.knora.org/ontology/knora-base#valueHasInteger".toSmartIri, - propertyPathOperator = None - ), - obj = XsdLiteral( - value = "3", - datatype = "http://www.w3.org/2001/XMLSchema#integer".toSmartIri - ), - namedGraph = None - ), StatementPattern( subj = QueryVariable(variableName = "thing"), pred = IriRef( From f3d8de6b62c72d979d303a405f772549f0baecc2 Mon Sep 17 00:00:00 2001 From: Sepideh Alassi Date: Mon, 1 Mar 2021 13:21:15 +0100 Subject: [PATCH 30/33] fix(gravsearch) fix the failing test --- .../gravsearch/prequery/TopologicalSortUtilSpec.scala | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtilSpec.scala b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtilSpec.scala index a8f7929b28..101c20f658 100644 --- a/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtilSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtilSpec.scala @@ -44,10 +44,9 @@ class TopologicalSortUtilSpec extends CoreSpec() { val allOrders: Set[Vector[Int]] = nodesToValues( TopologicalSortUtil - .findAllTopologicalOrders(graph)) + .findAllTopologicalOrderPermutations(graph)) val expectedOrders = Set( - Vector(2, 4, 5, 7), Vector(2, 4, 7, 5), Vector(2, 7, 4, 5) ) @@ -60,7 +59,7 @@ class TopologicalSortUtilSpec extends CoreSpec() { val allOrders: Set[Vector[Int]] = nodesToValues( TopologicalSortUtil - .findAllTopologicalOrders(graph)) + .findAllTopologicalOrderPermutations(graph)) assert(allOrders.isEmpty) } @@ -71,7 +70,7 @@ class TopologicalSortUtilSpec extends CoreSpec() { val allOrders: Set[Vector[Int]] = nodesToValues( TopologicalSortUtil - .findAllTopologicalOrders(graph)) + .findAllTopologicalOrderPermutations(graph)) assert(allOrders.isEmpty) } From 885bb33c09d959a64d87f9be1c6617b9fc839196 Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Mon, 1 Mar 2021 15:13:39 +0100 Subject: [PATCH 31/33] style(TopologicalSortUtil): Improve style a little bit. --- .../search/gravsearch/prequery/TopologicalSortUtil.scala | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtil.scala b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtil.scala index feebe27f0c..cef35e4470 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtil.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/TopologicalSortUtil.scala @@ -38,11 +38,13 @@ object TopologicalSortUtil { */ def findAllTopologicalOrderPermutations[T](graph: Graph[T, DiHyperEdge]): Set[Vector[Graph[T, DiHyperEdge]#NodeT]] = { type NodeT = Graph[T, DiHyperEdge]#NodeT + def findPermutations(listOfLists: List[Vector[NodeT]]): List[Vector[NodeT]] = { def makePermutations(next: Vector[NodeT], acc: List[Vector[NodeT]]): List[Vector[NodeT]] = { next.permutations.toList.flatMap(i => acc.map(j => j ++ i)) } + @scala.annotation.tailrec def makePermutationsRec(next: Vector[NodeT], rest: List[Vector[NodeT]], acc: List[Vector[NodeT]]): List[Vector[NodeT]] = { @@ -61,20 +63,23 @@ object TopologicalSortUtil { } // Accumulates topological orders. - val allOrders = graph.topologicalSort match { + val allOrders: Set[Vector[NodeT]] = graph.topologicalSort match { // Is there any topological order? case Right(topOrder) => - // Yes, find all valid permutations + // Yes. Find all valid permutations. val nodesOfLayers: List[Vector[NodeT]] = topOrder.toLayered.iterator.foldRight(List.empty[Vector[NodeT]]) { (layer, acc) => val layerNodes: Vector[NodeT] = layer._2.toVector layerNodes +: acc } + findPermutations(nodesOfLayers).toSet + case Left(_) => // No, The graph has a cycle, so don't try to sort it. Set.empty[Vector[NodeT]] } + allOrders.filter(_.nonEmpty) } } From f9fb4aa9c646c7cba769021f3c5caae9b9dec2d9 Mon Sep 17 00:00:00 2001 From: Sepideh Alassi Date: Mon, 1 Mar 2021 17:44:55 +0100 Subject: [PATCH 32/33] doc (gravsearch) documentation about optimization of gravsearches with topological sorting --- docs/03-apis/api-v2/query-language.md | 59 +++++++ .../design/api-v2/figures/query_graph.png | Bin 0 -> 45468 bytes docs/05-internals/design/api-v2/gravsearch.md | 152 +++++++++++++++++- 3 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 docs/05-internals/design/api-v2/figures/query_graph.png diff --git a/docs/03-apis/api-v2/query-language.md b/docs/03-apis/api-v2/query-language.md index 83338d4487..91c2c3639e 100644 --- a/docs/03-apis/api-v2/query-language.md +++ b/docs/03-apis/api-v2/query-language.md @@ -1211,3 +1211,62 @@ CONSTRUCT { } ORDER BY (?int) ``` + +## Query Optimization by Topological Sorting of Statements +The query performance of triplestores, such as Fuseki, highly depends on the order of query statements. +To increase the +speed of queries without losing the accuracy of results, we have defined an optimization based on Kahn's [topological +sorting algorithm](https://www.wikiwand.com/en/Topological_sorting) to automatically reorder the statements of a gravsearch +query. Currently, this optimization is defined under the `gravsearch-dependency-optimisation` +[feature toggle](../feature-toggles.md) that must be activated when sending the gravsearch query request to the API. + +Let's consider, the following gravsearch query: +```sparql +PREFIX beol: +PREFIX knora-api: + +CONSTRUCT { + ?letter knora-api:isMainResource true . + ?letter ?linkingProp1 ?person1 . + ?letter ?linkingProp2 ?person2 . + ?letter beol:creationDate ?date . +} WHERE { + ?letter beol:creationDate ?date . + + ?letter ?linkingProp1 ?person1 . + FILTER(?linkingProp1 = beol:hasAuthor || ?linkingProp1 = beol:hasRecipient ) + + ?letter ?linkingProp2 ?person2 . + FILTER(?linkingProp2 = beol:hasAuthor || ?linkingProp2 = beol:hasRecipient ) + + ?person1 beol:hasIAFIdentifier ?gnd1 . + ?gnd1 knora-api:valueAsString "(DE-588)118531379" . + + ?person2 beol:hasIAFIdentifier ?gnd2 . + ?gnd2 knora-api:valueAsString "(DE-588)118696149" . +} ORDER BY ?date +``` + +This query, as it is, would take a considerably long time with Fuseki. The query time would have been much less, if the +statements such as +``` + ?gnd1 knora-api:valueAsString "(DE-588)118531379" . + ?gnd2 knora-api:valueAsString "(DE-588)118696149" . +``` +were given at the top of the query followed by +``` + ?person1 beol:hasIAFIdentifier ?gnd1 . + ?person2 beol:hasIAFIdentifier ?gnd2 . + ?letter ?linkingProp1 ?person1 . + FILTER(?linkingProp1 = beol:hasAuthor || ?linkingProp1 = beol:hasRecipient ) + + ?letter ?linkingProp2 ?person2 . + FILTER(?linkingProp2 = beol:hasAuthor || ?linkingProp2 = beol:hasRecipient ) + ?letter beol:creationDate ?date . +``` + +To automatically rearrange the statements of the given query in the above order, upon receiving the gravsearch query, the +optimization algorithm converts the query to a graph and sorts it using topological sorting algorithm. From all valid +topological orders returned by the sorting algorithm, the best one is chosen and used to reorder the query statements +before converting the query to SPARQL and submitting it to the triplestore. + diff --git a/docs/05-internals/design/api-v2/figures/query_graph.png b/docs/05-internals/design/api-v2/figures/query_graph.png new file mode 100644 index 0000000000000000000000000000000000000000..a8db48a7e9d2950d0571fe19bc58843dd64e135e GIT binary patch literal 45468 zcmeFZgFs(jc9K(%s#lf`SSNNQcth-6|!ibeD9;fb_dZ zJ)fiJocH$^e7!Don3-ok``P=Bb+2`AUns~uz=M&)FfcIiBqi=CVqiew7#NrdIOoAT zj-k7O7#OfYQ!z0GNii{61v_gaQwu{342c&Jk=V*%ZKN#=M_(ye2rwDmIj0aIq!!=d zd=*u`OZZ0Nt^^LFo*1c`4NJ!JM;c#9E<@GS5CeYd1qC?Vs}u$g6$YF9@FGqi!3EVD zHTzxF3o`T0R_kMx&g}MGA{YvHhxKf^bIv1P)9Vrv&B)73$-SFBkBKFLfVw~+u{TIK zl9O*>=qc??u8mvXBrl|J3e-M3J@MwkW0n7mi6JG<8_zWNjdtwyOJmByn=}}G*FISn z(d*_+Q9tFnCE<&6miFlMrniF4Ay#r5(H+}6m^i(d4PnC;7ichwSZ316Y`#lS7rn>E zzN|)nw*u!e&1;`d@$WZ3l=1OZ!l>^K5xyWhhY*#+X8U^fChV;Ic6N*!V6o8$^_^4zaf^S|lC*JRQ5sqWs>xuP7G zRNV%Bh8;543}Jc&B8ts8Fjb>)y7vyIRQ)~Ds|N>OyhIufnSA! z;l}FHq3iW|2RYL>Mck}#HWU8a$gA+KeYN#lU9VM}cZ$Tqy-Y=O>@0=t_|gaOGj+3t zQIQHYA#ko^QrIkJDPbXmt!?Yxhx%x55xxk4x2AK14mKwj-=lQbstS7&b^{}@T)>rT zqUZL+&3o1CYwhOP4apa6x6?f>h%n>?F?|lLMvS+~E2StfdHX+xO2Te?nlxdbKR_w_ z-tNIX-+-kK5tzan^C8Ipqb{0%I4s~3#5HP= z;iY{2WZ|O+FFxy2=_!;Ai8d~cU+i@8rpk9x=77YH$@D(g-DJop0 zzhK6fby4L-mN?h@5ry~+3VOa}=<*HnptSc(CITg7{KQnj^GQ2B?O#@U`WMeF8tuC7 z;yH-8VV#hOH7-83U`x0{di%o17UmD(dW?6D?zG$?V=HHS66LJU{MNs;p-Nw@ux&_Y zRdJPnRnCH_jA}H9s^RsF4KGz3UKa(~GyU0Z+tiA+3b6`~ij@k{F>JvL*3X?kV9zRT zUiY|DN9ax9P3}!~Ml5n);YQSD!3z(uSRY?_eC{#LyH_@^*d(sp$Rc8SKJkfH@3t*T zVc?KN4Z;KAA$C3Vr6hGq<;`oRw=x4Ao;tkflRuOgpw$XH*P3Z?)m5nS^_X0pw3paB zg$@RE(Y=oX*No$b4w3TA8=47?EU__E}jcNi#z;(@h2y zvy98-7gMR}L#}06srhQ;LaA`+L&vPDWVdj)*N6P5C(Duj0h~7E^W>uB+OG&-)sio8 zp3k7qy1?c0`CVSxdNMNUl9?VC3#Ud*bj(~#09Q5FgA&dX?GnNgQuBV3_^)qz6*%q5 zgWk%HA_SzWWsf+;ywBU8w{9M}`idj3+a+H}X;eNoT_ALo~cvZCne6cPnldsqOwz1K0M>5C8Qlr~|+Z{J))ZVh=(yL{P#el^iRQXC` zulHjF$0kQB=a8Pw$fKbUvqgP+&M`fg=FO`*L0u`vz1fQkOE>Pj-G6tVJz41f_FNR$VW4TRr`$n37T4Jy>)$C`j#iRK=ooab9TI{&_F?HHn5fE!x%h^S)&^z!=}CF zL4~LSlHq}oz2VTcC?}{Bp3})5=Mm14-Vy0hWU-8KYR3p?w*7z_wpxHifaj%sDlW>8 zltEOZl)3^M0=98xaaR0FaZlpac+2_g?FwzDD_89*XQdopIq1xeOdmFNj?2yrEOM_} zEho&^eXI?B*Y?~fM_+rp!R5Pq&L;2D)|cK+j>(;j1sbFiQUi%y>u;phG2c1CQ)L}raSGViD8Tp-N-*LUqz1s+7ybG|E)F^h3N+G-CvUgfrwtoGC)YMC8;}3?;#--7^Kt8aE!VKW_if z6%h7O|6}$?ng{9+B(9HKpKdK^?F_}AkLtYKY5D^4Fejt)A$~H;`1z;L_mH zh#HIWla`biPGskYn@kM2KFQ3>OwBA`(CW|?JllqhLx*wYSd?{IkuJ=5VQ-_H*w#1% z0(IJ{WsOreBxt6wY|3mC>`yARDn1HYBcU6n{?7Nsn+Yak>!U@OH%XLXNSt#|`#-2f zzl*Nsm^2nQ>^FR6OlQP1H)a3TPVPD9J&pG}mFFvi?A@EVXA}zsyE&}QC!XKbF4Bcb zE=X(ba9s}VyHy$&kVDy@3Alm%^syAn~B;x9V_>>TjlxZKm<1zew{*3k1Fl zR3$;d^;t)YDK!)7YV@DA1->D9!X&0jtuC$|QMw$d7cdhzQ=UzvRvxh!nZtbb`|WSB z{)!QblBSd7`CQfY;@(WhW$w8)fA= z`s+{>QixMnTW^m$B;BojU!eEM{@eUK%Pxzd)`XQ#14_eG3%BQstsp5XXNBudA4o&>H79~VjY>6x3PR{!o23r$miOjTBv*C zs`Sv{!MWLh!;`9`_I>Y6qW(7gPn5j68%SMmg^ely^A(~4Y##I|h3@HwkGZd$Ub)AZ z$j&`zxI7$cERuHOa%A%T%WM6Y`maQyPfBKaQ7#%BeKjerDdIiXt!1s&gf2V1PwvcK z``*1LG8oUvuVF8>(&yj%srOcTOE+=CRZFt+dfs*sfu6#G+0)7td!FuYp-Inc#~)II z8p*mH*#)|_x9c3Xix%dH)UK#$=2hufuG_8gp9ZbKTdK#aV)H9?&DXXTO;_T_-4Cla zP{lkDYweo0Dp%A%oK|i5to7uQ^jg^D^u*PwSkL|bT0eLFeKp|*L3;O$y@Kt~iR2O8 zTrbNv)ot=9>^*0p*Pb5UJ{+6JJWJMz*wP6K3ChA;9-1QGI34VGT`CraYmA?CH+8ol z!%y>0k6uT{-6qS-rBSLf4=Qtv(#=7%n;iCbI_j1Zb<`c&!z+ zSbc4S2?4D zUeZf<1Z-{b9AZq3V0^!Rm4;@NPx#c3MsZaKV`S$xSM7&_=Wj*)9x6mO+HK0m_8oaI zZ6$IjC{IC7`OYq+3XlyY#MG1kq^oMEE@>nyi@^w9<6vM0m||doSD4_39QosNq`ir7UVv>^JUnPAzLqjWj6KjWWOI5L8s6kU@bq94> z8Ge0hOV-B*)_R7lE|xavOE3go_`yp{Lx;z-E|wNn_WUkF*MEG2AG}7t4ZlwN<0}s4 zLf6%06==n*?F?zTSUFkQt_#CxX=w%R42<{{@7@1(IrvTJx`~5>4L=<2?Ci|y%)x4H zXAHl?$Hxa}V~4Y|vw&}~*t=ReJa%ERvcK_jkiW*cXK1f)XKLeMYHdY}9`~`HwWEX3 z_3P*x{qN7uc^bNy{(UDa`(KX*9uSWH1b&B=4gSBe!KH%ecli}eT?{SM@0nVH7XlJyb= zG7npa?lmHONr;FENpUW!v)!?2?CFD&X>4IW#X5gp5P|v3CxnEMkQhR&ile}cBPk)F zECIujp=TUZC=^dEn&0Glf)W-SGk7x;`z=!X0NZFtL`#SVkSn$g8 z|9MjcZ8-Q)OoPP$_U{>?XNiNf!~3t<(E1?gp*TqYBF$Ff|DG>F%{TJ@dQJp}uL6vq z=AvO~2*ZD`1g%RO*MBSkdK&MDXBg*|Ib5zIcyRs)=yu#Cpy*hPRwa_=#KkByz^>n=nbctPZ##J^Ov^T!syl(za;0; zR}I=H9&?|%E*(`lu4vY}p`_|9O3eq$&f}4*=W3SbJpSl!xL*1ZN~ZnDINtH?sNCqA zf5ZhCosA}=#4`cP2S1%RX&VCLrWSPRxkMvu0aIk;UD`k;_yxl*E~gY%t+kR|eU(9*2Z=t~ z-*9f!9;Sbu9qU&PR0bw_j5}v{otVYW^ z%d98Yt;p}MJ+Xm(;QW2*0Z1W_%9z9#V+)!ims5lVeN0lB2c1k^pOkPJ59^`2EKDlOwH2R-N|`G3?MA{dZzFz#*vVXqdVN-V3>hi6@GzaVcal8P0fk zqk1VVHb*{DnAIivUNCRO0dP0gkV?Bbsqzm8x#bgP=gwbzB=?n9h22o}?<7Y%4-8iy z*+`6!-j~}+{S71q+69k%mkJ?oVP>Y{Hj0AKlAi8B5+;i;@5QZNE7G(Y)O~ds;7R>0 zQ_`}6L1q=h!q1@x5q&h6r^D^MuB{|5*!h^xq=%L#;8wQEf)81;Y>CwI@xjh`j~z5G z0FR>iuE?D%-9W#;9}p3TB}|z9L4g#k5{B<8L^Ha|exb{$lWS1@$$2~ruV&V%S1iW3_V=0-YCRbWEm)NSW=ATGJneum1~}6+?{ZNL8$58 z-wTG}%Z3Xuk^Nc-511pZysvSh=RotztNJi0*#yDiyz2ET!Qf9;l0=)B)}7Dz3_la= zryiu}V1|>SL&MK;d~JYq--1O>E3`W6fTg8P zMBV(~Hla5xNgZ;L0{)h{MI3BhAT67I^Tk*}*CHWQpOV$%gzIsBV%I~88@l2J)(%(l z%P$b!9%}GCXFm~)_ow%30|Y+L*yMzs3LdTr*A^ZHcSVC?u~DZPFHan`uQ!efPNmx; zGwKcorL>0M90F_-v9izOS?JuiN9c=fv4|_iDr}Q$H$ULspDN;QrsNXw1-Y_f#cpLF zm)myc)A6Mb$lq7e2f+s^{p=ba`C~?RuP6ERfK0o3u#m{p$s=)>)&f$+$`v6RaES_? zYcF2I*Mf%9{I&+ynE@RC5b$ z6D->!;m%4_eOXPk!3Be42V#GFENusbd5CC!};CwQKVAq3agDkTP(;5}o3%a$Hl$xWN9{p2?%2~GO5BC20I z3Wfdleeif5Xe(iA@j`EOC>FKb;x!6BJA=^I)SVws*Q~vZbp*ngiO?7M`5@$G*`JnYXl5qtyM>&Z zuNte*HowZ1j%0Oegpv&{2=DX7vOWF~Kw;>*yEke*I^P*Pz6P?#=OQSaM5{r!<+O@* z?|@hMcTf8y56Z6N<#57lK8UB#fgBC}TKE0eHWy%tD1Zejn{?gwR!z@gUn<$@66n4} z$@_?iPOjvZ^jqT7r;V4(t;Wrg9|V_;T2<%Lx5$4Lbrf2uy8UMXcYw}$4_4 zjDVPpT-6k%d+P=J6Ylw$3*Ue}TQvGX*XMCTT)K2tkhE_;m{*ET?b49oGV{WZn&$;} zFG{An{pNt6AsXKNTJ;jlNDt9&Lk5@`lZ*a>rrMXM}jvfU#|Wz4$nz-Zz4OR!fT>f7-=Prd0z&Ccu5 z)zAmjg%4;v`-HqUYWG(IneqzqJ=e^m0}52KRfkp7bPOGr`wDluMJCT*yz+=VO^%T% zX+>j@>rY(P`T#7hu-Pg)$`VT=#8N0YO%zg+ADk6rDu2c!QZGMae0ONVxo0f@-nkiJ!gD;iMiR6FIVErc^D$=I7SdTusir$PNZ z_+7VCP7b=g_vF_ECJ$bt)Hz#Q<>ChkGyti=53Dc^1Y z7p39aL&J)|n>NSaZQEBh)Khxz?G*sqDh9Yqqc=@<4O`^9?@DOqMCksccRf=W>Gng6 zoyhuQ!%E!4jL5kSuYG}D*f)`bHu zdqpKMAf#$76Tc8`oM7ASeb}2g(hjd*rQYcdfK(+o>ipBKu*~L)iH?3?DB=eQ-vqIM zOB$dM2(=EiHw*s^A^@&Gj&7{D z9)J5yfQq~{DrXfSq!-8D$Fnr@B4?e}USVEweAcPn53`Z@RQ!FXhlBMaAcKvs1&AC! zt`zGaVeXXC6l_hO{unSi)%fH@$=Q3tbs@r~E6jWMWxu2BZzvG#{`TcZUF{xyL)gH_ z=-(g!bg@|IiWld1E5Qd#sqFrJI$$kZ_r&alAE(tA?}P66()RHr9niHWrUI>NngzRFOULRu=J^+euIK8X@PlFaaY- zPqLrO^h!*bv<5{;<*jkwZ8XmND+D6E(9zFcAC8WG^bi30OXk`mt4Z>M?7;3T+e@0U zDzH?9MH36I_Pl%IzSKmfTY?L(>FE+V^C*v!n|E>>1Ma>nT~xRGRob#Ej&HrR{&)_~ zt@jCM|NDq}zN~fnM1u2#Yr1>hM4~3A#FAS5VMM(9dZksw7Tj}5+G%gtc=TIa*tq-g z4n8LIX(Y3TUeW(EzLi*wk1P-5ns*ByS%h{GjDJjU8Zl+vITbFx87#D??)Hw7C8@W_ zQ2J@6%!sm^z>WrCT4K;f@QF&kGuLT;Wlht@Avx z*qUit(l;&LI@npz0(e?x;p+zxO&N(=E)w2bp7cp8>Vw63=6|B7kl+T>Mi1mIy4}wU z^gp5ytpc0xRSu(}y|vM`Ho8RSp*vxAw7Jx$p}`)XprlNU#r{vsZR3Mjt6DA74(_1Q zxnGoK=NKKTd92>Mel!o1dZ%-{Cp2f@AM_G9LRmnpj`xbkJ|n1@_NT}Juz=g{n}o#g z-h<&=1L)29re-;`QEy?rrV+6Mz^^b@r+NVD73h5X^-J(BvCrV9JM`HJ<@+W9iU+^N zc?{nQ7@LJ)9+z|_K6(~7+rFWJ`JLgbgW8c*UXD~m6*7CJh^60vTL!u& z&Vm(*piwavW<8<o7+J=|H|E94_EJ-ehOX<~?#Vl9eVE`6k?iLLJYQy;ay-Al@JTd~* z=y#ut?JU=WU2Rv@iEwzL?)XY6sG_w1YA-r3d2&$k0K<11TJjn7^a7DKMi>q3v-3>6 z4bI{jE0mQ>+heN1ce<6h@{dwj0u)fpb|{9$bR>O#?OseKS%hDNTb@rMkQBJb! zP7q?1SbE}EHOTU!_A)9})wkp1XuosA8v#55|oi?@`>y0$qYA$Yxa3c;mhxGFy^0MzF`3}{$2-1cZI9r z$53~>P-RS1#s$aC+Jf9{Tt37q!F&V=M_(~2a%2^APrj+b>v9fd!zHNXqGZe}np+J2ba6DVi z&PRg*3v`kk-^mf!ybcj3Rfx@bHv)*x3urzQ6N}&SGNGdp}jQd>_@9tHm=&ROVmYvUIssx&a?gx6;nA@+iAy z>w(DOLka<>9HHGFF)Nh?fsLBtu{GCozIr$v@dAUl7)C`iSg-Z?lGEw!jSbHYO*l9F_=QK|73P7Hd@`?0L93S0AOO#YLs1n+L3N zgA4l9surcH)J;2oGdB%3#cB1BO2<6_pN3l}*e%XgX;X1cZeG^61(&s=p%zqn0_BP6 zIcoU{z_MFqL#K3i_75?QMoLVrDk}ZsdU3njZFEh1Q>7zE0Qwon7sD6+p46YGQ{9t3 z{LcB#F-KUTsheqUlf z$XZTbIX@rIXMcOALG@{qdjz3^K-Ph}V?K4=OQ{TE8j3#xJ_(fYa!p^PTW5D_f=kRt z#UNahMk>|pyqSKY!f_Jf7!mI}3eCI*RKeqqIGgu!E@d-SKM1B~A&Tp`sF>9f4S55* zci51!?%Cazh1FtRf8w-%F6i~Ugz)k)JL#?_VSvlcSJLF-m)%-k(TJRei+0E1clo~{ zQV8*y`Sha9ppCHlXuaC$^LZ+&Ub`?tG!%Sq`r~DS8N7PnJeU#mNixNT2EgEe@T7$c(2^kT)@=g%6U z?EeP9fk(qUIx%tS_UqXM)^iwUN`p=@9XlkFvF+eYk3aFG=rR6W2nZvZhjK zzQ*&#jbgKSvK2Y_e8r|S&#E1&a?D;ED*OoT!+nVBRgv%2@J0LC6Kwo67G^99nc#Ox zscz}nvV9JWFwJFb>OYZ;LKvhFh7>+pEzXM!^&?xpV>j+N*q_3(@H)f?-^Dy*9n?fo zy)2&`(=nutY!rhz&V`e2$FuM*oeOh==P2FJSYy0DQGyuu+#ffFy}ks{GW5>UGwPY3 znwa9W`l~c0t9_Yauc(D$xC>%de0qN`VG#$ALLB5ekOWw6Csnd=4aSmAGu+2W^0+_o zBFM)tYB@K1;yz5zY>P+{1v7cf0=?ncEjg(LmL65-CiC`^g3)=pY@U`ce6enNT~T*F za>i-z)R{59GU|*GPsjLHk_0)dSK1LZzjcJHb*oIeCB%X&)zSDngi6AIgkD0LseiBC zpNK`uAaD!%`fR!(eb4}$7d~$Foe4st&<~~jmHu*eIwW~^$5P{SojeR@Uq4o&#<1y@ z?x>@x6yyb{@5%08-w{YgRcFiV%}INZDgU|GhB5ueoU4}>1P55)iBTgP|@E~ z9w8IK@YEbANXIu@&;rCqe36d8su#6SgVb!G!}nj28-gUr!;=j9 z`4B5qf^z<1$5e{M`WS||0H;;WpcFp&;rb1ob9-{wKkDn@J9D2*ypMwrfhj>sESNEG(mrsuAv4XO4~ z%^f83tY5bJTEBYq$YAO5_^D$jg{GTCIV60UFLMwj%-{im@tb2Zp+%3?$@=`n2XxVP*4?a>?>)gfuhtL7!ld z^SEQA>2z3nf*WJm4i=-y+>?;hCt+S)l6Y{HO|fziv}gw6cc-i<^ITA0m4anZUwhf~ zQLW1GS^&N|5^bI{se9LJhD+M-FJlYe>W@#uN?LGv&h$Fx5950V3B7>K+J8PjA0zBp zMKPfMIMam)5Gl;anxR~idJscazNSitb}80;G|KB+dlH^X_G!yk(tG1vxG-;PPtXQX(J5^u|86o?YKG=>Pjf4@Ob%7wP8EyC}1UXk+mcJFP1a3%B@(3 zDE*kw8q>MV-a(!Sct_Yt-6{LbsEZtrBH z*;Co8RZX>^tn5$1+;U>^6|+DGAw%@lxZT>@8V@U^j#ss%5cZ6WA!&#-tEze8jy^xO zNFAr>*qP)$R>9m9pET*=TKDqUqPxw@Ct1@+|CO zD06<*GUI;KBSZE)Y^gtlr85E?%S}6wQ_@_aBo!&;S0c^Tp_~`10axO*KkmH1`E+bQ z1k?{Ad%lw6gSlFRC}jI8S0-&Lz8pI9umBFuP*JLWb-y3~XXf1{Dn^z2H3r9KK$KS;aH7D=6 ztmL%SM3(*pF3h{+F2_Dr%y0h8UR)MY8%)yIaq;_7VWOO+6nVgjTT9hBWKq#B1!&%6t|hR&e}&=Ms=$qJ-SbSaj>gWE~M9MFnM) zc$gRHpD#XnDjb#J1M>S9Y_if^v=Ba7x4Uv$iSPH;E3e)ZdLx23PMJWt+tIs=Z=s^) z)Uh7E{|6kyfT)n+sBGGu9v^U<4=}r<`qeaqN|znmFPk9{jr*}yRc+_-skP4;FNFW& zI`Au^2J0?Sjk0yLG;8Ij1dnaW@+`BOCtBHRe~+QJ>J zaVeMHVS&6i z;<^yO;I@ut%ic0;D5e&pMfm%{)RyJrPFm5k%Dc$uSFYELF?G7D^8Q5nzu2(@+^5%F zGO5k#$~8~LU+g8I+4Ne}fEHTk)&d%E^#LIoYl(EMc&&|;?lqjlA0m<$d}wqQIKfnW z2mXyYX(c)#lFz;nEk^%FxuJ&-QK2PnYzH${N5e}aHU_wvS9`GMs{YbUg5!Y3QVSrd zWJJ359)aq{sb57+10;i05D3dL&z&C{wXR>|_5?-y>X!#Jj=_`e8$#%?wrZ|>G{P|1 zd!B@9kc#%ZG0?%*qjV2GtF6+B9J~NzdRv6pj`MB4CAWcp5|clnP^)f|>S;Bpl{{Xw zRZo)W^rb6XH&PWDdTne|neuDCT2;=ywUgeB-|%Sh$D^2^sQ>mxdUde=wAAzLXtEr{ zpn?`xt&J?D*aafI%w8d~+nu&aG#FyG_?O>jcwXQP-B8nqClV=X@ zeIV^VA8kCg1Bm@})(l>QRsvATgt*J%LQKCT{@};wY9rfi$1&MYT|v;6WlAw0s-FKM zc(U8eF(Er4_Wv@4_Y0{Ip=E}ljEBj`(u83`Bqed|hJl_lM2R-O4FJ4FYO$kS1Z`)Y zx}Smi-a%9nm9uNQzdpVPWP=Hj&I94Z*@0}e?gY<04ZohVbQy&3Li?*fWzZj0x33)( zShez0(A1pBvI(dI(;m-)B<3J=wLQ0T3NQMA6SNNWvT&SC<9oMy#O%y^ymkj1Dp5rT zfw0?d#?n8g7jd^^!w&Cx69_E zv%c{}z}MbNV4~MVe)h;OYGK)klJwH~xbsip;1@}-O@<@BSp*PX8Q^1JWNB^n&%cb= zFZX3yg;L>Bh^V}YRiz(<2!f8H64@BeHbC9RPOP2mPdW#mc)WMZLqDH2qFER1q%AvYxyP`18+Q zLi^e){n?R#U)E*En@s?UwEG6gb&mUo4{G^=`qZ_%W3e&kHnuD3qHq_HT>;l9`t(nw z;E%_1Tp7q~Nh0--bmLzu>813o@7~?aKX9*2?{8k{PPE1&=VW6UC# zXI9R5y3`y-FJqtBPS$yrxopksK5lo1N6s$O3d2u@J*TildK`183;utDPaJ87qx$K+ z=La<9U)|AF4QMs80^+b_xsH_q`^uy+YsKTI+N(WkutknY}vfQ%azC(|(i&d#vq zCB6~ATyg8tC!*@#m}Ac0%bvf)zARdTn5j7+`>xFd!OH-lU)3N_^7&I9PG6J{WRpD5 zd#V9bRX4v-bL!K-LKy*>ufjzNvNV4_^V8;^w6S(x-qQ#%d!6OOe#cd^yHZy$o6!9M z|F|=Z5!^IYYL4s;=rXnv#cgXO$hd^*J$|@S!Qa)!_m6LbUwwITpBfESJeUAIV~*=% z!(f@Apk$8rj&{eo%aZgg1fUM%SRU&#P_94Y&o{XAuUis44@6|RAz`w#IUf}mD z3^_?hH-ZHTvuZ-G{_AQ4?K0#Y9ul?(lwA8-cqs%eHomu#J=MSc2Os)M<3y42UsnUF zq6d$n`{qM=Zm2)!O*Nr037n_mh{5DMX-MSb%*)%pzjD^!yJJg?g~@=($*jBu>u*{Kp=BI@!SqeEJ3Rw6COLV0<$kUR#%I=S(jb;c>gspZHTh30(>Ye zR}l?qNVmUeKM58-%;2&ZVjDH8@kcaby{6=w$*TljG3*D$T?Ij=JQC^F1`@UzD51;4 z=zcUOFDdR>+H>XlxC%-ZTqa$Twu%IqaVbtzXUvY-oIfZTac>{Z`1Aat>x|RZx#oQFO&FsxC!#fz0{d ziPdH4A<1#&M%|G%hRI^jyEIxOK!P8Sj?rmPfv~WYWC+&GVt4UNM9g4O)puZYk(W%e zoYVYvU`32cnIv6YW7;3P{{uTBD4}{3HM18*7mWCGHgB|9fZAWF0=z zO&e9B)MExZj`9{p&^?Smgul@`>#OIH|8AN|HOB^1T9%FtI4}jbg}Q~?j9Fes$D=d| zIuF}6%Ak6bH)Mq9ey=Kj?Z87K37O5WY8}9|AFT}Lm;b>h`P;P?baU(^^8LBKwbLkcGU!MH*FQ@yr8$ zMTb^Nv=iOd2}mb1i^jJdHCwzQn9wzJUmD~szn|OCr7LU=K$Hug#MCc}%yUF_?KaAE zUdUb|BJM+hSE-W!;l&?rhAd7Q8i{8($G91oS;?Rjf6_$n>2ucO7)bJ3VEjz9usQSV4$aC(T=#BlGXBTIcz|v? zjh9@;Np2gtTIF52>tn#|SAOz9cerjizJJd?u<6-Azex z9ofHAcoqOzj*q_D(>H^F87-$F+cYm`h(0+_v)*vyU)ceXgM$zB1#S zk?;MYVe|AM%{9IeSe-?c5F$F z&JJb{m``Pd(T5xZ5+*eurdP|GWJtN26KZ{|Fv^ry@d4c^V(E;;#UhT{ZDg}n2Z9)((2;=#zq$tX#7t8VI!5KG z%>yobkwl$Z8B=$V6SwXX=%9{ge4g)<=4QNXO}m79ni+Ov;DosGMx)(>8;gZUH=>+vB?pXoI^cgqgi{J**z` z1%{5;_2*K7r!Q+YUMGYg5o;+UPd^6WSJU_s$Mr?pjQ`TOYdt*?f?9~J-ClX;Q6qC* zlBAr)ksZv3iTNEqiy$3ML*9E)&=9Cl_wTJ#Zs*i6=nB_Y4Vt4FmNPN~+&70iw!t7` zfZlZhjf9K&zjPnO7--W|6~kKz%JXwX>(^S-!`SN%`S4Ph#d1*I*@x^FG{uxJJ}sSB zSFqAvT{pCXplcDa~&Hwi-6W5;9D>96)vQDvPQQ^$wA2P1kq+a4OLnw_o`F ztxQ?v5K-Tu4(^9@%F!eQ{iyZPd*gIV*tr>fHxBM4-NrJZGo>@1shW-ZxkFENfSutD zhSYU1W<+1z?A7<&>aP|4bO_(}Qctc|+qf}KJk=LC>gA%pbp2pWx_;dqFTzV_VIXn^ zbR-R;ZYl;}pUjpx*g86Z&y7s(GbdPJ>`?XE@X5q#PBHXJpODp-w~Y`VCbK9tdmeNQ-c?Q{t6vrD@dO5CMQxz$j=g#a=I10`Odqh5fp-5E z*&(_12LFR;3{Ld{SDTx6r`k6G(NDvQ0g}YrLfOx(@wj&gr2; z(8iN!;iTvkYWHVqT^r923G*st&=UUga8Apz_;98Y9oeIu-n-wN9rhSmlHSdOIOXIk zQ&0ak&)bzLN+VjlQ;H^b0W_3Q0(YIMcjohk+DTmV3)dh`#j~!Q(e<*Ccjy>Pv1adqL2XzlD~k zv?%ZIJT_ar#ejrpN{H&-<51%nGzV6ICltKXV**|I{kX^-$GbIosE)7n&+GM?VC9dZ z-6D?=w}m=6B3xcJ1VBo{Xf$umU0yR#G>ftOaj5B@EO^{LvzN!Ju|A}9SA{()w2t*a zO~&l^&~ls0{yM5=6WobwP&-ER`zQGMl!@r&+^Zp$pKn#3avLgmTw)usso(1YEUmfP%bta5%^*+FA^>o2>G4jQrvk-v zkPBoIsfsTMXA!m)MC|f$yLJ|4Rpkg+;8GoI01oOqP8EN*X*rycO3=68Tv9@e3l6cBgxwNy zd8-3lq8c~UKVSizo|Bn%0N;GvN3`#R}i>^>E=lFd5t2;hD%Fr=su8ao4UwBri7HOg3UL zEQG%KUM{v-G^9q-5}fLzx}ow&qIPw64rzbm4MB*H|6Akp93(x;pr3{lRH}|j6O>_^ zyo+P@-CU^fK0nczB7|6ty{x8<6s8k&q%NGbdW|NyEi<68@)GXOTKS}vae`}_<6=)ETKk4Q%`Q)qN5d;2 zI7%C{Fh-WKbPrpsOM_K(Usil(q-mXPcMVp`Woo4=9GBOJ~S>uUcF?vy1~E+iPb& zUXEUvx&xG0ldOwSxMQvG;bQW|TX6iqK`|x}&_=Re$TpkM=_^GLr;tP647ul!yYi{L z{JCN5&19V7*+@+2`@S7np)jMaxVFIB{&t#!Vk$uwHmV+v)xwrl^jQPVG=R-3SER*0 zlVGW!wR$qr8{ofm`1yYUL|B!Ypm1(q1IK%En{)b?x!@})l40t{nNdTO)wIB9ppU=N zMi3;$=U|b>-bT{Cz>NB&#*YELEHN{A>0>jl-I5IfRtMfS@6&^YaY|cs@7$8pV$U@b zf>ueyPMVpQRO&V)(oMb=4$dc2Fw0PWX)p9!)ee*zlK z#$=x)`3|Zd@iKzCTzzAF%2!=L^s0+{MX;pYp5T#(^kD}Pkl zkc~rp4}n`~BHRJaT(Z_~+lyZ65Y?k&bKkQ-pJ>8WU{uKbaW)hZsrjiT9d!6std-z| zkj`Jb28!_**4bYNoCqZH+XAP?Zh;6UZL^u;4>h4PuYmA+WVA(6|zl6Pq@ zLY6Op4qIa$e02TtA>`I}$Rm)Yu++Ss2g#wcvki z1pu5?iK_vPR=AwM(mT-|tLVb74pd{2!!N-&+D7MQO3BF-5IRDBpr_9Ymy9)M>o~;s z8-T92^ShE&^+wt<`mY@P$htUT;H1z;Wr}aN9zrx7WPr8+5C$f8fIXW>qfwq<5j3SAbQXYZ?+DXG zw}{yh_u#V&x$PE^Azd2Lh372N!XGr$1M99VRO^HRdiHmk4uMX1fG^2+ z9^WtG#tk#=q{wJHX?abh6?M*z7%S`HBY*;Mx6ED>^U^oz53suU58)U!)U7 z?0hm!5Myz>0}^2Me2lpTXgQXo=Qq5FMG8l*`n| zfb-Bnt)#|1t)k~I#W-R4m3K_(1`D|w!ntiy@iDma`} z8u4bdJ7^ovr__GD1wniUp3iA^5?l@?I5|ln3d2m{3y6^sZq z|3h{@oh8`Tq(HmD=m4cSqS@@Zf?L4}Loyvk3FMvG;630v=lzi2TYhYN69coA#V=pJ z0ay#1vNCj}9#*Q2xTM5pXJ17(Nl?A{KBFBoAMYuYy6WHxGf4?%Lb0oWd4I;`rZR9s zjV&WCqOoNpoxMK4hg}dCjyf2d7YaYv46ltDt#U@a#q7|ag*+yqWDqiPtjy*H79!T* zGZgC(B(M}NLn2gX;%HxJ^HX37*sEImq5OHLdKF@sap#wp+>Lez26kQm|ZtdkJ*5NDLHytTob7C86VP8^$Bype~UR=_na<>Hj@6@;U zN_?b|DqIMC0^w$;gsmn09VL%itBBFT1Sk1}w$VV(BZGJ`r}%Gf3UO9rB#=0 zf1+qxXtZn6x@x0wd*9AIK|i`nbgr5|-H;Ab*Ow$lP-@lP*6*$gmF`SeT`>RkLm|B{ zdcI)Qb-g*@}UaH)uoai z;d1S!#eh365>T`(w0YkovYxhHzwfPoT&4_3MJ4kHKFeXE%1zp8k%PcBSl&O7h_YwX zJAh3x1x$>Q_@rr);OTH)#Ev1cta|8B*lk)WpOgM|yTwLu&9`-xQ%rWGX4h zUR({t2CFTb$_YCc7F7Ipw6lP`R~BSUGeNu%T87yzt!1(0-pTEECV8}7y#s26S^JwW z&A#=b=POpBzA=6bW_HIVu(d5>cJv{4vvz3s#Y_F5ik7$eMNnHw`CJPnFBy3DX4x_k zW0nUQS%_+&$~_&p5F}Y%YHw;p8ZhY}ZN+1&+E?%B?Z0blxWG3yB|8q)s6FkW8o3Ew zXUw9U=aQ}v6i)X92dY0r7JZ8ee~=SQ^d?B;7j0df^W3%4k8<5t`dHp1`no^aAnKmM z)aN;DFtlSse@?ir7ne@Bw1JA$1kr%=>4|h42BmOAS)aheo;`;%Iwa+fh0=Qy*N2ja zdBOeJgA+~1i}A-Th)ILMWjVC8yfW_Rplq1dF#g0FNpuuUdLQ$)1JYGPRZ`ve2_i&w zwkU-qEJnZL4uQ+PG&1TOb6X!>OVaZV4o&^}+t}B=;GAAxqjX*2AaEhqWcT&eVymlG z?-WL2`f)g9lWvbn+t^|;d+C&9)83uVV5SdN5k#=I}+sW!x9Nk$>+3k*~ZZM&%0;b`ju{X4yQfsyO~?B zK+vYbLA#02w4Utbd{9^GjV&;%Mig6)6ny(5OTkQ?5q>_DG^cjKC1w;3VoRF|&KmAP ztBdLJYjLNm1;;U&xdtHVGJ;%Y7|#z9+*{NMjq+K)Pmg!AM7VzU`EfCAKuw~DwEL?x zi0=X&sU>H1DT}r#Yvt;-*^d(ssS@ZRwkkw2(mr||NY-UARf=ryfh1{tUfAL)<|V`x ze>|@BdBba>7VLZqZ_jmb1(LNp8zMFn=k9Og^?9YF=Pe4Bb1OTAV7=D-B~|2pycJ&x zk-$MirK{kO$HDU(=LhojYp4&`oNBY;?@3vPJG08%@>ne%iV}K*w{=E?{YJ^R5?JP# z@s(1bY;TbKcL?P&a@gnv>-8N}h{kYn~m#4KyRVkJc zuPdG?1ItzOMUNWi{u~pB=szo|V-_N+vA&^Gu%fLpPKYphxscOiKzAgC2 z4g_buBP&~mi zM0{RQ&PH>kiwXaQ8HcKS4YSRqo9-nSUZ+0g8Wy)J^#AY3uZL$)p^H$zCCXsRi-`a=b;=uIH$mco$hg<1ol z_JzR$I_JToG+P<+-AlN1<6rP)gx2QQHbh;xQyw>H`e1V;$9xD~%z{P=P0}X`gwCh? z5aO-vlhyq4-R7O8kV9<<{Hi-;eK@5y7DA^pG$DLeF|8|+IuL?NDPck( z?$px0Q=Yk0`jPk^7&)7CPSZ(C3Dm$M=G`1=Yl$AQUf&;kaa|_v5m8;QCpPMebc6Xz z!s9g}TN`@0*^pJt=I9{Rn~&DPVQj6JM|C6^x9H1vZbQ_1sp1LYX6FGKXGNQZCc*1}k1 zOR3H5g_CV)dt)q`4A)ZNNm z0{$FjGcD|h+J18wQ=d@0Il6U5SdFa1B5E!in3$CAy3Vx?KGc!Q-f$R{~C_k5L) zWL$_{)E)Y`3o&;^;u=_9^!X?21C=AA)%NoRg2IB-ex?kt&&q^|bn6I`Y!jod2%4OB zz3fZ0@wE{noCI>9=rv(m86gk28GEaJ!Z0RmEe2g%-jeH05G%H6m7 ztdiLdqR7N0wj2(&3}qk3JG0{53-tp#6IZoW_w~Nj`2DU-A$cR~>wZlW0f#{koCTQH ze~=lxX_yqFZu^*HH4tN6!_xPYq$Z%vQp-JoW5`h&^#!k5TKh09)CCh`8Rgfn*L56` z+7@2qm}kF#$0crYo}xU@?a)iZzJ0XRCSDV&l@@$V2k7PC$cTPeha#|}T7I=)+xCw-{h z194n%XLOeTv0l&o^PIz}m;IUx(ga^a!qO@j*#Yq5@d?G_Qu;%D+rYn_gyZ+aez0JZd zq38)!0*w&a^aBwUmqP~Oba90ci#(StU663vTG4M^V#~8q$s?XL4!CiT)7`XK;@E-G zw7r<(aHPv4Y(mW{Q*PwL;^fwzg!!IZB&s?+wq51jY7$k#i7>sYZ>2mHYTXjIzxR)P zrLWd~lTu6VncGVcb+DULHD$y;RDk9yG#Sg>RfyX$h*GWTbR>A~8m`5RVOVpmx9I;s z7-8-QM~vmgs)Z(M4a8izSg_w!9y;#0*qgd`6~D^JqjNVh`(I;HD_mNe*Zkck*%j|| z@n+?c)}&LL6v=Giwov&ErTr}j4zu2rUccvE0}2mM4(eMg@%2hia%@vPcCLmPykISO z-QV{B-LiMYlTV4#JzXsOkjfedLN(jq(-xg%9e&Q>hGeHSe#E4)b>}9(J;SZ?9Btlo zndmgRjKO>ZaRMPr!|{9Sp#p;W#k}1y=8{o{+Khe10NEd`_4MTXQQ^wt}Y{rq82F@LE^n%dPVJk}+rr zwK~aTC0+DE^yCqpI}OX=M=5@l`;KrA+fO;s-?se)n&Cjr$;Ru5pIHZb1}WwW(rqgH zE!`!qsJSeJ>Qs-)?Q)4~(l!Iy|2;}9gb0qC`kD1GdUQ%w1S087#m(;>pX3RsAxQmS{ zSV;aPr81pds+`s*skxGRc@)8G%gS({A0ug6DoU(P{Xb=qIu&a33%Q_#d09}ISMgj@8|R3B$Ax<7%$RiC2^e%RgdpBt3Iq!qJp~ zyzu>^sN|eq#9fj+IDA#9F>yv7+3OpP(QTW7y9t&hZ}etQ8XAiu%0U6r(*HdBO7z=fHCjC{K18 zLs%Wnle|}T=dBNXl7={yV$dz^c5?@1RV4%1h@@%55~F&7k@A}r?T(tOgX!&48KkUK zvuq;wBWo#KUrSL%Of>o2v`Ozb3A*cH7i?t`XJ7PC;EP;uBJ$}fwI~$ z+4tUc@%?gD3FVKswqB|&ZZ0%*pBKt~@4>fffYT@O_&Ci5^K9Ip1AlfLQF(Bdy!y$d$CUkoz8|LNxfSdOcGjIwbyE)HGtB|idypT+pER8 z6U=F#^c)AY7$%9C4}vFD1TD9}>#Z88sJY2dawig+(qGZs;9z$Ak`S(ir!ClbMA$W; zB8}(gv7JmADb%-!dbX&*zBHRbS-ZV2D>z*g5ae?Ac*ql@R^jAn?12%A%v;Rw+yHA@ z+IE{p{Bq$+c+EDgQeKt*-F(Ccb;Ll4_s!?KYww{MoC8VCxF14&VBDl1k+)H9ZSRBh zdn0s0)u`JfjiS0JMGyH9z*`0@dLxYeCtthSC%x!7tg_ON!` z^x^*gL~d*FvG7(h=e9OuxSQ&3#F7}JIBgD2JWEF;R})6>dVAo~lEzSys<@gLkGeqK zLj9qo^Sf~i9uC!5)GaTOj&r6T>JmQ5;nk9VceMT<)t7S+$rI`H>$_msoyIe@%;`SZ zN8@hdsi{X}4_&Zu29?Ildmfv(e7M>XTn@B`ZPqB0$LPhh{b1xwTmeZe#rnJiMi$M) znZR^BT1k2xEWdbnP%Iy?fGt;hmcHsB0Q9&9?6I<_aS_{xVkf( zPIVXR@rjHqw^F9FEM0Q7er1(SGV|_5?MeRC&d;6`z#eVQR)_BFz(8vdxVxK51h)9N^p9E%K)4;KX)Q2~~U`U|6YZKaqfvce!4h zGpEF&V>4PpBr*R)Q&RVBuxdjHnKv`ef&L|X135F>2;1q#R3E*d0?Y4GJhTaHVVXDS ztQoJ4E*vXSavDB~OxkDb#J7h$+#O7OEouEZk3`n|YE?~;5qdB!sKiXjQa&U8h@cS0 zN{&U-Re9KAixg9EVk0S;ee{x)>e=cRWnSvdD%A_vErz-j=hUPuH?)Cbx(~hhpaf-> zD&=|9uEK`e7g6~+b;&d7|I61EL%uFEsYC@knKiEHnmO>im_d48T)U0Vc zb)1bi(NQ)Kau{Jk#@$lm6n)s>BFC=q`$MjyE!wHWtRHa9Me!qxTpYTCd)$o)(w;KS zBxb?lTyy0;gCt({#ux^YLux-Zwl$Q5`|+yJ>pbT{Da7~QS1rov%nZ%jDKF(0WNKA1 zK1Mlw3ip@^_aptVMPO)rGpiub-&Wc^{o>qr`~Ch~BHyKbkI{_Ax&#g@ef9|nQ-FGV zlHf&0H26%2Tr%OQNqu-YF4L>LX7aU!AGhl99e%WFf3i!lkZU045fIzjH1)nY&izQC zfu&U{s#2yw=4G8xr1M5Ax~VqT{UxCqX}0=;UaL!<^EfwO(Cg2pXXzN7PO`l22h+&5 zTT=OYOW0Xu=8xYT3!Z&iSvQY5e+c#aTP z(rFv|5WJ3NaNnFifAiKy<&$`=+>~iSa}CO0yp>TnLJ4VX;S!ntz4dKTee>f!&YC+CETVF%6s9hMe>#4)^x83HCDOYxuf1>t>z{iNyjCmZcY94hn0z~-<(C6yiJ$KI zwOAK_MOia?R2B=1ENOfBEUBw{g?;8>tq#Eou4v2Z@!@*cDwU-QF{H|qzSzm*#(1gz zl69FhCpP=t z57>?k>c>p1f0w0AH{W=y8#-NJ^M6C?ddXbtq~#_`&oMl< z`}Pa^_np?>>q{g>8{v6?>e~pZ??55)aVgb|00&NmtV6YIyijbN(KGXEV*<(h8imI( z)tfL4L-`B{|Ed)ioc~b2(}}ygtAG`O2J%Zb)ZTc3jcef0qf|g*`4WIPqflYemg9YW zK=MBH6p6Zsl59r$7axg0fz2o0ft^2yGSs2W^Ne!hf*;ha?k;(2zY?c)rQ91$jBDQH_Y0o-B*$!CQYkCd&0cmtUnT+_4mpZ*9pgEZBV@3 z(s5?h&~9011u8%1T`zn>=z1r-zaVVTKmYcSp*uz5iV|TRX#GaYY^PVEEOKqLB3`p+ z<+lX6XnhusCSB`cn$eVh`$|NHV1d7X2c(>WF4}|kM$bKlro$bB2&gULwLi_ewE2gy zNhAiEB!z6A&G#Zta6v}&EU#+w9Rg~@WA{6M?HQE32qe~ihF`#ApKp&efT|EDsr)-V z%;TI#nsriH@0Qo&6MrgXRsWs~cjT;bxc)t;NA_|4U^5p|q!zgfX~5U{s?Aq$E?w>> z`wn^|^hC9*PF^@t`Z8{hcp8O%t9-*71S}OQ%{)z02nPvu`NKOAJe-O(dyJWiEz6V` z!jeR8_|wd+pChNUgY4T&h%EgbeVvcs_rh5y3o9a189`_9Wjlj91rnU4kNB0v6OD54 zn7D_gk^_az!Q7qwq0jCEF2Rj^cHeYROHVPsrvtUX*>ySlK~FLjhzxu+->KaMb%siO zNLm?=gtmQ3S<($bMr2GA=ZzF|ty|y1N@FV92r1ddo%8+gW;B~c^=?Y|l6@>Dl>_5e zFp{t?v0v={8vp9{tzVuv6*cy;ixsXve96vg+ZjE22p`ww0pe)sUfISpCdRXWUQsi$ zh!o&c-#_~eFnmoQinCHmmn{ZGcjb>%&HNbUKa8f40KN%RO=&YEd0z9&jT?3(l(HlRK|1eL+mYP~_T^R@pL6KJ%^{OL*R?@9RecR$w=*ObvdDaFz$ zHpMBSCUVp&dH-51f6n6N9@#ow?(*e0U+pDO;8jU^^s zH~Wr~HA&)5C~s6Hdl#6EH_W1y(wJ|erL&gKT zY2fumiY1>2&uel7+?BHwiaUABA`J~YFTfeQ#!vnHTB0OP79OnQ%vX)F zFFv||Eo&)-othYAX^j_7<$Aa3osuBX{=9$d#kH@GW0ejfGXZnp$IE_68^#=tif%H@~ zIHussQXSBE`V>TuGe2bh?kDETpaKloMA`izQ0o$EuBvZ6Wol0PF}p86A~Nygp;495yE6%qT;zkC!u z>iI_@Zeo)ng7v_;{)ux0@*_uwB!0wGO2A7dcq#_g(=i`4ka$%SnbJ3C@y+}p&Y$r9 zH(*$P8i#x^d!HSEi-u8zg~0UXiA$$35)phla2o}PtzD>bmy1L$CBc@>8j&gH_2fIr zU5}$O0@VY4`B;XCY<~H=pnK2!m%Dl{-3^70AoJBbtkA0AM!>yO$g`cm8DIfebjA03 zpl?!^K;kJ7NO8L^B>GuG(#cw=9bAfPixDd^@m_QIN_oPdn^u~Vu9s#xiyAQs*MOCL z^FF(4D&FqOhymn%N@DLj@f+-T4E(}Cq^t?h^D#lu{XT605h{CFQNHJ02Bq?K)aifC z6-}5cGD9ck0XtWP3jyiOTtolIO+>EgjM?NTx=->*Wa|xMT?r7~obuRKJ|`n+BmFcV zE5xY=`2G~CdH?AL#1U?bso~N?Ma)LXwmYBhH48x2*_&6g-y5tdrO})PX>xU#8GV*@ z^QFH&%{NJqeqtCTcYb`ryU2@_p~o#^$vtzQePB6|_~OY6<4WwZfN-0!&ns^t{x9-{ zplR`mpnsDuwtclxZ$6iP&V7o2FN}11o@=+w;CMenH0`0OvCI}<@BDemXK4_DesA81 z)|`vO@WyWdGI{xvMvqVx+0B!9`^10g7A_V` zx_S%lB1+yucwCrjQi5KYyn#%mrUBQI6D5;1_L(I1{TovBv0Bw$6YhIDeY_Qo_FL#f z*I79}+Ss8pdJgN9eAZ*bcq~%!FF~X3C4xvm%O&&w&-nLE0&XY9Jh*_$%j=3fVJRYu zy`VJc76Iwo2Xv=#@NXSZB@)sxO+{Q&-mVw3%BnM1oGE3)RCunvOh}Elh?BsR{r{(0 zh&Uix^wPWY|KfdkFCk49C|Y9xXm9>VALzBYXVP4zA-&=^kY|xXdYAFYJ4 zfg250XXFYjHfKvkU=pu@4ksUi12m|Y4Mbm`x2@j(8f#(qzaBtcuc_4VDIn54V@&u_ z0|lWH5Lr}E*Mw`^z3l*3bWt#+MaQcR2z(?Hm26jZ*K07#=&4H56nWHNEKkhg0DA0uc z3xNNS=?j?C`5;=V?{|v;|KB+?*%s1&j?xCH{_u;c;Km5vA2YB$U z3wRCc2nuvCidye16%iA{?<^;2k&C(e<8_b|xYw2UZ*%>7`uzRVH^lpO!y=RuZm$Oy zcy><>8sFz5fdoJ&Oo83cG+KU_{htpdK?V=?XQ4QO6MA`JC@+3>ymCmh*xU%=8O^Q2OxZ1P%vTw_5~Ko7{SO6ZS?Vy3?w9VcRYxv+<7!L6qSEW29Dl z&je;f#5t%-j6Z&Vbpwm8yl$HL`2ahOE=nr;n)2;yN)-(a9UToZ?rXePr(e^(N~NJo zWj0JSlVIBSBE@}^%}&d6TS1@tXHBGX7Ok8Ch)>Mx0vqK0ARf@ z4lo-Wrk|Q~#=}WI8up76_He_0eox*@=en(`Tn1y{FBDnX0Y6)4h34(+?$-Zsf9)<8 zoNEi6)+a*dqgMXuCGB>B-=)_TAXfe}0zvpMy*#Uvpx- zvF-0^nkv5zzP+&*5%}h^;KH_FpAg)rv4kP{ub&bb#4~k{+_~d*1ivN=e;b@F0s!06%BwcEfqo zxSRIz&wq^bLsKR^-SG0izmY$G@(w%8 zK4&EL7yq2$RR_6>b}&H5z^%a(f(9t7PT?Nze?Qs@oW)BCvqmnhK>*hgi9NTyJk-@8 zDFmKBF0l2YK%*T!UgkSQ-e=lDyJg7_2^A2sn*l#eJMlK^>OWuqlDL=k^p28h3h_x?P8xPF z;A}w)fhZ991B>mTY%qeY_Q>uvZu9>*ZX2O18xyh`n{+E&oUHZCit&5-$9wZ1 z#2%t|5ZlO0`q(4??Hpbhh|OT`-i?5LkL$vP3+MI>x&G(5q>$%Q(M=(Rp?}Wx(FP_J zGIrZwpwkBS=<5yka`t~rs$K?|iUFyTaWEBkaL5T!UtgWGR;0vZt%Pg_e9#RqA~M;W zIsbe18zR4GgvrJZMud@M7(({zgi7Ah4zy}n?P*~!H=+T@#k0e){-0D%WblaIpI~8o z$%ua$>84Snod9u;F+}gi00b}wiS-;nY`6iMz~)kiD3DAyDgX4G`Ohi+5LU=*F6TZ8 zA$!w#Z-}I?1^9u{rm^8@u(ZYq;@2zpAHuiM0bUsLYD)AQl7GG>9ON?+q5g=ATpkbTo2eJ}@=y!WvWS=ba~5BvPS;|e=@^cI949QbvgcQOWdx|boMh(b1?D~lS- z9vkT2J74ETFWyU8h=|8OK=waV&J%C*+XO-x+~b7RHzLge8ZFz zMJ%x_S=gIqzrVetu}-IJD>m}rW0-}=);NnXT(YR4cEAHoHAIS?~AAP)6foMymv z8%m0tY*3QDN6Q--=wk_V0ETM|cgQ3j)Pg%b0!a&*(%HLT8>&FMaxKNUF#MW->FG!5 zQRbR6rB}_2|BPvvN)30#C91VygFqgb77hlF!vxeDg_VI&AQK?*c?j|}w&U$aBXBV7E49(P!Q~O~ z5p!_4JMd9|U??NkQd8?)(ANL?y8mN)IpKW^XQ{Ue`ZEn3$g!Ll2-WCvseRt_hzAa_ zybx=>7wQeS#bZQb)I#$CrajFZhQpoZc^pf8W%JlnP8_Cq`nVTg|IkG$T1Ec=r?c%q zzv(?nATO+1dOm|PYq?|O@0xNJISM>UeV~af=1k{)uep$C^_=0}$-dT-SGmSt2wkrV z5sFf_@}4nXm)nVe+zu5c8dvhrU`=E5*S`@|*4iAtKG?D{krh98*?C8?A(BW$L{wfolWx-+(#5-X zcnq8pG$gGGwvBX*QtcFfJu*tRBWKt5vN#t#`aYNg*PKQKqkj80X-wEbKA!2y-#$Mg zv5=3;4)(bfi3}hu-Vxa*P!C#*Yr#b|{-OP93SI}$s$NBiWXB-1{@%ZiU?ijv zGd`8Sh@4_x(It#Pv#_ew67+lBJNlUkVaB*766s!zY}H^eYDW@3+>bK}4LNnq_}ikj zi?!i`vyM9z^g9OCVK)~4@cEkyVCFK13pvASUYWphtgNRUtWg=XagAS~w?)zQhM0kMa6=&e`-6^kApB5sjdW!ud*%^tf+uqEG`7%41AS4CZyT9Y!d@*$r zwLcqv-F;EfzYU;N#OU{XM5Z)Tr})VUP{5>l9XWUX(#GRh_;d!HPY$y0_XqNPo9~aj z)C?|0E?=p~S}IijNHI!lC+H{AR!n619OMPRzo!(QFkLCRZe3O{^R?kcSk8Rx-0fwl z3eDxu_sb!1df|duC#!Rt7rgn?_~nusDIVFExzk%xkb8|}Hs#c@MeemK@#ni6M95y6 zpn})WqWz!^f}IFRN<eN>^}2Tctd%fcL#J*eqb z>TP6xCR3%+<=y@9>PkCg*&fPrnu$`;iD0fyL*GFdN|J9g@^C@8YLN(p7c3$MWPf)% zbD5OE=ZLbrP>c`?5u*}QZr84KXSYY?c!2*3ln%MnPvY`7^U_xog$I?)Qg7$<2QD&* z+bOtuGdsHHRP!@Fc_~7Qb%b#J^__45c&(rDAUkI0vc2#_vmsOEhIgAXnG<{DG5~~~ z!S2`&pmXgz(gD=XJ|d=YV~L?pczZ=L5dn5|;eT%M2doX})0v;ZS(>W*iT;nTzSfpf zGHINi$8LS_vyaigPUGazCTKLTS1hygMPr~Lhh_ih8$54$+zd)l#y2zDZc^O~PgZwXnwj)Fc z-@@EpiR|F^`)aeW^IV(*|9#=1L-Z?P_ouM%spfwQh_2EYA?I%AJO0;qcTB?$3uZbc>StaJ_c~ zrJaS3G3xVpPd5KNu@C88hK2r5cj30E3~=zzg8QjI8`L?b0xYC}AnkU@OgxV`cp;7# zA=~W6XZXID=pk+UBaOzQKai2}Nx>iR7*PzYQ&aC=;ljaO6_0!W8c! zMfBN=?nAG`XtFq=Q_jVk0(bjUf^tL7$_uBZP;q^{u+(&(m{a3V2f>Vr69&4f(7O)# zhy(Bt-kl3Hq6B*9Gf>iuyf_6OQIV4}iX3qC849gROr)+O26qE;sYK$!@H0zIE1~!9 zvFcIzPw$52eY<&<`_GH2Z)+SnxsFU{6_R0)x-I+M2Z1RHsG2G}V=?EwF(eTj2CpN* zgMNK!D&gfqcYbZ$3MB+D?_IyB>cOl-;`ovS9F6mJ%6gGpX$r`2N)HD&-iZ`|2i7^) z4zi-Eqs_KC=wX$SWgo1LOT!MS$mE37Tmw>ECKlM#%oIMT$-vAoH%GC;yFTH~P0+o` z*^w$W0Wbc=Nj#Q6F+Pw4q2nKgn~MY$EMkz(T&M_ylSAa&w09>rbp2tz^*$#r zd|H)?hMvKjx#T9n=Mzt)$MFrpQW_TS;94F8N8Beb>%fMyh`8nY~t`O|EIiUp48eTqRdkk4CEoSKWa;RV!mMq3eI=mPXm6P_C} ziRVFVEgeYfpMK-W7g+GBaX~7LE2`{=jpw1(=dC{uH3G=6JTbrwUyhbv>YqH`EuBO9 z?zjqUe7h1*5R`h5t1uVs{PH0G8kNvvD(^akmN)Q#t>$;qHQ3IyuwY|Ao35{WVh@cW z6CW^<7tW`=j6Vu$5a;@c*a6J#_H5nThrOsnSo972?#{=c#t#1| zMh~G~&imc1!`h7i8~Z&oqzxMi02c38^Fp5;{&qhE(Vywz+{E3VZ*G&PK4@|ss+<@w zRXa>Bm5LUQU!z6Lw{viQRoyK?;JeqhBxw1P_hju`P72=9AYj!dyM`~CK0G1 zJAujH5r8O}fdY~4AB&gMEK0jjG+op-6qD7^iXiO|<0n7@4gznamj*=sgFpLo96KSF zuWZpm>}Zi-F^ooB+;II!pmkq6{N54czB`iRxc2d4wq78Q?j+ECO_03g%7t`=tW?V5 zUw(5S$H|JkhcbTBlB2GPjDjkk1S#$&9?}OS641i_b4mWJoV)(1hn3CPs$UM|%S~Se z=rr>D_<6E+bb>b9CSMSeP&DybQ~lyPruvoH#*qTjJ3J z<%pV9D;qAbg4Q@U-Jf`B$v#q?yUmNZ=qlq?U68her!U?xMf3|EKw*qP?s^V!&W7}- zH1%JU*t|mV8>^c^A9?{-+~$ybDIjHWd&nwt`7ZxSwx2kpF@Hat(>S~S($>&lkmoLH zId&67B3+PR<>U9Rrhe}cTlBzx0aH}_MQ_f4g&Dh(=Q+C)P*V7!%XK1nbY&Sx#2r2& ziC(LZLzjU`b9ns=-%2^~p7{3Q>_uWtnv&*`pbikF>nW>rW*tNJFlauY-n3bB>GYs zKqoXFtCYh)ptUl$r?|Oyao2Byfk#NrZXaF)MRojk#3;3dN75-Y8HoG!P;mBt-}-PK zxL-gQ4)XfC)mq>WL;B3;XdqXUM`igGdilD&>9wzare zFwb@CJg}@hLLrS_|KwN?iiaIbId&WGjPAa=BFlatHo&$!H(ZRb?!)D*ZvIm{MGfRTFTV2wxtM`_?wT_-_4WZz+ zyum4*#?qpG!D!O)>*o8=eA2Lga}9Efyh|sQ&3me&F_${r)?Fw4c`J_(mXn8aHXhCk zEM+7=3;e1Y!&|o`YvsAkyL;yLo~H9)lB*FWekwc#{rP*E%eUdFXh7|`9<8nIL%oI< z%IeRXKDf&p{%h3bAgvlJ#==NI&$_dR*jc!chPr5376nWaHzg!St#^Qfn(ukPhXd0n zu{m8juy=BYJip$APtCh_?r45CrLcT?m@;_&7LJDhU=Y$?m5TUe!1yzI+jF${NaXx3 z+?cb}@5MaK%y)6~m&Ov5d3f*UTw3*%>tT!BuYERZP2F<{VD*-fx(qGsaFY;eYH8;5 z$o01|t2i?y!f*hzBTs&EI1*hOb0Ox@A)3m_{sX8X<}LY>O~)OdzAfn^@MLv86{SSb zdSq`gnMPWC`U{_iwJX{vJ+ou!*d1YHYY>@A)|G~|V9`AmhqT;IEwte?ajt9kN-aj0 zo6_7Qt0$FXp=IVw1}i`CPf)vOqD*y4JltCCme>wcIr(2>3SyKn(R#+m5#M{GLl1J* zC#3#w1nE8&<*HRHb|%>}ucmNXlNCQx)|t_|fKk(3XJNzK8`$g_vf}-+ zF!^C^@7?MZE9FLl;;NztSuwFs7)&bfRjRsucT|(32q4MhHoNhp_)U@Uaizofm3g*V zOA9{^yQVMR<42;8@_1?rIx|V*J6KXnWTNN>{3dj;lkM@s!cL%dnuE60!G+`Xr~|C5 z|Aoim(@hpP6l)jUCtDqXj=r{<0^!83u2dMpjY;|g@GU7VKf_Ygd73cN$MPFzE)T~S zmYeJwS+(eXC{+DA0}ZZyNS+*e@=rn!)bW58j;0&IXpjOZUp%H$+(*K!_`>P=nU%cp z9+~&2FAODFlaPPWLL&@oyY)pnm1URV!AEP=N7IZ?K;G6HW7V@LRLJ5pR)Xw(;}yp} zGJfkHl31x6-6%=}YJM%JN%_~d^f|+!V0ofXcz<5;qmo&7N0?l&Rlqf?Wxa)nU9xAT z0X});%>o_%rBnOZGc)-NUC$uUUR$T_@T6yH(aPsn(V=vwFxHZI?4aclyL2i*VwufB zMPTJigZ4GkpB$488*m8vu*C1#{?*Mer{+$(Cu+GLZ*~ChMJhTg$f4Rtr=&P8D;d;? zCByMad52N>kCvKDbS#!?w110MP7K*%rcY7+7FijVP$RBtuZ`O|bqpufu)^wBTKK|B_i`D4{twwU3EH^GY1>FvjoQJ^Iyp9+mOm6$uB!%*!DQ@diQeT4)CZ$mt0%wFWzsnieiFWWm(xv*o0TbimGTE|1D zN9O*umsQ{_Ddw!5G1FdIacwL)&ABA>?B(0Wi3sl2w2D^KFILybxF>%%@+El>xtAKq z;3Pg6D9*?VsHmudeg!?YD{SjM7S-~5iAZbR^EU;Chp#b=o5cVAMn6_a*Vg0e*v?^~ z_4Kh$#cIHZ{KCr>R#gY2^y7_pDqnNY_HNiBb5;6r z`kGor|LS<>!9qpdyiIW0P*lp3q2kSsQ!k$ldC}csC|Ot^Dyib$l?p7M@*x;=2pK%c zS3i!1*1!BJJE?gMmdn8z)`dCk(E^63op8FbFI@W-7zb$?&NMMmixABM)ydEThTUsB zw;i74^K;~^)8;JuV{(dS1kpe5lY2Z0-V7ReVWVqfD=77xXG0s0yR9wd2V{IQ zyBsL*fl$iHX10{gM9bz@I?@_|GU;a*Y?^QVj-GkG;zBY&NCo~H_-b5TLB^#(=H$Sp zcJBya`i_&=R?32{eQ0ZI>GpdRJ2@~l9fHd>9*DT+DwBhpTDZ;O?O3g$8kd}>pNnW@ z($Nu~nvm}+5VKrC%DgMJUNzb2J8k9G&Gw?!m1<=gjs9kf1o@PCkK-*Yd05DU#ieRG zpoTQo@v@PA)8F`AyB>F3Dvrs%zYn#?4T}2-r9I64C5Ooh`pW*+AxB9fDSF(pF?z(( zjN03?dh>lj!i^@!juqa0GQ&u>;SZqb<*P|=+>zLN=K0Cmk#v!JKE5iH^h?f%0IN$^ z(9IX0+pMr(Y;+vwDxvl53BSx@mP@H~y6q@Elb;8Bh9bZ$%+M_NxF8r;%&#SnAeqm< zs`;6_+N;DFt^Pn-=ilsNIJ5mbp{*s9w-uTSt1VjjvunF7OiiCZC^(AcxmdAAi4hfk zcqj&aD&h);MGds(*eaR^uukNjiqqMLuCnL=0sL2Hu4wYUaX$0++5u|Wlp|J7FPsmHT! zR!4UTkYoEwUOS*eQPQQNKA!zv0A%?6^3Lsm@=R9`LFIUho`^l9FU8n$%A)=CEjfQ& z&Gl9*1-oIS@Z9`C1dKp3$Kl_&XSMn4&GVB_76-GI9G0Q`C*5WUC37DX50KyS+^7wi zuT}f4+0;vOrJ|)LVFe+9BPGD=mRfu?2gPG-NW%e7GXN$;E;tSbFJHz+dUl$-&u_(VM;>rw5~BwSMamuAQ#U_T<_+M; zvGl;XJL>oGmWl9TVhOF(4j*g%o=VkxHpdt9B&hT8 zndyLUI7L;`(%)Yr`As9deo14H*@p_6Qi=Dmr62HU)TZzjFP!dka#9`=cQESt=@}3f zXfu!zEroY$z_(F&vsRza;i7}Pvf33)Z%)_=p@`4dZVU{F&i|?v$&w1x+pK&S?#h*A zFuZ{p%MN$Ri42*9p1c*TB1@-3Jh{@k`ie|u_zaH7le z)4&1iBvp&WQWK4&rP4Zbst#?>qu3jxkgac3QOTxrzSJsO?!EI#Xc z8R-sZGnPYlyrp<%GI&(NevDn(MX!NlKQ#ui3vM=LY@>R)TMj`=fJz?{tbumDNNNTk zXgW(Cibh6F_Z-H{+}oNkf@z%^Wu~SKJKIcdzwP9w&pMFVm@|fTrj^WMaB+4YKK)aW z%JBB0IBe|rl6B6ow`BM_$GC5q#wmiWxihju^Sj}5V?#B&#WBW(0~N-2rF^VCtV7zU z09U9KVQCPkNrOBR_u6(rIf94)S0o3&V)rA5}h;wk#;f z?fuc1WqTP$)W4+9|Ji(jcekl)^vPVzDtLDR_uU$ngb{Lz&w zW(C*pAG>#!oOf94{`SO!bRX%x6uBtm0S~lR=_Qv%=iaIpbW`atBzW%#wNQ-p?CRpp z-D{OdFPWSaJFTHr5I&t9$__9x?*zPUJf%W|bgO)w8{-{**8TWoo}w8d;juJLR_RzT ztI|&~6jBT&<;H>i0kO4$5t!<>f9P% z3)qWTcFTJ+4)-4t)`|uhn4@X-H=3B;cCqWTXI{3+nEc>q(+L+H3_BI_G}`u$9>JUZ zY(|!M1y>gOfhwu1=t2eG=yeW0k3hPG9#I_w%SF<8@ zF_!*fd<8b2*y^3^`ra>>=}wUorBry{DUUDzX4x0EK9 zMbIe~>!=y^!F)~pS3%Th4EU3Y7>ItIMxUd1`mXfVa{9Z@ z3poA|WxN9y8mcIMME7h$oKr|Fw7S@lbAS95)lrFsf1RDMM~UMy^pCLn(5nBPOEUFLe}l z+c;q+cBmwUNz&|-+vFC}=Zs{RU4+rjbtqj(r6If8p+tK>Yp|2u`TzXUKRz>FpZ8tw zTI*TwYyG~@^P7D3OOG@wF5KHEEsdgw5m{&S#!hWll*}e4v$`T%4Wkm%$)Is(YJR0D zhPvBWs`QyP62_G#(DwtkFT8k3B6;q7*c zWt^jmp$G7JAg%yvZJ3 z5E>jbNhM*R4b+nBx$1dSdRMHS8e0h5Y{F}7ye&MJnsOU(uFX>m>h)&i-cD1`~o zu}k-3OeL{8(gORMmNO(_@~+f7aVanzSL#)|P@pR@(xSG4Cyz}MAU|#{$NjMw;SQDc zh}(;L>B`GEL5>e0b-#PI`3mV$!DYsFC?OR#rEinQkOJwc%EM*{9R?qRE5V(CR*h7y zPz1qh@2uu(|L>T~$llC(HQnXnR!&cL4&CcC8AP9qghpZ)cTl`MOx>xxJ$<{zOb7@^ z@$$ULWo)Gp&O4h}9pNgu1a^|a3;PR55*#+ps~=%1f}nC9U#xZxyP2O+BBQttm#D%T zap9@a0l-p?eaVXB?mo-2OM`P1L+A33nY@>9wGj2$qAHMu ziE3C7m@*iutKgpY(}Bv7bJ+1yr6@8$?u7mFs32y4x0f$aQe!y}z)zC{jFvsTHizY@ zuSkHX;5Y5ns^jIr;FqymE3c@g-?rUWwWEHhW?Z-ma)Mf0d#>e~ATZEb4|5`(LmLn6 zgp`fQ1~CmLC-Q~+Xs;rUZiVsosl6NZu*~~f){OvV!iXi(o&KrFc9K91W=aS!Iblso zYpW1GNfm*A1xT|B2I6&(f`S4U3<~-q*{?oG(DSj#!4{!*GrNKYHo2)kL+(50${%1-m| zw=yVpB2hEL8#}@W#Xa~_DiM9I4ihB>(49KtoiY27LaxmbP|?M@fIj_2;|VykoYK9J zU$pipB4-XOv8hu9{{<9=GHr|boSd%itPpBC{mqcE|JMGQYZgNlCK z=?}kqujk)cd(!Z9!#R;5={0k4;Cc11j4`7KY!z2!x- z#s+9*Sy{D?a=fanf(A|r(lGbEnKDrt_d9%MCELCtL4C~t9mAu1;8AE{NrR?-SKkbh z>(Hu~Z$-gyq43vPD1~(nH256Y&*$ZWsp0VuW@?Add9mojb@K!a(*BVpBN(CtK|+xa zZJW6@aV<<+dD5^-Uw+&=M#C;a^KruK0FHdppd5lCzCalCp&(dB15utUgqA`ilptgR zM}IQwP$qWSunCeTBcF5iV2lkyLr^ts18@Vx_Fyt4ZN=??>jJjvp{(T+Jj$#>9!}6~Cq0$rFCGB!s95-Xvrv-QfScmd=!c;MDO+!CU?3hJ4 zol*7)X|*{@6VO*>+2Xrs|AK-*fw1Kqh+vE}ZjO5Jcn-+DB~-qo28R-mxT(_GT=3_9 z(Xu7L(D#OrZ1^%_iYkWQCGKqTqjz=J$vxT2*|TW}zVQI`j#VJ4AHY(nUQT3D*eMP$ zV~9W!iQ?>Q+ILl)m#>NRrZrBq{R1o=jPGa~cHXtJa*U z%83Aq_$;1Qj9d)nq;Qs_h+!a;Is>i{N3lWTfuUB+rzb5QwfXNZ^2Dq6Sf@gToWnUZQQ1afD(3)3QxS?LBi!53}N&AXtg%dJzBe+7VS~;#dEeQ2UxBS47Ok z9;Efi`HTAk-eF2pjO6N#MS~1YT`O<7+HEBR<7Y=2Q$A#fds2d@$4l5isq2Yf}7$OtM1d667%9#Iq|!k6kD z1(&uFMtP9gGK=Ym$x`>^pL8LA%vj;$7{W$>rIpDP)7%qWe{F&ECV+yw6=O#$$v?#2 zQ9hcOXVwACTA3zg#%88BHCy)RY)lZNfeUckneUI11aUqTOLGyU-6xOri7J~v_knQ8UC}= zcJZYtUa*w(A2IssEfH{A;3!eVQvZK6j0^)cc607IGvAH#mlj6(73mD3{Ne+%Eg>L= zeA0L6=Nno035&d6Y%J3yzTU88s3ri6j-ZtP;#&d^6`^7IL>mMh{{1NK7OZcLW*&=^ ztY!!N+c5uatcl--`OAq!`4-KK +PREFIX knora-api: + +CONSTRUCT { + ?letter knora-api:isMainResource true . + ?letter ?linkingProp1 ?person1 . + ?letter ?linkingProp2 ?person2 . + ?letter beol:creationDate ?date . +} WHERE { + ?letter beol:creationDate ?date . + + ?letter ?linkingProp1 ?person1 . + FILTER(?linkingProp1 = beol:hasAuthor || ?linkingProp1 = beol:hasRecipient ) + + ?letter ?linkingProp2 ?person2 . + FILTER(?linkingProp2 = beol:hasAuthor || ?linkingProp2 = beol:hasRecipient ) + + ?person1 beol:hasIAFIdentifier ?gnd1 . + ?gnd1 knora-api:valueAsString "(DE-588)118531379" . + + ?person2 beol:hasIAFIdentifier ?gnd2 . + ?gnd2 knora-api:valueAsString "(DE-588)118696149" . +} ORDER BY ?date +``` + +takes a very long time with Fuseki. The query time would have been much less, if the +statements with literal object values that are not dependent on any other query statement, such as +``` + ?gnd1 knora-api:valueAsString "(DE-588)118531379" . + ?gnd2 knora-api:valueAsString "(DE-588)118696149" . +``` +were given at the top of the query followed by +``` + ?person1 beol:hasIAFIdentifier ?gnd1 . + ?person2 beol:hasIAFIdentifier ?gnd2 . + ?letter ?linkingProp1 ?person1 . + FILTER(?linkingProp1 = beol:hasAuthor || ?linkingProp1 = beol:hasRecipient ) + + ?letter ?linkingProp2 ?person2 . + FILTER(?linkingProp2 = beol:hasAuthor || ?linkingProp2 = beol:hasRecipient ) + ?letter beol:creationDate ?date . +``` +Since we cannot expect clients to know about performance of triplestores in order to write efficient queries, we have +implemented an optimization method to automatically rearrange the statements of the given queries. +Upon receiving the gravsearch query, the algorithm converts the query to a graph by defining graph components for every +statement pattern where subject of the statement defines the origin node, predicate defines a directed edge, and object +defines the target node. +For the query above, this conversion would result in the following graph: +![query_graph](figures/query_graph.png) + +The [Graph for Scala](http://www.scala-graph.org/) library is used to construct the graph and sort it using the +topological sorting algorithm. The sorting algorithm returns the nodes of the graph ordered in several layers where the +root element `?letter` is in layer 0, `[?date, ?person1, ?person2]` are in layer 1, `[?gnd1, ?gnd2]` in layer 2, and the +leaf nodes `[(DE-588)118531379, (DE-588)118696149]` are given in the last layer (i.e. layer 3). +According to Kahn's algorithm, there are multiple valid permutations of the topological order, the graph in the example + above has 24 valid permutations of topological order, such as (nodes are ordered from left to right with the highest + order to the lowest): +`(?letter, ?date, ?person2, ?person1, ?gnd2, ?gnd1, (DE-588)118696149, (DE-588)118531379)` +and + `(?letter, ?date, ?person1, ?person2, ?gnd1, ?gnd2, (DE-588)118531379, (DE-588)118696149)`. + +From all valid topological order, one is chosen based on certain criteria; for example, the leaf should node should not +belong to a statement that has predicate `rdf:type`. Once the best order is chosen, it is used to re-arrange the query +statements. Starting from the last leaf node, i.e. +`(DE-588)118696149`, the method finds the statement pattern which has this node as its object and brings this statement +to the top of the query. This re-arrangement continues so that the statements with the least dependencies on other +statements are all brought to the top of the query. Resulting in + +```sparql +PREFIX beol: +PREFIX knora-api: + +CONSTRUCT { + ?letter knora-api:isMainResource true . + ?letter ?linkingProp1 ?person1 . + ?letter ?linkingProp2 ?person2 . + ?letter beol:creationDate ?date . +} WHERE { + ?gnd2 knora-api:valueAsString "(DE-588)118696149" . + ?gnd1 knora-api:valueAsString "(DE-588)118531379" . + ?person2 beol:hasIAFIdentifier ?gnd2 . + ?person1 beol:hasIAFIdentifier ?gnd1 . + ?letter ?linkingProp2 ?person2 . + ?letter ?linkingProp1 ?person1 . + ?letter beol:creationDate ?date . + FILTER(?linkingProp1 = beol:hasAuthor || ?linkingProp1 = beol:hasRecipient ) + FILTER(?linkingProp2 = beol:hasAuthor || ?linkingProp2 = beol:hasRecipient ) +} ORDER BY ?date +``` + +Note that position of the above given filter statements does not play a significant role in the optimization of the performance. +In case a gravsearch query contains statements given in `UnionPattern`, `OptionalPattern`, `MinusPattern`, or +`FilterNotExistsPattern`, they are reordered accordingly +by defining a graph per block. For example, the following query with a `UNION` + +```sparql +{ + ?thing anything:hasRichtext ?richtext . + FILTER knora-api:matchText(?richtext, "test") + ?thing anything:hasInteger ?int . + ?int knora-api:intValueAsInt 1 . +} +UNION +{ + ?thing anything:hasText ?text . + FILTER knora-api:matchText(?text, "test") + ?thing anything:hasInteger ?int . + ?int knora-api:intValueAsInt 3 . +} +``` +would result in one graph per block of the `UNION`. Each graph is then sorted and the statements of its correspoding +block are re-arranged with respect to the topological order of graph, resulting in: +```sparql +{ + ?int knora-api:intValueAsInt 1 . + ?thing anything:hasRichtext ?richtext . + ?thing anything:hasInteger ?int . + FILTER(knora-api:matchText(?richtext, "test")) +} UNION { + ?int knora-api:intValueAsInt 3 . + ?thing anything:hasText ?text . + ?thing anything:hasInteger ?int . + FILTER(knora-api:matchText(?text, "test")) +} +``` + +###Cyclic Graphs +The topological sorting algorithm can only be used for DAGs (directed acyclic graphs). Therefore, in case a +gravsearch query contains statements that result in a cyclic graph, for example +``` +PREFIX anything: +PREFIX knora-api: + +CONSTRUCT { + ?thing knora-api:isMainResource true . +} WHERE { + ?thing anything:hasOtherThing ?thing1 . + ?thing1 anything:hasOtherThing ?thing2 . + ?thing2 anything:hasOtherThing ?thing . + +``` + +the algorithm tries to break the cycles in order to sort the graph. If breaking a cycle was not possible, the query +statements are not reordered and the query is submitted to the triplestore as given. From b2272c6f4766fb6810ff15ff2e983dd5be5ae2bf Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Tue, 2 Mar 2021 10:24:05 +0100 Subject: [PATCH 33/33] style(docs): Improve style a bit. --- docs/03-apis/api-v2/query-language.md | 35 +++++---- docs/05-internals/design/api-v2/gravsearch.md | 76 +++++++++++-------- 2 files changed, 61 insertions(+), 50 deletions(-) diff --git a/docs/03-apis/api-v2/query-language.md b/docs/03-apis/api-v2/query-language.md index 91c2c3639e..5ed2b651f0 100644 --- a/docs/03-apis/api-v2/query-language.md +++ b/docs/03-apis/api-v2/query-language.md @@ -1212,15 +1212,17 @@ CONSTRUCT { ORDER BY (?int) ``` -## Query Optimization by Topological Sorting of Statements -The query performance of triplestores, such as Fuseki, highly depends on the order of query statements. -To increase the -speed of queries without losing the accuracy of results, we have defined an optimization based on Kahn's [topological -sorting algorithm](https://www.wikiwand.com/en/Topological_sorting) to automatically reorder the statements of a gravsearch -query. Currently, this optimization is defined under the `gravsearch-dependency-optimisation` -[feature toggle](../feature-toggles.md) that must be activated when sending the gravsearch query request to the API. - -Let's consider, the following gravsearch query: +## Query Optimization by Dependency + +The query performance of triplestores, such as Fuseki, is highly dependent on the order of query +patterns. To improve performance, Gravsearch automatically reorders the +statement patterns in the WHERE clause according to their dependencies on each other, to minimise +the number of possible matches for each pattern. +This optimization can be controlled using `gravsearch-dependency-optimisation` +[feature toggle](../feature-toggles.md), which is turned on by default. + +Consider the following Gravsearch query: + ```sparql PREFIX beol: PREFIX knora-api: @@ -1247,13 +1249,16 @@ CONSTRUCT { } ORDER BY ?date ``` -This query, as it is, would take a considerably long time with Fuseki. The query time would have been much less, if the -statements such as +Gravsearch optimises the performance of this query by moving these statements +to the top of the WHERE clause: + ``` ?gnd1 knora-api:valueAsString "(DE-588)118531379" . ?gnd2 knora-api:valueAsString "(DE-588)118696149" . ``` -were given at the top of the query followed by + +The rest of the WHERE clause then reads: + ``` ?person1 beol:hasIAFIdentifier ?gnd1 . ?person2 beol:hasIAFIdentifier ?gnd2 . @@ -1264,9 +1269,3 @@ were given at the top of the query followed by FILTER(?linkingProp2 = beol:hasAuthor || ?linkingProp2 = beol:hasRecipient ) ?letter beol:creationDate ?date . ``` - -To automatically rearrange the statements of the given query in the above order, upon receiving the gravsearch query, the -optimization algorithm converts the query to a graph and sorts it using topological sorting algorithm. From all valid -topological orders returned by the sorting algorithm, the best one is chosen and used to reorder the query statements -before converting the query to SPARQL and submitting it to the triplestore. - diff --git a/docs/05-internals/design/api-v2/gravsearch.md b/docs/05-internals/design/api-v2/gravsearch.md index 9d352f2c47..dc6728302d 100644 --- a/docs/05-internals/design/api-v2/gravsearch.md +++ b/docs/05-internals/design/api-v2/gravsearch.md @@ -335,6 +335,7 @@ the method `optimiseQueryPatterns` inherited from `WhereTransformer`. For exampl Lucene queries to the beginning of the block in which they occur. ## Query Optimization by Topological Sorting of Statements + GraphDB seems to have inherent algorithms to optimize the query time, however query performance of Fuseki highly depends on the order of the query statements. For example, a query such as the one below: @@ -364,13 +365,16 @@ CONSTRUCT { } ORDER BY ?date ``` -takes a very long time with Fuseki. The query time would have been much less, if the -statements with literal object values that are not dependent on any other query statement, such as +takes a very long time with Fuseki. The performance of this query can be improved +by moving up the statements with literal objects that are not dependent on any other statement: + ``` ?gnd1 knora-api:valueAsString "(DE-588)118531379" . ?gnd2 knora-api:valueAsString "(DE-588)118696149" . ``` -were given at the top of the query followed by + +The rest of the query then reads: + ``` ?person1 beol:hasIAFIdentifier ?gnd1 . ?person2 beol:hasIAFIdentifier ?gnd2 . @@ -381,31 +385,35 @@ were given at the top of the query followed by FILTER(?linkingProp2 = beol:hasAuthor || ?linkingProp2 = beol:hasRecipient ) ?letter beol:creationDate ?date . ``` + Since we cannot expect clients to know about performance of triplestores in order to write efficient queries, we have implemented an optimization method to automatically rearrange the statements of the given queries. -Upon receiving the gravsearch query, the algorithm converts the query to a graph by defining graph components for every -statement pattern where subject of the statement defines the origin node, predicate defines a directed edge, and object -defines the target node. -For the query above, this conversion would result in the following graph: +Upon receiving the Gravsearch query, the algorithm converts the query to a graph. For each statement pattern, +the subject of the statement is the origin node, the predicate is a directed edge, and the object +is the target node. For the query above, this conversion would result in the following graph: + ![query_graph](figures/query_graph.png) -The [Graph for Scala](http://www.scala-graph.org/) library is used to construct the graph and sort it using the -topological sorting algorithm. The sorting algorithm returns the nodes of the graph ordered in several layers where the +The [Graph for Scala](http://www.scala-graph.org/) library is used to construct the graph and sort it using [Kahn's +topological sorting algorithm](https://en.wikipedia.org/wiki/Topological_sorting#Kahn's_algorithm). + +The algorithm returns the nodes of the graph ordered in several layers, where the root element `?letter` is in layer 0, `[?date, ?person1, ?person2]` are in layer 1, `[?gnd1, ?gnd2]` in layer 2, and the leaf nodes `[(DE-588)118531379, (DE-588)118696149]` are given in the last layer (i.e. layer 3). -According to Kahn's algorithm, there are multiple valid permutations of the topological order, the graph in the example - above has 24 valid permutations of topological order, such as (nodes are ordered from left to right with the highest - order to the lowest): -`(?letter, ?date, ?person2, ?person1, ?gnd2, ?gnd1, (DE-588)118696149, (DE-588)118531379)` -and - `(?letter, ?date, ?person1, ?person2, ?gnd1, ?gnd2, (DE-588)118531379, (DE-588)118696149)`. - -From all valid topological order, one is chosen based on certain criteria; for example, the leaf should node should not -belong to a statement that has predicate `rdf:type`. Once the best order is chosen, it is used to re-arrange the query +According to Kahn's algorithm, there are multiple valid permutations of the topological order. The graph in the example + above has 24 valid permutations of topological order. Here are two of them (nodes are ordered from left to right with the highest + order to the lowest): + +- `(?letter, ?date, ?person2, ?person1, ?gnd2, ?gnd1, (DE-588)118696149, (DE-588)118531379)` +- `(?letter, ?date, ?person1, ?person2, ?gnd1, ?gnd2, (DE-588)118531379, (DE-588)118696149)`. + +From all valid topological orders, one is chosen based on certain criteria; for example, the leaf should node should not +belong to a statement that has predicate `rdf:type`, since that could match all resources of the specified type. +Once the best order is chosen, it is used to re-arrange the query statements. Starting from the last leaf node, i.e. -`(DE-588)118696149`, the method finds the statement pattern which has this node as its object and brings this statement -to the top of the query. This re-arrangement continues so that the statements with the least dependencies on other -statements are all brought to the top of the query. Resulting in +`(DE-588)118696149`, the method finds the statement pattern which has this node as its object, and brings this statement +to the top of the query. This rearrangement continues so that the statements with the fewest dependencies on other +statements are all brought to the top of the query. The resulting query is as follows: ```sparql PREFIX beol: @@ -429,10 +437,11 @@ CONSTRUCT { } ORDER BY ?date ``` -Note that position of the above given filter statements does not play a significant role in the optimization of the performance. -In case a gravsearch query contains statements given in `UnionPattern`, `OptionalPattern`, `MinusPattern`, or -`FilterNotExistsPattern`, they are reordered accordingly -by defining a graph per block. For example, the following query with a `UNION` +Note that position of the FILTER statements does not play a significant role in the optimization. + +If a Gravsearch query contains statements in `UNION`, `OPTIONAL`, `MINUS`, or +`FILTER NOT EXISTS`, they are reordered +by defining a graph per block. For example, consider the following query with `UNION`: ```sparql { @@ -449,8 +458,9 @@ UNION ?int knora-api:intValueAsInt 3 . } ``` -would result in one graph per block of the `UNION`. Each graph is then sorted and the statements of its correspoding -block are re-arranged with respect to the topological order of graph, resulting in: +This would result in one graph per block of the `UNION`. Each graph is then sorted, and the statements of its +block are rearranged according to the topological order of graph. This is the result: + ```sparql { ?int knora-api:intValueAsInt 1 . @@ -465,9 +475,11 @@ block are re-arranged with respect to the topological order of graph, resulting } ``` -###Cyclic Graphs -The topological sorting algorithm can only be used for DAGs (directed acyclic graphs). Therefore, in case a -gravsearch query contains statements that result in a cyclic graph, for example +### Cyclic Graphs + +The topological sorting algorithm can only be used for DAGs (directed acyclic graphs). However, +a Gravsearch query can contains statements that result in a cyclic graph, e.g.: + ``` PREFIX anything: PREFIX knora-api: @@ -481,5 +493,5 @@ CONSTRUCT { ``` -the algorithm tries to break the cycles in order to sort the graph. If breaking a cycle was not possible, the query -statements are not reordered and the query is submitted to the triplestore as given. +In this case, the algorithm tries to break the cycles in order to sort the graph. If this is not possible, +the query statements are not reordered.