Content Search: use AND operator by default and don't apply mask for phrases

CHANGELOG-DEV.md

@@ -25,6 +25,7 @@ HumHub Changelog
 - Fix #6552: When a JS file has `module.initOnAjaxLoad = true;`, if the `initOnAjaxUrls` contains multiple params in the URL, the `init` function is not triggered
 - Fix #7006: Disable `mustChangePassword` check for impersonated access tokens
 - Fix #7004: Fix people filter by group
+- Enh #6979: Content Search: use AND operator by default and don't apply mask for phrases
 - Fix #7011: Fixed performance issue in `Members::getPrivilegedUserIds`
 - Enh #7010: Rich text tables: Vertical align top instead of middle
 - Enh #5310: Mobile - Zooming into pictures

protected/humhub/libs/SearchQuery.php

@@ -13,81 +13,118 @@
  * Parses a given search query into relevant terms.
  *
  * Examples:
- * - `Apple Banana`, `Apple OR Banana`
- * - `Apple AND Banana`, `+Apple +Banana`
- * - `Apple AND Banana OR Grape`, `+Apple +Banana Grape`
+ * - `Apple Banana`, `+Apple +Banana`
  * - `"Apple pie" "Banana bread"`
  * - `Apple NOT Banana`, `Apple -Banana`
  *
- *
  * @since 1.16
  */
 class SearchQuery
 {
     /**
-     * @var string[] All terms without operator
-     * @readonly
-     */
-    public array $orTerms;
-
-    /**
-     * @var string[] All terms which are required, with AND or + operator
+     * All terms which are required, without operator or with AND or + operator
+     *
+     * This is DEFAULT term (for all keyword without operator)
+     *
+     * Keyword samples how they are stored here:
+     *   'Apple' => ['Apple*']
+     *   'AND Apple' => ['AND*', 'Apple*']
+     *   '+Apple' => ['Apple*']
+     *   '"Apple"' => ['Apple']
+     *   '+"Apple"' => ['Apple']
+     *   '+"Apple Banana"' => ['Apple Banana']
+     *   'Apple pie AND Banana' => ['Apple*', 'pie*', 'AND*', 'Banana*']
+     *   '"Apple pie" Banana' => ['Apple pie', 'Banana*']
+     * @var string[] $terms
      * @readonly
      */
-    public array $andTerms;
+    public array $terms = [];
 
     /**
-     * @var string[] All terms which should excluded, with NOT or - operator
+     * All terms which should excluded, with NOT or - operator
+     *
+     * Keyword samples how they are stored here:
+     *   'NOT Apple' => ['Apple*']
+     *   '-Apple' => ['Apple*']
+     *   'NOT "Apple"' => ['Apple']
+     *   '-"Apple"' => ['Apple']
+     *   '-"Apple Banana"' => ['Apple Banana']
+     *   '-Apple -pie NOT Banana' => ['Apple*', 'pie*', 'Banana*']
+     *   '-"Apple pie" -Banana' => ['Apple pie', 'Banana*']
+     * @var string[] $notTerms
      * @readonly
      */
-    public array $notTerms;
+    public array $notTerms = [];
 
     /**
      * @param $query string The search query to parse
      */
     public function __construct(string $query)
     {
-        $notTerms = [];
-        $orTerms = [];
-        $andTerms = [];
-
-        preg_match_all(
-            '/(?|((?i)AND )?((?i)OR )?((?i)NOT )?[\+\-]?"([^"]+)"|(((?i)AND )?((?i)OR )?((?i)NOT )?\S+))/',
-            $query,
-            $result,
-            PREG_PATTERN_ORDER,
-        );
-
-        if (!empty($result[0]) && is_array($result[0])) {
-            foreach ($result[0] as $i => $term) {
-
-                // Remove quotation marks
-                $term = str_replace('"', '', $term);
-
-                if (str_starts_with($term, '+') || str_starts_with($term, 'AND ')) {
-
-                    /**
-                     * Special Case: In search queries like "Apple AND Banana", Apple should
-                     * become a `AND` term too.
-                     */
-                    if ($i === 1 && str_starts_with($term, 'AND ') && count($orTerms) === 1) {
-                        $andTerms = $orTerms;
-                        $orTerms = [];
-                    }
-
-                    $andTerms[] = preg_replace('/^\+?((?i)AND )?/', '', $term);
-                } elseif (str_starts_with($term, '-') || str_starts_with($term, 'NOT ')) {
-                    $notTerms[] = preg_replace('/^\-?((?i)NOT )?/', '', $term);
-                } else {
-                    $orTerms[] = preg_replace('/^((?i)OR )?/', '', $term);
-                }
+        foreach ($this->parseQuery($query) as $term) {
+            if ($this->isNotTerm($term)) {
+                $this->addNotTerm($term);
+            } else {
+                $this->addTerm($term);
             }
         }
+    }
+
+    protected function parseQuery(string $query): array
+    {
+        preg_match_all('/(?|((?i)NOT )?[\+\-]*"([^"]+)"|(((?i)NOT )?\S+))/', $query, $result);
 
-        $this->notTerms = array_filter($notTerms);
-        $this->orTerms = array_filter($orTerms);
-        $this->andTerms = array_filter($andTerms);
+        return !empty($result[0]) && is_array($result[0]) ? $result[0] : [];
     }
 
+    protected function isNotTerm(string $term): bool
+    {
+        return str_starts_with($term, '-') || str_starts_with($term, 'NOT ');
+    }
+
+    protected function addNotTerm(string $term)
+    {
+        $term = preg_replace('/^(-*|NOT )/', '', $term);
 
+        if ($term = $this->prepareTerm($term)) {
+            $this->notTerms[] = $term;
+        }
+    }
+
+    protected function addTerm(string $term)
+    {
+        $term = ltrim($term, '+');
+
+        if ($term = $this->prepareTerm($term)) {
+            $this->terms[] = $term;
+        }
+    }
+
+    /**
+     * Prepare a term before using
+     *
+     * @param string $term
+     * @return string|null NULL - if the term must not be used at all
+     */
+    protected function prepareTerm(string $term): ?string
+    {
+        if (!$this->isPhrase($term)) {
+            // A not quoted term should be searched as wildcard by default
+            // The last char '*' is used as flag for wildcard
+            $term = rtrim($term, '*"') . '*';
+        }
+
+        // Remove all rest quotes around each term
+        $term = trim($term, '"');
+
+        // Remove the chars if they stay after first quote like `"---Apple"`, `"++++Banana"`
+        $term = ltrim($term, '+-');
+
+        return $term === '' || $term === '*' ? null : $term;
+    }
+
+    protected function isPhrase(string $term): bool
+    {
+        return str_starts_with($term, '"') && str_ends_with($term, '"');
+    }
 }

protected/humhub/modules/comment/resources/js/humhub.comment.js

@@ -254,8 +254,8 @@ humhub.module('comment', function (module, require, $) {
 
             // Highlight currently searching keywords in the loaded comments
             const contentSearchKeyword = $('.container-contents .form-search input[name=keyword]');
-            if (contentSearchKeyword.length && contentSearchKeyword.val().length) {
-                contentSearchKeyword.val().split(' ').forEach((keyword) => $html.highlight(keyword))
+            if (contentSearchKeyword.length) {
+                additions.highlightWords($html, contentSearchKeyword.val());
             }
         }).catch(function (err) {
             module.log.error(err, true);

protected/humhub/modules/content/migrations/m240203_112155_search.php

@@ -24,7 +24,7 @@ public function safeUp()
         $this->safeAddForeignKey('fk_content_fulltext', 'content_fulltext', 'content_id', 'content', 'id', 'CASCADE', 'CASCADE');
 
         try {
-            $this->execute("ALTER TABLE content_fulltext ADD FULLTEXT INDEX ftx (contents ASC, comments ASC, files ASC)");
+            $this->execute('ALTER TABLE content_fulltext ADD FULLTEXT INDEX ftx (contents, comments, files)');
 
             Yii::$app->queue->push(new SearchRebuildIndex());
 

protected/humhub/modules/content/resources/js/humhub.content.highlight.js

@@ -1,13 +1,14 @@
 humhub.module('content.highlight', function (module, require, $) {
     const Widget = require('ui.widget').Widget;
     const event = require('event');
+    const highlightWords = require('ui.additions').highlightWords;
 
     const layout = $('.layout-content-container');
 
     const init = function () {
         event.on('humhub:modules:content:highlight:afterInit', () => highlight());
         layout.find('[data-ui-widget="ui.richtext.prosemirror.RichText"]')
-            .on('afterRender', (obj) => highlight($(obj.target)));
+            .on('afterRender', (obj) => highlight(obj.target));
 
         const wallStream = Widget.instance('[data-ui-widget="stream.wall.WallStream"]');
         if (wallStream) {
@@ -20,7 +21,7 @@ humhub.module('content.highlight', function (module, require, $) {
             if (typeof object === 'undefined') {
                 object = layout;
             }
-            module.config.keyword.split(' ').forEach((keyword) => object.highlight(keyword));
+            highlightWords(object, module.config.keyword);
         }
     }
 

protected/humhub/modules/content/search/driver/MysqlDriver.php

@@ -24,6 +24,15 @@
 
 class MysqlDriver extends AbstractDriver
 {
+    /**
+     * Minimum word length for "And Terms",
+     * Words with less length are handled as "Or Terms"
+     * NOTE: Using of the config value mysql.ft_min_word_len doesn't work properly.
+     *
+     * @var int $minAndTermLength
+     */
+    public int $minAndTermLength = 3;
+
     public function purge(): void
     {
         ContentFulltext::deleteAll();
@@ -128,14 +137,11 @@ private function createMysqlFullTextQuery(SearchQuery $query, array $matchFields
     {
         $againstQuery = '';
 
-        foreach ($query->andTerms as $keyword) {
-            $againstQuery .= '+' . rtrim($keyword, '*') . '* ';
-        }
-        foreach ($query->orTerms as $keyword) {
-            $againstQuery .= rtrim($keyword, '*') . '* ';
+        foreach ($query->terms as $term) {
+            $againstQuery .= '+' . $this->prepareTerm($term) . ' ';
         }
-        foreach ($query->notTerms as $keyword) {
-            $againstQuery .= '-' . $keyword . ' ';
+        foreach ($query->notTerms as $term) {
+            $againstQuery .= '-' . $this->prepareTerm($term) . ' ';
         }
 
         return sprintf(
@@ -145,6 +151,12 @@ private function createMysqlFullTextQuery(SearchQuery $query, array $matchFields
         );
     }
 
+    protected function prepareTerm(string $term): string
+    {
+        // Wrap a keyword in quotes to avoid error with the special chars in the sql MATCH-AGAINST expression
+        return preg_match('#[^\p{L}\d\*’\'`\-\_]#', $term) ? '"' . $term . '"' : $term;
+    }
+
     protected function addQueryFilterVisibility(ActiveQuery $query): ActiveQuery
     {
         $query->andWhere(['content.state' => Content::STATE_PUBLISHED]);

protected/humhub/modules/content/search/driver/ZendLucenceDriver.php

@@ -24,8 +24,10 @@
 use ZendSearch\Lucene\Index;
 use ZendSearch\Lucene\Index\Term;
 use ZendSearch\Lucene\Lucene;
+use ZendSearch\Lucene\Search\Query\AbstractQuery;
 use ZendSearch\Lucene\Search\Query\Boolean;
 use ZendSearch\Lucene\Search\Query\MultiTerm;
+use ZendSearch\Lucene\Search\Query\Phrase;
 use ZendSearch\Lucene\Search\Query\Range;
 use ZendSearch\Lucene\Search\Query\Term as TermQuery;
 use ZendSearch\Lucene\Search\Query\Wildcard;
@@ -155,23 +157,35 @@ public function search(SearchRequest $request): ResultSet
         return $resultSet;
     }
 
+    protected function prepareTerm(string $term): AbstractQuery
+    {
+        $term = mb_strtolower($term);
+
+        if (str_contains($term, ' ')) {
+            return new Phrase(explode(' ', $term));
+        }
+
+        if (str_ends_with($term, '*')) {
+            return new Wildcard(new Term($term));
+        }
+
+        return new TermQuery(new Term($term));
+    }
+
     protected function buildSearchQuery(SearchRequest $request): Boolean
     {
         $query = new Boolean();
 
         Wildcard::setMinPrefixLength(0);
 
         $keywordQuery = new Boolean();
-        foreach ($request->getSearchQuery()->orTerms as $term) {
-            $keywordQuery->addSubquery(new Wildcard(new Term(mb_strtolower($term) . '*')), null);
-        }
 
-        foreach ($request->getSearchQuery()->andTerms as $term) {
-            $keywordQuery->addSubquery(new Wildcard(new Term(mb_strtolower($term) . '*')), true);
+        foreach ($request->getSearchQuery()->terms as $term) {
+            $keywordQuery->addSubquery($this->prepareTerm($term), true);
         }
 
         foreach ($request->getSearchQuery()->notTerms as $term) {
-            $keywordQuery->addSubquery(new TermQuery(new Term(mb_strtolower($term))), false);
+            $keywordQuery->addSubquery($this->prepareTerm($term), false);
         }
 
         if (count($keywordQuery->getSubqueries())) {

protected/humhub/modules/content/tests/codeception/unit/search/AbstractDriverTestSuite.php

@@ -5,6 +5,7 @@
 use humhub\modules\content\models\Content;
 use humhub\modules\content\Module;
 use humhub\modules\content\search\driver\AbstractDriver;
+use humhub\modules\content\search\driver\MysqlDriver;
 use humhub\modules\content\search\ResultSet;
 use humhub\modules\content\search\SearchRequest;
 use humhub\modules\content\services\ContentSearchService;
@@ -45,16 +46,23 @@ public function testKeywords()
         (new Post($space, Content::VISIBILITY_PUBLIC, ['message' => 'Something Other']))->save();
         (new Post($space, Content::VISIBILITY_PUBLIC, ['message' => 'Marabru Leav Test X']))->save();
 
-        // Test Multiple AND Keywords
-        #$this->assertCount(1, $this->getSearchResultByKeyword('Marabru')->results);
+        $this->assertCount(1, $this->getSearchResultByKeyword('"Marabru" Tes')->results);
+        $this->assertCount(0, $this->getSearchResultByKeyword('"Marabr" Test')->results);
+        $this->assertCount(1, $this->getSearchResultByKeyword('"Marabru Leav" "Leav Test"')->results);
+        $this->assertCount(1, $this->getSearchResultByKeyword('"Something Other"')->results);
+        $this->assertCount(0, $this->getSearchResultByKeyword('Some Test')->results);
+        $this->assertCount(1, $this->getSearchResultByKeyword('Some -Test')->results);
 
-        $this->assertCount(1, $this->getSearchResultByKeyword('Marabru Leav Abcd')->results);
+        $this->assertCount(1, $this->getSearchResultByKeyword('Marabru Leav')->results);
+        $this->assertCount(1, $this->getSearchResultByKeyword('Marabru Leav NOT Abcd')->results);
+        $this->assertCount(0, $this->getSearchResultByKeyword('Marabru -Leav')->results);
         $this->assertCount(0, $this->getSearchResultByKeyword('+Marabru +Leav* +Abcd')->results);
         $this->assertCount(0, $this->getSearchResultByKeyword('Marabru Leav +Abcd')->results);
 
         $this->assertCount(1, $this->getSearchResultByKeyword('Something -Marabru')->results);
+        $this->assertCount(1, $this->getSearchResultByKeyword('Something -Marab')->results);
 
-        // Wildcards
+        // Wildcards (it is applied automatically even if the char `*` is not typed)
         $this->assertCount(1, $this->getSearchResultByKeyword('Marabr*')->results);
     }
 

protected/humhub/modules/stream/resources/js/humhub.stream.SimpleStream.js

@@ -17,6 +17,7 @@ humhub.module('stream.SimpleStream', function (module, require, $) {
     var content = require('content');
     var Url = require('ui.filter').Url;
     var Widget = require('ui.widget').Widget;
+    var highlightWords = require('ui.additions').highlightWords;
 
     /**
      * Simple stream component can be used for static streams without load logic (only reload single content).
@@ -58,7 +59,7 @@ humhub.module('stream.SimpleStream', function (module, require, $) {
                 widgets.on('afterInit', function() {
                     if (!$(this).data('isHighlighted')) {
                         $(this).data('isHighlighted', true);
-                        that.highlightInput.val().split(' ').forEach((keyword) => $(this).highlight(keyword));
+                        highlightWords(this, that.highlightInput.val());
                     }
                 });
             }