[backend] nil operator handles empty strings (#6517)

OpenCTI-Platform · Apr 3, 2024 · eebc89c · eebc89c
1 parent cff70c3
commit eebc89c
Show file tree

Hide file tree

Showing 2 changed files with 105 additions and 31 deletions.
diff --git a/opencti-platform/opencti-graphql/src/database/engine.js b/opencti-platform/opencti-graphql/src/database/engine.js
@@ -1683,43 +1683,70 @@ const buildLocalMustFilter = async (validFilter) => {
       throw UnsupportedError('Filter must have only one field', { keys: arrayKeys });
     } else {
       const schemaKey = schemaAttributesDefinition.getAttributeByName(R.head(arrayKeys));
-      valuesFiltering.push(schemaKey?.type === 'string' && schemaKey?.format === 'text' ? {
-        bool: {
-          must_not: {
-            wildcard: {
-              [R.head(arrayKeys)]: '*'
-            }
-          },
-        }
-      } : {
-        bool: {
-          must_not: {
-            exists: {
-              field: R.head(arrayKeys)
-            }
+      valuesFiltering.push(schemaKey?.type === 'string' && schemaKey?.format === 'text'
+        ? { // text filters: use wildcard
+          bool: {
+            must_not: {
+              wildcard: {
+                [R.head(arrayKeys)]: '*'
+              }
+            },
           }
-        }
-      });
+        } : { // other filters: nil <-> (field doesn't exist) OR (field = empty string)
+          bool: {
+            should: [{
+              bool: {
+                must_not: {
+                  exists: {
+                    field: R.head(arrayKeys)
+                  }
+                }
+              }
+            }, {
+              multi_match: {
+                fields: arrayKeys.map((k) => `${isDateNumericOrBooleanAttribute(k) || k === '_id' || isObjectFlatAttribute(k) ? k : `${k}.keyword`}`),
+                query: '',
+              },
+            }],
+            minimum_should_match: 1,
+          }
+        });
     }
   } else if (operator === 'not_nil') {
     if (arrayKeys.length > 1) {
       throw UnsupportedError('Filter must have only one field', { keys: arrayKeys });
     } else {
       const schemaKey = schemaAttributesDefinition.getAttributeByName(R.head(arrayKeys));
-      valuesFiltering.push(schemaKey?.type === 'string' && schemaKey?.format === 'text' ? {
-        bool: {
-          must:
-            {
-              wildcard: {
-                [R.head(arrayKeys)]: '*'
-              }
-            },
-        }
-      } : {
-        exists: {
-          field: R.head(arrayKeys)
-        }
-      });
+      valuesFiltering.push(
+        schemaKey?.type === 'string' && schemaKey?.format === 'text'
+          ? { // text filters: use wildcard
+            bool: {
+              must: {
+                wildcard: {
+                  [R.head(arrayKeys)]: '*'
+                }
+              },
+            }
+          } : { // other filters: not_nil <-> (field exists) AND (field != empty string)
+            bool: {
+              should: [{
+                exists: {
+                  field: R.head(arrayKeys)
+                }
+              }, {
+                bool: {
+                  must_not: {
+                    multi_match: {
+                      fields: arrayKeys.map((k) => `${isDateNumericOrBooleanAttribute(k) || k === '_id' || isObjectFlatAttribute(k) ? k : `${k}.keyword`}`),
+                      query: '',
+                    },
+                  }
+                }
+              }],
+              minimum_should_match: 2,
+            }
+          }
+      );
     }
   }
   // 03. Handle values according to the operator

diff --git a/opencti-platform/opencti-graphql/tests/02-integration/01-database/filterGroup-test.js b/opencti-platform/opencti-graphql/tests/02-integration/01-database/filterGroup-test.js
@@ -175,6 +175,7 @@ describe('Complex filters combinations for elastic queries', () => {
     const REPORT4 = {
       input: {
         name: 'Report4',
+        description: '', // empty string
         stix_id: report4StixId,
         published: '2023-09-15T00:51:35.000Z',
         objectMarking: [marking2StixId],
@@ -659,7 +660,7 @@ describe('Complex filters combinations for elastic queries', () => {
     expect(queryResult.data.reports.edges.map((n) => n.node.name).includes('Report1')).toBeTruthy();
     expect(queryResult.data.reports.edges.map((n) => n.node.name)).includes('A demo report for testing purposes').toBeTruthy();
   });
-  it('should list entities according to filters: filter with \'nil\' operator', async () => {
+  it('should list entities according to filters: filter with \'nil\' and \'not_nil\' operators', async () => {
     // test for 'nil': objectMarking is empty
     let queryResult = await queryAsAdmin({
       query: REPORT_LIST_QUERY,
@@ -703,6 +704,52 @@ describe('Complex filters combinations for elastic queries', () => {
     expect(queryResult.data.reports.edges.length).toEqual(4);
     expect(queryResult.data.reports.edges.map((n) => n.node.name).includes('Report3')).toBeFalsy();
   });
+  it('should list entities according to filters: \'nil\' / \'not_nil\' operators should take empty string into account', async () => {
+    // description is empty
+    let queryResult = await queryAsAdmin({
+      query: REPORT_LIST_QUERY,
+      variables: {
+        first: 10,
+        filters: {
+          mode: 'and',
+          filters: [
+            {
+              key: 'description',
+              operator: 'nil',
+              values: [],
+              mode: 'or',
+            }
+          ],
+          filterGroups: [],
+        },
+      }
+    });
+    expect(queryResult.data.reports.edges.length).toEqual(2);
+    expect(queryResult.data.reports.edges.map((n) => n.node.name).includes('Report3')).toBeTruthy(); // description is empty string
+    expect(queryResult.data.reports.edges.map((n) => n.node.name).includes('Report4')).toBeTruthy(); // description is null
+    // description is not empty
+    queryResult = await queryAsAdmin({
+      query: REPORT_LIST_QUERY,
+      variables: {
+        first: 10,
+        filters: {
+          mode: 'and',
+          filters: [
+            {
+              key: 'description',
+              operator: 'not_nil',
+              values: [],
+              mode: 'or',
+            }
+          ],
+          filterGroups: [],
+        },
+      }
+    });
+    expect(queryResult.data.reports.edges.length).toEqual(3); // 'Report1', 'Report2', 'A demo for testing purpose'
+    expect(queryResult.data.reports.edges.map((n) => n.node.name).includes('Report1')).toBeTruthy();
+    expect(queryResult.data.reports.edges.map((n) => n.node.name).includes('Report2')).toBeTruthy();
+  });
   it('should list entities according to filters: aggregation with filters', async () => {
     // count the number of entities with each marking
     const distributionArgs = {