[8.12] [Bug][Investigations] - Fix slow timeline queries (#176838) (#…

…176956) # Backport This will backport the following commits from `main` to `8.12`: - [[Bug][Investigations] - Fix slow timeline queries (#176838)](#176838)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  Co-authored-by: Michael Olorunnisola <michael.olorunnisola@elastic.co>
elastic · Feb 14, 2024 · f5bd489 · f5bd489
1 parent e446672
commit f5bd489
Show file tree

Hide file tree

Showing 8 changed files with 434 additions and 0 deletions.
diff --git a/x-pack/plugins/timelines/common/api/search_strategy/timeline/eql.test.ts b/x-pack/plugins/timelines/common/api/search_strategy/timeline/eql.test.ts
@@ -0,0 +1,109 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { timelineEqlRequestOptionsSchema } from './eql';
+import { mockBaseTimelineRequest } from './mocks/base_timeline_request';
+
+const mockEqlRequestOptions = {
+  ...mockBaseTimelineRequest,
+  filterQuery: 'sequence\n[any where true]\n[any where true]',
+  eventCategoryField: 'event.category',
+  tiebreakerField: '',
+  fieldRequested: [
+    '@timestamp',
+    'message',
+    'event.category',
+    'event.action',
+    'host.name',
+    'source.ip',
+    'destination.ip',
+    'user.name',
+    '@timestamp',
+    'kibana.alert.workflow_status',
+    'kibana.alert.workflow_tags',
+    'kibana.alert.workflow_assignee_ids',
+    'kibana.alert.group.id',
+    'kibana.alert.original_time',
+    'kibana.alert.building_block_type',
+    'kibana.alert.rule.from',
+    'kibana.alert.rule.name',
+    'kibana.alert.rule.to',
+    'kibana.alert.rule.uuid',
+    'kibana.alert.rule.rule_id',
+    'kibana.alert.rule.type',
+    'kibana.alert.suppression.docs_count',
+    'kibana.alert.original_event.kind',
+    'kibana.alert.original_event.module',
+    'file.path',
+    'file.Ext.code_signature.subject_name',
+    'file.Ext.code_signature.trusted',
+    'file.hash.sha256',
+    'host.os.family',
+    'event.code',
+    'process.entry_leader.entity_id',
+  ],
+  language: 'eql',
+  pagination: {
+    activePage: 0,
+    querySize: 25,
+  },
+  runtimeMappings: {},
+  size: 100,
+  sort: [
+    {
+      direction: 'asc',
+      esTypes: ['date'],
+      field: '@timestamp',
+      type: 'date',
+    },
+  ],
+  timerange: {
+    from: '2018-02-12T20:39:22.229Z',
+    interval: '12h',
+    to: '2024-02-13T20:39:22.229Z',
+  },
+  timestampField: '@timestamp',
+};
+
+describe('timelineEqlRequestOptionsSchema', () => {
+  it('should correctly parse the last eql request object without unknown fields', () => {
+    expect(timelineEqlRequestOptionsSchema.parse(mockEqlRequestOptions)).toEqual(
+      mockEqlRequestOptions
+    );
+  });
+
+  it('should correctly parse the last eql request object and remove unknown fields', () => {
+    const invalidEqlRequest = {
+      ...mockEqlRequestOptions,
+      unknownField: 'should-be-removed',
+    };
+    expect(timelineEqlRequestOptionsSchema.parse(invalidEqlRequest)).toEqual(mockEqlRequestOptions);
+  });
+
+  it('should correctly error if an incorrect field type is provided for a schema key', () => {
+    const invalidEqlRequest = {
+      ...mockEqlRequestOptions,
+      fieldRequested: 123,
+    };
+
+    expect(() => {
+      timelineEqlRequestOptionsSchema.parse(invalidEqlRequest);
+    }).toThrowErrorMatchingInlineSnapshot(`
+      "[
+        {
+          \\"code\\": \\"invalid_type\\",
+          \\"expected\\": \\"array\\",
+          \\"received\\": \\"number\\",
+          \\"path\\": [
+            \\"fieldRequested\\"
+          ],
+          \\"message\\": \\"Expected array, received number\\"
+        }
+      ]"
+    `);
+  });
+});
diff --git a/x-pack/plugins/timelines/common/api/search_strategy/timeline/events_all.test.ts b/x-pack/plugins/timelines/common/api/search_strategy/timeline/events_all.test.ts
@@ -0,0 +1,76 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { timelineEventsAllSchema } from './events_all';
+import { mockBaseTimelineRequest } from './mocks/base_timeline_request';
+
+const mockEventsAllRequest = {
+  ...mockBaseTimelineRequest,
+  factoryQueryType: 'eventsAll',
+  excludeEcsData: false,
+  pagination: { activePage: 0, querySize: 25 },
+  fieldRequested: [
+    '@timestamp',
+    '_index',
+    'message',
+    'host.name',
+    'event.module',
+    'agent.type',
+    'event.dataset',
+    'event.action',
+    'user.name',
+    'source.ip',
+    'destination.ip',
+  ],
+  sort: [
+    {
+      field: '@timestamp',
+      type: 'date',
+      direction: 'desc',
+      esTypes: [],
+    },
+  ],
+  fields: [],
+  language: 'kuery',
+};
+
+describe('timelineEventsAllSchema', () => {
+  it('should correctly parse the events request object', () => {
+    expect(timelineEventsAllSchema.parse(mockEventsAllRequest)).toEqual(mockEventsAllRequest);
+  });
+
+  it('should correctly parse the events request object and remove unknown fields', () => {
+    const invalidEventsRequest = {
+      ...mockEventsAllRequest,
+      unknownField: 'shouldBeRemoved',
+    };
+    expect(timelineEventsAllSchema.parse(invalidEventsRequest)).toEqual(mockEventsAllRequest);
+  });
+
+  it('should correctly error if an incorrect field type is provided for a schema key', () => {
+    const invalidEventsRequest = {
+      ...mockEventsAllRequest,
+      excludeEcsData: 'notABoolean',
+    };
+
+    expect(() => {
+      timelineEventsAllSchema.parse(invalidEventsRequest);
+    }).toThrowErrorMatchingInlineSnapshot(`
+      "[
+        {
+          \\"code\\": \\"invalid_type\\",
+          \\"expected\\": \\"boolean\\",
+          \\"received\\": \\"string\\",
+          \\"path\\": [
+            \\"excludeEcsData\\"
+          ],
+          \\"message\\": \\"Expected boolean, received string\\"
+        }
+      ]"
+    `);
+  });
+});
diff --git a/x-pack/plugins/timelines/common/api/search_strategy/timeline/events_details.test.ts b/x-pack/plugins/timelines/common/api/search_strategy/timeline/events_details.test.ts
@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { timelineEventsDetailsSchema } from './events_details';
+
+const mockEventsDetails = {
+  entityType: 'events',
+  indexName: 'test-large-index',
+  eventId: 'enfXnY0Byt9Ce9tO1aWh',
+  factoryQueryType: 'eventsDetails',
+  runtimeMappings: {},
+};
+
+describe('timelineEventsDetailsSchema', () => {
+  it('should correctly parse the event details request schema', () => {
+    expect(timelineEventsDetailsSchema.parse(mockEventsDetails)).toEqual(mockEventsDetails);
+  });
+
+  it('should correctly parse the event details request schema and remove unknown fields', () => {
+    const invalidEventsDetailsRequest = {
+      ...mockEventsDetails,
+      unknownField: 'should-be-removed',
+    };
+    expect(timelineEventsDetailsSchema.parse(invalidEventsDetailsRequest)).toEqual(
+      mockEventsDetails
+    );
+  });
+
+  it('should correctly error if an incorrect field type is provided for a schema key', () => {
+    const invalidEventsDetailsRequest = {
+      ...mockEventsDetails,
+      indexName: 123,
+    };
+
+    expect(() => {
+      timelineEventsDetailsSchema.parse(invalidEventsDetailsRequest);
+    }).toThrowErrorMatchingInlineSnapshot(`
+      "[
+        {
+          \\"code\\": \\"invalid_type\\",
+          \\"expected\\": \\"string\\",
+          \\"received\\": \\"number\\",
+          \\"path\\": [
+            \\"indexName\\"
+          ],
+          \\"message\\": \\"Expected string, received number\\"
+        }
+      ]"
+    `);
+  });
+});
diff --git a/x-pack/plugins/timelines/common/api/search_strategy/timeline/events_last_event_time.test.ts b/x-pack/plugins/timelines/common/api/search_strategy/timeline/events_last_event_time.test.ts
@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { timelineEventsLastEventTimeRequestSchema } from './events_last_event_time';
+import { mockBaseTimelineRequest } from './mocks/base_timeline_request';
+
+const mockEventsLastEventTimeRequest = {
+  ...mockBaseTimelineRequest,
+  // Remove fields that are omitted in the schema
+  runtimeMappings: undefined,
+  filterQuery: undefined,
+  timerange: undefined,
+  // Add eventsLastEventTime specific fields
+  factoryQueryType: 'eventsLastEventTime',
+  indexKey: 'hosts',
+  details: {},
+};
+
+describe('timelineEventsLastEventTimeRequestSchema', () => {
+  it('should correctly parse the last event time request object without unknown fields', () => {
+    expect(timelineEventsLastEventTimeRequestSchema.parse(mockEventsLastEventTimeRequest)).toEqual(
+      mockEventsLastEventTimeRequest
+    );
+  });
+
+  it('should correctly parse the last event time request object and remove unknown fields', () => {
+    const invalidEventsDetailsRequest = {
+      ...mockEventsLastEventTimeRequest,
+      unknownField: 'should-be-removed',
+    };
+    expect(timelineEventsLastEventTimeRequestSchema.parse(invalidEventsDetailsRequest)).toEqual(
+      mockEventsLastEventTimeRequest
+    );
+  });
+
+  it('should correctly error if an incorrect field type is provided for a schema key', () => {
+    const invalidEventsDetailsRequest = {
+      ...mockEventsLastEventTimeRequest,
+      indexKey: 'unknown-key',
+    };
+
+    expect(() => {
+      timelineEventsLastEventTimeRequestSchema.parse(invalidEventsDetailsRequest);
+    }).toThrowErrorMatchingInlineSnapshot(`
+      "[
+        {
+          \\"received\\": \\"unknown-key\\",
+          \\"code\\": \\"invalid_enum_value\\",
+          \\"options\\": [
+            \\"hostDetails\\",
+            \\"hosts\\",
+            \\"users\\",
+            \\"userDetails\\",
+            \\"ipDetails\\",
+            \\"network\\"
+          ],
+          \\"path\\": [
+            \\"indexKey\\"
+          ],
+          \\"message\\": \\"Invalid enum value. Expected 'hostDetails' | 'hosts' | 'users' | 'userDetails' | 'ipDetails' | 'network', received 'unknown-key'\\"
+        }
+      ]"
+    `);
+  });
+});
diff --git a/x-pack/plugins/timelines/common/api/search_strategy/timeline/kpi.test.ts b/x-pack/plugins/timelines/common/api/search_strategy/timeline/kpi.test.ts
@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { timelineKpiRequestOptionsSchema } from './kpi';
+import { mockBaseTimelineRequest } from './mocks/base_timeline_request';
+
+const mockKpiRequest = {
+  ...mockBaseTimelineRequest,
+  factoryQueryType: 'eventsKpi',
+};
+
+describe('timelineKpiRequestOptionsSchema', () => {
+  it('should correctly parse the events kpi request object', () => {
+    expect(timelineKpiRequestOptionsSchema.parse(mockKpiRequest)).toEqual(mockKpiRequest);
+  });
+
+  it('should correctly parse the events kpi request object and remove unknown fields', () => {
+    const invalidKpiRequest = {
+      ...mockKpiRequest,
+      unknownField: 'shouldBeRemoved',
+    };
+    expect(timelineKpiRequestOptionsSchema.parse(invalidKpiRequest)).toEqual(mockKpiRequest);
+  });
+
+  it('should correctly error if an incorrect field type is provided for a schema key', () => {
+    const invalidKpiRequest = {
+      ...mockKpiRequest,
+      factoryQueryType: 'someOtherType',
+    };
+
+    expect(() => {
+      timelineKpiRequestOptionsSchema.parse(invalidKpiRequest);
+    }).toThrowErrorMatchingInlineSnapshot(`
+      "[
+        {
+          \\"received\\": \\"someOtherType\\",
+          \\"code\\": \\"invalid_literal\\",
+          \\"expected\\": \\"eventsKpi\\",
+          \\"path\\": [
+            \\"factoryQueryType\\"
+          ],
+          \\"message\\": \\"Invalid literal value, expected \\\\\\"eventsKpi\\\\\\"\\"
+        }
+      ]"
+    `);
+  });
+});
diff --git a/x-pack/plugins/timelines/common/api/search_strategy/timeline/mocks/base_timeline_request.ts b/x-pack/plugins/timelines/common/api/search_strategy/timeline/mocks/base_timeline_request.ts
@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const mockBaseTimelineRequest = {
+  id: 'Fnh1dVQ4SDRTUldtRXpUcDEwZXliWHcdZXdlWVBFWkVSWHVIdzY4a19JbFRvUTozMzgzNzk=',
+  defaultIndex: ['*-large-index'],
+  filterQuery:
+    '{"bool":{"must":[],"filter":[{"bool":{"should":[{"exists":{"field":"host.name"}}],"minimum_should_match":1}},{"range":{"@timestamp":{"gte":"2019-02-13T15:39:10.392Z","lt":"2024-02-14T04:59:59.999Z","format":"strict_date_optional_time"}}}],"should":[],"must_not":[]}}',
+  runtimeMappings: {},
+  timerange: {
+    interval: '12h',
+    from: '2019-02-13T15:39:10.392Z',
+    to: '2024-02-14T04:59:59.999Z',
+  },
+  entityType: 'events',
+};