forked from elastic/kibana
/
non_ecs_fields.ts
377 lines (301 loc) · 11.7 KB
/
non_ecs_fields.ts
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import expect from 'expect';
import {
deleteAllRules,
deleteAllAlerts,
getPreviewAlerts,
getRuleForSignalTesting,
previewRule,
} from '../../utils';
import { dataGeneratorFactory, enhanceDocument } from '../../utils/data_generator';
import { FtrProviderContext } from '../../common/ftr_provider_context';
const getQueryRule = (docIdToQuery: string) => ({
...getRuleForSignalTesting(['ecs_non_compliant']),
query: `id: "${docIdToQuery}"`,
});
// eslint-disable-next-line import/no-default-export
export default ({ getService }: FtrProviderContext) => {
const supertest = getService('supertest');
const esArchiver = getService('esArchiver');
const es = getService('es');
const log = getService('log');
const { indexListOfDocuments } = dataGeneratorFactory({
es,
index: 'ecs_non_compliant',
log,
});
/**
* test helper:
* 1. index document with auto generated ID
* 2. run preview query rule that targets document by ID
* 3. return created preview alert and errors logs
*/
const indexAndCreatePreviewAlert = async (document: Record<string, unknown>) => {
const enhancedDocument = enhanceDocument({ document });
await indexListOfDocuments([enhancedDocument]);
const { previewId, logs } = await previewRule({
supertest,
rule: getQueryRule(enhancedDocument.id),
});
const previewAlerts = await getPreviewAlerts({ es, previewId });
return {
alertSource: previewAlerts?.[0]?._source,
errors: logs[0].errors,
};
};
// FAILING ES PROMOTION: https://github.com/elastic/kibana/issues/154277
describe('Non ECS fields in alert document source', () => {
before(async () => {
await esArchiver.load(
'x-pack/test/functional/es_archives/security_solution/ecs_non_compliant'
);
});
after(async () => {
await esArchiver.unload(
'x-pack/test/functional/es_archives/security_solution/ecs_non_compliant'
);
await deleteAllAlerts(supertest, log, es);
await deleteAllRules(supertest, log);
});
// source agent.name is object, ECS mapping for agent.name is keyword
it('should remove source object field from alert if ECS field mapping is keyword', async () => {
const document = {
agent: {
name: {
first: 'test name 1',
},
version: 'test-1',
},
};
const { errors, alertSource } = await indexAndCreatePreviewAlert(document);
expect(errors).toEqual([]);
// valid ECS field is not getting removed
expect(alertSource).toHaveProperty('agent.version', 'test-1');
// invalid ECS field 'agent.name' is getting removed
expect(alertSource).not.toHaveProperty('agent.name');
});
// source container.image is keyword, ECS mapping for container.image is object
it('should remove source keyword field from alert if ECS field mapping is object', async () => {
const document = {
container: {
name: 'test-image',
image: 'test-1',
},
};
const { errors, alertSource } = await indexAndCreatePreviewAlert(document);
expect(errors).toEqual([]);
// invalid ECS field 'container.image' is getting removed
expect(alertSource).not.toHaveProperty('container.image');
expect(alertSource).toHaveProperty('container.name', 'test-image');
});
// source agent.type is long, ECS mapping for agent.type is keyword
it('should not remove source long field from alert if ECS field mapping is keyword', async () => {
const document = {
agent: {
type: 13,
},
};
const { errors, alertSource } = await indexAndCreatePreviewAlert(document);
expect(errors).toEqual([]);
// long value should be indexed as keyword
expect(alertSource).toHaveProperty('agent.type', 13);
});
// source client.ip is keyword, ECS mapping for client.ip is ip
it('should remove source non ip field from alert if ECS field mapping is ip', async () => {
const document = {
client: {
ip: 'non-valid-ip',
name: 'test name',
},
};
const { errors, alertSource } = await indexAndCreatePreviewAlert(document);
expect(errors).toEqual([]);
// invalid ECS field is getting removed
expect(alertSource).not.toHaveProperty('client.ip');
expect(alertSource).toHaveProperty('client.name', 'test name');
});
it('should not remove valid ips from ECS source field', async () => {
const ip = [
'127.0.0.1',
'::afff:4567:890a',
'::',
'::11.22.33.44',
'1111:2222:3333:4444:AAAA:BBBB:CCCC:DDDD',
];
const document = { client: { ip } };
const { errors, alertSource } = await indexAndCreatePreviewAlert(document);
expect(errors).toEqual([]);
expect(alertSource).toHaveProperty('client.ip', ip);
});
// source event.created is boolean, ECS mapping for client.created is date
it('should remove source non date field from alert if ECS field mapping is date', async () => {
const document = {
event: {
end: true,
},
};
const { errors, alertSource } = await indexAndCreatePreviewAlert(document);
expect(errors).toEqual([]);
// invalid ECS field is getting removed
// event properties getting flattened, so we make sure event.created was removed
expect(alertSource).not.toHaveProperty(['event.end']);
expect(alertSource).not.toHaveProperty('event.end');
});
it('should not remove valid dates from ECS source field', async () => {
const validDates = [
'2015-01-01T12:10:30.666Z',
'2015-01-01T12:10:30.666',
'2015-01-01T12:10:30Z',
'2015-01-01T12:10:30',
'2015-01-01T12:10Z',
'2015-01-01T12:10',
'2015-01-01T12Z',
'2015-01-01T12',
'2015-01-01',
'2015-01',
'2015-01-02T',
123.3,
'23242',
-1,
'-1',
0,
'0',
];
const document = {
event: {
created: validDates,
},
};
const { errors, alertSource } = await indexAndCreatePreviewAlert(document);
expect(errors).toEqual([]);
// event properties getting flattened
expect(alertSource).toHaveProperty(['event.created'], validDates);
});
// source threat.enrichments is keyword, ECS mapping for threat.enrichments is nested
it('should remove source array of keywords field from alert if ECS field mapping is nested', async () => {
const document = {
threat: {
enrichments: ['non-valid-threat-1', 'non-valid-threat-2'],
'indicator.port': 443,
},
};
const { errors, alertSource } = await indexAndCreatePreviewAlert(document);
expect(errors).toEqual([]);
// invalid ECS field is getting removed
expect(alertSource).toHaveProperty('threat.enrichments', []);
expect(alertSource).toHaveProperty(['threat', 'indicator.port'], 443);
});
// source client.bytes is text, ECS mapping for client.bytes is long
it('should remove source text field from alert if ECS field mapping is long', async () => {
const document = {
client: {
nat: {
port: '3000',
},
bytes: 'conflict',
},
};
const { errors, alertSource } = await indexAndCreatePreviewAlert(document);
expect(errors).toEqual([]);
// invalid ECS field is getting removed
expect(alertSource).not.toHaveProperty('client.bytes');
// ensures string numeric field is indexed
expect(alertSource).toHaveProperty('client.nat.port', '3000');
});
// we don't validate it because geo_point is very complex type with many various representations: array, different object, string with few valid patterns
// more on geo_point type https://www.elastic.co/guide/en/elasticsearch/reference/current/geo-point.html
it('should fail creating alert when ECS field mapping is geo_point', async () => {
const document = {
client: {
geo: {
name: 'test',
location: 'test test',
},
},
};
const { errors } = await indexAndCreatePreviewAlert(document);
expect(errors[0]).toContain(
'Bulk Indexing of signals failed: [1:1193] failed to parse field [client.geo.location] of type [geo_point]'
);
});
it('should strip invalid boolean values and left valid ones', async () => {
const document = {
dll: {
code_signature: {
valid: ['non-valid', 'true', 'false', [true, false], '', 'False', 'True', 1],
},
},
};
const { errors, alertSource } = await indexAndCreatePreviewAlert(document);
expect(errors).toEqual([]);
// invalid ECS values is getting removed
expect(alertSource).toHaveProperty('dll.code_signature.valid', [
'true',
'false',
[true, false],
'',
]);
});
// dll.code_signature.valid is boolean in ECS mapping
it('should strip conflicting ECS mappings boolean field', async () => {
const document = {
dll: {
code_signature: {
valid: 'False',
},
},
};
const { errors, alertSource } = await indexAndCreatePreviewAlert(document);
expect(errors).toEqual([]);
// invalid ECS field is getting removed
expect(alertSource).not.toHaveProperty('dll.code_signature.valid');
});
describe('multi-fields', () => {
it('should not add multi field .text to ecs compliant nested source', async () => {
const document = {
process: {
command_line: 'string longer than 10 characters',
},
};
const { errors, alertSource } = await indexAndCreatePreviewAlert(document);
expect(errors).toEqual([]);
expect(alertSource).toHaveProperty('process', document.process);
expect(alertSource).not.toHaveProperty('process.command_line.text');
});
it('should not add multi field .text to ecs compliant flattened source', async () => {
const document = {
'process.command_line': 'string longer than 10 characters',
};
const { errors, alertSource } = await indexAndCreatePreviewAlert(document);
expect(errors).toEqual([]);
expect(alertSource?.['process.command_line']).toEqual(document['process.command_line']);
expect(alertSource).not.toHaveProperty('process.command_line.text');
});
it('should not add multi field .text to ecs non compliant nested source', async () => {
const document = {
nonEcs: {
command_line: 'string longer than 10 characters',
},
};
const { errors, alertSource } = await indexAndCreatePreviewAlert(document);
expect(errors).toEqual([]);
expect(alertSource).toHaveProperty('nonEcs', document.nonEcs);
expect(alertSource).not.toHaveProperty('nonEcs.command_line.text');
});
it('should not add multi field .text to ecs non compliant flattened source', async () => {
const document = {
'nonEcs.command_line': 'string longer than 10 characters',
};
const { errors, alertSource } = await indexAndCreatePreviewAlert(document);
expect(errors).toEqual([]);
expect(alertSource?.['nonEcs.command_line']).toEqual(document['nonEcs.command_line']);
expect(alertSource).not.toHaveProperty('nonEcs.command_line.text');
});
});
});
};