From d53d1a8b75e2649078c6eaef29ef7bfa52ef5c85 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 10:31:09 +0930 Subject: [PATCH] [7.17](backport #38709) x-pack/filebeat/module/o365: fix handling of o365.audit.OperationProperties (#38751) * x-pack/filebeat/module/o365: fix handling of o365.audit.OperationProperties (#38709) In some cases, this field may be a string. This results in a mapping failure. So in cases where the field is a string, conditionally parse out the JSON. Use the ingest pipeline rather than modifying the js processor due to clarity. (cherry picked from commit 8105935e24fb12a2120488290015a38f143d8471) # Conflicts: # x-pack/filebeat/module/o365/fields.go * remove irrelevant changelog entries * fix conflict * fix test expectation --------- Co-authored-by: Dan Kortschak --- CHANGELOG.next.asciidoc | 1 + .../module/o365/audit/_meta/fields.yml | 30 +++++++ .../module/o365/audit/ingest/pipeline.yml | 10 +++ .../module/o365/audit/test/stringly-json.log | 1 + .../test/stringly-json.log-expected.json | 80 +++++++++++++++++++ x-pack/filebeat/module/o365/fields.go | 2 +- 6 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 x-pack/filebeat/module/o365/audit/test/stringly-json.log create mode 100644 x-pack/filebeat/module/o365/audit/test/stringly-json.log-expected.json diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 6fb12603dd4..4872e1d2885 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -37,6 +37,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Filebeat* +- Fix handling of un-parsed JSON in O365 module. {issue}37800[37800] {pull}38709[38709] *Heartbeat* diff --git a/x-pack/filebeat/module/o365/audit/_meta/fields.yml b/x-pack/filebeat/module/o365/audit/_meta/fields.yml index e107c3a2376..47a06857574 100644 --- a/x-pack/filebeat/module/o365/audit/_meta/fields.yml +++ b/x-pack/filebeat/module/o365/audit/_meta/fields.yml @@ -7,6 +7,9 @@ - name: AADGroupId type: keyword + - name: Activity + type: keyword + - name: Actor type: array fields: @@ -110,6 +113,9 @@ - name: ExceptionInfo.* type: object + - name: Experience + type: keyword + - name: ExtendedProperties.* type: object @@ -215,9 +221,21 @@ - name: ObjectId type: keyword + - name: ObjectDisplayName + type: keyword + + - name: ObjectType + type: keyword + - name: Operation type: keyword + - name: OperationId + type: keyword + + - name: OperationProperties + type: object + - name: OrganizationId type: keyword @@ -239,6 +257,9 @@ - name: RecordType type: keyword + - name: RequestId + type: keyword + - name: ResultStatus type: keyword @@ -305,6 +326,9 @@ - name: TemplateTypeId type: keyword + - name: Timestamp + type: keyword + - name: UniqueSharingId type: keyword @@ -329,5 +353,11 @@ - name: Workload type: keyword + - name: WorkspaceId + type: keyword + + - name: WorkspaceName + type: keyword + - name: YammerNetworkId type: keyword diff --git a/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml b/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml index c2d1a74c030..57692734fd1 100644 --- a/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml +++ b/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml @@ -7,6 +7,16 @@ processors: - user_agent: field: user_agent.original ignore_missing: true + - json: + tag: json-extract-stringly-OperationProperties + field: o365.audit.OperationProperties + if: ctx.o365?.audit?.OperationProperties instanceof String + on_failure: + - remove: + field: o365.audit.OperationProperties + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' # URL - uri_parts: field: url.original diff --git a/x-pack/filebeat/module/o365/audit/test/stringly-json.log b/x-pack/filebeat/module/o365/audit/test/stringly-json.log new file mode 100644 index 00000000000..4b8a3225b6b --- /dev/null +++ b/x-pack/filebeat/module/o365/audit/test/stringly-json.log @@ -0,0 +1 @@ +{"Activity":"CreateArtifact","WorkspaceName":"obszar_robaczy","OrganizationId":"53d83e1d-xxx-xxx-84e9-01ec5045dd81","Operation":"CreateArtifact","Id":"a4420e70-b7a1-xxx-xxx-11e3364acd22","CreationTime":"2024-01-30T14:23:40","Timestamp":"2024-01-30T14:22:50","UserId":"username@domain.pl","ClientIP":"81.2.69.144","RecordType":20,"ResultStatus":"InProgress","ObjectDisplayName":"test_lakehouse","OperationId":"a84f7f73-xxxx-xxxx-8cf3-094f69c23756","Experience":"Lakehouse","WorkspaceId":"91dad513-xxxx-xxxx-94bb-f5cbf305691c","ObjectId":"0e00d1cf-825a-4d78-98ff-8a8199357669","UserAgent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36","Workload":"PowerBI","RequestId":"fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b","OperationProperties":"[{\"Name\":\"SystemArtifactType\",\"Value\":\"None\"}]","ObjectType":"Lakehouse","UserType":0,"UserKey":"xxxxxxxx"} diff --git a/x-pack/filebeat/module/o365/audit/test/stringly-json.log-expected.json b/x-pack/filebeat/module/o365/audit/test/stringly-json.log-expected.json new file mode 100644 index 00000000000..c4f98391753 --- /dev/null +++ b/x-pack/filebeat/module/o365/audit/test/stringly-json.log-expected.json @@ -0,0 +1,80 @@ +[ + { + "@timestamp": "2024-01-30T14:23:40.000Z", + "client.address": "81.2.69.144", + "client.ip": "81.2.69.144", + "event.action": "CreateArtifact", + "event.category": "web", + "event.code": "PowerBIAudit", + "event.dataset": "o365.audit", + "event.id": "a4420e70-b7a1-xxx-xxx-11e3364acd22", + "event.kind": "event", + "event.module": "o365", + "event.outcome": "success", + "event.provider": "PowerBI", + "event.type": "info", + "fileset.name": "audit", + "host.id": "53d83e1d-xxx-xxx-84e9-01ec5045dd81", + "host.name": "domain.pl", + "input.type": "log", + "log.offset": 0, + "network.type": "ipv4", + "o365.audit.Activity": "CreateArtifact", + "o365.audit.ClientIP": "81.2.69.144", + "o365.audit.CreationTime": "2024-01-30T14:23:40", + "o365.audit.Experience": "Lakehouse", + "o365.audit.Id": "a4420e70-b7a1-xxx-xxx-11e3364acd22", + "o365.audit.ObjectDisplayName": "test_lakehouse", + "o365.audit.ObjectId": "0e00d1cf-825a-4d78-98ff-8a8199357669", + "o365.audit.ObjectType": "Lakehouse", + "o365.audit.Operation": "CreateArtifact", + "o365.audit.OperationId": "a84f7f73-xxxx-xxxx-8cf3-094f69c23756", + "o365.audit.OperationProperties": [ + { + "Name": "SystemArtifactType", + "Value": "None" + } + ], + "o365.audit.OrganizationId": "53d83e1d-xxx-xxx-84e9-01ec5045dd81", + "o365.audit.RecordType": 20, + "o365.audit.RequestId": "fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b", + "o365.audit.ResultStatus": "InProgress", + "o365.audit.Timestamp": "2024-01-30T14:22:50", + "o365.audit.UserAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "o365.audit.UserId": "username@domain.pl", + "o365.audit.UserKey": "xxxxxxxx", + "o365.audit.UserType": 0, + "o365.audit.Workload": "PowerBI", + "o365.audit.WorkspaceId": "91dad513-xxxx-xxxx-94bb-f5cbf305691c", + "o365.audit.WorkspaceName": "obszar_robaczy", + "organization.id": "53d83e1d-xxx-xxx-84e9-01ec5045dd81", + "related.ip": "81.2.69.144", + "related.user": "username", + "service.type": "o365", + "source.as.number": 20712, + "source.as.organization.name": "Andrews & Arnold Ltd", + "source.geo.city_name": "Abingdon", + "source.geo.continent_name": "Europe", + "source.geo.country_iso_code": "GB", + "source.geo.country_name": "United Kingdom", + "source.geo.location.lat": 51.7095, + "source.geo.location.lon": -1.3614, + "source.geo.region_iso_code": "GB-OXF", + "source.geo.region_name": "Oxfordshire", + "source.ip": "81.2.69.144", + "tags": [ + "forwarded" + ], + "user.domain": "domain.pl", + "user.email": "username@domain.pl", + "user.id": "username@domain.pl", + "user.name": "username", + "user_agent.device.name": "Other", + "user_agent.name": "Chrome", + "user_agent.original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "user_agent.os.full": "Windows 10", + "user_agent.os.name": "Windows", + "user_agent.os.version": "10", + "user_agent.version": "120.0.0.0" + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/o365/fields.go b/x-pack/filebeat/module/o365/fields.go index 781720386fa..7d57ba62cc4 100644 --- a/x-pack/filebeat/module/o365/fields.go +++ b/x-pack/filebeat/module/o365/fields.go @@ -19,5 +19,5 @@ func init() { // AssetO365 returns asset data. // This is the base64 encoded zlib format compressed contents of module/o365. func AssetO365() string { - return "eJzUmUFv40YMhe/7K+ZcYHMp2oMPBVw7WRiNEyNyuuipGEu0wno0VDmUE++vL0bawoljd7N5vjRHK/z0huK8IaWPbkO7kZMff/7pg3PGFmjkbtdrLskNv1WUSuXWWOLI/fLBOefmUnWB3FrUPfhYBY61C1Int1ZpnkVffHBuzRSqNOrjhr+PLvqGhnte+K5ie3bROdu1NHK1Ste++L2ite+C/dkDR27tQ6KDf3gldP931cs4FOjmPvqaGormxouZ69X0S7l4Ef96EftljMfTT1nsrDq45bCQDe0eRffXjjNKEz0a7lX97uDKMTV71mx6cOFNSvbxy11L7yG8WMtEotGTwTmZteOqUkoJ5NwnUljMH75pCEcFUruMxraDMTDgmuPmeG5fFt5pwpFy+R4RLbZv2jZw6fOWn3Jqg9/d+AaT8y8PkvWlUxqXxluaslKund3lliKYrMun8sHHmuZkfurNX/xwlCWrv6i0/0ZNvFEteugs3yFmEpiigc9vgMziWgpTjjVMWuAE2HAm0uQD5Tghu+LBhSjapMMD7SS5i18LFKuliahSgCt9ojSIYWTbTbpk0txH/rsjRE3eFFg0ltSp3IhNOZnyqjPqt/xR2kokkI/f2O79GQFahqroTdes6Hh78TZIXgeW2x5RSKcl5n/Ut3fZMADzu3wyihVVC5WW1JgSCNPow7gsIde4UmnGbfv+cumbUOz4Q/berMlHJ1vx4BWREEuusg8jUuLwSK6lRm2yRxW7ZNQkUJL6gQNh0lTKa169v0pmRg1Q7UM4CJhId8IXgzzrAU7Gg0Vu1GBFcc3JfvWJltS0wRudiQZTJhJOTJFvRsxKiSABKO8cv2QLYB7yE8abid49+vMTZIAPNiPyzFkwsJi557CSp0/dGSC3j5F07pORjssy7+VzSOupZwPdL24AEOV+6d1j8ddwwCTnUvGaXzYoAA7zy9v+DshOum1J+ykBQGjtI3+B55XnHDAtyjVHbxzrgnSL9NcLr74hw2pmIYHL3ZTMc3h36Q4QJMF3VIpWmOXdUeqCFeatAxrqgmJi4y3laWFKRmVfPGkWy9BVBKyxb3EXwtHO8CamoJTAoi5oS8oGvMwp2IDnlaPvNQAAcDAc4q84UD/aJchq9jDMIAbOXf+SZQvmB90KXduK2pLLDUFOvvRa0/HW/f/3oWJYzBm+VAyg3KPd6hmm8Fc4zE2X5MGJKROwzvH5pIRkeujqs/9yrCFQIh3Xp17PvRmBSviNANPOAKw0fifFvPIzrZAUfBbdBPEAYfgkd0P2KLr5hpR/AgAA//9RmUaQ" + return "eJzUmUFv4zYQhe/7K3gusLkU7cGHAq6dXRiNYyNyuuipoKWxMjVFaocjJ95fX5DarRNH7qZ+vjRHK/PpkZx55FDvzZb2IxN+/Pmnd8Yoq6ORWWw2XJLpf6solsKtcvAj88s7Y4yZh6pzZDZBzIP1lWNfGxfqaDYSmmfRV++M2TC5Ko5yXP/33njbUP/OK9tVrM8eGqP7lkamltC1L36vaGM7p39m4MhsrIt09A+vhB7+PmQZxwLN3HpbU0NezXg5M1lNHsrVi/jXgzgMYzyefkxiZ9XRK/uBbGn/GOTwbJhRKu9Y9xAhyGC4FbHH4KHxHFiz6dGDNyk5xK/2LZ1DeDGWSfBKTwrOapBZO64qoRhBzn0kgcX8YZuGcJQj0WuvrHsYAwNu2G+H5/Zl4p0mDKTLfxHRYpXXto5Lm0xjyrF1dn9rG0zONx4k60snlF2BpiyUcmd/vSMPTtb1U/lgfU1zUju1aq9+GGSF9V9U6r+jJlapDgJY1sQxeQXXr4fM/CYUKuxrmLTECbDhTEKTtqRhQnLFowc+SBOPt8ST5M5/TVAslyZBhByc6ROhXgwjZTfpoobm3vPnjhA1qSiwaGxSp+E26JSjCq87pVzyg7R1CI6s/0655z0CtAyRILdds6bh48XbIGkc2NxmRBE6KTH/o3xATIYBmN/1U0vC5DEtSr6iaimhJVGmCAlSEm/duCwh5/kgoRm37fkpl4/C2BaK1O+sSdsva/FgBZHgS66SlyNSfL8kN6FGrTajin1UaiIoSWzPgTBxGsobXp+fJTOlBsj2PhwETEJ3wltdeHaOOBkPJrlSgyXFDUf91UZaUdM6q3QhGkyZBHeiE30zYlYGDxKA9E7xK1YHzkNaYfxAkt0j78EgA1zYhEh9a8HAYOaW3To8fewuAFk8epK5jUoyLstUy5eQlqkXA90vbwEQpTPX2a3113DAJOeh4g2/PKAAOMwvF/kNSCX1hIs0+T0Kq6hFS5IbnwsgoGn5Bjks89lLvJDaev6Ca3rGAVdKuGZvlX1dkOyQHmZpxTakWE0tg+NyPyW17M4u7R6CTPAdlUEqLIHv6HNH2E57R7FzWqjVDuhZCvKRlXeUmropKZU5/+LMl66rCNCXu4hlYK8XuDArKEawLgrakUCfCQpWYMlT9L04AAD27338B3aUu+cI2ecBhnlMz7nLd2E7cH7QUujaNoiuuNwSVJkrKzUNd0f/v+9J/WAu8EGpB6Vj8EIucNHxCocZ8oos2JQmAnY4f96MQjPNDUW1zfBN1JsQfe+VLJx9jWhJCzSuT13EvhmBSviNAN9PACy7fifB7PYTrZEp+BRk64IFCbG1JZSY/0CwSus/BN+SPgbZfkfP3wEAAP//y2fq1g==" }