[7.17](backport #38709) x-pack/filebeat/module/o365: fix handling of …

…o365.audit.OperationProperties (#38751)

* x-pack/filebeat/module/o365: fix handling of o365.audit.OperationProperties (#38709)

In some cases, this field may be a string. This results in a mapping
failure. So in cases where the field is a string, conditionally parse
out the JSON.

Use the ingest pipeline rather than modifying the js processor due to
clarity.

(cherry picked from commit 8105935)

# Conflicts:
#	x-pack/filebeat/module/o365/fields.go

* remove irrelevant changelog entries
* fix conflict
* fix test expectation

---------

Co-authored-by: Dan Kortschak <dan.kortschak@elastic.co>

CHANGELOG.next.asciidoc

@@ -37,6 +37,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Filebeat*
 
+- Fix handling of un-parsed JSON in O365 module. {issue}37800[37800] {pull}38709[38709]
 
 *Heartbeat*
 

x-pack/filebeat/module/o365/audit/_meta/fields.yml

@@ -7,6 +7,9 @@
     - name: AADGroupId
       type: keyword
 
+    - name: Activity
+      type: keyword
+
     - name: Actor
       type: array
       fields:
@@ -110,6 +113,9 @@
     - name: ExceptionInfo.*
       type: object
 
+    - name: Experience
+      type: keyword
+
     - name: ExtendedProperties.*
       type: object
 
@@ -215,9 +221,21 @@
     - name: ObjectId
       type: keyword
 
+    - name: ObjectDisplayName
+      type: keyword
+
+    - name: ObjectType
+      type: keyword
+
     - name: Operation
       type: keyword
 
+    - name: OperationId
+      type: keyword
+
+    - name: OperationProperties
+      type: object
+
     - name: OrganizationId
       type: keyword
 
@@ -239,6 +257,9 @@
     - name: RecordType
       type: keyword
 
+    - name: RequestId
+      type: keyword
+
     - name: ResultStatus
       type: keyword
 
@@ -305,6 +326,9 @@
     - name: TemplateTypeId
       type: keyword
 
+    - name: Timestamp
+      type: keyword
+
     - name: UniqueSharingId
       type: keyword
 
@@ -329,5 +353,11 @@
     - name: Workload
       type: keyword
 
+    - name: WorkspaceId
+      type: keyword
+
+    - name: WorkspaceName
+      type: keyword
+
     - name: YammerNetworkId
       type: keyword

x-pack/filebeat/module/o365/audit/ingest/pipeline.yml

@@ -7,6 +7,16 @@ processors:
   - user_agent:
       field: user_agent.original
       ignore_missing: true
+  - json:
+      tag: json-extract-stringly-OperationProperties
+      field: o365.audit.OperationProperties
+      if: ctx.o365?.audit?.OperationProperties instanceof String
+      on_failure:
+        - remove:
+            field: o365.audit.OperationProperties
+        - append:
+            field: error.message
+            value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   # URL
   - uri_parts:
       field: url.original

x-pack/filebeat/module/o365/audit/test/stringly-json.log

@@ -0,0 +1 @@
+{"Activity":"CreateArtifact","WorkspaceName":"obszar_robaczy","OrganizationId":"53d83e1d-xxx-xxx-84e9-01ec5045dd81","Operation":"CreateArtifact","Id":"a4420e70-b7a1-xxx-xxx-11e3364acd22","CreationTime":"2024-01-30T14:23:40","Timestamp":"2024-01-30T14:22:50","UserId":"username@domain.pl","ClientIP":"81.2.69.144","RecordType":20,"ResultStatus":"InProgress","ObjectDisplayName":"test_lakehouse","OperationId":"a84f7f73-xxxx-xxxx-8cf3-094f69c23756","Experience":"Lakehouse","WorkspaceId":"91dad513-xxxx-xxxx-94bb-f5cbf305691c","ObjectId":"0e00d1cf-825a-4d78-98ff-8a8199357669","UserAgent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36","Workload":"PowerBI","RequestId":"fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b","OperationProperties":"[{\"Name\":\"SystemArtifactType\",\"Value\":\"None\"}]","ObjectType":"Lakehouse","UserType":0,"UserKey":"xxxxxxxx"}

x-pack/filebeat/module/o365/audit/test/stringly-json.log-expected.json

@@ -0,0 +1,80 @@
+[
+    {
+        "@timestamp": "2024-01-30T14:23:40.000Z",
+        "client.address": "81.2.69.144",
+        "client.ip": "81.2.69.144",
+        "event.action": "CreateArtifact",
+        "event.category": "web",
+        "event.code": "PowerBIAudit",
+        "event.dataset": "o365.audit",
+        "event.id": "a4420e70-b7a1-xxx-xxx-11e3364acd22",
+        "event.kind": "event",
+        "event.module": "o365",
+        "event.outcome": "success",
+        "event.provider": "PowerBI",
+        "event.type": "info",
+        "fileset.name": "audit",
+        "host.id": "53d83e1d-xxx-xxx-84e9-01ec5045dd81",
+        "host.name": "domain.pl",
+        "input.type": "log",
+        "log.offset": 0,
+        "network.type": "ipv4",
+        "o365.audit.Activity": "CreateArtifact",
+        "o365.audit.ClientIP": "81.2.69.144",
+        "o365.audit.CreationTime": "2024-01-30T14:23:40",
+        "o365.audit.Experience": "Lakehouse",
+        "o365.audit.Id": "a4420e70-b7a1-xxx-xxx-11e3364acd22",
+        "o365.audit.ObjectDisplayName": "test_lakehouse",
+        "o365.audit.ObjectId": "0e00d1cf-825a-4d78-98ff-8a8199357669",
+        "o365.audit.ObjectType": "Lakehouse",
+        "o365.audit.Operation": "CreateArtifact",
+        "o365.audit.OperationId": "a84f7f73-xxxx-xxxx-8cf3-094f69c23756",
+        "o365.audit.OperationProperties": [
+            {
+                "Name": "SystemArtifactType",
+                "Value": "None"
+            }
+        ],
+        "o365.audit.OrganizationId": "53d83e1d-xxx-xxx-84e9-01ec5045dd81",
+        "o365.audit.RecordType": 20,
+        "o365.audit.RequestId": "fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b",
+        "o365.audit.ResultStatus": "InProgress",
+        "o365.audit.Timestamp": "2024-01-30T14:22:50",
+        "o365.audit.UserAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+        "o365.audit.UserId": "username@domain.pl",
+        "o365.audit.UserKey": "xxxxxxxx",
+        "o365.audit.UserType": 0,
+        "o365.audit.Workload": "PowerBI",
+        "o365.audit.WorkspaceId": "91dad513-xxxx-xxxx-94bb-f5cbf305691c",
+        "o365.audit.WorkspaceName": "obszar_robaczy",
+        "organization.id": "53d83e1d-xxx-xxx-84e9-01ec5045dd81",
+        "related.ip": "81.2.69.144",
+        "related.user": "username",
+        "service.type": "o365",
+        "source.as.number": 20712,
+        "source.as.organization.name": "Andrews & Arnold Ltd",
+        "source.geo.city_name": "Abingdon",
+        "source.geo.continent_name": "Europe",
+        "source.geo.country_iso_code": "GB",
+        "source.geo.country_name": "United Kingdom",
+        "source.geo.location.lat": 51.7095,
+        "source.geo.location.lon": -1.3614,
+        "source.geo.region_iso_code": "GB-OXF",
+        "source.geo.region_name": "Oxfordshire",
+        "source.ip": "81.2.69.144",
+        "tags": [
+            "forwarded"
+        ],
+        "user.domain": "domain.pl",
+        "user.email": "username@domain.pl",
+        "user.id": "username@domain.pl",
+        "user.name": "username",
+        "user_agent.device.name": "Other",
+        "user_agent.name": "Chrome",
+        "user_agent.original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+        "user_agent.os.full": "Windows 10",
+        "user_agent.os.name": "Windows",
+        "user_agent.os.version": "10",
+        "user_agent.version": "120.0.0.0"
+    }
+]