[azure] rename azure.platformlogs.properties as .raw when it contains a string value (#11732)

Mitigates #11729 If the `azure.platformlogs.properties` field contains a string instead of the expected object, the pipeline renames it as `azure.platformlogs.properties.raw` field. This allows: - Elasticsearch to index the log events instead of dropping them (status 400) - Users to customize parsing of the `.raw` field in the `logs-azure.platformlogs@custom pipeline` This avoids dropping log events while we build a complete and more robust [invalid JSON](#11729) handling solution with #11728

Author: zmoog

packages/azure/changelog.yml

@@ -1,3 +1,8 @@
+- version: "1.19.2"
+ changes:
+ - description: Rename the `properties` field to `properties.raw` to avoid parse errors when the `properties` field contains a string.
+ type: bugfix
+ link: https://github.com/elastic/integrations/pull/11732
 - version: "1.19.1"
  changes:
  - description: Fix an error and clarify the docs about the Storage Account container.

packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-json-raw.log

@@ -0,0 +1 @@
+{"time": "2023-03-07T22:19:49Z","resourceId": "/SUBSCRIPTIONS/0E073EC1-C22F-4488-ADDE-DA35ED609CCD/RESOURCEGROUPS/MBRANCA-MALFORMED-JSON-RG/PROVIDERS/MICROSOFT.WEB/SITES/MBRANCA-HELLO-WORLD2","category": "FunctionAppLogs","operationName": "Microsoft.Web/sites/functions/log","level": "Informational","location": "East US","properties": "{'appName':'mbranca-hello-world2','roleInstance':'A6CE8668-638138213605792171','message':'Executing Functions.hello (Reason=This function was programmatically called via the host APIs., Id=0738eec6-ad5e-48f9-a949-5ac36ba84161)','category':'Function.hello','hostVersion':'4.15.1.1','functionInvocationId':'0738eec6-ad5e-48f9-a949-5ac36ba84161','functionName':'Functions.hello','hostInstanceId':'9eb66127-a244-467e-b6a2-01879ad19da2','level':'Information','levelId': 2,'processId': 55,'eventId': 1,'eventName':'FunctionStarted'}"}

packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-json-raw.log-expected.json

@@ -0,0 +1,44 @@
+{
+ "expected": [
+ {
+ "@timestamp": "2023-03-07T22:19:49.000Z",
+ "azure": {
+ "platformlogs": {
+ "category": "FunctionAppLogs",
+ "event_category": "Administrative",
+ "operation_name": "Microsoft.Web/sites/functions/log",
+ "properties": {
+ "raw": "{'appName':'mbranca-hello-world2','roleInstance':'A6CE8668-638138213605792171','message':'Executing Functions.hello (Reason=This function was programmatically called via the host APIs., Id=0738eec6-ad5e-48f9-a949-5ac36ba84161)','category':'Function.hello','hostVersion':'4.15.1.1','functionInvocationId':'0738eec6-ad5e-48f9-a949-5ac36ba84161','functionName':'Functions.hello','hostInstanceId':'9eb66127-a244-467e-b6a2-01879ad19da2','level':'Information','levelId': 2,'processId': 55,'eventId': 1,'eventName':'FunctionStarted'}"
+ }
+ },
+ "resource": {
+ "group": "MBRANCA-MALFORMED-JSON-RG",
+ "id": "/SUBSCRIPTIONS/0E073EC1-C22F-4488-ADDE-DA35ED609CCD/RESOURCEGROUPS/MBRANCA-MALFORMED-JSON-RG/PROVIDERS/MICROSOFT.WEB/SITES/MBRANCA-HELLO-WORLD2",
+ "name": "MBRANCA-HELLO-WORLD2",
+ "provider": "MICROSOFT.WEB/SITES"
+ },
+ "subscription_id": "0E073EC1-C22F-4488-ADDE-DA35ED609CCD"
+ },
+ "cloud": {
+ "provider": "azure"
+ },
+ "ecs": {
+ "version": "8.11.0"
+ },
+ "event": {
+ "action": "Microsoft.Web/sites/functions/log",
+ "kind": "event",
+ "original": "{\"time\": \"2023-03-07T22:19:49Z\",\"resourceId\": \"/SUBSCRIPTIONS/0E073EC1-C22F-4488-ADDE-DA35ED609CCD/RESOURCEGROUPS/MBRANCA-MALFORMED-JSON-RG/PROVIDERS/MICROSOFT.WEB/SITES/MBRANCA-HELLO-WORLD2\",\"category\": \"FunctionAppLogs\",\"operationName\": \"Microsoft.Web/sites/functions/log\",\"level\": \"Informational\",\"location\": \"East US\",\"properties\": \"{'appName':'mbranca-hello-world2','roleInstance':'A6CE8668-638138213605792171','message':'Executing Functions.hello (Reason=This function was programmatically called via the host APIs., Id=0738eec6-ad5e-48f9-a949-5ac36ba84161)','category':'Function.hello','hostVersion':'4.15.1.1','functionInvocationId':'0738eec6-ad5e-48f9-a949-5ac36ba84161','functionName':'Functions.hello','hostInstanceId':'9eb66127-a244-467e-b6a2-01879ad19da2','level':'Information','levelId': 2,'processId': 55,'eventId': 1,'eventName':'FunctionStarted'}\"}"
+ },
+ "geo": {
+ "name": "East US"
+ },
+ "log": {
+ "level": "Informational"
+ },
+ "tags": [
+ "preserve_original_event"
+ ]
+ }
+ ]
+}

packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml

@@ -54,6 +54,12 @@ processors:
  field: tags
  value: ["preserve_original_event"]
  ignore_failure: true
+ - rename:
+ field: azure.platformlogs.properties
+ if: "ctx.azure?.platformlogs?.properties instanceof String"
+ target_field: azure.platformlogs.properties.raw
+ ignore_missing: true
+ description: 'Rename the field to `properties.raw` to avoid parse errors with the `properties` containing a string.'
  - rename:
  field: azure.platformlogs.identity
  if: "ctx.azure?.platformlogs?.identity instanceof String"

packages/azure/manifest.yml

@@ -1,6 +1,6 @@
 name: azure
 title: Azure Logs
-version: 1.19.1
+version: 1.19.2
 description: This Elastic integration collects logs from Azure
 type: integration
 icons: