Azure SignInLogs - support additional category types

Add support and tests for ManagedIdentitySignInLogs, NonInteractiveUserSignInLogs, and ServicePrincipalSignInLogs. The pipeline will process any logs that have category of /.*SignInLogs$/. It previously only processed logs that matched a category of /^SignInLogs$/. Changes - Convert azure field names from camel case to snake case to be consistent with our other fields. Previous this was done on field by field basis with rename processors. Now a script processor does it recursively on all fields. - Populate user_agent fields. - Flatten the key/value objects under azure.signinlogs.properties.authentication_processing_details. - Populate event.id with azure.signinlogs.properties.id. - Set source.address.

Author: andrewkroh

packages/azure/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.10.0"
+ changes:
+ - description: signinlogs - Add support for ManagedIdentitySignInLogs, NonInteractiveUserSignInLogs, and ServicePrincipalSignInLogs.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/1721
 - version: "0.9.2"
  changes:
  - description: Prevent pipeline script error

packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-signinlogs-raw.log-config.yml renamed to packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-common-config.yml

@@ -0,0 +1 @@
+{"Level":4,"category":"ManagedIdentitySignInLogs","correlationId":"22222222-92d0-4887-9ead-46258539a699","durationMs":0,"operationName":"Sign-in activity","operationVersion":"1.0","properties":{"appId":"22222222-b540-4792-a2a2-81818990a95b","correlationId":"22222222-92d0-4887-9ead-46258539a699","createdDateTime":"2021-01-23T20:44:29.7688982+00:00","flaggedForReview":false,"id":"22222222-0b57-4b77-bf1a-317a88591a00","ipAddress":"","isInteractive":false,"location":{"city":"","countryOrRegion":"","geoCoordinates":{"latitude":0,"longitude":0},"state":""},"processingTimeInMilliseconds":0,"resourceDisplayName":"Windows Azure Service Management API","resourceId":"22222222-ba00-4fd7-ba43-dac1f8f63013","riskDetail":"none","riskLevelAggregated":"low","riskLevelDuringSignIn":"low","riskState":"none","servicePrincipalId":"22222222-864d-4e00-9882-ff649530f186","servicePrincipalName":"ASC provisioning Dependency agent for Linux","status":{"errorCode":0},"tokenIssuerType":"AzureAD","userId":null},"resourceId":"/tenants/tenantId/providers/Microsoft.aadiam","resultSignature":"None","resultType":"0","tenantId":"tenantId","time":"2021-01-23T20:44:29.7688982Z"}

packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-managed-identity.log

@@ -0,0 +1,78 @@
+{
+ "expected": [
+ {
+ "geo": {
+ "country_name": "",
+ "city_name": "",
+ "location": {
+ "lon": 0,
+ "lat": 0
+ }
+ },
+ "cloud": {
+ "provider": "azure"
+ },
+ "@timestamp": "2021-01-23T20:44:29.768Z",
+ "ecs": {
+ "version": "1.12.0"
+ },
+ "log": {
+ "level": "4"
+ },
+ "event": {
+ "duration": 0,
+ "ingested": "2021-09-28T19:32:36.351618100Z",
+ "original": "{\"Level\":4,\"category\":\"ManagedIdentitySignInLogs\",\"correlationId\":\"22222222-92d0-4887-9ead-46258539a699\",\"durationMs\":0,\"operationName\":\"Sign-in activity\",\"operationVersion\":\"1.0\",\"properties\":{\"appId\":\"22222222-b540-4792-a2a2-81818990a95b\",\"correlationId\":\"22222222-92d0-4887-9ead-46258539a699\",\"createdDateTime\":\"2021-01-23T20:44:29.7688982+00:00\",\"flaggedForReview\":false,\"id\":\"22222222-0b57-4b77-bf1a-317a88591a00\",\"ipAddress\":\"\",\"isInteractive\":false,\"location\":{\"city\":\"\",\"countryOrRegion\":\"\",\"geoCoordinates\":{\"latitude\":0,\"longitude\":0},\"state\":\"\"},\"processingTimeInMilliseconds\":0,\"resourceDisplayName\":\"Windows Azure Service Management API\",\"resourceId\":\"22222222-ba00-4fd7-ba43-dac1f8f63013\",\"riskDetail\":\"none\",\"riskLevelAggregated\":\"low\",\"riskLevelDuringSignIn\":\"low\",\"riskState\":\"none\",\"servicePrincipalId\":\"22222222-864d-4e00-9882-ff649530f186\",\"servicePrincipalName\":\"ASC provisioning Dependency agent for Linux\",\"status\":{\"errorCode\":0},\"tokenIssuerType\":\"AzureAD\",\"userId\":null},\"resourceId\":\"/tenants/tenantId/providers/Microsoft.aadiam\",\"resultSignature\":\"None\",\"resultType\":\"0\",\"tenantId\":\"tenantId\",\"time\":\"2021-01-23T20:44:29.7688982Z\"}",
+ "kind": "event",
+ "action": "Sign-in activity",
+ "id": "22222222-0b57-4b77-bf1a-317a88591a00",
+ "category": [
+ "authentication"
+ ],
+ "type": [
+ "info"
+ ],
+ "outcome": "success"
+ },
+ "tags": [
+ "preserve_original_event"
+ ],
+ "azure": {
+ "tenant_id": "tenantId",
+ "correlation_id": "22222222-92d0-4887-9ead-46258539a699",
+ "signinlogs": {
+ "operation_name": "Sign-in activity",
+ "result_type": "0",
+ "operation_version": "1.0",
+ "category": "ManagedIdentitySignInLogs",
+ "result_signature": "None",
+ "properties": {
+ "risk_level_aggregated": "low",
+ "is_interactive": false,
+ "flagged_for_review": false,
+ "service_principal_id": "22222222-864d-4e00-9882-ff649530f186",
+ "created_at": "2021-01-23T20:44:29.7688982+00:00",
+ "risk_level_during_signin": "low",
+ "risk_detail": "none",
+ "resource_display_name": "Windows Azure Service Management API",
+ "risk_state": "none",
+ "token_issuer_type": "AzureAD",
+ "processing_time_ms": 0,
+ "resource_id": "22222222-ba00-4fd7-ba43-dac1f8f63013",
+ "correlation_id": "22222222-92d0-4887-9ead-46258539a699",
+ "id": "22222222-0b57-4b77-bf1a-317a88591a00",
+ "service_principal_name": "ASC provisioning Dependency agent for Linux",
+ "app_id": "22222222-b540-4792-a2a2-81818990a95b",
+ "status": {
+ "error_code": 0
+ }
+ }
+ },
+ "resource": {
+ "provider": "Microsoft.aadiam",
+ "id": "/tenants/tenantId/providers/Microsoft.aadiam"
+ }
+ }
+ }
+ ]
+}

packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-managed-identity.log-expected.json

@@ -0,0 +1 @@
+{"Level":4,"callerIpAddress":"11.22.33.44","category":"NonInteractiveUserSignInLogs","correlationId":"22222222-18ab-4afa-aa79-21af67c8b108","durationMs":0,"identity":"Hello World","location":"DE","operationName":"Sign-in activity","operationVersion":"1.0","properties":{"appDisplayName":"Microsoft Teams","appId":"22222222-bce4-4aaf-ab1b-5451cc387264","appliedConditionalAccessPolicies":[{"conditionsNotSatisfied":0,"conditionsSatisfied":7,"displayName":"01 - Require Windows Hybrid AD Joined Device","enforcedGrantControls":["RequireDomainJoinedDevice"],"enforcedSessionControls":[],"id":"22222222-b7da-4d9e-ae41-779c5c256ac8","result":"success"},{"conditionsNotSatisfied":2,"conditionsSatisfied":1,"displayName":"05 - MFA für Gäste","enforcedGrantControls":["Mfa"],"enforcedSessionControls":[],"id":"22222222-e960-42e6-ae3a-355df7e475d5","result":"notApplied"},{"conditionsNotSatisfied":12,"conditionsSatisfied":19,"displayName":"02 - Mobile Device Policy","enforcedGrantControls":["Block"],"enforcedSessionControls":[],"id":"22222222-877a-4100-a0cf-5a589f2da3ad","result":"notApplied"},{"conditionsNotSatisfied":16,"conditionsSatisfied":3,"displayName":"04 - Block Legacy Authentication","enforcedGrantControls":["Block"],"enforcedSessionControls":[],"id":"22222222-8e59-4055-87b1-b54a055a7ca5","result":"notApplied"},{"conditionsNotSatisfied":1,"conditionsSatisfied":0,"displayName":"06 - Enterprise Apps","enforcedGrantControls":["Block"],"enforcedSessionControls":[],"id":"22222222-39cb-4ec4-8ed2-ac1352d260ba","result":"notApplied"},{"conditionsNotSatisfied":2,"conditionsSatisfied":1,"displayName":"03 - Require MFA for Admins","enforcedGrantControls":["Mfa"],"enforcedSessionControls":[],"id":"22222222-ea2f-4502-abb7-3689a1b0da41","result":"notApplied"},{"conditionsNotSatisfied":1,"conditionsSatisfied":0,"displayName":"07 - PowerAutomate Pilot","enforcedGrantControls":["Block"],"enforcedSessionControls":[],"id":"22222222-8b95-43cb-8e7d-69e34704ab56","result":"notApplied"},{"conditionsNotSatisfied":2,"conditionsSatisfied":1,"displayName":"02c - Mobile Device Policy Device Compliance","enforcedGrantControls":["RequireCompliantDevice"],"enforcedSessionControls":[],"id":"22222222-ff75-460a-800c-7fe88bd9c877","result":"notApplied"},{"conditionsNotSatisfied":2,"conditionsSatisfied":1,"displayName":"02d - MacOS","enforcedGrantControls":["Block"],"enforcedSessionControls":[],"id":"22222222-9886-4897-b2e2-a096cd37bac3","result":"notApplied"}],"authenticationDetails":[],"authenticationProcessingDetails":[{"key":"Is Client Capable","value":"True"},{"key":"IsCAEToken","value":"False"}],"authenticationRequirement":"singleFactorAuthentication","authenticationRequirementPolicies":[],"autonomousSystemNumber":3320,"clientAppUsed":"Mobile Apps and Desktop clients","conditionalAccessStatus":"success","correlationId":"22222222-18ab-4afa-aa79-21af67c8b108","createdDateTime":"2021-07-30T11:20:59.7789167+00:00","crossTenantAccessType":"none","deviceDetail":{"browser":"Edge 18.1836","deviceId":"22222222-1e7a-44dc-8bc9-5736d8e2b063","displayName":"ABCDEFG","operatingSystem":"Windows 10","trustType":"Hybrid Azure AD joined"},"flaggedForReview":false,"homeTenantId":"22222222-902d-4dea-8026-5a790862fede","id":"22222222-fb7b-4f83-bf74-3876f9ef3900","ipAddress":"11.22.33.44","isInteractive":false,"isTenantRestricted":false,"location":{"city":"Hannover","countryOrRegion":"DE","geoCoordinates":{"latitude":50.12345678912345,"longitude":9.123456789123456},"state":"Niedersachsen"},"networkLocationDetails":[{"networkNames":["Hannover"],"networkType":"trustedNamedLocation"}],"originalRequestId":"22222222-fb7b-4f83-bf74-3876f9ef3900","privateLinkDetails":{},"processingTimeInMilliseconds":65,"resourceDisplayName":"Office 365 Exchange Online","resourceId":"22222222-0000-0ff1-ce00-000000000000","resourceTenantId":"22222222-902d-4dea-8026-5a790862fede","riskDetail":"none","riskEventTypes":[],"riskEventTypes_v2":[],"riskLevelAggregated":"none","riskLevelDuringSignIn":"none","riskState":"none","servicePrincipalId":"","ssoExtensionVersion":"","status":{"errorCode":0},"tokenIssuerName":"","tokenIssuerType":"AzureAD","userAgent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.18363","userDisplayName":"Hello World","userId":"22222222-473d-4f4e-a526-ff54e71afe84","userPrincipalName":"hello.world@company.de","userType":"Member"},"resourceId":"/tenants/22222222-902d-4dea-8026-5a790862fede/providers/Microsoft.aadiam","resultSignature":"None","resultType":"0","tenantId":"22222222-902d-4dea-8026-5a790862fede","time":"2021-07-30T11:20:59.7789167Z"}