Merge branch 'main' into chore/event-original-part3

Author: taylor-swanson

.buildkite/pipeline.schedule-weekly.yml

@@ -21,7 +21,7 @@ steps:
  env:
  SERVERLESS: "false"
  FORCE_CHECK_ALL: "true"
- STACK_VERSION: 8.19.6-SNAPSHOT
+ STACK_VERSION: 8.19.7-SNAPSHOT
  PUBLISH_COVERAGE_REPORTS: "false"
  ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "true"
  depends_on:

.buildkite/pipeline.serverless.yml

@@ -109,6 +109,8 @@ steps:
  - label: ":github: Report failed tests"
  key: report-failed-tests
  command: ".buildkite/scripts/report_issues.sh"
+ env:
+ CI_MAX_TESTS_REPORTED: 30
  agents:
  image: "${LINUX_AGENT_IMAGE}"
  cpu: "8"

packages/aws/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "4.4.0"
+ changes:
+ - description: Prefer set with copy_from.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/15279
 - version: "4.3.2"
  changes:
  - description: Update transform to filter out document containing an error.message from AWS Config, AWS Inspector, and AWS Security Hub latest indexes.

packages/aws/data_stream/vpcflow/elasticsearch/ingest_pipeline/default.yml

@@ -284,11 +284,11 @@ processors:
  - set:
  if: ctx.aws?.vpcflow?.account_id != null
  field: cloud.account.id
- value: '{{aws.vpcflow.account_id}}'
+ copy_from: aws.vpcflow.account_id
  - set:
- if: 'ctx?.aws?.vpcflow?.instance_id != null && ctx.aws.vpcflow.instance_id != "-"'
+ if: 'ctx.aws?.vpcflow?.instance_id != null && ctx.aws.vpcflow.instance_id != "-"'
  field: cloud.instance.id
- value: '{{aws.vpcflow.instance_id}}'
+ copy_from: aws.vpcflow.instance_id
  - convert:
  field: aws.vpcflow.packets_lost_no_route
  type: long

packages/aws/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 3.3.2
 name: aws
 title: AWS
-version: "4.3.2"
+version: "4.4.0"
 description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent.
 type: integration
 categories:

packages/azure_app_service/_dev/build/docs/README.md

@@ -1,62 +1,34 @@
 # Azure App Service Integration
 
-The Azure App Service logs integration retrieves different types of logs categories from Azure App Service.
+## Overview
+
 Azure App Service provides different logging to help you track, monitor, and debug your web application.
 
+## What data does this integration collect?
+
+This integration currently collects one data stream: App Service Logs.
+
+The Azure App Service logs integration retrieves different types of logs categories from Azure App Service:
+
 - HTTPLogs help monitor application health, performance and usage patterns.
 - AuditLogs provide insights when publishing users successfully log on via one of the App Service publishing protocols.
 - IPSecAuditLogs are generated through your application and pushed to Azure Monitoring.
 - PlatformLogs are generated through AppService platform for your application.
 - ConsoleLogs are generated from application or container.
-- AppLogs are generated through your application (ex. logging capabilities)
-
-## Data streams
-
-This integration currently collects one data stream:
-
-- App Service Logs
+- AppLogs are generated through your application (for example, logging capabilities).
 
-## Requirements
+## What do I need to use this integration?
 
-### Credentials
+Refer to the [Azure Logs](https://docs.elastic.co/integrations/azure) page for more information on requirements and setup instructions.
 
-`eventhub` :
-_string_
-Is the fully managed, real-time data ingestion service.
-
-`consumer_group` :
-_string_
-The publish/subscribe mechanism of Event Hubs is enabled through consumer groups. A consumer group is a view (state, position, or offset) of an entire event hub. Consumer groups enable multiple consuming applications to each have a separate view of the event stream, and to read the stream independently at their own pace and with their own offsets.
-Default value: `$Default`
-
-`connection_string` :
-_string_
-The connection string required to communicate with Event Hubs, steps here https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string.
-
-A Blob Storage account is required in order to store/retrieve/update the offset or state of the eventhub messages. This means that after stopping the filebeat azure module it can start back up at the spot that it stopped processing messages.
-
-`storage_account` :
-_string_
-The name of the storage account the state/offsets will be stored and updated.
-
-`storage_account_key` :
-_string_
-The storage account key, this key will be used to authorize access to data in your storage account.
+## App Service Logs
 
-`resource_manager_endpoint` :
-_string_
-Optional, by default we are using the azure public environment, to override, users can provide a specific resource manager endpoint in order to use a different azure environment.
-Ex:
-https://management.chinacloudapi.cn/ for azure ChinaCloud
-https://management.microsoftazure.de/ for azure GermanCloud
-https://management.azure.com/ for azure PublicCloud
-https://management.usgovcloudapi.net/ for azure USGovernmentCloud
-Users can also use this in case of a Hybrid Cloud model, where one may define their own endpoints.
+Collects different types of logs from Azure App Service via Event Hub.
 
-## App Service Logs
+{{event "app_service_logs"}}
 
 **ECS Field Reference**
 
-Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields.
+Refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields.
 
 {{fields "app_service_logs"}}

packages/azure_app_service/changelog.yml

@@ -1,4 +1,34 @@
 # newer versions go on top
+- version: "1.0.0"
+ changes:
+ - description: Make Azure App Service Integration GA
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/15845
+- version: "0.10.2"
+ changes:
+ - description: Review Azure App Service Integration documentation.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/15838
+- version: "0.10.1"
+ changes:
+ - description: Fix dashboard summary
+ type: bugfix
+ link: https://github.com/elastic/integrations/pull/15827
+- version: "0.10.0"
+ changes:
+ - description: Fix json parsing and missing azure.subscription_id issues
+ type: bugfix
+ link: https://github.com/elastic/integrations/pull/15591
+- version: "0.9.0"
+ changes:
+ - description: Update readme
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/15784
+- version: "0.8.0"
+ changes:
+ - description: Add dashboard for log categories Application Logs, Audit Logs, Console Logs, HTTP Logs, IPsec Audit Logs and Platform Logs.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/15487
 - version: "0.7.0"
  changes:
  - description: Add a flag `fips_compatible` to control whether the package is allowed in the ECH FedRAMP High environment.

packages/azure_app_service/data_stream/app_service_logs/_dev/test/pipeline/test-appserviceapplogs-raw.log-expected.json

@@ -13,9 +13,13 @@
  "result_description": "Exception on /favicon.ico [GET]"
  },
  "resource": {
- "id": "/SUBSCRIPTIONS/0E072EC1-C22F-44L8-ADDE-DA36ED609CCD/RESOURCEGROUPS/LUCIAN.DEACONESCU_RG_6914/PROVIDERS/MICROSOFT.WEB/SITES/MANGO-TREE-3004D00656084194B08980B8DB637B86"
+ "id": "/SUBSCRIPTIONS/0E072EC1-C22F-44L8-ADDE-DA36ED609CCD/RESOURCEGROUPS/LUCIAN.DEACONESCU_RG_6914/PROVIDERS/MICROSOFT.WEB/SITES/MANGO-TREE-3004D00656084194B08980B8DB637B86",
+ "provider": "MICROSOFT.WEB/SITES/MANGO-TREE-3004D00656084194B08980B8DB637B86"
  }
  },
+ "cloud": {
+ "provider": "azure"
+ },
  "ecs": {
  "version": "8.11.0"
  },
@@ -39,9 +43,13 @@
  "result_description": "hi there"
  },
  "resource": {
- "id": "/SUBSCRIPTIONS/0E0733C1-C22F-4408-ADDE-DA35XD609CCD/RESOURCEGROUPS/LUCIAN.DEACONESCU_RG_6914/PROVIDERS/MICROSOFT.WEB/SITES/MANGO-TREE-3004D00656084194B08980B8DB637B86"
+ "id": "/SUBSCRIPTIONS/0E0733C1-C22F-4408-ADDE-DA35XD609CCD/RESOURCEGROUPS/LUCIAN.DEACONESCU_RG_6914/PROVIDERS/MICROSOFT.WEB/SITES/MANGO-TREE-3004D00656084194B08980B8DB637B86",
+ "provider": "MICROSOFT.WEB/SITES/MANGO-TREE-3004D00656084194B08980B8DB637B86"
  }
  },
+ "cloud": {
+ "provider": "azure"
+ },
  "ecs": {
  "version": "8.11.0"
  },

packages/azure_app_service/data_stream/app_service_logs/_dev/test/pipeline/test-appserviceauditlogs-raw.log-expected.json

@@ -15,8 +15,18 @@
  }
  },
  "resource": {
- "id": "/SUBSCRIPTIONS/12CABCB5-36E8-104F-A3D2-1DC9982F45CA/RESOURCEGROUPS/USER-TEST/PROVIDERS/MICROSOFT.WEB/SITES/USER-TEST-APP"
- }
+ "group": "USER-TEST",
+ "id": "/SUBSCRIPTIONS/12CABCB5-36E8-104F-A3D2-1DC9982F45CA/RESOURCEGROUPS/USER-TEST/PROVIDERS/MICROSOFT.WEB/SITES/USER-TEST-APP",
+ "name": "USER-TEST-APP",
+ "provider": "MICROSOFT.WEB/SITES"
+ },
+ "subscription_id": "12CABCB5-36E8-104F-A3D2-1DC9982F45CA"
+ },
+ "cloud": {
+ "account": {
+ "id": "12CABCB5-36E8-104F-A3D2-1DC9982F45CA"
+ },
+ "provider": "azure"
  },
  "ecs": {
  "version": "8.11.0"

packages/azure_app_service/data_stream/app_service_logs/_dev/test/pipeline/test-appserviceconsolelogs-raw.log-expected.json

@@ -16,8 +16,18 @@
  "result_description": " Request for index page received\n\n"
  },
  "resource": {
- "id": "/SUBSCRIPTIONS/0E073EC1-C22F-4488-ADDE-DA35ED609CCD/RESOURCEGROUPS/LUCIAN.DEACONESCU_RG_6914/PROVIDERS/MICROSOFT.WEB/SITES/MANGO-TREE-3004D00656084194B08980B8DB637B86"
- }
+ "group": "LUCIAN.DEACONESCU_RG_6914",
+ "id": "/SUBSCRIPTIONS/0E073EC1-C22F-4488-ADDE-DA35ED609CCD/RESOURCEGROUPS/LUCIAN.DEACONESCU_RG_6914/PROVIDERS/MICROSOFT.WEB/SITES/MANGO-TREE-3004D00656084194B08980B8DB637B86",
+ "name": "MANGO-TREE-3004D00656084194B08980B8DB637B86",
+ "provider": "MICROSOFT.WEB/SITES"
+ },
+ "subscription_id": "0E073EC1-C22F-4488-ADDE-DA35ED609CCD"
+ },
+ "cloud": {
+ "account": {
+ "id": "0E073EC1-C22F-4488-ADDE-DA35ED609CCD"
+ },
+ "provider": "azure"
  },
  "ecs": {
  "version": "8.11.0"
@@ -45,8 +55,18 @@
  "result_description": " 169.254.129.1 - - [16/Feb/2023:08:28:44 +0000] \"GET / HTTP/1.1\" 200 1469 \"https://sandbox-92-3.reactblade.portal.azure.net/\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36\"\n\n"
  },
  "resource": {
- "id": "/SUBSCRIPTIONS/0E073EC1-C22F-4488-ADDE-DA35ED609CCD/RESOURCEGROUPS/LUCIAN.DEACONESCU_RG_6914/PROVIDERS/MICROSOFT.WEB/SITES/MANGO-TREE-3004D00656084194B08980B8DB637B86"
- }
+ "group": "LUCIAN.DEACONESCU_RG_6914",
+ "id": "/SUBSCRIPTIONS/0E073EC1-C22F-4488-ADDE-DA35ED609CCD/RESOURCEGROUPS/LUCIAN.DEACONESCU_RG_6914/PROVIDERS/MICROSOFT.WEB/SITES/MANGO-TREE-3004D00656084194B08980B8DB637B86",
+ "name": "MANGO-TREE-3004D00656084194B08980B8DB637B86",
+ "provider": "MICROSOFT.WEB/SITES"
+ },
+ "subscription_id": "0E073EC1-C22F-4488-ADDE-DA35ED609CCD"
+ },
+ "cloud": {
+ "account": {
+ "id": "0E073EC1-C22F-4488-ADDE-DA35ED609CCD"
+ },
+ "provider": "azure"
  },
  "ecs": {
  "version": "8.11.0"