rapid7_insightvm: canonicalize host.name to lower case and map subdomain to host.hostname (elastic#9665)

Author: efd6

packages/rapid7_insightvm/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.10.0"
+ changes:
+ - description: Canonicalize `host.name` to lower case and map subdomain to `host.hostname`.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/9665
 - version: "1.9.0"
  changes:
  - description: Set sensitive values as secret.

packages/rapid7_insightvm/data_stream/asset/_dev/test/pipeline/test-asset.log

@@ -1,3 +1,3 @@
 {"assessed_for_policies":false,"assessed_for_vulnerabilities":true,"critical_vulnerabilities":0,"exploits":0,"id":"452534235-25a7-40a3-9321-28ce0b5cc90e-default-asset-199","ip":"10.1.0.128","last_assessed_for_vulnerabilities":"2020-03-20T19:19:42.611Z","last_scan_end":"2020-03-20T19:19:42.611Z","last_scan_start":"2020-03-20T19:18:13.611Z","malware_kits":0,"moderate_vulnerabilities":2,"os_architecture":"x86_64","os_description":"CentOS Linux 2.6.18","os_family":"Linux","os_name":"Linux","os_system_name":"CentOS Linux","os_type":"General","os_vendor":"CentOS","os_version":"2.6.18","risk_score":0,"severe_vulnerabilities":0,"tags":[{"name":"lab","type":"SITE"}],"total_vulnerabilities":2,"new":[],"remediated":[]}
-{"assessed_for_policies":false,"assessed_for_vulnerabilities":true,"critical_vulnerabilities":1,"exploits":9,"host_name":"host.domain.com","id":"452534235-25a7-40a3-9321-28ce0b5cc90e-default-asset-198","ip":"10.4.24.164","last_scan_end":"2020-03-20T19:12:39.766Z","last_scan_start":"2020-03-20T19:05:06.766Z","malware_kits":0,"moderate_vulnerabilities":11,"os_architecture":"","os_description":"Ubuntu Linux 12.04","os_family":"Linux","os_name":"Linux","os_system_name":"Ubuntu Linux","os_type":"","os_vendor":"Ubuntu","os_version":"12.04","risk_score":12251.76171875,"severe_vulnerabilities":16,"tags":[{"name":"all_assets2","type":"CUSTOM"},{"name":"all_assets","type":"CUSTOM"},{"name":"Linux","type":"CUSTOM"},{"name":"docker hosts","type":"SITE"},{"name":"lab","type":"SITE"}],"total_vulnerabilities":28,"new":[],"remediated":[],"unique_identifiers":{"id":"4421d73dfe04f594df731e6bcd8156a","source":"R7 Agent"}}
+{"assessed_for_policies":false,"assessed_for_vulnerabilities":true,"critical_vulnerabilities":1,"exploits":9,"host_name":"HOST.domain.com","id":"452534235-25a7-40a3-9321-28ce0b5cc90e-default-asset-198","ip":"10.4.24.164","last_scan_end":"2020-03-20T19:12:39.766Z","last_scan_start":"2020-03-20T19:05:06.766Z","malware_kits":0,"moderate_vulnerabilities":11,"os_architecture":"","os_description":"Ubuntu Linux 12.04","os_family":"Linux","os_name":"Linux","os_system_name":"Ubuntu Linux","os_type":"","os_vendor":"Ubuntu","os_version":"12.04","risk_score":12251.76171875,"severe_vulnerabilities":16,"tags":[{"name":"all_assets2","type":"CUSTOM"},{"name":"all_assets","type":"CUSTOM"},{"name":"Linux","type":"CUSTOM"},{"name":"docker hosts","type":"SITE"},{"name":"lab","type":"SITE"}],"total_vulnerabilities":28,"new":[],"remediated":[],"unique_identifiers":{"id":"4421d73dfe04f594df731e6bcd8156a","source":"R7 Agent"}}
 {"data":[],"metadata":{"number":0,"size":0,"totalResources":2195,"totalPages":2195,"cursor":null},"links":[{"href":"https://us.api.insight.rapid7.com:443/vm/v4/integration/assets?page=0&size=2","rel":"first"},{"href":"https://us.api.insight.rapid7.com:443/vm/v4/integration/assets?page=0&size=2","rel":"self"},{"href":"https://us.api.insight.rapid7.com:443/vm/v4/integration/assets?page=1097&size=2","rel":"last"}]}

packages/rapid7_insightvm/data_stream/asset/_dev/test/pipeline/test-asset.log-expected.json

@@ -85,12 +85,13 @@
  "host"
  ],
  "kind": "state",
- "original": "{\"assessed_for_policies\":false,\"assessed_for_vulnerabilities\":true,\"critical_vulnerabilities\":1,\"exploits\":9,\"host_name\":\"host.domain.com\",\"id\":\"452534235-25a7-40a3-9321-28ce0b5cc90e-default-asset-198\",\"ip\":\"10.4.24.164\",\"last_scan_end\":\"2020-03-20T19:12:39.766Z\",\"last_scan_start\":\"2020-03-20T19:05:06.766Z\",\"malware_kits\":0,\"moderate_vulnerabilities\":11,\"os_architecture\":\"\",\"os_description\":\"Ubuntu Linux 12.04\",\"os_family\":\"Linux\",\"os_name\":\"Linux\",\"os_system_name\":\"Ubuntu Linux\",\"os_type\":\"\",\"os_vendor\":\"Ubuntu\",\"os_version\":\"12.04\",\"risk_score\":12251.76171875,\"severe_vulnerabilities\":16,\"tags\":[{\"name\":\"all_assets2\",\"type\":\"CUSTOM\"},{\"name\":\"all_assets\",\"type\":\"CUSTOM\"},{\"name\":\"Linux\",\"type\":\"CUSTOM\"},{\"name\":\"docker hosts\",\"type\":\"SITE\"},{\"name\":\"lab\",\"type\":\"SITE\"}],\"total_vulnerabilities\":28,\"new\":[],\"remediated\":[],\"unique_identifiers\":{\"id\":\"4421d73dfe04f594df731e6bcd8156a\",\"source\":\"R7 Agent\"}}",
+ "original": "{\"assessed_for_policies\":false,\"assessed_for_vulnerabilities\":true,\"critical_vulnerabilities\":1,\"exploits\":9,\"host_name\":\"HOST.domain.com\",\"id\":\"452534235-25a7-40a3-9321-28ce0b5cc90e-default-asset-198\",\"ip\":\"10.4.24.164\",\"last_scan_end\":\"2020-03-20T19:12:39.766Z\",\"last_scan_start\":\"2020-03-20T19:05:06.766Z\",\"malware_kits\":0,\"moderate_vulnerabilities\":11,\"os_architecture\":\"\",\"os_description\":\"Ubuntu Linux 12.04\",\"os_family\":\"Linux\",\"os_name\":\"Linux\",\"os_system_name\":\"Ubuntu Linux\",\"os_type\":\"\",\"os_vendor\":\"Ubuntu\",\"os_version\":\"12.04\",\"risk_score\":12251.76171875,\"severe_vulnerabilities\":16,\"tags\":[{\"name\":\"all_assets2\",\"type\":\"CUSTOM\"},{\"name\":\"all_assets\",\"type\":\"CUSTOM\"},{\"name\":\"Linux\",\"type\":\"CUSTOM\"},{\"name\":\"docker hosts\",\"type\":\"SITE\"},{\"name\":\"lab\",\"type\":\"SITE\"}],\"total_vulnerabilities\":28,\"new\":[],\"remediated\":[],\"unique_identifiers\":{\"id\":\"4421d73dfe04f594df731e6bcd8156a\",\"source\":\"R7 Agent\"}}",
  "type": [
  "info"
  ]
  },
  "host": {
+ "hostname": "host",
  "id": "452534235-25a7-40a3-9321-28ce0b5cc90e-default-asset-198",
  "ip": [
  "10.4.24.164"
@@ -113,7 +114,7 @@
  "assessed_for_vulnerabilities": true,
  "critical_vulnerabilities": 1,
  "exploits": 9,
- "host_name": "host.domain.com",
+ "host_name": "HOST.domain.com",
  "id": "452534235-25a7-40a3-9321-28ce0b5cc90e-default-asset-198",
  "ip": "10.4.24.164",
  "last_scan_end": "2020-03-20T19:12:39.766Z",
@@ -162,7 +163,8 @@
  },
  "related": {
  "hosts": [
- "host.domain.com"
+ "host.domain.com",
+ "host"
  ],
  "ip": [
  "10.4.24.164"

packages/rapid7_insightvm/data_stream/asset/elasticsearch/ingest_pipeline/default.yml

@@ -115,10 +115,28 @@ processors:
  field: host.name
  copy_from: rapid7.insightvm.asset.host_name
  ignore_empty_value: true
+ - lowercase:
+ field: host.name
+ ignore_missing: true
+ - append:
+ field: related.hosts
+ value: '{{{host.name}}}'
+ if: ctx.host?.name != null
+ allow_duplicates: false
+ - script:
+ lang: painless
+ if: ctx.host?.name != null
+ source: |
+ int idx = ctx.host.name.indexOf(".");
+ if (idx == -1) {
+ ctx.host.hostname = ctx.host.name;
+ } else {
+ ctx.host.hostname = ctx.host.name.substring(0, idx);
+ }
  - append:
  field: related.hosts
- value: '{{{rapid7.insightvm.asset.host_name}}}'
- if: ctx.rapid7?.insightvm?.asset?.host_name != null
+ value: '{{{host.hostname}}}'
+ if: ctx.host?.hostname != null
  allow_duplicates: false
  - rename:
  field: json.id

packages/rapid7_insightvm/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: rapid7_insightvm
 title: Rapid7 InsightVM
-version: "1.9.0"
+version: "1.10.0"
 source:
  license: "Elastic-2.0"
 description: Collect logs from Rapid7 InsightVM with Elastic Agent.