Add data_stream.dataset option for custom aws-cloudwatch log input (#2560)

kaiyan-sheng · web-flow · commit 38da5971f780 · 2022-02-09T07:31:58.000-07:00
diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.11.2"
+  changes:
+    - description: Add data_stream.dataset option for custom aws-cloudwatch log input
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/2560
 - version: "1.11.1"
   changes:
     - description: Update permission list
diff --git a/packages/aws/data_stream/cloudwatch_logs/_dev/test/pipeline/test-cloudwatch-ec2.log-expected.json b/packages/aws/data_stream/cloudwatch_logs/_dev/test/pipeline/test-cloudwatch-ec2.log-expected.json
@@ -1,109 +1,97 @@
 {
     "expected": [
         {
-            "@timestamp": "2020-02-20T07:01:01.000Z",
+            "cloud": {
+                "provider": "aws"
+            },
             "ecs": {
                 "version": "8.0.0"
             },
+            "message": "2020-02-20T07:01:01.000Z Feb 20 07:01:01 ip-172-31-81-156 systemd: Stopping User Slice of root.",
             "event": {
-                "ingested": "2022-01-09T23:41:38.962436254Z",
+                "kind": "event",
                 "original": "2020-02-20T07:01:01.000Z Feb 20 07:01:01 ip-172-31-81-156 systemd: Stopping User Slice of root."
             },
-            "aws": {
-                "cloudwatch": {
-                    "message": "ip-172-31-81-156 systemd: Stopping User Slice of root."
-                }
-            },
             "tags": [
                 "preserve_original_event"
             ]
         },
         {
-            "@timestamp": "2020-02-20T07:02:18.000Z",
+            "cloud": {
+                "provider": "aws"
+            },
             "ecs": {
                 "version": "8.0.0"
             },
+            "message": "2020-02-20T07:02:18.000Z Feb 20 07:02:18 ip-172-31-81-156 dhclient[3000]: XMT: Solicit on eth0, interval 125240ms.",
             "event": {
-                "ingested": "2022-01-09T23:41:38.962442522Z",
+                "kind": "event",
                 "original": "2020-02-20T07:02:18.000Z Feb 20 07:02:18 ip-172-31-81-156 dhclient[3000]: XMT: Solicit on eth0, interval 125240ms."
             },
-            "aws": {
-                "cloudwatch": {
-                    "message": "ip-172-31-81-156 dhclient[3000]: XMT: Solicit on eth0, interval 125240ms."
-                }
-            },
             "tags": [
                 "preserve_original_event"
             ]
         },
         {
-            "@timestamp": "2020-02-20T07:02:37.000Z",
+            "cloud": {
+                "provider": "aws"
+            },
             "ecs": {
                 "version": "8.0.0"
             },
+            "message": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: DHCPREQUEST on eth0 to 172.31.80.1 port 67 (xid=0x4575af22)",
             "event": {
-                "ingested": "2022-01-09T23:41:38.962444166Z",
+                "kind": "event",
                 "original": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: DHCPREQUEST on eth0 to 172.31.80.1 port 67 (xid=0x4575af22)"
             },
-            "aws": {
-                "cloudwatch": {
-                    "message": "ip-172-31-81-156 dhclient[2898]: DHCPREQUEST on eth0 to 172.31.80.1 port 67 (xid=0x4575af22)"
-                }
-            },
             "tags": [
                 "preserve_original_event"
             ]
         },
         {
-            "@timestamp": "2020-02-20T07:02:37.000Z",
+            "cloud": {
+                "provider": "aws"
+            },
             "ecs": {
                 "version": "8.0.0"
             },
+            "message": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: DHCPACK from 172.31.80.1 (xid=0x4575af22)",
             "event": {
-                "ingested": "2022-01-09T23:41:38.962445580Z",
+                "kind": "event",
                 "original": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: DHCPACK from 172.31.80.1 (xid=0x4575af22)"
             },
-            "aws": {
-                "cloudwatch": {
-                    "message": "ip-172-31-81-156 dhclient[2898]: DHCPACK from 172.31.80.1 (xid=0x4575af22)"
-                }
-            },
             "tags": [
                 "preserve_original_event"
             ]
         },
         {
-            "@timestamp": "2020-02-20T07:02:37.000Z",
+            "cloud": {
+                "provider": "aws"
+            },
             "ecs": {
                 "version": "8.0.0"
             },
+            "message": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: bound to 172.31.81.156 -- renewal in 1599 seconds.",
             "event": {
-                "ingested": "2022-01-09T23:41:38.962446977Z",
+                "kind": "event",
                 "original": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: bound to 172.31.81.156 -- renewal in 1599 seconds."
             },
-            "aws": {
-                "cloudwatch": {
-                    "message": "ip-172-31-81-156 dhclient[2898]: bound to 172.31.81.156 -- renewal in 1599 seconds."
-                }
-            },
             "tags": [
                 "preserve_original_event"
             ]
         },
         {
-            "@timestamp": "2020-02-20T07:02:37.000Z",
+            "cloud": {
+                "provider": "aws"
+            },
             "ecs": {
                 "version": "8.0.0"
             },
+            "message": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 ec2net: [get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s",
             "event": {
-                "ingested": "2022-01-09T23:41:38.962448339Z",
+                "kind": "event",
                 "original": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 ec2net: [get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s"
             },
-            "aws": {
-                "cloudwatch": {
-                    "message": "ip-172-31-81-156 ec2net: [get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s"
-                }
-            },
             "tags": [
                 "preserve_original_event"
             ]
diff --git a/packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs
@@ -1,3 +1,6 @@
+data_stream:
+dataset: {{data_stream.dataset}}
+
 {{#unless log_group_name}}
 {{#unless log_group_name_prefix}}
 {{#if log_group_arn }}
diff --git a/packages/aws/data_stream/cloudwatch_logs/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/cloudwatch_logs/elasticsearch/ingest_pipeline/default.yml
@@ -1,37 +1,20 @@
 ---
-description: "Pipeline for CloudWatch logs"
+description: "Pipeline for logs ingested from CloudWatch"
 
 processors:
-  - set:
-      field: event.ingested
-      value: '{{_ingest.timestamp}}'
   - set:
       field: ecs.version
       value: '8.0.0'
-  - rename:
-      field: message
-      target_field: event.original
-      ignore_missing: true
-  - grok:
-      field: event.original
-      patterns:
-        - '%{TIMESTAMP_ISO8601:_tmp.timestamp} %{SYSLOGTIMESTAMP:_tmp.syslog_timestamp} %{GREEDYDATA:aws.cloudwatch.message}'
-        - '%{TIMESTAMP_ISO8601:_tmp.timestamp} %{GREEDYDATA:aws.cloudwatch.message}'
-  - date:
-      field: _tmp.timestamp
-      target_field: "@timestamp"
-      ignore_failure: true
-      formats:
-        - ISO8601
-  - remove:
-      field:
-        - _tmp
-      ignore_missing: true
-  - remove:
+  - set:
       field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
+      copy_from: message
+      override: false
+  - set:
+      field: cloud.provider
+      value: aws
+  - set:
+      field: event.kind
+      value: event
 on_failure:
   - set:
       field: error.message
diff --git a/packages/aws/data_stream/cloudwatch_logs/fields/ecs.yml b/packages/aws/data_stream/cloudwatch_logs/fields/ecs.yml
@@ -2,5 +2,9 @@
   name: ecs.version
 - external: ecs
   name: error.message
+- name: message
+  external: ecs
 - external: ecs
   name: tags
+- name: event.ingested
+  external: ecs
diff --git a/packages/aws/data_stream/cloudwatch_logs/manifest.yml b/packages/aws/data_stream/cloudwatch_logs/manifest.yml
@@ -175,3 +175,12 @@ streams:
         type: bool
         multi: false
         default: false
+      - name: data_stream.dataset
+        type: text
+        required: true
+        default: generic
+        show_user: false
+        title: Dataset name
+        description: >
+          Set the name for your dataset. Changing the dataset will send the data to a different index. You can't use `-` in the name of a dataset and only valid characters for [Elasticsearch index names](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html).
+
diff --git a/packages/aws/docs/cloudwatch.md b/packages/aws/docs/cloudwatch.md
@@ -32,6 +32,7 @@ setup already.
 | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword |
 | error.message | Error message. | match_only_text |
 | event.dataset | Event dataset | constant_keyword |
+| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred.  It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date |
 | event.module | Event module | constant_keyword |
 | host.architecture | Operating system architecture. | keyword |
 | host.containerized | If the host is a container. | boolean |
@@ -49,6 +50,7 @@ setup already.
 | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword |
 | host.os.version | Operating system version as a raw string. | keyword |
 | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword |
+| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text |
 | tags | List of keywords used to tag each event. | keyword |
 
 
diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml
@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: aws
 title: AWS
-version: 1.11.1
+version: 1.11.2
 license: basic
 description: Collect logs and metrics from Amazon Web Services with Elastic Agent.
 type: integration
