Add leader election variable in kube state metrics integration

packages/kubernetes/_dev/build/docs/events.md

@@ -7,6 +7,10 @@
 This is the `event` dataset of the Kubernetes package. It collects Kubernetes events
 related metrics.
 
+If Leader Election is activated (default behaviour) only the `elastic agent` which holds the leadership lock
+will retrieve events related metrics.
+This is relevant in multi-node kubernetes cluster and prevents duplicate data.
+
 {{event "event"}}
 
 {{fields "event"}}

packages/kubernetes/_dev/build/docs/kube-apiserver.md

@@ -11,8 +11,12 @@ This metricset needs access to the `apiserver` component of Kubernetes, accessib
 by any POD via the `kubernetes.default` service or via environment
 variables (`KUBERNETES_SERVICE_HOST` and `KUBERNETES_SERVICE_PORT`).
 
+If Leader Election is activated (default behaviour) only the `elastic agent` which holds the leadership lock
+will retrieve metrics from the `apiserver`.
+This is relevant in multi-node kubernetes cluster and prevents duplicate data.
+
 When the API uses https, the pod will need to authenticate using its default token and trust
-the server using the appropiate CA file.
+the server using the appropriate CA file.
 
 Configuration example using https and token based authentication:
 

packages/kubernetes/_dev/build/docs/kube-state-metrics.md

@@ -2,6 +2,10 @@
 
 ## Metrics
 
+If Leader Election is activated (default behaviour) only the `elastic agent` which holds the leadership lock
+will retrieve metrics from the `kube_state_metrics`.
+This is relevant in multi-node kubernetes cluster and prevents duplicate data.
+
 ### state_container
 
 This is the `state_container` dataset of the Kubernetes package. It collects container related

packages/kubernetes/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.8.0"
+  changes:
+    - description: Leverage leader election in kubernetes integration
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1321
 - version: "0.7.0"
   changes:
     - description: Add _meta information to Kubernetes fields

packages/kubernetes/data_stream/apiserver/agent/stream/stream.yml.hbs

@@ -5,6 +5,10 @@ hosts:
 {{/each}}
 period: {{period}}
 
+{{#if leaderelection}}
+condition: ${kubernetes_leaderelection.leader} == true
+{{/if}}
+
 {{#if bearer_token_file}}
 bearer_token_file: {{bearer_token_file}}
 ssl.certificate_authorities:

packages/kubernetes/data_stream/apiserver/manifest.yml

@@ -19,6 +19,13 @@ streams:
         show_user: true
         default:
           - https://${env.KUBERNETES_SERVICE_HOST}:${env.KUBERNETES_SERVICE_PORT}
+      - name: leaderelection
+        type: bool
+        title: Leader Election
+        multi: false
+        required: true
+        show_user: true
+        default: true
       - name: period
         type: text
         title: Period

packages/kubernetes/data_stream/event/agent/stream/stream.yml.hbs

@@ -1,3 +1,6 @@
 metricsets: ["event"]
 period: {{period}}
 add_metadata: {{add_metadata}}
+{{#if leaderelection}}
+condition: ${kubernetes_leaderelection.leader} == true
+{{/if}}

packages/kubernetes/data_stream/event/manifest.yml

@@ -18,5 +18,12 @@ streams:
         required: true
         show_user: true
         default: true
+      - name: leaderelection
+        type: bool
+        title: Leader Election
+        multi: false
+        required: true
+        show_user: true
+        default: true
     title: Kubernetes Event metrics
     description: Collect Kubernetes Event metrics

packages/kubernetes/data_stream/state_container/agent/stream/stream.yml.hbs

@@ -5,3 +5,6 @@ hosts:
   - {{this}}
 {{/each}}
 period: {{period}}
+{{#if leaderelection}}
+condition: ${kubernetes_leaderelection.leader} == true
+{{/if}}

packages/kubernetes/data_stream/state_container/manifest.yml

@@ -20,6 +20,13 @@ streams:
         show_user: true
         default:
           - kube-state-metrics:8080
+      - name: leaderelection
+        type: bool
+        title: Leader Election
+        multi: false
+        required: true
+        show_user: true
+        default: true
       - name: period
         type: text
         title: Period

packages/kubernetes/data_stream/state_cronjob/agent/stream/stream.yml.hbs

@@ -5,3 +5,7 @@ hosts:
   - {{this}}
 {{/each}}
 period: {{period}}
+{{#if leaderelection}}
+condition: ${kubernetes_leaderelection.leader} == true
+{{/if}}
+

packages/kubernetes/data_stream/state_cronjob/manifest.yml

@@ -20,6 +20,13 @@ streams:
         show_user: true
         default:
           - kube-state-metrics:8080
+      - name: leaderelection
+        type: bool
+        title: Leader Election
+        multi: false
+        required: true
+        show_user: true
+        default: true
       - name: period
         type: text
         title: Period

packages/kubernetes/data_stream/state_daemonset/agent/stream/stream.yml.hbs

@@ -5,3 +5,6 @@ hosts:
   - {{this}}
 {{/each}}
 period: {{period}}
+{{#if leaderelection}}
+condition: ${kubernetes_leaderelection.leader} == true
+{{/if}}

packages/kubernetes/data_stream/state_daemonset/manifest.yml

@@ -20,6 +20,13 @@ streams:
         show_user: true
         default:
           - kube-state-metrics:8080
+      - name: leaderelection
+        type: bool
+        title: Leader Election
+        multi: false
+        required: true
+        show_user: true
+        default: true
       - name: period
         type: text
         title: Period

packages/kubernetes/data_stream/state_deployment/agent/stream/stream.yml.hbs

@@ -5,3 +5,6 @@ hosts:
   - {{this}}
 {{/each}}
 period: {{period}}
+{{#if leaderelection}}
+condition: ${kubernetes_leaderelection.leader} == true
+{{/if}}

packages/kubernetes/data_stream/state_deployment/manifest.yml

@@ -20,6 +20,13 @@ streams:
         show_user: true
         default:
           - kube-state-metrics:8080
+      - name: leaderelection
+        type: bool
+        title: Leader Election
+        multi: false
+        required: true
+        show_user: true
+        default: true
       - name: period
         type: text
         title: Period

packages/kubernetes/data_stream/state_node/agent/stream/stream.yml.hbs

@@ -5,3 +5,6 @@ hosts:
   - {{this}}
 {{/each}}
 period: {{period}}
+{{#if leaderelection}}
+condition: ${kubernetes_leaderelection.leader} == true
+{{/if}}

packages/kubernetes/data_stream/state_node/manifest.yml

@@ -20,6 +20,13 @@ streams:
         show_user: true
         default:
           - kube-state-metrics:8080
+      - name: leaderelection
+        type: bool
+        title: Leader Election
+        multi: false
+        required: true
+        show_user: true
+        default: true
       - name: period
         type: text
         title: Period

packages/kubernetes/data_stream/state_persistentvolume/agent/stream/stream.yml.hbs

@@ -5,3 +5,7 @@ hosts:
   - {{this}}
 {{/each}}
 period: {{period}}
+{{#if leaderelection}}
+condition: ${kubernetes_leaderelection.leader} == true
+{{/if}}
+

packages/kubernetes/data_stream/state_persistentvolume/manifest.yml

@@ -20,6 +20,13 @@ streams:
         show_user: true
         default:
           - kube-state-metrics:8080
+      - name: leaderelection
+        type: bool
+        title: Leader Election
+        multi: false
+        required: true
+        show_user: true
+        default: true
       - name: period
         type: text
         title: Period

packages/kubernetes/data_stream/state_persistentvolumeclaim/agent/stream/stream.yml.hbs

@@ -5,3 +5,6 @@ hosts:
   - {{this}}
 {{/each}}
 period: {{period}}
+{{#if leaderelection}}
+condition: ${kubernetes_leaderelection.leader} == true
+{{/if}}

packages/kubernetes/data_stream/state_persistentvolumeclaim/manifest.yml

@@ -20,6 +20,13 @@ streams:
         show_user: true
         default:
           - kube-state-metrics:8080
+      - name: leaderelection
+        type: bool
+        title: Leader Election
+        multi: false
+        required: true
+        show_user: true
+        default: true
       - name: period
         type: text
         title: Period

packages/kubernetes/data_stream/state_pod/agent/stream/stream.yml.hbs

@@ -5,3 +5,6 @@ hosts:
   - {{this}}
 {{/each}}
 period: {{period}}
+{{#if leaderelection}}
+condition: ${kubernetes_leaderelection.leader} == true
+{{/if}}

packages/kubernetes/data_stream/state_pod/manifest.yml

@@ -20,6 +20,13 @@ streams:
         show_user: true
         default:
           - kube-state-metrics:8080
+      - name: leaderelection
+        type: bool
+        title: Leader Election
+        multi: false
+        required: true
+        show_user: true
+        default: true
       - name: period
         type: text
         title: Period

packages/kubernetes/data_stream/state_replicaset/agent/stream/stream.yml.hbs

@@ -5,3 +5,6 @@ hosts:
   - {{this}}
 {{/each}}
 period: {{period}}
+{{#if leaderelection}}
+condition: ${kubernetes_leaderelection.leader} == true
+{{/if}}

packages/kubernetes/data_stream/state_replicaset/manifest.yml

@@ -20,6 +20,13 @@ streams:
         show_user: true
         default:
           - kube-state-metrics:8080
+      - name: leaderelection
+        type: bool
+        title: Leader Election
+        multi: false
+        required: true
+        show_user: true
+        default: true
       - name: period
         type: text
         title: Period

packages/kubernetes/data_stream/state_resourcequota/agent/stream/stream.yml.hbs

@@ -5,3 +5,6 @@ hosts:
   - {{this}}
 {{/each}}
 period: {{period}}
+{{#if leaderelection}}
+condition: ${kubernetes_leaderelection.leader} == true
+{{/if}}

packages/kubernetes/data_stream/state_resourcequota/manifest.yml

@@ -20,6 +20,13 @@ streams:
         show_user: true
         default:
           - kube-state-metrics:8080
+      - name: leaderelection
+        type: bool
+        title: Leader Election
+        multi: false
+        required: true
+        show_user: true
+        default: true
       - name: period
         type: text
         title: Period

packages/kubernetes/data_stream/state_service/agent/stream/stream.yml.hbs

@@ -5,3 +5,6 @@ hosts:
   - {{this}}
 {{/each}}
 period: {{period}}
+{{#if leaderelection}}
+condition: ${kubernetes_leaderelection.leader} == true
+{{/if}}

packages/kubernetes/data_stream/state_service/manifest.yml

@@ -20,6 +20,13 @@ streams:
         show_user: true
         default:
           - kube-state-metrics:8080
+      - name: leaderelection
+        type: bool
+        title: Leader Election
+        multi: false
+        required: true
+        show_user: true
+        default: true
       - name: period
         type: text
         title: Period

packages/kubernetes/data_stream/state_statefulset/agent/stream/stream.yml.hbs

@@ -5,3 +5,6 @@ hosts:
   - {{this}}
 {{/each}}
 period: {{period}}
+{{#if leaderelection}}
+condition: ${kubernetes_leaderelection.leader} == true
+{{/if}}

packages/kubernetes/data_stream/state_statefulset/manifest.yml

@@ -20,6 +20,13 @@ streams:
         show_user: true
         default:
           - kube-state-metrics:8080
+      - name: leaderelection
+        type: bool
+        title: Leader Election
+        multi: false
+        required: true
+        show_user: true
+        default: true
       - name: period
         type: text
         title: Period

packages/kubernetes/data_stream/state_storageclass/agent/stream/stream.yml.hbs

@@ -5,3 +5,6 @@ hosts:
   - {{this}}
 {{/each}}
 period: {{period}}
+{{#if leaderelection}}
+condition: ${kubernetes_leaderelection.leader} == true
+{{/if}}

packages/kubernetes/data_stream/state_storageclass/manifest.yml

@@ -20,6 +20,13 @@ streams:
         show_user: true
         default:
           - kube-state-metrics:8080
+      - name: leaderelection
+        type: bool
+        title: Leader Election
+        multi: false
+        required: true
+        show_user: true
+        default: true
       - name: period
         type: text
         title: Period

packages/kubernetes/docs/events.md

@@ -7,6 +7,10 @@
 This is the `event` dataset of the Kubernetes package. It collects Kubernetes events
 related metrics.
 
+If Leader Election is activated (default behaviour) only the `elastic agent` which holds the leadership lock
+will retrieve events related metrics.
+This is relevant in multi-node kubernetes cluster and prevents duplicate data.
+
 An example event for `event` looks as following:
 
 ```json

packages/kubernetes/docs/kube-apiserver.md

@@ -11,8 +11,12 @@ This metricset needs access to the `apiserver` component of Kubernetes, accessib
 by any POD via the `kubernetes.default` service or via environment
 variables (`KUBERNETES_SERVICE_HOST` and `KUBERNETES_SERVICE_PORT`).
 
+If Leader Election is activated (default behaviour) only the `elastic agent` which holds the leadership lock
+will retrieve metrics from the `apiserver`.
+This is relevant in multi-node kubernetes cluster and prevents duplicate data.
+
 When the API uses https, the pod will need to authenticate using its default token and trust
-the server using the appropiate CA file.
+the server using the appropriate CA file.
 
 Configuration example using https and token based authentication: