Skip to content

Commit e70e471

Browse files
legoguy1000endorama
legoguy1000
authored and
endorama
committed
[GCP] Add GCP Billing Metricset (#2141)
This commit has been changed to remove changes to README included in #2141 to allow documentation changes from #2842 Changes to packages/gcp/data_stream/audit/manifest.yml has been removed too.
1 parent 7d0d9cd commit e70e471

19 files changed

+1184
-31
lines changed

‎packages/gcp/changelog.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,9 @@
3939
- description: Add 8.0.0 version constraint
4040
type: enhancement
4141
link: https://github.com/elastic/integrations/pull/2251
42+
- description: Add GCP Billing Metricset
43+
type: enhancement
44+
link: https://github.com/elastic/integrations/pull/2141
4245
- version: "1.1.2"
4346
changes:
4447
- description: Update Title and Description.

‎packages/gcp/data_stream/billing/agent/stream/stream.yml.hbs

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
metricsets: ["billing"]
2+
period: {{period}}
3+
project_id: {{project_id}}
4+
{{#if credentials_file}}
5+
credentials_file: {{credentials_file}}
6+
{{/if}}
7+
{{#if credentials_json}}
8+
credentials_json: {{credentials_json}}
9+
{{/if}}
10+
dataset_id: {{dataset_id}}
11+
table_pattern: {{table_pattern}}
12+
cost_type: {{cost_type}}

‎packages/gcp/data_stream/billing/fields/agent.yml

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
- name: cloud
2+
title: Cloud
3+
group: 2
4+
description: Fields related to the cloud or infrastructure the events are coming from.
5+
footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.'
6+
type: group
7+
fields:
8+
- name: account.id
9+
level: extended
10+
type: keyword
11+
ignore_above: 1024
12+
description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment.
13+
14+
Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.'
15+
example: 666777888999
16+
- name: availability_zone
17+
level: extended
18+
type: keyword
19+
ignore_above: 1024
20+
description: Availability zone in which this host is running.
21+
example: us-east-1c
22+
- name: instance.id
23+
level: extended
24+
type: keyword
25+
ignore_above: 1024
26+
description: Instance ID of the host machine.
27+
example: i-1234567890abcdef0
28+
- name: instance.name
29+
level: extended
30+
type: keyword
31+
ignore_above: 1024
32+
description: Instance name of the host machine.
33+
- name: machine.type
34+
level: extended
35+
type: keyword
36+
ignore_above: 1024
37+
description: Machine type of the host machine.
38+
example: t2.medium
39+
- name: provider
40+
level: extended
41+
type: keyword
42+
ignore_above: 1024
43+
description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean.
44+
example: aws
45+
- name: region
46+
level: extended
47+
type: keyword
48+
ignore_above: 1024
49+
description: Region in which this host is running.
50+
example: us-east-1
51+
- name: project.id
52+
type: keyword
53+
description: Name of the project in Google Cloud.
54+
- name: image.id
55+
type: keyword
56+
description: Image ID for the cloud instance.
57+
- name: container
58+
title: Container
59+
group: 2
60+
description: 'Container fields are used for meta information about the specific container that is the source of information.
61+
62+
These fields help correlate data based containers from any runtime.'
63+
type: group
64+
fields:
65+
- name: id
66+
level: core
67+
type: keyword
68+
ignore_above: 1024
69+
description: Unique container id.
70+
- name: image.name
71+
level: extended
72+
type: keyword
73+
ignore_above: 1024
74+
description: Name of the image the container was built on.
75+
- name: labels
76+
level: extended
77+
type: object
78+
object_type: keyword
79+
description: Image labels.
80+
- name: name
81+
level: extended
82+
type: keyword
83+
ignore_above: 1024
84+
description: Container name.
85+
- name: host
86+
title: Host
87+
group: 2
88+
description: 'A host is defined as a general computing instance.
89+
90+
ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.'
91+
type: group
92+
fields:
93+
- name: architecture
94+
level: core
95+
type: keyword
96+
ignore_above: 1024
97+
description: Operating system architecture.
98+
example: x86_64
99+
- name: domain
100+
level: extended
101+
type: keyword
102+
ignore_above: 1024
103+
description: 'Name of the domain of which the host is a member.
104+
105+
For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.'
106+
example: CONTOSO
107+
default_field: false
108+
- name: hostname
109+
level: core
110+
type: keyword
111+
ignore_above: 1024
112+
description: 'Hostname of the host.
113+
114+
It normally contains what the `hostname` command returns on the host machine.'
115+
- name: id
116+
level: core
117+
type: keyword
118+
ignore_above: 1024
119+
description: 'Unique host id.
120+
121+
As hostname is not always unique, use values that are meaningful in your environment.
122+
123+
Example: The current usage of `beat.name`.'
124+
- name: ip
125+
level: core
126+
type: ip
127+
description: Host ip addresses.
128+
- name: mac
129+
level: core
130+
type: keyword
131+
ignore_above: 1024
132+
description: Host mac addresses.
133+
- name: name
134+
level: core
135+
type: keyword
136+
ignore_above: 1024
137+
description: 'Name of the host.
138+
139+
It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.'
140+
- name: os.family
141+
level: extended
142+
type: keyword
143+
ignore_above: 1024
144+
description: OS family (such as redhat, debian, freebsd, windows).
145+
example: debian
146+
- name: os.kernel
147+
level: extended
148+
type: keyword
149+
ignore_above: 1024
150+
description: Operating system kernel version as a raw string.
151+
example: 4.4.0-112-generic
152+
- name: os.name
153+
level: extended
154+
type: keyword
155+
ignore_above: 1024
156+
multi_fields:
157+
- name: text
158+
type: text
159+
norms: false
160+
default_field: false
161+
description: Operating system name, without the version.
162+
example: Mac OS X
163+
- name: os.platform
164+
level: extended
165+
type: keyword
166+
ignore_above: 1024
167+
description: Operating system platform (such centos, ubuntu, windows).
168+
example: darwin
169+
- name: os.version
170+
level: extended
171+
type: keyword
172+
ignore_above: 1024
173+
description: Operating system version as a raw string.
174+
example: 10.14.1
175+
- name: type
176+
level: core
177+
type: keyword
178+
ignore_above: 1024
179+
description: 'Type of host.
180+
181+
For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.'
182+
- name: containerized
183+
type: boolean
184+
description: >
185+
If the host is a container.
186+
187+
- name: os.build
188+
type: keyword
189+
example: "18D109"
190+
description: >
191+
OS build information.
192+
193+
- name: os.codename
194+
type: keyword
195+
example: "stretch"
196+
description: >
197+
OS codename, if any.
198+

‎packages/gcp/data_stream/billing/fields/base-fields.yml

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
- name: data_stream.type
2+
type: constant_keyword
3+
description: Data stream type.
4+
- name: data_stream.dataset
5+
type: constant_keyword
6+
description: Data stream dataset.
7+
- name: data_stream.namespace
8+
type: constant_keyword
9+
description: Data stream namespace.
10+
- name: '@timestamp'
11+
type: date
12+
description: Event timestamp.
13+
- name: event.module
14+
type: constant_keyword
15+
description: Event module
16+
value: gcp
17+
- name: event.dataset
18+
type: constant_keyword
19+
description: Event dataset
20+
value: gcp.billing

‎packages/gcp/data_stream/billing/fields/ecs.yml

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
- external: ecs
2+
name: cloud
3+
- external: ecs
4+
name: cloud.account.id
5+
- external: ecs
6+
name: cloud.account.name
7+
- external: ecs
8+
name: cloud.availability_zone
9+
- external: ecs
10+
name: cloud.instance.id
11+
- external: ecs
12+
name: cloud.machine.type
13+
- external: ecs
14+
name: cloud.provider
15+
- external: ecs
16+
name: cloud.region
17+
- external: ecs
18+
name: ecs.version
19+
- external: ecs
20+
name: error
21+
- external: ecs
22+
name: error.message
23+
- external: ecs
24+
name: service.type

‎packages/gcp/data_stream/billing/fields/fields.yml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
- name: gcp.billing
2+
type: group
3+
description: Google Cloud Billing metrics
4+
fields:
5+
- name: cost_type
6+
type: keyword
7+
description: Cost types include regular, tax, adjustment, and rounding_error.
8+
- name: invoice_month
9+
type: keyword
10+
description: Billing report month.
11+
- name: project_id
12+
type: keyword
13+
description: Project ID of the billing report belongs to.
14+
- name: project_name
15+
type: keyword
16+
description: Project Name of the billing report belongs to.
17+
- name: total
18+
type: float
19+
description: Total billing amount.
20+
- name: billing_account_id
21+
type: keyword
22+
description: Project Billing Account ID.

‎packages/gcp/data_stream/billing/manifest.yml

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
title: "GCP Billing Metrics"
2+
type: metrics
3+
streams:
4+
- input: gcp/metrics
5+
title: GCP Billing Metrics
6+
description: Collect GCP Billing Metrics
7+
vars:
8+
- name: period
9+
type: text
10+
title: Period
11+
default: 24h
12+
- name: dataset_id
13+
type: text
14+
title: Dataset ID
15+
multi: false
16+
required: true
17+
show_user: true
18+
description: "Dataset ID that points to the top-level container which contains the actual billing tables."
19+
- name: table_pattern
20+
type: text
21+
title: Table pattern
22+
multi: false
23+
required: true
24+
show_user: true
25+
description: "Daily cost detail billing table name prefix."
26+
default: gcp_billing_export_v1
27+
- name: cost_type
28+
type: text
29+
title: Cost Type
30+
multi: false
31+
required: true
32+
show_user: true
33+
description: "The type of cost this line item represents: regular, tax, adjustment, or rounding error"
34+
default: regular

‎packages/gcp/data_stream/billing/sample_event.json

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
{
2+
"@timestamp": "2017-10-12T08:05:34.853Z",
3+
"cloud": {
4+
"account": {
5+
"id": "01475F-5B1080-1137E7"
6+
},
7+
"project": {
8+
"id": "elastic-bi",
9+
"name": "elastic-containerlib-prod"
10+
},
11+
"provider": "gcp"
12+
},
13+
"event": {
14+
"dataset": "gcp.billing",
15+
"duration": 115000,
16+
"module": "gcp"
17+
},
18+
"gcp": {
19+
"billing": {
20+
"billing_account_id": "01475F-5B1080-1137E7",
21+
"cost_type": "regular",
22+
"invoice_month": "202106",
23+
"project_id": "containerlib-prod-12763",
24+
"project_name": "elastic-containerlib-prod",
25+
"total": 4717.170681
26+
}
27+
},
28+
"metricset": {
29+
"name": "billing",
30+
"period": 10000
31+
},
32+
"service": {
33+
"type": "gcp"
34+
}
35+
}

‎packages/gcp/data_stream/firewall/manifest.yml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,13 @@ streams:
2525
required: true
2626
show_user: false
2727
default: false
28+
- name: alternative_host
29+
type: text
30+
title: Alternative host
31+
multi: false
32+
required: false
33+
show_user: false
34+
description: "GCP Alternative host"
2835
- name: tags
2936
type: text
3037
title: Tags

‎packages/gcp/data_stream/vpcflow/manifest.yml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,13 @@ streams:
2525
required: true
2626
show_user: false
2727
default: false
28+
- name: alternative_host
29+
type: text
30+
title: Alternative host
31+
multi: false
32+
required: false
33+
show_user: false
34+
description: "GCP Alternative host"
2835
- name: tags
2936
type: text
3037
title: Tags

0 commit comments

Comments
 (0)