From 1d556798f8ce964698ed1725cba7514290c3d4bd Mon Sep 17 00:00:00 2001 From: stahlco Date: Tue, 11 Nov 2025 08:03:41 +0100 Subject: [PATCH 1/8] TimeScaleDB Deployment, Added example secrets --- k8s/kafka/kafka.yml | 3 + .../timescaledb-secrets.example..yml | 9 +++ k8s/timescaledb/timescaledb.yml | 56 +++++++++++++++++++ 3 files changed, 68 insertions(+) create mode 100644 k8s/timescaledb/timescaledb-secrets.example..yml create mode 100644 k8s/timescaledb/timescaledb.yml diff --git a/k8s/kafka/kafka.yml b/k8s/kafka/kafka.yml index 167da47..ef65b92 100644 --- a/k8s/kafka/kafka.yml +++ b/k8s/kafka/kafka.yml @@ -48,6 +48,9 @@ spec: transaction.state.log.min.isr: 1 default.replication.factor: 1 min.insync.replicas: 1 + auto.create.topics.enable: true + compression.type: gzip + entityOperator: topicOperator: {} userOperator: {} diff --git a/k8s/timescaledb/timescaledb-secrets.example..yml b/k8s/timescaledb/timescaledb-secrets.example..yml new file mode 100644 index 0000000..1ddd1c7 --- /dev/null +++ b/k8s/timescaledb/timescaledb-secrets.example..yml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secrets +metadata: + name: timescaledb-secret +type: Opaque +stringData: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: password + POSTGRES_DB: timescaledb diff --git a/k8s/timescaledb/timescaledb.yml b/k8s/timescaledb/timescaledb.yml new file mode 100644 index 0000000..45efa96 --- /dev/null +++ b/k8s/timescaledb/timescaledb.yml @@ -0,0 +1,56 @@ + +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: timescaledb-pvc +spec: + accessModes: + - ReadWriteOnce + storageClassName: standard + resources: + requests: + storage: 10Gi # please chang that an tune that +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: timescaledb +spec: + replicas: 1 + selector: + matchLabels: + app: timescaledb + template: + metadata: + labels: + app: timescaledb + spec: + containers: + - name: timescaledb + image: timescale/timescaledb:latest-pg16 # pg16 because there might be some instabilities with pg17 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 5432 + envFrom: + - secretRef: + name: timescaledb-secret + volumeMounts: + - name: timescaledb-pvc + mountPath: /var/lib/postgresql/data + volumes: + - name: timescaledb-pvc + persistentVolumeClaim: + claimName: timescaledb-pvc +--- +apiVersion: v1 +kind: Service +metadata: + name: timescaledb +spec: + selector: + app: timescaledb + ports: + - name: postgres + port: 5432 + targetPort: 5432 + type: ClusterIP \ No newline at end of file From 5834d7c4b56bbc5d862da3c24959899a38fbf8c0 Mon Sep 17 00:00:00 2001 From: stahlco Date: Tue, 11 Nov 2025 08:10:29 +0100 Subject: [PATCH 2/8] Added strimzi/kafkaconnector, pulls image from private aws repo --- k8s/kafka-connect/kafka-connect.yml | 31 +++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 k8s/kafka-connect/kafka-connect.yml diff --git a/k8s/kafka-connect/kafka-connect.yml b/k8s/kafka-connect/kafka-connect.yml new file mode 100644 index 0000000..d1fa1df --- /dev/null +++ b/k8s/kafka-connect/kafka-connect.yml @@ -0,0 +1,31 @@ +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaConnect +metadata: + name: my-connect-cluster + namespace: test + annotations: + strimzi.io/use-connector-resources: "true" +spec: + version: 4.1.0 + replicas: 1 + bootstrapServers: my-cluster-kafka-bootstrap:9092 + image: public.ecr.aws/r9b1k0m0/kafka-connect:latest + resources: + requests: + memory: 1Gi + cpu: 500m + limits: + memory: 2Gi + cpu: 1 + config: + group.id: connect-cluster + offset.storage.topic: connect-cluster-offsets + config.storage.topic: connect-cluster-configs + status.storage.topic: connect-cluster-status + + config.storage.replication.factor: -1 + offset.storage.replication.factor: -1 + status.storage.replication.factor: -1 + jvmOptions: + "-Xmx": "2G" + "-Xms": "512M" \ No newline at end of file From fdf6fd4ad00c24fba79df7105f0aa2637961120c Mon Sep 17 00:00:00 2001 From: stahlco Date: Tue, 11 Nov 2025 15:14:06 +0100 Subject: [PATCH 3/8] Added Telegraf for Influx --- k8s/influxdb/influx.yml | 5 ++ k8s/telegraf/telegraf.yml | 123 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+) create mode 100644 k8s/telegraf/telegraf.yml diff --git a/k8s/influxdb/influx.yml b/k8s/influxdb/influx.yml index 71ba748..624baf2 100644 --- a/k8s/influxdb/influx.yml +++ b/k8s/influxdb/influx.yml @@ -32,10 +32,15 @@ spec: volumeMounts: - name: influxdb-storage mountPath: /var/lib/influxdb + - name: influx-config + mountPath: /etc/influx/influx.conf volumes: - name: influxdb-storage persistentVolumeClaim: claimName: influxdb-pvc + - name: influx-config + configMap: + name: influx-config --- apiVersion: v1 kind: Service diff --git a/k8s/telegraf/telegraf.yml b/k8s/telegraf/telegraf.yml new file mode 100644 index 0000000..e7a9abf --- /dev/null +++ b/k8s/telegraf/telegraf.yml @@ -0,0 +1,123 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: processors-star +data: + fhir.star: | + def apply(metric): + + if not metric.fields: + return metric + + fhir = metric.fields + + new_metric = metric + new_metric.measurement = fhir.get("resourceType", "FHIR-Resource") + new_metric.tags["device"] = fhir.get("device", "") + new_metric.tags["user"] = fhir.get("user", "") + + code_coding = fhir.get("code", {}).get("coding", []) + if len(code_coding) > 0: + new_metric.tags["code"] = code_coding[0].get("code", "") + new_metric.tags["system"] = code_coding[0].get("system", "") + new_metric.tags["display"] = code_coding[0].get("display", "") + + value_quantity = fhir.get("valueQuantity", {}) + new_metric.fields["value"] = value_quantity.get("value", 0) + new_metric.fields["unit"] = value_quantity.get("unit", "") + new_metric.fields["status"] = fhir.get("status", "") + + if "effectiveDateTime" in fhir: + new_metric.time = fhir["effectiveDateTime"] + + return new_metric +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: telegraf-config +data: + telegraf.conf: | + [global_tags] + + [agent] + interval = "10s" + round_interval = true + metric_batch_size = 1000 + metric_buffer_limit = 10000 + collection_jitter = "1s" + flush_interval = "5s" + flush_jitter = "0s" + precision = "" + debug = true + quiet = false + logfile = "" + hostname = "" + omit_hostname = false + + [[inputs.kafka_consumer]] + brokers = ["my-cluster-kafka-bootstrap:9092"] + topics = ["wearables-fhir"] + max_message_len = 1000000 + consumer_fetch_default = "1MB" + version = "4.0.0" + data_format = "influx" + + [[inputs.cpu]] + percpu = true + totalcpu = true + collect_cpu_time = false + report_active = false + + [[outputs.influxdb_v2]] + urls = ["http://influx-service.test.svc.cluster.local:8086"] + token = "WWC4CqjzlKvgQs_XGX9L-9tan8YfMvqXDCmYbKf4l80Hekp1a3_jJbZToWfY1mGKMUuA8CATNlI9Jap5MUZz-g==" + organization = "test" + bucket = "test" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: telegraf + labels: + app: telegraf +spec: + replicas: 1 + selector: + matchLabels: + app: telegraf + template: + metadata: + labels: + app: telegraf + spec: + serviceAccountName: telegraf + containers: + - name: telegraf + image: telegraf:1.36 # supports Kafka >= v4.x + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + volumeMounts: + - name: processors-star + mountPath: /etc/telegraf/fhir.star + subPath: fhir.star + - name: telegraf-config + mountPath: /etc/telegraf/telegraf.conf + subPath: telegraf.conf + ports: + - containerPort: 8125 + name: metrics + protocol: UDP + volumes: + - name: telegraf-config + configMap: + name: telegraf-config + - name: processors-star + configMap: + name: processors-star \ No newline at end of file From e92f779908fca22334c7bd7898e3e3e242834aac Mon Sep 17 00:00:00 2001 From: stahlco Date: Tue, 11 Nov 2025 15:23:24 +0100 Subject: [PATCH 4/8] Removed unnecessary ConfigMap --- k8s/influxdb/influx.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/k8s/influxdb/influx.yml b/k8s/influxdb/influx.yml index 624baf2..8a936c0 100644 --- a/k8s/influxdb/influx.yml +++ b/k8s/influxdb/influx.yml @@ -38,9 +38,6 @@ spec: - name: influxdb-storage persistentVolumeClaim: claimName: influxdb-pvc - - name: influx-config - configMap: - name: influx-config --- apiVersion: v1 kind: Service From ca905d0b6bc0dea3105a59639bafa833dac133de Mon Sep 17 00:00:00 2001 From: stahlco Date: Tue, 11 Nov 2025 15:27:49 +0100 Subject: [PATCH 5/8] Removed unnecessary ConfigMap #2 --- k8s/influxdb/influx.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/k8s/influxdb/influx.yml b/k8s/influxdb/influx.yml index 8a936c0..71ba748 100644 --- a/k8s/influxdb/influx.yml +++ b/k8s/influxdb/influx.yml @@ -32,8 +32,6 @@ spec: volumeMounts: - name: influxdb-storage mountPath: /var/lib/influxdb - - name: influx-config - mountPath: /etc/influx/influx.conf volumes: - name: influxdb-storage persistentVolumeClaim: From e36c1364a487e0794691277173c2caa3849b09a1 Mon Sep 17 00:00:00 2001 From: stahlco Date: Tue, 11 Nov 2025 15:31:24 +0100 Subject: [PATCH 6/8] Removed unnecessary ConfigMap in Telegraf #2 --- k8s/telegraf/telegraf.yml | 42 +-------------------------------------- 1 file changed, 1 insertion(+), 41 deletions(-) diff --git a/k8s/telegraf/telegraf.yml b/k8s/telegraf/telegraf.yml index e7a9abf..3120db2 100644 --- a/k8s/telegraf/telegraf.yml +++ b/k8s/telegraf/telegraf.yml @@ -1,39 +1,5 @@ apiVersion: v1 kind: ConfigMap -metadata: - name: processors-star -data: - fhir.star: | - def apply(metric): - - if not metric.fields: - return metric - - fhir = metric.fields - - new_metric = metric - new_metric.measurement = fhir.get("resourceType", "FHIR-Resource") - new_metric.tags["device"] = fhir.get("device", "") - new_metric.tags["user"] = fhir.get("user", "") - - code_coding = fhir.get("code", {}).get("coding", []) - if len(code_coding) > 0: - new_metric.tags["code"] = code_coding[0].get("code", "") - new_metric.tags["system"] = code_coding[0].get("system", "") - new_metric.tags["display"] = code_coding[0].get("display", "") - - value_quantity = fhir.get("valueQuantity", {}) - new_metric.fields["value"] = value_quantity.get("value", 0) - new_metric.fields["unit"] = value_quantity.get("unit", "") - new_metric.fields["status"] = fhir.get("status", "") - - if "effectiveDateTime" in fhir: - new_metric.time = fhir["effectiveDateTime"] - - return new_metric ---- -apiVersion: v1 -kind: ConfigMap metadata: name: telegraf-config data: @@ -104,9 +70,6 @@ spec: cpu: 500m memory: 256Mi volumeMounts: - - name: processors-star - mountPath: /etc/telegraf/fhir.star - subPath: fhir.star - name: telegraf-config mountPath: /etc/telegraf/telegraf.conf subPath: telegraf.conf @@ -117,7 +80,4 @@ spec: volumes: - name: telegraf-config configMap: - name: telegraf-config - - name: processors-star - configMap: - name: processors-star \ No newline at end of file + name: telegraf-config \ No newline at end of file From 5b8fcc55debdb754fd4024588047f0d5c4c1912e Mon Sep 17 00:00:00 2001 From: linusgustafsson Date: Tue, 11 Nov 2025 15:49:47 +0100 Subject: [PATCH 7/8] fixed resource bug --- k8s/telegraf/telegraf.yml | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/k8s/telegraf/telegraf.yml b/k8s/telegraf/telegraf.yml index 3120db2..de37b8a 100644 --- a/k8s/telegraf/telegraf.yml +++ b/k8s/telegraf/telegraf.yml @@ -23,7 +23,7 @@ data: [[inputs.kafka_consumer]] brokers = ["my-cluster-kafka-bootstrap:9092"] - topics = ["wearables-fhir"] + topics = ["wearables-lp"] max_message_len = 1000000 consumer_fetch_default = "1MB" version = "4.0.0" @@ -37,7 +37,7 @@ data: [[outputs.influxdb_v2]] urls = ["http://influx-service.test.svc.cluster.local:8086"] - token = "WWC4CqjzlKvgQs_XGX9L-9tan8YfMvqXDCmYbKf4l80Hekp1a3_jJbZToWfY1mGKMUuA8CATNlI9Jap5MUZz-g==" + token = "p-5-DcHD6PYruOiqQ4EDZaBM19GNW7C10n1puRQl3Jvyso3aDtgGo04h2KIvY1brwGNstWeLcuRqhuDAh3ypwQ==" organization = "test" bucket = "test" --- @@ -62,13 +62,6 @@ spec: - name: telegraf image: telegraf:1.36 # supports Kafka >= v4.x imagePullPolicy: IfNotPresent - resources: - requests: - cpu: 100m - memory: 128Mi - limits: - cpu: 500m - memory: 256Mi volumeMounts: - name: telegraf-config mountPath: /etc/telegraf/telegraf.conf @@ -80,4 +73,9 @@ spec: volumes: - name: telegraf-config configMap: - name: telegraf-config \ No newline at end of file + name: telegraf-config +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: telegraf From 85e88fbbb2ec826b50a3b151be67024f16a6418e Mon Sep 17 00:00:00 2001 From: pineapple the fruit dude <150360654+stahlco@users.noreply.github.com> Date: Tue, 11 Nov 2025 17:19:36 +0100 Subject: [PATCH 8/8] Remove InfluxDB URLs and token from configuration Clear sensitive information from InfluxDB output configuration. --- k8s/telegraf/telegraf.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/k8s/telegraf/telegraf.yml b/k8s/telegraf/telegraf.yml index de37b8a..8f1a021 100644 --- a/k8s/telegraf/telegraf.yml +++ b/k8s/telegraf/telegraf.yml @@ -36,8 +36,8 @@ data: report_active = false [[outputs.influxdb_v2]] - urls = ["http://influx-service.test.svc.cluster.local:8086"] - token = "p-5-DcHD6PYruOiqQ4EDZaBM19GNW7C10n1puRQl3Jvyso3aDtgGo04h2KIvY1brwGNstWeLcuRqhuDAh3ypwQ==" + urls = [""] + token = "" organization = "test" bucket = "test" ---