From 0695ed168deb80afd9285cf55672f541b96a0c56 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Thu, 3 Nov 2022 15:12:13 +0100 Subject: [PATCH 1/2] Support HDFS connections --- CHANGELOG.md | 2 ++ deploy/crd/hivecluster.crd.yaml | 9 ++++++ deploy/helm/hive-operator/crds/crds.yaml | 9 ++++++ deploy/manifests/crds.yaml | 9 ++++++ rust/crd/src/lib.rs | 9 ++++++ rust/operator-binary/src/controller.rs | 38 +++++++++++++++++++++++- 6 files changed, 75 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab644928..bb9283d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ All notable changes to this project will be documented in this file. - PVCs for data storage, cpu and memory limits are now configurable ([#242]). - Orphaned resources are deleted ([#254]) +- Support HDFS connections ([#264]) ### Changed @@ -15,6 +16,7 @@ All notable changes to this project will be documented in this file. [#242]: https://github.com/stackabletech/hive-operator/pull/242 [#254]: https://github.com/stackabletech/hive-operator/pull/254 +[#264]: https://github.com/stackabletech/hive-operator/pull/264 ## [0.7.0] - 2022-09-06 diff --git a/deploy/crd/hivecluster.crd.yaml b/deploy/crd/hivecluster.crd.yaml index 0fdedc1f..13c59e75 100644 --- a/deploy/crd/hivecluster.crd.yaml +++ b/deploy/crd/hivecluster.crd.yaml @@ -22,6 +22,15 @@ spec: properties: spec: properties: + hdfs: + nullable: true + properties: + configMap: + description: Name of the discovery-configmap providing information about the HDFS cluster + type: string + required: + - configMap + type: object metastore: nullable: true properties: diff --git a/deploy/helm/hive-operator/crds/crds.yaml b/deploy/helm/hive-operator/crds/crds.yaml index 5b0d9f02..af3ea4f2 100644 --- a/deploy/helm/hive-operator/crds/crds.yaml +++ b/deploy/helm/hive-operator/crds/crds.yaml @@ -24,6 +24,15 @@ spec: properties: spec: properties: + hdfs: + nullable: true + properties: + configMap: + description: Name of the discovery-configmap providing information about the HDFS cluster + type: string + required: + - configMap + type: object metastore: nullable: true properties: diff --git a/deploy/manifests/crds.yaml b/deploy/manifests/crds.yaml index 2611a323..fab4b044 100644 --- a/deploy/manifests/crds.yaml +++ b/deploy/manifests/crds.yaml @@ -25,6 +25,15 @@ spec: properties: spec: properties: + hdfs: + nullable: true + properties: + configMap: + description: Name of the discovery-configmap providing information about the HDFS cluster + type: string + required: + - configMap + type: object metastore: nullable: true properties: diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index df5d191e..16d37981 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -67,6 +67,8 @@ pub struct HiveClusterSpec { pub metastore: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] pub s3: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub hdfs: Option, /// Specify the type of the created kubernetes service. /// This attribute will be removed in a future release when listener-operator is finished. /// Use with caution. @@ -74,6 +76,13 @@ pub struct HiveClusterSpec { pub service_type: Option, } +#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct HdfsConnection { + /// Name of the discovery-configmap providing information about the HDFS cluster + pub config_map: String, +} + #[derive(strum::Display)] #[strum(serialize_all = "camelCase")] pub enum HiveRole { diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index b4d4c80d..be3f0790 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -9,6 +9,7 @@ use stackable_hive_crd::{ JVM_HEAP_FACTOR, LOG_4J_PROPERTIES, METRICS_PORT, METRICS_PORT_NAME, STACKABLE_CONFIG_DIR, STACKABLE_RW_CONFIG_DIR, }; +use stackable_operator::k8s_openapi::api::core::v1::VolumeMount; use stackable_operator::kube::Resource; use stackable_operator::{ builder::{ @@ -550,6 +551,20 @@ fn build_metastore_rolegroup_statefulset( } } + if let Some(hdfs) = &hive.spec.hdfs { + pod_builder.add_volume( + VolumeBuilder::new("hdfs-site") + .with_config_map(&hdfs.config_map) + .build(), + ); + container_builder.add_volume_mounts(vec![VolumeMount { + name: "hdfs-site".to_string(), + mount_path: format!("{STACKABLE_CONFIG_DIR}/hdfs-site.xml"), + sub_path: Some("hdfs-site.xml".to_string()), + ..VolumeMount::default() + }]); + } + if let Some(s3_conn) = s3_connection { if let Some(credentials) = &s3_conn.credentials { pod_builder.add_volume(credentials.to_volume("s3-credentials")); @@ -607,7 +622,28 @@ fn build_metastore_rolegroup_statefulset( .join(" "), s3_connection, )) - .add_volume_mount("config", STACKABLE_CONFIG_DIR) + .add_volume_mounts(vec![ + // We have to mount every config file individually, so that we can add additional config files + // such as hdfs-site.xml as well + VolumeMount { + name: "config".to_string(), + mount_path: format!("{STACKABLE_CONFIG_DIR}/hive-env.sh"), + sub_path: Some("hive-env.sh".to_string()), + ..VolumeMount::default() + }, + VolumeMount { + name: "config".to_string(), + mount_path: format!("{STACKABLE_CONFIG_DIR}/hive-site.xml"), + sub_path: Some("hive-site.xml".to_string()), + ..VolumeMount::default() + }, + VolumeMount { + name: "config".to_string(), + mount_path: format!("{STACKABLE_CONFIG_DIR}/log4j.properties"), + sub_path: Some("log4j.properties".to_string()), + ..VolumeMount::default() + } + ]) .add_volume_mount("rwconfig", STACKABLE_RW_CONFIG_DIR) .add_container_port(HIVE_PORT_NAME, HIVE_PORT.into()) .add_container_port(METRICS_PORT_NAME, METRICS_PORT.into()) From b01247ae60c928c12433c752243b63f6987190b0 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Thu, 3 Nov 2022 15:50:40 +0100 Subject: [PATCH 2/2] linter --- rust/operator-binary/src/controller.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index be3f0790..c2526b04 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -554,8 +554,8 @@ fn build_metastore_rolegroup_statefulset( if let Some(hdfs) = &hive.spec.hdfs { pod_builder.add_volume( VolumeBuilder::new("hdfs-site") - .with_config_map(&hdfs.config_map) - .build(), + .with_config_map(&hdfs.config_map) + .build(), ); container_builder.add_volume_mounts(vec![VolumeMount { name: "hdfs-site".to_string(), @@ -642,7 +642,7 @@ fn build_metastore_rolegroup_statefulset( mount_path: format!("{STACKABLE_CONFIG_DIR}/log4j.properties"), sub_path: Some("log4j.properties".to_string()), ..VolumeMount::default() - } + }, ]) .add_volume_mount("rwconfig", STACKABLE_RW_CONFIG_DIR) .add_container_port(HIVE_PORT_NAME, HIVE_PORT.into())