[Merged by Bors] - Support HDFS connections

CHANGELOG.md

@@ -8,13 +8,15 @@ All notable changes to this project will be documented in this file.
 
 - PVCs for data storage, cpu and memory limits are now configurable ([#242]).
 - Orphaned resources are deleted ([#254])
+- Support HDFS connections ([#264])
 
 ### Changed
 
 - `operator-rs` `0.22.0` -> `0.25.2` ([#254])
 
 [#242]: https://github.com/stackabletech/hive-operator/pull/242
 [#254]: https://github.com/stackabletech/hive-operator/pull/254
+[#264]: https://github.com/stackabletech/hive-operator/pull/264
 
 ## [0.7.0] - 2022-09-06
 

deploy/crd/hivecluster.crd.yaml

@@ -22,6 +22,15 @@ spec:
         properties:
           spec:
             properties:
+              hdfs:
+                nullable: true
+                properties:
+                  configMap:
+                    description: Name of the discovery-configmap providing information about the HDFS cluster
+                    type: string
+                required:
+                - configMap
+                type: object
               metastore:
                 nullable: true
                 properties:

deploy/helm/hive-operator/crds/crds.yaml

@@ -24,6 +24,15 @@ spec:
           properties:
             spec:
               properties:
+                hdfs:
+                  nullable: true
+                  properties:
+                    configMap:
+                      description: Name of the discovery-configmap providing information about the HDFS cluster
+                      type: string
+                  required:
+                    - configMap
+                  type: object
                 metastore:
                   nullable: true
                   properties:

deploy/manifests/crds.yaml

@@ -25,6 +25,15 @@ spec:
           properties:
             spec:
               properties:
+                hdfs:
+                  nullable: true
+                  properties:
+                    configMap:
+                      description: Name of the discovery-configmap providing information about the HDFS cluster
+                      type: string
+                  required:
+                    - configMap
+                  type: object
                 metastore:
                   nullable: true
                   properties:

rust/crd/src/lib.rs

@@ -67,13 +67,22 @@ pub struct HiveClusterSpec {
     pub metastore: Option<Role<MetaStoreConfig>>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub s3: Option<S3ConnectionDef>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub hdfs: Option<HdfsConnection>,
     /// Specify the type of the created kubernetes service.
     /// This attribute will be removed in a future release when listener-operator is finished.
     /// Use with caution.
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub service_type: Option<ServiceType>,
 }
 
+#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct HdfsConnection {
+    /// Name of the discovery-configmap providing information about the HDFS cluster
+    pub config_map: String,
+}
+
 #[derive(strum::Display)]
 #[strum(serialize_all = "camelCase")]
 pub enum HiveRole {

rust/operator-binary/src/controller.rs

@@ -9,6 +9,7 @@ use stackable_hive_crd::{
     JVM_HEAP_FACTOR, LOG_4J_PROPERTIES, METRICS_PORT, METRICS_PORT_NAME, STACKABLE_CONFIG_DIR,
     STACKABLE_RW_CONFIG_DIR,
 };
+use stackable_operator::k8s_openapi::api::core::v1::VolumeMount;
 use stackable_operator::kube::Resource;
 use stackable_operator::{
     builder::{
@@ -550,6 +551,20 @@ fn build_metastore_rolegroup_statefulset(
         }
     }
 
+    if let Some(hdfs) = &hive.spec.hdfs {
+        pod_builder.add_volume(
+            VolumeBuilder::new("hdfs-site")
+                .with_config_map(&hdfs.config_map)
+                .build(),
+        );
+        container_builder.add_volume_mounts(vec![VolumeMount {
+            name: "hdfs-site".to_string(),
+            mount_path: format!("{STACKABLE_CONFIG_DIR}/hdfs-site.xml"),
+            sub_path: Some("hdfs-site.xml".to_string()),
+            ..VolumeMount::default()
+        }]);
+    }
+
     if let Some(s3_conn) = s3_connection {
         if let Some(credentials) = &s3_conn.credentials {
             pod_builder.add_volume(credentials.to_volume("s3-credentials"));
@@ -607,7 +622,28 @@ fn build_metastore_rolegroup_statefulset(
                 .join(" "),
             s3_connection,
         ))
-        .add_volume_mount("config", STACKABLE_CONFIG_DIR)
+        .add_volume_mounts(vec![
+            // We have to mount every config file individually, so that we can add additional config files
+            // such as hdfs-site.xml as well
+            VolumeMount {
+                name: "config".to_string(),
+                mount_path: format!("{STACKABLE_CONFIG_DIR}/hive-env.sh"),
+                sub_path: Some("hive-env.sh".to_string()),
+                ..VolumeMount::default()
+            },
+            VolumeMount {
+                name: "config".to_string(),
+                mount_path: format!("{STACKABLE_CONFIG_DIR}/hive-site.xml"),
+                sub_path: Some("hive-site.xml".to_string()),
+                ..VolumeMount::default()
+            },
+            VolumeMount {
+                name: "config".to_string(),
+                mount_path: format!("{STACKABLE_CONFIG_DIR}/log4j.properties"),
+                sub_path: Some("log4j.properties".to_string()),
+                ..VolumeMount::default()
+            },
+        ])
         .add_volume_mount("rwconfig", STACKABLE_RW_CONFIG_DIR)
         .add_container_port(HIVE_PORT_NAME, HIVE_PORT.into())
         .add_container_port(METRICS_PORT_NAME, METRICS_PORT.into())