From d0d3a383255def2d14fb43cf90e7556ce165c754 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 3 Apr 2025 16:05:08 +0200 Subject: [PATCH 1/9] inject vector aggregator address as env into config file --- .../src/product_logging/framework.rs | 35 ++++++++++++------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/crates/stackable-operator/src/product_logging/framework.rs b/crates/stackable-operator/src/product_logging/framework.rs index 4d946609b..7d26bdd40 100644 --- a/crates/stackable-operator/src/product_logging/framework.rs +++ b/crates/stackable-operator/src/product_logging/framework.rs @@ -31,6 +31,8 @@ const SHUTDOWN_FILE: &str = "shutdown"; /// File name of the Vector config file pub const VECTOR_CONFIG_FILE: &str = "vector.yaml"; +/// Key in the discovery ConfigMap that holds the vector aggregator address +const VECTOR_AGGREGATOR_CM_KEY: &str = "ADDRESS"; #[derive(Debug, Snafu)] pub enum LoggingError { @@ -678,7 +680,6 @@ pub fn create_logback_config( /// # } /// # /// # let logging = fragment::validate::>(default_logging()).unwrap(); -/// # let vector_aggregator_address = "vector-aggregator:6000"; /// # let role_group = RoleGroupRef { /// # cluster: ObjectRef::::new("test-cluster"), /// # role: "role".into(), @@ -702,7 +703,6 @@ pub fn create_logback_config( /// product_logging::framework::VECTOR_CONFIG_FILE, /// product_logging::framework::create_vector_config( /// &role_group, -/// vector_aggregator_address, /// vector_log_config, /// ), /// ); @@ -712,7 +712,6 @@ pub fn create_logback_config( /// ``` pub fn create_vector_config( role_group: &RoleGroupRef, - vector_aggregator_address: &str, config: Option<&AutomaticContainerLogConfig>, ) -> String where @@ -1330,7 +1329,7 @@ sinks: inputs: - extended_logs type: vector - address: {vector_aggregator_address} + address: $VECTOR_AGGREGATOR_ADDRESS "#, namespace = role_group.cluster.namespace.clone().unwrap_or_default(), cluster_name = role_group.cluster.name, @@ -1419,14 +1418,20 @@ sinks: /// ); /// /// if logging.enable_vector_agent { -/// pod_builder.add_container(product_logging::framework::vector_container( -/// &resolved_product_image, -/// "config", -/// "log", -/// logging.containers.get(&Container::Vector), -/// resources, -/// ).unwrap()); -/// } +/// if let Some(vector_aggregator_config_map_name) = spec +/// .cluster_config +/// .vector_aggregator_config_map_name +/// .to_owned() +/// { +/// pod_builder.add_container(product_logging::framework::vector_container( +/// &resolved_product_image, +/// "config", +/// "log", +/// logging.containers.get(&Container::Vector), +/// resources, +/// vector_aggregator_config_map_name, +/// ).unwrap()); +/// } /// /// pod_builder.build().unwrap(); /// ``` @@ -1436,6 +1441,7 @@ pub fn vector_container( log_volume_name: &str, log_config: Option<&ContainerLogConfig>, resources: ResourceRequirements, + vector_aggregator_config_map_name: &str, ) -> Result { let log_level = if let Some(ContainerLogConfig { choice: Some(ContainerLogConfigChoice::Automatic(automatic_log_config)), @@ -1473,6 +1479,11 @@ kill $vector_pid " )]) .add_env_var("VECTOR_LOG", log_level.to_vector_literal()) + .add_env_var_from_config_map( + "VECTOR_AGGREGATOR_ADDRESS", + vector_aggregator_config_map_name, + VECTOR_AGGREGATOR_CM_KEY, + ) .add_volume_mount(config_volume_name, STACKABLE_CONFIG_DIR) .context(AddVolumeMountsSnafu)? .add_volume_mount(log_volume_name, STACKABLE_LOG_DIR) From 9d4617c291a2b6b47ebeef7c5941111552925a73 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 3 Apr 2025 16:17:01 +0200 Subject: [PATCH 2/9] add changelog entry --- crates/stackable-operator/CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/stackable-operator/CHANGELOG.md b/crates/stackable-operator/CHANGELOG.md index bfc1d698d..fb1cd473c 100644 --- a/crates/stackable-operator/CHANGELOG.md +++ b/crates/stackable-operator/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +### Added + +- Inject vector aggregator address into vector config file using an environment variable ([#1000]). + +[#1000]: https://github.com/stackabletech/operator-rs/pull/1000 + ## [0.87.5] - 2025-03-19 ### Fixed From 58c958fe73ae09408596ebf5931b78848a6dd22c Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 3 Apr 2025 17:02:50 +0200 Subject: [PATCH 3/9] fix rustdoc --- .../src/product_logging/framework.rs | 63 +++++++++++-------- 1 file changed, 38 insertions(+), 25 deletions(-) diff --git a/crates/stackable-operator/src/product_logging/framework.rs b/crates/stackable-operator/src/product_logging/framework.rs index ee210fc9e..14f297df6 100644 --- a/crates/stackable-operator/src/product_logging/framework.rs +++ b/crates/stackable-operator/src/product_logging/framework.rs @@ -1418,20 +1418,21 @@ sinks: /// ); /// /// if logging.enable_vector_agent { -/// if let Some(vector_aggregator_config_map_name) = spec -/// .cluster_config -/// .vector_aggregator_config_map_name -/// .to_owned() +/// if let Some(vector_aggregator_config_map_name) = spec +/// .cluster_config +/// .vector_aggregator_config_map_name +/// .to_owned() /// { -/// pod_builder.add_container(product_logging::framework::vector_container( -/// &resolved_product_image, -/// "config", -/// "log", -/// logging.containers.get(&Container::Vector), -/// resources, -/// vector_aggregator_config_map_name, -/// ).unwrap()); -/// } +/// pod_builder.add_container(product_logging::framework::vector_container( +/// &resolved_product_image, +/// "config", +/// "log", +/// logging.containers.get(&Container::Vector), +/// resources, +/// vector_aggregator_config_map_name, +/// ).unwrap()); +/// } +/// } /// /// pod_builder.build().unwrap(); /// ``` @@ -1566,9 +1567,12 @@ mod tests { #[test] fn log4j2_config() { let log_config = AutomaticContainerLogConfig { - loggers: vec![("ROOT".to_string(), LoggerConfig { - level: LogLevel::INFO, - })] + loggers: vec![( + "ROOT".to_string(), + LoggerConfig { + level: LogLevel::INFO, + }, + )] .into_iter() .collect::>(), console: Some(AppenderConfig { @@ -1598,15 +1602,24 @@ mod tests { fn log4j2_config_with_additional_loggers() { let log_config = AutomaticContainerLogConfig { loggers: vec![ - ("ROOT".to_string(), LoggerConfig { - level: LogLevel::INFO, - }), - ("test".to_string(), LoggerConfig { - level: LogLevel::INFO, - }), - ("test_2".to_string(), LoggerConfig { - level: LogLevel::DEBUG, - }), + ( + "ROOT".to_string(), + LoggerConfig { + level: LogLevel::INFO, + }, + ), + ( + "test".to_string(), + LoggerConfig { + level: LogLevel::INFO, + }, + ), + ( + "test_2".to_string(), + LoggerConfig { + level: LogLevel::DEBUG, + }, + ), ] .into_iter() .collect::>(), From 2a2d4b3cb67eadbffd149b3f72c93dba8c070b3a Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 3 Apr 2025 17:13:18 +0200 Subject: [PATCH 4/9] run cargo fmt --- .../src/product_logging/framework.rs | 36 +++++++------------ 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/crates/stackable-operator/src/product_logging/framework.rs b/crates/stackable-operator/src/product_logging/framework.rs index 14f297df6..732f8ac6a 100644 --- a/crates/stackable-operator/src/product_logging/framework.rs +++ b/crates/stackable-operator/src/product_logging/framework.rs @@ -1567,12 +1567,9 @@ mod tests { #[test] fn log4j2_config() { let log_config = AutomaticContainerLogConfig { - loggers: vec![( - "ROOT".to_string(), - LoggerConfig { - level: LogLevel::INFO, - }, - )] + loggers: vec![("ROOT".to_string(), LoggerConfig { + level: LogLevel::INFO, + })] .into_iter() .collect::>(), console: Some(AppenderConfig { @@ -1602,24 +1599,15 @@ mod tests { fn log4j2_config_with_additional_loggers() { let log_config = AutomaticContainerLogConfig { loggers: vec![ - ( - "ROOT".to_string(), - LoggerConfig { - level: LogLevel::INFO, - }, - ), - ( - "test".to_string(), - LoggerConfig { - level: LogLevel::INFO, - }, - ), - ( - "test_2".to_string(), - LoggerConfig { - level: LogLevel::DEBUG, - }, - ), + ("ROOT".to_string(), LoggerConfig { + level: LogLevel::INFO, + }), + ("test".to_string(), LoggerConfig { + level: LogLevel::INFO, + }), + ("test_2".to_string(), LoggerConfig { + level: LogLevel::DEBUG, + }), ] .into_iter() .collect::>(), From 20c1b7e5bc70b6b46826be3165066cc41287329a Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 3 Apr 2025 20:18:03 +0200 Subject: [PATCH 5/9] fix doctest --- crates/stackable-operator/src/product_logging/framework.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/crates/stackable-operator/src/product_logging/framework.rs b/crates/stackable-operator/src/product_logging/framework.rs index 732f8ac6a..853ca6b8a 100644 --- a/crates/stackable-operator/src/product_logging/framework.rs +++ b/crates/stackable-operator/src/product_logging/framework.rs @@ -1385,6 +1385,7 @@ sinks: /// # image_pull_policy: "Always".into(), /// # pull_secrets: None, /// # }; +/// # let vector_aggregator_config_map_name = "vector-aggregator-discovery"; /// /// let mut pod_builder = PodBuilder::new(); /// pod_builder.metadata(ObjectMetaBuilder::default().build()); @@ -1418,11 +1419,6 @@ sinks: /// ); /// /// if logging.enable_vector_agent { -/// if let Some(vector_aggregator_config_map_name) = spec -/// .cluster_config -/// .vector_aggregator_config_map_name -/// .to_owned() -/// { /// pod_builder.add_container(product_logging::framework::vector_container( /// &resolved_product_image, /// "config", @@ -1431,7 +1427,6 @@ sinks: /// resources, /// vector_aggregator_config_map_name, /// ).unwrap()); -/// } /// } /// /// pod_builder.build().unwrap(); From 58db8d0c68160ac56880e07564c5e117bd1c1b38 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 4 Apr 2025 16:17:39 +0200 Subject: [PATCH 6/9] Update crates/stackable-operator/CHANGELOG.md Co-authored-by: Siegfried Weber --- crates/stackable-operator/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/stackable-operator/CHANGELOG.md b/crates/stackable-operator/CHANGELOG.md index db2bb638d..ade1b3e15 100644 --- a/crates/stackable-operator/CHANGELOG.md +++ b/crates/stackable-operator/CHANGELOG.md @@ -6,7 +6,7 @@ All notable changes to this project will be documented in this file. ### Added -- Inject vector aggregator address into vector config file using an environment variable ([#1000]). +- BREAKING: Inject vector aggregator address into vector config file using an environment variable ([#1000]). [#1000]: https://github.com/stackabletech/operator-rs/pull/1000 From e55387e054d6024603940f7026d60fcb01cfdfeb Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 4 Apr 2025 16:27:11 +0200 Subject: [PATCH 7/9] move env name into const --- crates/stackable-operator/src/product_logging/framework.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/stackable-operator/src/product_logging/framework.rs b/crates/stackable-operator/src/product_logging/framework.rs index 853ca6b8a..b6ee5b467 100644 --- a/crates/stackable-operator/src/product_logging/framework.rs +++ b/crates/stackable-operator/src/product_logging/framework.rs @@ -33,6 +33,8 @@ const SHUTDOWN_FILE: &str = "shutdown"; pub const VECTOR_CONFIG_FILE: &str = "vector.yaml"; /// Key in the discovery ConfigMap that holds the vector aggregator address const VECTOR_AGGREGATOR_CM_KEY: &str = "ADDRESS"; +/// Name of the env var in the vector container that holds the vector aggregator address +const VECTOR_AGGREGATOR_ENV_NAME: &str = "VECTOR_AGGREGATOR_ADDRESS"; #[derive(Debug, Snafu)] pub enum LoggingError { @@ -1329,7 +1331,7 @@ sinks: inputs: - extended_logs type: vector - address: $VECTOR_AGGREGATOR_ADDRESS + address: ${VECTOR_AGGREGATOR_ENV_NAME} "#, namespace = role_group.cluster.namespace.clone().unwrap_or_default(), cluster_name = role_group.cluster.name, @@ -1476,7 +1478,7 @@ kill $vector_pid )]) .add_env_var("VECTOR_LOG", log_level.to_vector_literal()) .add_env_var_from_config_map( - "VECTOR_AGGREGATOR_ADDRESS", + VECTOR_AGGREGATOR_ENV_NAME, vector_aggregator_config_map_name, VECTOR_AGGREGATOR_CM_KEY, ) From b2d6e34e477b7c043923c4e98d668340ec444263 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Mon, 7 Apr 2025 09:11:30 +0200 Subject: [PATCH 8/9] Update crates/stackable-operator/src/product_logging/framework.rs Co-authored-by: Siegfried Weber --- .../src/product_logging/framework.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/stackable-operator/src/product_logging/framework.rs b/crates/stackable-operator/src/product_logging/framework.rs index b6ee5b467..226604f68 100644 --- a/crates/stackable-operator/src/product_logging/framework.rs +++ b/crates/stackable-operator/src/product_logging/framework.rs @@ -1421,14 +1421,14 @@ sinks: /// ); /// /// if logging.enable_vector_agent { -/// pod_builder.add_container(product_logging::framework::vector_container( -/// &resolved_product_image, -/// "config", -/// "log", -/// logging.containers.get(&Container::Vector), -/// resources, -/// vector_aggregator_config_map_name, -/// ).unwrap()); +/// pod_builder.add_container(product_logging::framework::vector_container( +/// &resolved_product_image, +/// "config", +/// "log", +/// logging.containers.get(&Container::Vector), +/// resources, +/// vector_aggregator_config_map_name, +/// ).unwrap()); /// } /// /// pod_builder.build().unwrap(); From 0ed8339715e79ffa704025ec0931b98f3bbe2507 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 4 Apr 2025 16:27:11 +0200 Subject: [PATCH 9/9] move env name into const --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ff786d846..25214635c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -724,9 +724,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.4.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cfac68e08048ae1883171632c2aef3ebc555621ae56fbccce1cbf22dd7f058" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" dependencies = [ "powerfmt", ]