From 5d389a231cfdca9ffdb785ea54d925a3da5ea57b Mon Sep 17 00:00:00 2001 From: Satyam Singh Date: Mon, 27 Feb 2023 12:39:35 +0530 Subject: [PATCH 1/5] Change resolved reason for Alertmanager --- server/src/alerts/target.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/alerts/target.rs b/server/src/alerts/target.rs index 2a8924486..819078058 100644 --- a/server/src/alerts/target.rs +++ b/server/src/alerts/target.rs @@ -331,6 +331,8 @@ impl CallableTarget for AlertManager { AlertState::Resolved => { let alert = &mut alert[0]; alert["labels"]["status"] = "resolved".into(); + alert["annotations"]["reason"] = + serde_json::Value::String(payload.default_resolved_string()); alert["endsAt"] = Utc::now() .to_rfc3339_opts(chrono::SecondsFormat::Millis, true) .into(); From 5c0ce0e152bccc16c639bdae8b38d7d193f85bdc Mon Sep 17 00:00:00 2001 From: Satyam Singh Date: Mon, 27 Feb 2023 13:27:39 +0530 Subject: [PATCH 2/5] Add symbol for Contains and NotContains --- server/src/alerts/rule.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/src/alerts/rule.rs b/server/src/alerts/rule.rs index 82b68f89d..ce37c8e4e 100644 --- a/server/src/alerts/rule.rs +++ b/server/src/alerts/rule.rs @@ -408,8 +408,11 @@ pub mod base { Exact, #[serde(alias = "!=")] NotExact, + #[serde(alias = "=%")] Contains, + #[serde(alias = "!%")] NotContains, + // =~ and !~ reserved for regex } impl Default for StringOperator { From f3fb78aeb69fb676bf3788b018e4e82b8e6b012e Mon Sep 17 00:00:00 2001 From: Satyam Singh Date: Mon, 27 Feb 2023 13:28:01 +0530 Subject: [PATCH 3/5] Add version to alerts --- server/src/alerts/mod.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/server/src/alerts/mod.rs b/server/src/alerts/mod.rs index 7779b56c3..2abde0ea7 100644 --- a/server/src/alerts/mod.rs +++ b/server/src/alerts/mod.rs @@ -33,9 +33,17 @@ use self::target::Target; #[derive(Default, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Alerts { + pub version: AlertVerison, pub alerts: Vec, } +#[derive(Default, Debug, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum AlertVerison { + #[default] + V1, +} + #[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Alert { From fa51265115c9c2d231104d933b00a9dcc8c57b0e Mon Sep 17 00:00:00 2001 From: Satyam Singh Date: Mon, 27 Feb 2023 14:24:49 +0530 Subject: [PATCH 4/5] Add labels --- server/src/alerts/mod.rs | 20 ++++++++++++++++++-- server/src/alerts/target.rs | 22 ++++++++++++++++++---- server/src/utils/json.rs | 20 ++++++++++++++++++++ 3 files changed, 56 insertions(+), 6 deletions(-) diff --git a/server/src/alerts/mod.rs b/server/src/alerts/mod.rs index 2abde0ea7..7237a0827 100644 --- a/server/src/alerts/mod.rs +++ b/server/src/alerts/mod.rs @@ -62,7 +62,7 @@ impl Alert { match resolves { AlertState::Listening | AlertState::Firing => (), alert_state @ (AlertState::SetToFiring | AlertState::Resolved) => { - let context = self.get_context(stream_name, alert_state); + let context = self.get_context(stream_name, alert_state, &self.rule); ALERTS_STATES .with_label_values(&[ context.stream.as_str(), @@ -77,8 +77,20 @@ impl Alert { } } - fn get_context(&self, stream_name: String, alert_state: AlertState) -> Context { + fn get_context(&self, stream_name: String, alert_state: AlertState, rule: &Rule) -> Context { let deployment_id = storage::StorageMetadata::global().deployment_id; + let additional_labels = + serde_json::to_value(rule).expect("rule is perfectly deserializable"); + let mut flatten_additional_labels = serde_json::json!({}); + flatten_json::flatten( + &additional_labels, + &mut flatten_additional_labels, + Some("rule".to_string()), + false, + Some("_"), + ) + .expect("can be flattened"); + Context::new( stream_name, self.name.clone(), @@ -86,6 +98,7 @@ impl Alert { self.rule.trigger_reason(), alert_state, deployment_id, + flatten_additional_labels, ) } } @@ -102,6 +115,7 @@ pub struct Context { reason: String, alert_state: AlertState, deployment_id: uid::Uid, + additional_labels: serde_json::Value, } impl Context { @@ -112,6 +126,7 @@ impl Context { reason: String, alert_state: AlertState, deployment_id: uid::Uid, + additional_labels: serde_json::Value, ) -> Self { Self { stream, @@ -120,6 +135,7 @@ impl Context { reason, alert_state, deployment_id, + additional_labels, } } diff --git a/server/src/alerts/target.rs b/server/src/alerts/target.rs index 819078058..02ffa577b 100644 --- a/server/src/alerts/target.rs +++ b/server/src/alerts/target.rs @@ -30,6 +30,8 @@ use humantime_serde::re::humantime; use reqwest::ClientBuilder; use serde::{Deserialize, Serialize}; +use crate::utils::json; + use super::{AlertState, CallableTarget, Context}; #[derive(Debug, Clone, Copy, Serialize, Deserialize)] @@ -313,7 +315,7 @@ impl CallableTarget for AlertManager { .build() .expect("Client can be constructed on this system"); - let mut alert = serde_json::json!([{ + let mut alerts = serde_json::json!([{ "labels": { "alertname": payload.alert_name, "stream": payload.stream, @@ -325,11 +327,23 @@ impl CallableTarget for AlertManager { } }]); + let alert = &mut alerts[0]; + + alert["labels"].as_object_mut().expect("is object").extend( + payload + .additional_labels + .as_object() + .expect("is object") + .iter() + // filter non null values for alertmanager and only pass strings + .filter(|(_, value)| !value.is_null()) + .map(|(k, value)| (k.to_owned(), json::convert_to_string(&value))), + ); + // fill in status label accordingly match payload.alert_state { - AlertState::SetToFiring => alert[0]["labels"]["status"] = "firing".into(), + AlertState::SetToFiring => alert["labels"]["status"] = "firing".into(), AlertState::Resolved => { - let alert = &mut alert[0]; alert["labels"]["status"] = "resolved".into(); alert["annotations"]["reason"] = serde_json::Value::String(payload.default_resolved_string()); @@ -340,7 +354,7 @@ impl CallableTarget for AlertManager { _ => unreachable!(), }; - if let Err(e) = client.post(&self.endpoint).json(&alert).send().await { + if let Err(e) = client.post(&self.endpoint).json(&alerts).send().await { log::error!("Couldn't make call to alertmanager, error: {}", e) } } diff --git a/server/src/utils/json.rs b/server/src/utils/json.rs index f1d536d40..ef5f75ae5 100644 --- a/server/src/utils/json.rs +++ b/server/src/utils/json.rs @@ -40,3 +40,23 @@ pub fn merge(value: &mut Value, fields: impl Iterator) { } } } + +pub fn convert_to_string(value: &Value) -> Value { + match value { + Value::Null => Value::String("null".to_owned()), + Value::Bool(b) => Value::String(b.to_string()), + Value::Number(n) => Value::String(n.to_string()), + Value::String(s) => Value::String(s.to_owned()), + Value::Array(v) => { + let new_vec = v.iter().map(convert_to_string).collect(); + Value::Array(new_vec) + } + Value::Object(map) => { + let new_map = map + .iter() + .map(|(k, v)| (k.clone(), convert_to_string(v))) + .collect(); + Value::Object(new_map) + } + } +} From ab0b452962a27015450c390f6cc609d3545f7610 Mon Sep 17 00:00:00 2001 From: Satyam Singh Date: Mon, 27 Feb 2023 17:03:27 +0530 Subject: [PATCH 5/5] Clippy Fix --- server/src/alerts/target.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/alerts/target.rs b/server/src/alerts/target.rs index 02ffa577b..2ae0fadb0 100644 --- a/server/src/alerts/target.rs +++ b/server/src/alerts/target.rs @@ -337,7 +337,7 @@ impl CallableTarget for AlertManager { .iter() // filter non null values for alertmanager and only pass strings .filter(|(_, value)| !value.is_null()) - .map(|(k, value)| (k.to_owned(), json::convert_to_string(&value))), + .map(|(k, value)| (k.to_owned(), json::convert_to_string(value))), ); // fill in status label accordingly