Clean up alerts and add labels for alertmanager target (#315)

- Add versioning to alert json
- Add symbols =% (contains) and !% (doesn't contain) variant of string rule
- Fix resolved annotation for alertmanager
- Add additional labels as discussed in #311

Fixes #311

Author: trueleo

server/src/alerts/mod.rs

@@ -33,9 +33,17 @@ use self::target::Target;
 #[derive(Default, Debug, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct Alerts {
+    pub version: AlertVerison,
     pub alerts: Vec<Alert>,
 }
 
+#[derive(Default, Debug, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum AlertVerison {
+    #[default]
+    V1,
+}
+
 #[derive(Debug, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct Alert {
@@ -54,7 +62,7 @@ impl Alert {
         match resolves {
             AlertState::Listening | AlertState::Firing => (),
             alert_state @ (AlertState::SetToFiring | AlertState::Resolved) => {
-                let context = self.get_context(stream_name, alert_state);
+                let context = self.get_context(stream_name, alert_state, &self.rule);
                 ALERTS_STATES
                     .with_label_values(&[
                         context.stream.as_str(),
@@ -69,15 +77,28 @@ impl Alert {
         }
     }
 
-    fn get_context(&self, stream_name: String, alert_state: AlertState) -> Context {
+    fn get_context(&self, stream_name: String, alert_state: AlertState, rule: &Rule) -> Context {
         let deployment_id = storage::StorageMetadata::global().deployment_id;
+        let additional_labels =
+            serde_json::to_value(rule).expect("rule is perfectly deserializable");
+        let mut flatten_additional_labels = serde_json::json!({});
+        flatten_json::flatten(
+            &additional_labels,
+            &mut flatten_additional_labels,
+            Some("rule".to_string()),
+            false,
+            Some("_"),
+        )
+        .expect("can be flattened");
+
         Context::new(
             stream_name,
             self.name.clone(),
             self.message.clone(),
             self.rule.trigger_reason(),
             alert_state,
             deployment_id,
+            flatten_additional_labels,
         )
     }
 }
@@ -94,6 +115,7 @@ pub struct Context {
     reason: String,
     alert_state: AlertState,
     deployment_id: uid::Uid,
+    additional_labels: serde_json::Value,
 }
 
 impl Context {
@@ -104,6 +126,7 @@ impl Context {
         reason: String,
         alert_state: AlertState,
         deployment_id: uid::Uid,
+        additional_labels: serde_json::Value,
     ) -> Self {
         Self {
             stream,
@@ -112,6 +135,7 @@ impl Context {
             reason,
             alert_state,
             deployment_id,
+            additional_labels,
         }
     }
 

server/src/alerts/rule.rs

@@ -408,8 +408,11 @@ pub mod base {
             Exact,
             #[serde(alias = "!=")]
             NotExact,
+            #[serde(alias = "=%")]
             Contains,
+            #[serde(alias = "!%")]
             NotContains,
+            // =~ and !~ reserved for regex
         }
 
         impl Default for StringOperator {

server/src/alerts/target.rs

@@ -30,6 +30,8 @@ use humantime_serde::re::humantime;
 use reqwest::ClientBuilder;
 use serde::{Deserialize, Serialize};
 
+use crate::utils::json;
+
 use super::{AlertState, CallableTarget, Context};
 
 #[derive(Debug, Clone, Copy, Serialize, Deserialize)]
@@ -313,7 +315,7 @@ impl CallableTarget for AlertManager {
             .build()
             .expect("Client can be constructed on this system");
 
-        let mut alert = serde_json::json!([{
+        let mut alerts = serde_json::json!([{
           "labels": {
             "alertname": payload.alert_name,
             "stream": payload.stream,
@@ -325,20 +327,34 @@ impl CallableTarget for AlertManager {
           }
         }]);
 
+        let alert = &mut alerts[0];
+
+        alert["labels"].as_object_mut().expect("is object").extend(
+            payload
+                .additional_labels
+                .as_object()
+                .expect("is object")
+                .iter()
+                // filter non null values for alertmanager and only pass strings
+                .filter(|(_, value)| !value.is_null())
+                .map(|(k, value)| (k.to_owned(), json::convert_to_string(value))),
+        );
+
         // fill in status label accordingly
         match payload.alert_state {
-            AlertState::SetToFiring => alert[0]["labels"]["status"] = "firing".into(),
+            AlertState::SetToFiring => alert["labels"]["status"] = "firing".into(),
             AlertState::Resolved => {
-                let alert = &mut alert[0];
                 alert["labels"]["status"] = "resolved".into();
+                alert["annotations"]["reason"] =
+                    serde_json::Value::String(payload.default_resolved_string());
                 alert["endsAt"] = Utc::now()
                     .to_rfc3339_opts(chrono::SecondsFormat::Millis, true)
                     .into();
             }
             _ => unreachable!(),
         };
 
-        if let Err(e) = client.post(&self.endpoint).json(&alert).send().await {
+        if let Err(e) = client.post(&self.endpoint).json(&alerts).send().await {
             log::error!("Couldn't make call to alertmanager, error: {}", e)
         }
     }

server/src/utils/json.rs

@@ -40,3 +40,23 @@ pub fn merge(value: &mut Value, fields: impl Iterator<Item = (String, Value)>) {
         }
     }
 }
+
+pub fn convert_to_string(value: &Value) -> Value {
+    match value {
+        Value::Null => Value::String("null".to_owned()),
+        Value::Bool(b) => Value::String(b.to_string()),
+        Value::Number(n) => Value::String(n.to_string()),
+        Value::String(s) => Value::String(s.to_owned()),
+        Value::Array(v) => {
+            let new_vec = v.iter().map(convert_to_string).collect();
+            Value::Array(new_vec)
+        }
+        Value::Object(map) => {
+            let new_map = map
+                .iter()
+                .map(|(k, v)| (k.clone(), convert_to_string(v)))
+                .collect();
+            Value::Object(new_map)
+        }
+    }
+}