Add extra Alertmanager labels (#316)

This adds label for Parseable instance address in 
alertmanager.

Part of #311 

Co-authored-by: Nitish Tiwari <[email protected]>

Author: syepes

server/src/alerts/mod.rs

@@ -26,6 +26,7 @@ pub mod target;
 use crate::metrics::ALERTS_STATES;
 use crate::storage;
 use crate::utils::uid;
+use crate::CONFIG;
 
 pub use self::rule::Rule;
 use self::target::Target;
@@ -66,8 +67,8 @@ impl Alert {
                 ALERTS_STATES
                     .with_label_values(&[
                         context.stream.as_str(),
-                        context.alert_name.as_str(),
-                        context.alert_state.to_string().as_str(),
+                        context.alert_info.alert_name.as_str(),
+                        context.alert_info.alert_state.to_string().as_str(),
                     ])
                     .inc();
                 for target in &self.targets {
@@ -78,7 +79,13 @@ impl Alert {
     }
 
     fn get_context(&self, stream_name: String, alert_state: AlertState, rule: &Rule) -> Context {
+        let deployment_instance = format!(
+            "{}://{}",
+            CONFIG.parseable.get_scheme(),
+            CONFIG.parseable.address
+        );
         let deployment_id = storage::StorageMetadata::global().deployment_id;
+        let deployment_mode = storage::StorageMetadata::global().mode.to_string();
         let additional_labels =
             serde_json::to_value(rule).expect("rule is perfectly deserializable");
         let mut flatten_additional_labels = serde_json::json!({});
@@ -93,11 +100,13 @@ impl Alert {
 
         Context::new(
             stream_name,
-            self.name.clone(),
-            self.message.clone(),
-            self.rule.trigger_reason(),
-            alert_state,
-            deployment_id,
+            AlertInfo::new(
+                self.name.clone(),
+                self.message.clone(),
+                rule.trigger_reason(),
+                alert_state,
+            ),
+            DeploymentInfo::new(deployment_instance, deployment_id, deployment_mode),
             flatten_additional_labels,
         )
     }
@@ -110,44 +119,86 @@ pub trait CallableTarget {
 #[derive(Debug, Clone)]
 pub struct Context {
     stream: String,
-    alert_name: String,
-    message: String,
-    reason: String,
-    alert_state: AlertState,
-    deployment_id: uid::Uid,
+    alert_info: AlertInfo,
+    deployment_info: DeploymentInfo,
     additional_labels: serde_json::Value,
 }
 
 impl Context {
     pub fn new(
         stream: String,
-        alert_name: String,
-        message: String,
-        reason: String,
-        alert_state: AlertState,
-        deployment_id: uid::Uid,
+        alert_info: AlertInfo,
+        deployment_info: DeploymentInfo,
         additional_labels: serde_json::Value,
     ) -> Self {
         Self {
             stream,
-            alert_name,
-            message,
-            reason,
-            alert_state,
-            deployment_id,
+            alert_info,
+            deployment_info,
             additional_labels,
         }
     }
 
     fn default_alert_string(&self) -> String {
         format!(
             "{} triggered on {}\nMessage: {}\nFailing Condition: {}",
-            self.alert_name, self.stream, self.message, self.reason
+            self.alert_info.alert_name,
+            self.stream,
+            self.alert_info.message,
+            self.alert_info.reason
         )
     }
 
     fn default_resolved_string(&self) -> String {
-        format!("{} on {} is now resolved ", self.alert_name, self.stream)
+        format!(
+            "{} on {} is now resolved ",
+            self.alert_info.alert_name, self.stream
+        )
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct AlertInfo {
+    alert_name: String,
+    message: String,
+    reason: String,
+    alert_state: AlertState,
+}
+
+impl AlertInfo {
+    pub fn new(
+        alert_name: String,
+        message: String,
+        reason: String,
+        alert_state: AlertState,
+    ) -> Self {
+        Self {
+            alert_name,
+            message,
+            reason,
+            alert_state,
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct DeploymentInfo {
+    deployment_instance: String,
+    deployment_id: uid::Uid,
+    deployment_mode: String,
+}
+
+impl DeploymentInfo {
+    pub fn new(
+        deployment_instance: String,
+        deployment_id: uid::Uid,
+        deployment_mode: String,
+    ) -> Self {
+        Self {
+            deployment_instance,
+            deployment_id,
+            deployment_mode,
+        }
     }
 }
 

server/src/alerts/target.rs

@@ -55,7 +55,7 @@ pub struct Target {
 impl Target {
     pub fn call(&self, context: Context) {
         let timeout = &self.timeout;
-        let resolves = context.alert_state;
+        let resolves = context.alert_info.alert_state;
         let mut state = timeout.state.lock().unwrap();
 
         match resolves {
@@ -129,7 +129,7 @@ impl Target {
                     // Send and alert stating that this alert will only work once it has seen a RESOLVE
                     state.lock().unwrap().timed_out = false;
                     let mut context = alert_context;
-                    context.message = format!(
+                    context.alert_info.message = format!(
                         "Triggering alert did not resolve itself after {times} retries, This alert is paused until it resolves");
                     // Send and exit this task.
                     call_target(target, context);
@@ -229,7 +229,7 @@ impl CallableTarget for SlackWebHook {
             .build()
             .expect("Client can be constructed on this system");
 
-        let alert = match payload.alert_state {
+        let alert = match payload.alert_info.alert_state {
             AlertState::SetToFiring => {
                 serde_json::json!({ "text": payload.default_alert_string() })
             }
@@ -267,7 +267,7 @@ impl CallableTarget for OtherWebHook {
             .build()
             .expect("Client can be constructed on this system");
 
-        let alert = match payload.alert_state {
+        let alert = match payload.alert_info.alert_state {
             AlertState::SetToFiring => payload.default_alert_string(),
             AlertState::Resolved => payload.default_resolved_string(),
             _ => unreachable!(),
@@ -317,13 +317,15 @@ impl CallableTarget for AlertManager {
 
         let mut alerts = serde_json::json!([{
           "labels": {
-            "alertname": payload.alert_name,
+            "alertname": payload.alert_info.alert_name,
             "stream": payload.stream,
-            "deployment_id": payload.deployment_id
+            "deployment_instance": payload.deployment_info.deployment_instance,
+            "deployment_id": payload.deployment_info.deployment_id,
+            "deployment_mode": payload.deployment_info.deployment_mode
             },
           "annotations": {
-            "message": payload.message,
-            "reason": payload.reason
+            "message": payload.alert_info.message,
+            "reason": payload.alert_info.reason
           }
         }]);
 
@@ -341,7 +343,7 @@ impl CallableTarget for AlertManager {
         );
 
         // fill in status label accordingly
-        match payload.alert_state {
+        match payload.alert_info.alert_state {
             AlertState::SetToFiring => alert["labels"]["status"] = "firing".into(),
             AlertState::Resolved => {
                 alert["labels"]["status"] = "resolved".into();