Skip to content

Add extra AM labels #316

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Feb 27, 2023
Merged

Add extra AM labels #316

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7e1d3d2
Add extra AM labels
syepes Feb 27, 2023
4362447
Fix clippy
nitisht Feb 27, 2023
b90d942
Fmt fix
nitisht Feb 27, 2023
18738a1
Merge pull request #5 from nitisht/add-am-extra-labels
syepes Feb 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 75 additions & 24 deletions server/src/alerts/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ pub mod target;
use crate::metrics::ALERTS_STATES;
use crate::storage;
use crate::utils::uid;
use crate::CONFIG;

pub use self::rule::Rule;
use self::target::Target;
Expand Down Expand Up @@ -66,8 +67,8 @@ impl Alert {
ALERTS_STATES
.with_label_values(&[
context.stream.as_str(),
context.alert_name.as_str(),
context.alert_state.to_string().as_str(),
context.alert_info.alert_name.as_str(),
context.alert_info.alert_state.to_string().as_str(),
])
.inc();
for target in &self.targets {
Expand All @@ -78,7 +79,13 @@ impl Alert {
}

fn get_context(&self, stream_name: String, alert_state: AlertState, rule: &Rule) -> Context {
let deployment_instance = format!(
"{}://{}",
CONFIG.parseable.get_scheme(),
CONFIG.parseable.address
);
let deployment_id = storage::StorageMetadata::global().deployment_id;
let deployment_mode = storage::StorageMetadata::global().mode.to_string();
let additional_labels =
serde_json::to_value(rule).expect("rule is perfectly deserializable");
let mut flatten_additional_labels = serde_json::json!({});
Expand All @@ -93,11 +100,13 @@ impl Alert {

Context::new(
stream_name,
self.name.clone(),
self.message.clone(),
self.rule.trigger_reason(),
alert_state,
deployment_id,
AlertInfo::new(
self.name.clone(),
self.message.clone(),
rule.trigger_reason(),
alert_state,
),
DeploymentInfo::new(deployment_instance, deployment_id, deployment_mode),
flatten_additional_labels,
)
}
Expand All @@ -110,44 +119,86 @@ pub trait CallableTarget {
#[derive(Debug, Clone)]
pub struct Context {
stream: String,
alert_name: String,
message: String,
reason: String,
alert_state: AlertState,
deployment_id: uid::Uid,
alert_info: AlertInfo,
deployment_info: DeploymentInfo,
additional_labels: serde_json::Value,
}

impl Context {
pub fn new(
stream: String,
alert_name: String,
message: String,
reason: String,
alert_state: AlertState,
deployment_id: uid::Uid,
alert_info: AlertInfo,
deployment_info: DeploymentInfo,
additional_labels: serde_json::Value,
) -> Self {
Self {
stream,
alert_name,
message,
reason,
alert_state,
deployment_id,
alert_info,
deployment_info,
additional_labels,
}
}

fn default_alert_string(&self) -> String {
format!(
"{} triggered on {}\nMessage: {}\nFailing Condition: {}",
self.alert_name, self.stream, self.message, self.reason
self.alert_info.alert_name,
self.stream,
self.alert_info.message,
self.alert_info.reason
)
}

fn default_resolved_string(&self) -> String {
format!("{} on {} is now resolved ", self.alert_name, self.stream)
format!(
"{} on {} is now resolved ",
self.alert_info.alert_name, self.stream
)
}
}

#[derive(Debug, Clone)]
pub struct AlertInfo {
alert_name: String,
message: String,
reason: String,
alert_state: AlertState,
}

impl AlertInfo {
pub fn new(
alert_name: String,
message: String,
reason: String,
alert_state: AlertState,
) -> Self {
Self {
alert_name,
message,
reason,
alert_state,
}
}
}

#[derive(Debug, Clone)]
pub struct DeploymentInfo {
deployment_instance: String,
deployment_id: uid::Uid,
deployment_mode: String,
}

impl DeploymentInfo {
pub fn new(
deployment_instance: String,
deployment_id: uid::Uid,
deployment_mode: String,
) -> Self {
Self {
deployment_instance,
deployment_id,
deployment_mode,
}
}
}

Expand Down
20 changes: 11 additions & 9 deletions server/src/alerts/target.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ pub struct Target {
impl Target {
pub fn call(&self, context: Context) {
let timeout = &self.timeout;
let resolves = context.alert_state;
let resolves = context.alert_info.alert_state;
let mut state = timeout.state.lock().unwrap();

match resolves {
Expand Down Expand Up @@ -129,7 +129,7 @@ impl Target {
// Send and alert stating that this alert will only work once it has seen a RESOLVE
state.lock().unwrap().timed_out = false;
let mut context = alert_context;
context.message = format!(
context.alert_info.message = format!(
"Triggering alert did not resolve itself after {times} retries, This alert is paused until it resolves");
// Send and exit this task.
call_target(target, context);
Expand Down Expand Up @@ -229,7 +229,7 @@ impl CallableTarget for SlackWebHook {
.build()
.expect("Client can be constructed on this system");

let alert = match payload.alert_state {
let alert = match payload.alert_info.alert_state {
AlertState::SetToFiring => {
serde_json::json!({ "text": payload.default_alert_string() })
}
Expand Down Expand Up @@ -267,7 +267,7 @@ impl CallableTarget for OtherWebHook {
.build()
.expect("Client can be constructed on this system");

let alert = match payload.alert_state {
let alert = match payload.alert_info.alert_state {
AlertState::SetToFiring => payload.default_alert_string(),
AlertState::Resolved => payload.default_resolved_string(),
_ => unreachable!(),
Expand Down Expand Up @@ -317,13 +317,15 @@ impl CallableTarget for AlertManager {

let mut alerts = serde_json::json!([{
"labels": {
"alertname": payload.alert_name,
"alertname": payload.alert_info.alert_name,
"stream": payload.stream,
"deployment_id": payload.deployment_id
"deployment_instance": payload.deployment_info.deployment_instance,
"deployment_id": payload.deployment_info.deployment_id,
"deployment_mode": payload.deployment_info.deployment_mode
},
"annotations": {
"message": payload.message,
"reason": payload.reason
"message": payload.alert_info.message,
"reason": payload.alert_info.reason
}
}]);

Expand All @@ -341,7 +343,7 @@ impl CallableTarget for AlertManager {
);

// fill in status label accordingly
match payload.alert_state {
match payload.alert_info.alert_state {
AlertState::SetToFiring => alert["labels"]["status"] = "firing".into(),
AlertState::Resolved => {
alert["labels"]["status"] = "resolved".into();
Expand Down