bugfix and alerts retry

Author: parmesant

src/alerts/mod.rs

@@ -659,7 +659,7 @@ impl AlertConfig {
 
                 // validate condition config
                 let agg1 = &self.aggregates.aggregate_config[0];
-                let agg2 = &self.aggregates.aggregate_config[0];
+                let agg2 = &self.aggregates.aggregate_config[1];
 
                 validate_condition_config(&agg1.conditions)?;
                 validate_condition_config(&agg2.conditions)?;

src/sync.rs

@@ -246,16 +246,25 @@ pub async fn alert_runtime(mut rx: mpsc::Receiver<AlertTask>) -> Result<(), anyh
                 let alert = alert.clone();
                 let id = alert.id;
                 let handle = tokio::spawn(async move {
+                    let mut retry_counter = 0;
+                    let mut sleep_duration = alert.get_eval_frequency();
                     loop {
                         match alerts_utils::evaluate_alert(&alert).await {
-                            Ok(_) => {}
+                            Ok(_) => {
+                                retry_counter = 0;
+                            }
                             Err(err) => {
-                                error!("Error while evaluation- {err}");
-                                break;
+                                warn!("Error while evaluation- {}\nRetrying after sleeping for 1 minute", err);
+                                sleep_duration = 1;
+                                retry_counter += 1;
+
+                                if retry_counter > 3 {
+                                    error!("Alert with id {} failed to evaluate after 3 retries with err- {}", id, err);
+                                    break;
+                                }
                             }
                         }
-                        tokio::time::sleep(Duration::from_secs(alert.get_eval_frequency() * 60))
-                            .await;
+                        tokio::time::sleep(Duration::from_secs(sleep_duration * 60)).await;
                     }
                 });