API Call Alerts

Fails e2e run when alerts are firing.

Adds prometheus alerts for excessive API calls from operator-controller or catalogd, as well as summary graphs to match.

Signed-off-by: Daniel Franz <[email protected]>

Author: dtfranz

.github/workflows/e2e.yaml

@@ -35,21 +35,6 @@ jobs:
       - name: Run e2e tests
         run: ARTIFACT_PATH=/tmp/artifacts make test-e2e
 
-      - name: alerts-check
-        # Grab all current alerts, filtering out pending, and print the GH actions warning string
-        # containing the alert name and description.
-        #
-        # NOTE: Leaving this as annotating-only instead of failing the run until we have some more
-        # finely-tuned alerts.
-        run: |
-          if [[ -s /tmp/artifacts/alerts.out ]]; then \
-            jq -r 'if .state=="firing" then
-              "::error title=Prometheus Alert Firing::\(.labels.alertname): \(.annotations.description)"
-            elif .state=="pending" then
-              "::warning title=Prometheus Alert Pending::\(.labels.alertname): \(.annotations.description)"
-            end' /tmp/artifacts/alerts.out
-          fi
-
       - uses: actions/upload-artifact@v4
         if: failure()
         with:

config/overlays/prometheus/prometheus_rule.yaml

@@ -57,3 +57,15 @@ spec:
       keep_firing_for: 1d
       annotations:
         description: "catalogd using high cpu resources for 5 minutes: {{ $value | printf \"%.2f\" }}%"
+    - alert: operator-controller-api-call-rate
+      expr: sum(rate(rest_client_requests_total{job=~"operator-controller-service"}[5m])) > 10
+      for: 5m
+      keep_firing_for: 1d
+      annotations:
+        description: "operator-controller making excessive API calls for 5 minutes: {{ $value | printf \"%.2f\" }}/sec"
+    - alert: catalogd-api-call-rate
+      expr: sum(rate(rest_client_requests_total{job=~"catalogd-service"}[5m])) > 5
+      for: 5m
+      keep_firing_for: 1d
+      annotations:
+        description: "catalogd making excessive API calls for 5 minutes: {{ $value | printf \"%.2f\" }}/sec"

test/e2e/e2e_suite_test.go

@@ -2,6 +2,7 @@ package e2e
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"testing"
 
@@ -40,7 +41,11 @@ func TestMain(m *testing.M) {
 
 	res := m.Run()
 	err = utils.PrintSummary(testSummaryOutputEnvVar)
-	utilruntime.Must(err)
+	if err != nil {
+		// Fail the run if alerts are found
+		fmt.Printf("%s", err)
+		os.Exit(1)
+	}
 	os.Exit(res)
 }
 

test/utils/summary.go

@@ -42,25 +42,31 @@ type xychart struct {
 }
 
 type githubSummary struct {
-	client api.Client
-	Pods   []string
+	client       api.Client
+	firingAlerts bool
+	Pods         []string
 }
 
 func NewSummary(c api.Client, pods ...string) githubSummary {
 	return githubSummary{
-		client: c,
-		Pods:   pods,
+		client:       c,
+		Pods:         pods,
+		firingAlerts: false,
 	}
 }
 
+func (s *githubSummary) FiringAlerts() bool {
+	return s.firingAlerts
+}
+
 // PerformanceQuery queries the prometheus server and generates a mermaid xychart with the data.
 // title  - Display name of the xychart
 // pod    - Pod name with which to filter results from prometheus
 // query  - Prometheus query
 // yLabel - Label of the Y axis i.e. "KB/s", "MB", etc.
 // scaler - Constant by which to scale the results. For instance, cpu usage is more human-readable
 // as "mCPU" vs "CPU", so we scale the results by a factor of 1,000.
-func (s githubSummary) PerformanceQuery(title, pod, query string, yLabel string, scaler float64) (string, error) {
+func (s *githubSummary) PerformanceQuery(title, pod, query, yLabel string, scaler float64) (string, error) {
 	v1api := v1.NewAPI(s.client)
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
@@ -115,7 +121,7 @@ func (s githubSummary) PerformanceQuery(title, pod, query string, yLabel string,
 
 // Alerts queries the prometheus server for alerts and generates markdown output for anything found.
 // If no alerts are found, the alerts section will contain only "None." in the final output.
-func (s githubSummary) Alerts() (string, error) {
+func (s *githubSummary) Alerts() (string, error) {
 	v1api := v1.NewAPI(s.client)
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
@@ -136,6 +142,7 @@ func (s githubSummary) Alerts() (string, error) {
 			switch a.State {
 			case v1.AlertStateFiring:
 				firingAlerts = append(firingAlerts, aConv)
+				s.firingAlerts = true
 			case v1.AlertStatePending:
 				pendingAlerts = append(pendingAlerts, aConv)
 				// Ignore AlertStateInactive; the alerts endpoint doesn't return them
@@ -173,27 +180,31 @@ func executeTemplate(templateFile string, obj any) (string, error) {
 // template. This allows us to add or change queries (hopefully) without needing to
 // touch code. The summary will be output to a file supplied by the env target.
 func PrintSummary(envTarget string) error {
+	path := os.Getenv(envTarget)
+	if path == "" {
+		fmt.Printf("No summary output specified; skipping")
+		return nil
+	}
+
 	client, err := api.NewClient(api.Config{
 		Address: defaultPromUrl,
 	})
 	if err != nil {
-		fmt.Printf("Error creating prometheus client: %v\n", err)
-		os.Exit(1)
+		return err
 	}
 
 	summary := NewSummary(client, "operator-controller", "catalogd")
-	summaryMarkdown, err := executeTemplate(summaryTemplate, summary)
+	summaryMarkdown, err := executeTemplate(summaryTemplate, &summary)
 	if err != nil {
 		return err
 	}
-	if path := os.Getenv(envTarget); path != "" {
-		err = os.WriteFile(path, []byte(summaryMarkdown), 0o600)
-		if err != nil {
-			return err
-		}
-		fmt.Printf("Test summary output to %s successful\n", envTarget)
-	} else {
-		fmt.Printf("No summary output specified; skipping")
+	err = os.WriteFile(path, []byte(summaryMarkdown), 0o600)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("Test summary output to %s successful\n", envTarget)
+	if summary.FiringAlerts() {
+		return fmt.Errorf("Alert(s) encountered during test run; see summary for details")
 	}
 	return nil
 }

test/utils/templates/summary.md.tmpl

@@ -11,6 +11,13 @@
 
 #### CPU Usage
 {{$.PerformanceQuery "CPU Usage" $pod `rate(container_cpu_usage_seconds_total{pod=~"%s.*",container="manager"}[5m])[5m:]` "mCPU" 1000}}
+
+#### API Queries Total
+{{$.PerformanceQuery "API Queries Total" $pod `sum(rest_client_requests_total{job=~"%s.*"})[5m:]` "# queries" 1}}
+
+#### API Query Rate
+{{$.PerformanceQuery "API Queries/sec" $pod `sum(rate(rest_client_requests_total{job=~"%s.*"}[5m]))[5m:]` "per sec" 1}}
+
 {{end}}
 {{- end}}