pkg/ruler: Add external_labels option (#4499)

* pkg/ruler: Add external_labels option

This option can be used to add a set of labels to all alerts emitted by the
ruler.

Signed-off-by: Benoît Knecht <[email protected]>

* docs/configuration: Document external_labels option

Let the documentation generator understand the `labels.Labels` field, and
generate `config-file-reference.md` with

```console
$ go run tools/doc-generator \
> docs/configuration/config-file-reference.template > docs/configuration/config-file-reference.md
```

Signed-off-by: Benoît Knecht <[email protected]>

Author: BenoitKnecht

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## master / unreleased
 
+* [FEATURE] Ruler: Add `external_labels` option to tag all alerts with a given set of labels.
+
 ## 1.12.0 in pgoress
 
 * [CHANGE] Changed default for `-ingester.min-ready-duration` from 1 minute to 15 seconds. #4539

docs/configuration/config-file-reference.md

@@ -1181,6 +1181,9 @@ The `ruler_config` configures the Cortex ruler.
 # CLI flag: -ruler.external.url
 [external_url: <url> | default = ]
 
+# Labels to add to all alerts.
+[external_labels: <map of string to string> | default = ]
+
 ruler_client:
   # gRPC client max receive message size (bytes).
   # CLI flag: -ruler.client.grpc-max-recv-msg-size

pkg/ruler/manager.go

@@ -148,7 +148,7 @@ func (r *DefaultMultiTenantManager) syncRulesToManager(ctx context.Context, user
 			go manager.Run()
 			r.userManagers[user] = manager
 		}
-		err = manager.Update(r.cfg.EvaluationInterval, files, nil, r.cfg.ExternalURL.String())
+		err = manager.Update(r.cfg.EvaluationInterval, files, r.cfg.ExternalLabels, r.cfg.ExternalURL.String())
 		if err != nil {
 			r.lastReloadSuccessful.WithLabelValues(user).Set(0)
 			level.Error(r.logger).Log("msg", "unable to update rule manager", "user", user, "err", err)

pkg/ruler/notifier.go

@@ -128,6 +128,9 @@ func buildNotifierConfig(rulerConfig *Config) (*config.Config, error) {
 	}
 
 	promConfig := &config.Config{
+		GlobalConfig: config.GlobalConfig{
+			ExternalLabels: rulerConfig.ExternalLabels,
+		},
 		AlertingConfig: config.AlertingConfig{
 			AlertmanagerConfigs: amConfigs,
 		},

pkg/ruler/notifier_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/dns"
+	"github.com/prometheus/prometheus/model/labels"
 	"github.com/stretchr/testify/require"
 
 	"github.com/cortexproject/cortex/pkg/util"
@@ -220,6 +221,38 @@ func TestBuildNotifierConfig(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "with external labels",
+			cfg: &Config{
+				AlertmanagerURL: "http://alertmanager.default.svc.cluster.local/alertmanager",
+				ExternalLabels: []labels.Label{
+					{Name: "region", Value: "us-east-1"},
+				},
+			},
+			ncfg: &config.Config{
+				AlertingConfig: config.AlertingConfig{
+					AlertmanagerConfigs: []*config.AlertmanagerConfig{
+						{
+							APIVersion: "v1",
+							Scheme:     "http",
+							PathPrefix: "/alertmanager",
+							ServiceDiscoveryConfigs: discovery.Configs{
+								discovery.StaticConfig{
+									{
+										Targets: []model.LabelSet{{"__address__": "alertmanager.default.svc.cluster.local"}},
+									},
+								},
+							},
+						},
+					},
+				},
+				GlobalConfig: config.GlobalConfig{
+					ExternalLabels: []labels.Label{
+						{Name: "region", Value: "us-east-1"},
+					},
+				},
+			},
+		},
 	}
 
 	for _, tt := range tests {

pkg/ruler/ruler.go

@@ -17,6 +17,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/model/rulefmt"
 	"github.com/prometheus/prometheus/notifier"
 	promRules "github.com/prometheus/prometheus/rules"
@@ -71,6 +72,8 @@ const (
 type Config struct {
 	// This is used for template expansion in alerts; must be a valid URL.
 	ExternalURL flagext.URLValue `yaml:"external_url"`
+	// Labels to add to all alerts
+	ExternalLabels labels.Labels `yaml:"external_labels,omitempty" doc:"nocli|description=Labels to add to all alerts."`
 	// GRPC Client configuration.
 	ClientTLSConfig grpcclient.Config `yaml:"ruler_client"`
 	// How frequently to evaluate rules by default.

tools/doc-generator/parser.go

@@ -257,6 +257,8 @@ func getFieldType(t reflect.Type) (string, error) {
 		return "string", nil
 	case "[]*relabel.Config":
 		return "relabel_config...", nil
+	case "labels.Labels":
+		return "map of string to string", nil
 	}
 
 	// Fallback to auto-detection of built-in data types