Add config option to omit exporting rule_group label on ruler metrics (#4571)

zaharidichev · web-flow · commit 88790f9576fc · 2021-12-02T16:30:16.000Z
* add config option to omit exporting rule_group label on ruler metrics

Signed-off-by: Zahari Dichev &lt;zaharidichev@gmail.com&gt;

* add changelog entry

Signed-off-by: Zahari Dichev &lt;zaharidichev@gmail.com&gt;
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,7 @@
 ## master / unreleased
 
 * [CHANGE] Changed default for `-ingester.min-ready-duration` from 1 minute to 15 seconds. #4539
+* [ENHANCEMENT] Ruler: Add `ruler.disable-rule-group-label` to disable the `rule_group` label on exported metrics. #4571
 * [ENHANCEMENT] Query federation: improve performance in MergeQueryable by memoizing labels. #4502
 * [ENHANCEMENT] Added new ring related config `-ingester.readiness-check-ring-health` when enabled the readiness probe will succeed only after all instances are ACTIVE and healthy in the ring, this is enabled by default. #4539
 * [CHANGE] query-frontend: Do not print anything in the logs of `query-frontend` if a in-progress query has been canceled (context canceled). #4562
diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md
@@ -1650,6 +1650,10 @@ ring:
 # an info level log message.
 # CLI flag: -ruler.query-stats-enabled
 [query_stats_enabled: <boolean> | default = false]
+
+# Disable the rule_group label on exported metrics
+# CLI flag: -ruler.disable-rule-group-label
+[disable_rule_group_label: <boolean> | default = false]
 ```
 
 ### `ruler_storage_config`
diff --git a/pkg/ruler/manager.go b/pkg/ruler/manager.go
@@ -53,7 +53,7 @@ func NewDefaultMultiTenantManager(cfg Config, managerFactory ManagerFactory, reg
 		return nil, err
 	}
 
-	userManagerMetrics := NewManagerMetrics()
+	userManagerMetrics := NewManagerMetrics(cfg.DisableRuleGroupLabel)
 	if reg != nil {
 		reg.MustRegister(userManagerMetrics)
 	}
diff --git a/pkg/ruler/manager_metrics.go b/pkg/ruler/manager_metrics.go
@@ -9,7 +9,8 @@ import (
 // ManagerMetrics aggregates metrics exported by the Prometheus
 // rules package and returns them as Cortex metrics
 type ManagerMetrics struct {
-	regs *util.UserRegistries
+	regs                  *util.UserRegistries
+	disableRuleGroupLabel bool
 
 	EvalDuration         *prometheus.Desc
 	IterationDuration    *prometheus.Desc
@@ -25,9 +26,14 @@ type ManagerMetrics struct {
 }
 
 // NewManagerMetrics returns a ManagerMetrics struct
-func NewManagerMetrics() *ManagerMetrics {
+func NewManagerMetrics(disableRuleGroupLabel bool) *ManagerMetrics {
+	commonLabels := []string{"user"}
+	if !disableRuleGroupLabel {
+		commonLabels = append(commonLabels, "rule_group")
+	}
 	return &ManagerMetrics{
-		regs: util.NewUserRegistries(),
+		regs:                  util.NewUserRegistries(),
+		disableRuleGroupLabel: disableRuleGroupLabel,
 
 		EvalDuration: prometheus.NewDesc(
 			"cortex_prometheus_rule_evaluation_duration_seconds",
@@ -44,55 +50,55 @@ func NewManagerMetrics() *ManagerMetrics {
 		IterationsMissed: prometheus.NewDesc(
 			"cortex_prometheus_rule_group_iterations_missed_total",
 			"The total number of rule group evaluations missed due to slow rule group evaluation.",
-			[]string{"user", "rule_group"},
+			commonLabels,
 			nil,
 		),
 		IterationsScheduled: prometheus.NewDesc(
 			"cortex_prometheus_rule_group_iterations_total",
 			"The total number of scheduled rule group evaluations, whether executed or missed.",
-			[]string{"user", "rule_group"},
+			commonLabels,
 			nil,
 		),
 		EvalTotal: prometheus.NewDesc(
 			"cortex_prometheus_rule_evaluations_total",
 			"The total number of rule evaluations.",
-			[]string{"user", "rule_group"},
+			commonLabels,
 			nil,
 		),
 		EvalFailures: prometheus.NewDesc(
 			"cortex_prometheus_rule_evaluation_failures_total",
 			"The total number of rule evaluation failures.",
-			[]string{"user", "rule_group"},
+			commonLabels,
 			nil,
 		),
 		GroupInterval: prometheus.NewDesc(
 			"cortex_prometheus_rule_group_interval_seconds",
 			"The interval of a rule group.",
-			[]string{"user", "rule_group"},
+			commonLabels,
 			nil,
 		),
 		GroupLastEvalTime: prometheus.NewDesc(
 			"cortex_prometheus_rule_group_last_evaluation_timestamp_seconds",
 			"The timestamp of the last rule group evaluation in seconds.",
-			[]string{"user", "rule_group"},
+			commonLabels,
 			nil,
 		),
 		GroupLastDuration: prometheus.NewDesc(
 			"cortex_prometheus_rule_group_last_duration_seconds",
 			"The duration of the last rule group evaluation.",
-			[]string{"user", "rule_group"},
+			commonLabels,
 			nil,
 		),
 		GroupRules: prometheus.NewDesc(
 			"cortex_prometheus_rule_group_rules",
 			"The number of rules.",
-			[]string{"user", "rule_group"},
+			commonLabels,
 			nil,
 		),
 		GroupLastEvalSamples: prometheus.NewDesc(
 			"cortex_prometheus_last_evaluation_samples",
 			"The number of samples returned during the last rule group evaluation.",
-			[]string{"user", "rule_group"},
+			commonLabels,
 			nil,
 		),
 	}
@@ -126,21 +132,24 @@ func (m *ManagerMetrics) Describe(out chan<- *prometheus.Desc) {
 // Collect implements the Collector interface
 func (m *ManagerMetrics) Collect(out chan<- prometheus.Metric) {
 	data := m.regs.BuildMetricFamiliesPerUser()
-
+	labels := []string{}
+	if !m.disableRuleGroupLabel {
+		labels = append(labels, "rule_group")
+	}
 	// WARNING: It is important that all metrics generated in this method are "Per User".
 	// Thanks to that we can actually *remove* metrics for given user (see RemoveUserRegistry).
 	// If same user is later re-added, all metrics will start from 0, which is fine.
 
 	data.SendSumOfSummariesPerUser(out, m.EvalDuration, "prometheus_rule_evaluation_duration_seconds")
 	data.SendSumOfSummariesPerUser(out, m.IterationDuration, "prometheus_rule_group_duration_seconds")
 
-	data.SendSumOfCountersPerUserWithLabels(out, m.IterationsMissed, "prometheus_rule_group_iterations_missed_total", "rule_group")
-	data.SendSumOfCountersPerUserWithLabels(out, m.IterationsScheduled, "prometheus_rule_group_iterations_total", "rule_group")
-	data.SendSumOfCountersPerUserWithLabels(out, m.EvalTotal, "prometheus_rule_evaluations_total", "rule_group")
-	data.SendSumOfCountersPerUserWithLabels(out, m.EvalFailures, "prometheus_rule_evaluation_failures_total", "rule_group")
-	data.SendSumOfGaugesPerUserWithLabels(out, m.GroupInterval, "prometheus_rule_group_interval_seconds", "rule_group")
-	data.SendSumOfGaugesPerUserWithLabels(out, m.GroupLastEvalTime, "prometheus_rule_group_last_evaluation_timestamp_seconds", "rule_group")
-	data.SendSumOfGaugesPerUserWithLabels(out, m.GroupLastDuration, "prometheus_rule_group_last_duration_seconds", "rule_group")
-	data.SendSumOfGaugesPerUserWithLabels(out, m.GroupRules, "prometheus_rule_group_rules", "rule_group")
-	data.SendSumOfGaugesPerUserWithLabels(out, m.GroupLastEvalSamples, "prometheus_rule_group_last_evaluation_samples", "rule_group")
+	data.SendSumOfCountersPerUserWithLabels(out, m.IterationsMissed, "prometheus_rule_group_iterations_missed_total", labels...)
+	data.SendSumOfCountersPerUserWithLabels(out, m.IterationsScheduled, "prometheus_rule_group_iterations_total", labels...)
+	data.SendSumOfCountersPerUserWithLabels(out, m.EvalTotal, "prometheus_rule_evaluations_total", labels...)
+	data.SendSumOfCountersPerUserWithLabels(out, m.EvalFailures, "prometheus_rule_evaluation_failures_total", labels...)
+	data.SendSumOfGaugesPerUserWithLabels(out, m.GroupInterval, "prometheus_rule_group_interval_seconds", labels...)
+	data.SendSumOfGaugesPerUserWithLabels(out, m.GroupLastEvalTime, "prometheus_rule_group_last_evaluation_timestamp_seconds", labels...)
+	data.SendSumOfGaugesPerUserWithLabels(out, m.GroupLastDuration, "prometheus_rule_group_last_duration_seconds", labels...)
+	data.SendSumOfGaugesPerUserWithLabels(out, m.GroupRules, "prometheus_rule_group_rules", labels...)
+	data.SendSumOfGaugesPerUserWithLabels(out, m.GroupLastEvalSamples, "prometheus_rule_group_last_evaluation_samples", labels...)
 }
diff --git a/pkg/ruler/manager_metrics_test.go b/pkg/ruler/manager_metrics_test.go
@@ -12,10 +12,10 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestManagerMetrics(t *testing.T) {
+func TestManagerMetricsWithRuleGroupLabel(t *testing.T) {
 	mainReg := prometheus.NewPedanticRegistry()
 
-	managerMetrics := NewManagerMetrics()
+	managerMetrics := NewManagerMetrics(false)
 	mainReg.MustRegister(managerMetrics)
 	managerMetrics.AddUserRegistry("user1", populateManager(1))
 	managerMetrics.AddUserRegistry("user2", populateManager(10))
@@ -134,6 +134,104 @@ cortex_prometheus_rule_group_rules{rule_group="group_two",user="user3"} 100000
 	require.NoError(t, err)
 }
 
+func TestManagerMetricsWithoutRuleGroupLabel(t *testing.T) {
+	mainReg := prometheus.NewPedanticRegistry()
+
+	managerMetrics := NewManagerMetrics(true)
+	mainReg.MustRegister(managerMetrics)
+	managerMetrics.AddUserRegistry("user1", populateManager(1))
+	managerMetrics.AddUserRegistry("user2", populateManager(10))
+	managerMetrics.AddUserRegistry("user3", populateManager(100))
+
+	managerMetrics.AddUserRegistry("user4", populateManager(1000))
+	managerMetrics.RemoveUserRegistry("user4")
+
+	//noinspection ALL
+	err := testutil.GatherAndCompare(mainReg, bytes.NewBufferString(`
+# HELP cortex_prometheus_last_evaluation_samples The number of samples returned during the last rule group evaluation.
+# TYPE cortex_prometheus_last_evaluation_samples gauge
+cortex_prometheus_last_evaluation_samples{user="user1"} 2000
+cortex_prometheus_last_evaluation_samples{user="user2"} 20000
+cortex_prometheus_last_evaluation_samples{user="user3"} 200000
+# HELP cortex_prometheus_rule_evaluation_duration_seconds The duration for a rule to execute.
+# TYPE cortex_prometheus_rule_evaluation_duration_seconds summary
+cortex_prometheus_rule_evaluation_duration_seconds{user="user1",quantile="0.5"} 1
+cortex_prometheus_rule_evaluation_duration_seconds{user="user1",quantile="0.9"} 1
+cortex_prometheus_rule_evaluation_duration_seconds{user="user1",quantile="0.99"} 1
+cortex_prometheus_rule_evaluation_duration_seconds_sum{user="user1"} 1
+cortex_prometheus_rule_evaluation_duration_seconds_count{user="user1"} 1
+cortex_prometheus_rule_evaluation_duration_seconds{user="user2",quantile="0.5"} 10
+cortex_prometheus_rule_evaluation_duration_seconds{user="user2",quantile="0.9"} 10
+cortex_prometheus_rule_evaluation_duration_seconds{user="user2",quantile="0.99"} 10
+cortex_prometheus_rule_evaluation_duration_seconds_sum{user="user2"} 10
+cortex_prometheus_rule_evaluation_duration_seconds_count{user="user2"} 1
+cortex_prometheus_rule_evaluation_duration_seconds{user="user3",quantile="0.5"} 100
+cortex_prometheus_rule_evaluation_duration_seconds{user="user3",quantile="0.9"} 100
+cortex_prometheus_rule_evaluation_duration_seconds{user="user3",quantile="0.99"} 100
+cortex_prometheus_rule_evaluation_duration_seconds_sum{user="user3"} 100
+cortex_prometheus_rule_evaluation_duration_seconds_count{user="user3"} 1
+# HELP cortex_prometheus_rule_evaluation_failures_total The total number of rule evaluation failures.
+# TYPE cortex_prometheus_rule_evaluation_failures_total counter
+cortex_prometheus_rule_evaluation_failures_total{user="user1"} 2
+cortex_prometheus_rule_evaluation_failures_total{user="user2"} 20
+cortex_prometheus_rule_evaluation_failures_total{user="user3"} 200
+# HELP cortex_prometheus_rule_evaluations_total The total number of rule evaluations.
+# TYPE cortex_prometheus_rule_evaluations_total counter
+cortex_prometheus_rule_evaluations_total{user="user1"} 2
+cortex_prometheus_rule_evaluations_total{user="user2"} 20
+cortex_prometheus_rule_evaluations_total{user="user3"} 200
+# HELP cortex_prometheus_rule_group_duration_seconds The duration of rule group evaluations.
+# TYPE cortex_prometheus_rule_group_duration_seconds summary
+cortex_prometheus_rule_group_duration_seconds{user="user1",quantile="0.01"} 1
+cortex_prometheus_rule_group_duration_seconds{user="user1",quantile="0.05"} 1
+cortex_prometheus_rule_group_duration_seconds{user="user1",quantile="0.5"} 1
+cortex_prometheus_rule_group_duration_seconds{user="user1",quantile="0.9"} 1
+cortex_prometheus_rule_group_duration_seconds{user="user1",quantile="0.99"} 1
+cortex_prometheus_rule_group_duration_seconds_sum{user="user1"} 1
+cortex_prometheus_rule_group_duration_seconds_count{user="user1"} 1
+cortex_prometheus_rule_group_duration_seconds{user="user2",quantile="0.01"} 10
+cortex_prometheus_rule_group_duration_seconds{user="user2",quantile="0.05"} 10
+cortex_prometheus_rule_group_duration_seconds{user="user2",quantile="0.5"} 10
+cortex_prometheus_rule_group_duration_seconds{user="user2",quantile="0.9"} 10
+cortex_prometheus_rule_group_duration_seconds{user="user2",quantile="0.99"} 10
+cortex_prometheus_rule_group_duration_seconds_sum{user="user2"} 10
+cortex_prometheus_rule_group_duration_seconds_count{user="user2"} 1
+cortex_prometheus_rule_group_duration_seconds{user="user3",quantile="0.01"} 100
+cortex_prometheus_rule_group_duration_seconds{user="user3",quantile="0.05"} 100
+cortex_prometheus_rule_group_duration_seconds{user="user3",quantile="0.5"} 100
+cortex_prometheus_rule_group_duration_seconds{user="user3",quantile="0.9"} 100
+cortex_prometheus_rule_group_duration_seconds{user="user3",quantile="0.99"} 100
+cortex_prometheus_rule_group_duration_seconds_sum{user="user3"} 100
+cortex_prometheus_rule_group_duration_seconds_count{user="user3"} 1
+# HELP cortex_prometheus_rule_group_iterations_missed_total The total number of rule group evaluations missed due to slow rule group evaluation.
+# TYPE cortex_prometheus_rule_group_iterations_missed_total counter
+cortex_prometheus_rule_group_iterations_missed_total{user="user1"} 2
+cortex_prometheus_rule_group_iterations_missed_total{user="user2"} 20
+cortex_prometheus_rule_group_iterations_missed_total{user="user3"} 200
+# HELP cortex_prometheus_rule_group_iterations_total The total number of scheduled rule group evaluations, whether executed or missed.
+# TYPE cortex_prometheus_rule_group_iterations_total counter
+cortex_prometheus_rule_group_iterations_total{user="user1"} 2
+cortex_prometheus_rule_group_iterations_total{user="user2"} 20
+cortex_prometheus_rule_group_iterations_total{user="user3"} 200
+# HELP cortex_prometheus_rule_group_last_duration_seconds The duration of the last rule group evaluation.
+# TYPE cortex_prometheus_rule_group_last_duration_seconds gauge
+cortex_prometheus_rule_group_last_duration_seconds{user="user1"} 2000
+cortex_prometheus_rule_group_last_duration_seconds{user="user2"} 20000
+cortex_prometheus_rule_group_last_duration_seconds{user="user3"} 200000
+# HELP cortex_prometheus_rule_group_last_evaluation_timestamp_seconds The timestamp of the last rule group evaluation in seconds.
+# TYPE cortex_prometheus_rule_group_last_evaluation_timestamp_seconds gauge
+cortex_prometheus_rule_group_last_evaluation_timestamp_seconds{user="user1"} 2000
+cortex_prometheus_rule_group_last_evaluation_timestamp_seconds{user="user2"} 20000
+cortex_prometheus_rule_group_last_evaluation_timestamp_seconds{user="user3"} 200000
+# HELP cortex_prometheus_rule_group_rules The number of rules.
+# TYPE cortex_prometheus_rule_group_rules gauge
+cortex_prometheus_rule_group_rules{user="user1"} 2000
+cortex_prometheus_rule_group_rules{user="user2"} 20000
+cortex_prometheus_rule_group_rules{user="user3"} 200000
+`))
+	require.NoError(t, err)
+}
+
 func populateManager(base float64) *prometheus.Registry {
 	r := prometheus.NewRegistry()
 
@@ -265,7 +363,7 @@ func newGroupMetrics(r prometheus.Registerer) *groupMetrics {
 func TestMetricsArePerUser(t *testing.T) {
 	mainReg := prometheus.NewPedanticRegistry()
 
-	managerMetrics := NewManagerMetrics()
+	managerMetrics := NewManagerMetrics(true)
 	mainReg.MustRegister(managerMetrics)
 	managerMetrics.AddUserRegistry("user1", populateManager(1))
 	managerMetrics.AddUserRegistry("user2", populateManager(10))
diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go
@@ -115,7 +115,8 @@ type Config struct {
 
 	RingCheckPeriod time.Duration `yaml:"-"`
 
-	EnableQueryStats bool `yaml:"query_stats_enabled"`
+	EnableQueryStats      bool `yaml:"query_stats_enabled"`
+	DisableRuleGroupLabel bool `yaml:"disable_rule_group_label"`
 }
 
 // Validate config and returns error on failure
@@ -179,6 +180,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 	f.Var(&cfg.DisabledTenants, "ruler.disabled-tenants", "Comma separated list of tenants whose rules this ruler cannot evaluate. If specified, a ruler that would normally pick the specified tenant(s) for processing will ignore them instead. Subject to sharding.")
 
 	f.BoolVar(&cfg.EnableQueryStats, "ruler.query-stats-enabled", false, "Report the wall time for ruler queries to complete as a per user metric and as an info level log message.")
+	f.BoolVar(&cfg.DisableRuleGroupLabel, "ruler.disable-rule-group-label", false, "Disable the rule_group label on exported metrics")
 
 	cfg.RingCheckPeriod = 5 * time.Second
 }

Original file line number	Diff line number	Diff line change
`@@ -53,7 +53,7 @@ func NewDefaultMultiTenantManager(cfg Config, managerFactory ManagerFactory, reg`
`53`	`53`	`return nil, err`
`54`	`54`	`}`
`55`	`55`
`56`		`- userManagerMetrics := NewManagerMetrics()`
	`56`	`+ userManagerMetrics := NewManagerMetrics(cfg.DisableRuleGroupLabel)`
`57`	`57`	`if reg != nil {`
`58`	`58`	`reg.MustRegister(userManagerMetrics)`
`59`	`59`	`}`
Original file line number	Diff line number	Diff line change
`@@ -115,7 +115,8 @@ type Config struct {`
`115`	`115`
`116`	`116`	RingCheckPeriod time.Duration `yaml:"-"`
`117`	`117`
`118`		- EnableQueryStats bool `yaml:"query_stats_enabled"`
	`118`	+ EnableQueryStats bool `yaml:"query_stats_enabled"`
	`119`	+ DisableRuleGroupLabel bool `yaml:"disable_rule_group_label"`
`119`	`120`	`}`
`120`	`121`
`121`	`122`	`// Validate config and returns error on failure`
`@@ -179,6 +180,7 @@ func (cfg Config) RegisterFlags(f flag.FlagSet) {`
`179`	`180`	`f.Var(&cfg.DisabledTenants, "ruler.disabled-tenants", "Comma separated list of tenants whose rules this ruler cannot evaluate. If specified, a ruler that would normally pick the specified tenant(s) for processing will ignore them instead. Subject to sharding.")`
`180`	`181`
`181`	`182`	`f.BoolVar(&cfg.EnableQueryStats, "ruler.query-stats-enabled", false, "Report the wall time for ruler queries to complete as a per user metric and as an info level log message.")`
	`183`	`+ f.BoolVar(&cfg.DisableRuleGroupLabel, "ruler.disable-rule-group-label", false, "Disable the rule_group label on exported metrics")`
`182`	`184`
`183`	`185`	`cfg.RingCheckPeriod = 5 * time.Second`
`184`	`186`	`}`