Support metric_relabel_configs in distributor

Signed-off-by: Julien Pivotto <[email protected]>

Author: Julien Pivotto

CHANGELOG.md

@@ -51,6 +51,7 @@
 * [FEATURE] Shuffle sharding: added support for shuffle-sharding queriers in the query-frontend. When configured (`-frontend.max-queriers-per-tenant` globally, or using per-tenant limit `max_queriers_per_tenant`), each tenants's requests will be handled by different set of queriers. #3113 #3257
 * [FEATURE] Shuffle sharding: added support for shuffle-sharding ingesters on the read path. When ingesters shuffle-sharding is enabled and `-querier.shuffle-sharding-ingesters-lookback-period` is set, queriers will fetch in-memory series from the minimum set of required ingesters, selecting only ingesters which may have received series since 'now - lookback period'. #3252
 * [FEATURE] Query-frontend: added `compression` config to support results cache with compression. #3217
+* [FEATURE] Added support for applying Prometheus relabel configs on series received by the distributor. A `metric_relabel_configs` field has been added to the per-tenant limits configuration. #3329
 * [ENHANCEMENT] Allow to specify multiple comma-separated Cortex services to `-target` CLI option (or its respective YAML config option). For example, `-target=all,compactor` can be used to start Cortex single-binary with compactor as well. #3275
 * [ENHANCEMENT] Expose additional HTTP configs for the S3 backend client. New flag are listed below: #3244
   - `-blocks-storage.s3.http.idle-conn-timeout`

docs/configuration/config-file-reference.md

@@ -24,6 +24,7 @@ To specify which configuration file to load, pass the `-config.file` flag at the
 * `<string>`: a regular string
 * `<url>`: an URL
 * `<prefix>`: a CLI flag prefix based on the context (look at the parent configuration block to see which CLI flags prefix should be used)
+* `<relabel_config>`: a [prometheus relabeling configuration](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config).
 * `<time>`: a timestamp, with available formats: `2006-01-20` (midnight, local timezone), `2006-01-20T15:04` (local timezone), and RFC 3339 formats: `2006-01-20T15:04:05Z` (UTC) or `2006-01-20T15:04:05+07:00` (explicit timezone)
 
 ### Use environment variables in the configuration
@@ -2799,6 +2800,11 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s
 # CLI flag: -distributor.ingestion-tenant-shard-size
 [ingestion_tenant_shard_size: <int> | default = 0]
 
+# List of metric relabel configurations. Note that in most situations, it is
+# more effective to use metrics relabeling directly in the Prometheus server,
+# e.g. remote_write.write_relabel_configs.
+[metric_relabel_configs: <relabel_config...> | default = ]
+
 # The maximum number of series for which a query can fetch samples from each
 # ingester. This limit is enforced only in the ingesters (when querying samples
 # not flushed to the storage yet) and it's a per-instance limit. This limit is

docs/configuration/config-file-reference.template

@@ -24,6 +24,7 @@ To specify which configuration file to load, pass the `-config.file` flag at the
 * `<string>`: a regular string
 * `<url>`: an URL
 * `<prefix>`: a CLI flag prefix based on the context (look at the parent configuration block to see which CLI flags prefix should be used)
+* `<relabel_config>`: a [prometheus relabeling configuration](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config).
 * `<time>`: a timestamp, with available formats: `2006-01-20` (midnight, local timezone), `2006-01-20T15:04` (local timezone), and RFC 3339 formats: `2006-01-20T15:04:05Z` (UTC) or `2006-01-20T15:04:05+07:00` (explicit timezone)
 
 ### Use environment variables in the configuration

pkg/distributor/distributor.go

@@ -15,6 +15,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/pkg/labels"
+	"github.com/prometheus/prometheus/pkg/relabel"
 	"github.com/prometheus/prometheus/scrape"
 	"github.com/weaveworks/common/httpgrpc"
 	"github.com/weaveworks/common/instrument"
@@ -447,6 +448,11 @@ func (d *Distributor) Push(ctx context.Context, req *client.WriteRequest) (*clie
 			latestSampleTimestampMs = util.Max64(latestSampleTimestampMs, ts.Samples[len(ts.Samples)-1].TimestampMs)
 		}
 
+		if mrc := d.limits.MetricRelabelConfigs(userID); len(mrc) > 0 {
+			l := relabel.Process(client.FromLabelAdaptersToLabels(ts.Labels), mrc...)
+			ts.Labels = client.FromLabelsToLabelAdapters(l)
+		}
+
 		// If we found both the cluster and replica labels, we only want to include the cluster label when
 		// storing series in Cortex. If we kept the replica label we would end up with another series for the same
 		// series we're trying to dedupe when HA tracking moves over to a different replica.

pkg/distributor/distributor_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/pkg/labels"
+	"github.com/prometheus/prometheus/pkg/relabel"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/weaveworks/common/httpgrpc"
@@ -1720,6 +1721,80 @@ func TestSortLabels(t *testing.T) {
 	})
 }
 
+func TestDistributor_Push_Relabel(t *testing.T) {
+	ctx = user.InjectOrgID(context.Background(), "user")
+
+	type testcase struct {
+		inputSeries          labels.Labels
+		expectedSeries       labels.Labels
+		metricRelabelConfigs []*relabel.Config
+	}
+
+	cases := []testcase{
+		// No relabel config.
+		{
+			inputSeries: labels.Labels{
+				{Name: "__name__", Value: "foo"},
+				{Name: "cluster", Value: "one"},
+			},
+			expectedSeries: labels.Labels{
+				{Name: "__name__", Value: "foo"},
+				{Name: "cluster", Value: "one"},
+			},
+		},
+		{
+			inputSeries: labels.Labels{
+				{Name: "__name__", Value: "foo"},
+				{Name: "cluster", Value: "one"},
+			},
+			expectedSeries: labels.Labels{
+				{Name: "__name__", Value: "foo"},
+				{Name: "cluster", Value: "two"},
+			},
+			metricRelabelConfigs: []*relabel.Config{
+				{
+					SourceLabels: []model.LabelName{"cluster"},
+					Action:       relabel.DefaultRelabelConfig.Action,
+					Regex:        relabel.DefaultRelabelConfig.Regex,
+					TargetLabel:  "cluster",
+					Replacement:  "two",
+				},
+			},
+		},
+	}
+
+	for _, tc := range cases {
+		var err error
+		var limits validation.Limits
+		flagext.DefaultValues(&limits)
+		limits.MetricRelabelConfigs = tc.metricRelabelConfigs
+
+		ds, ingesters, r := prepare(t, prepConfig{
+			numIngesters:     2,
+			happyIngesters:   2,
+			numDistributors:  1,
+			shardByAllLabels: true,
+			limits:           &limits,
+		})
+		defer stopAll(ds, r)
+
+		// Push the series to the distributor
+		req := mockWriteRequest(tc.inputSeries, 1, 1)
+		_, err = ds[0].Push(ctx, req)
+		require.NoError(t, err)
+
+		// Since each test pushes only 1 series, we do expect the ingester
+		// to have received exactly 1 series
+		for i := range ingesters {
+			timeseries := ingesters[i].series()
+			assert.Equal(t, 1, len(timeseries))
+			for _, v := range timeseries {
+				assert.Equal(t, tc.expectedSeries, client.FromLabelAdaptersToLabels(v.Labels))
+			}
+		}
+	}
+}
+
 func countMockIngestersCalls(ingesters []mockIngester, name string) int {
 	count := 0
 	for i := 0; i < len(ingesters); i++ {

pkg/util/validation/limits.go

@@ -5,6 +5,8 @@ import (
 	"flag"
 	"time"
 
+	"github.com/prometheus/prometheus/pkg/relabel"
+
 	"github.com/cortexproject/cortex/pkg/util/flagext"
 )
 
@@ -46,6 +48,7 @@ type Limits struct {
 	EnforceMetadataMetricName bool                `yaml:"enforce_metadata_metric_name"`
 	EnforceMetricName         bool                `yaml:"enforce_metric_name"`
 	IngestionTenantShardSize  int                 `yaml:"ingestion_tenant_shard_size"`
+	MetricRelabelConfigs      []*relabel.Config   `yaml:"metric_relabel_configs,omitempty" doc:"nocli|description=List of metric relabel configurations. Note that in most situations, it is more effective to use metrics relabeling directly in the Prometheus server, e.g. remote_write.write_relabel_configs."`
 
 	// Ingester enforced limits.
 	// Series
@@ -372,6 +375,11 @@ func (o *Overrides) EvaluationDelay(userID string) time.Duration {
 	return o.getOverridesForUser(userID).RulerEvaluationDelay
 }
 
+// MetricRelabelConfigs returns the metric relabel configs for a given user.
+func (o *Overrides) MetricRelabelConfigs(userID string) []*relabel.Config {
+	return o.getOverridesForUser(userID).MetricRelabelConfigs
+}
+
 // RulerTenantShardSize returns shard size (number of rulers) used by this tenant when using shuffle-sharding strategy.
 func (o *Overrides) RulerTenantShardSize(userID string) int {
 	return o.getOverridesForUser(userID).RulerTenantShardSize

tools/doc-generator/parser.go

@@ -193,6 +193,16 @@ func parseConfig(block *configBlock, cfg interface{}, flags map[uintptr]*flag.Fl
 		if err != nil {
 			return nil, errors.Wrapf(err, "config=%s.%s", t.PkgPath(), t.Name())
 		}
+		if fieldFlag == nil {
+			block.Add(&configEntry{
+				kind:      "field",
+				name:      fieldName,
+				required:  isFieldRequired(field),
+				fieldDesc: getFieldDescription(field, ""),
+				fieldType: fieldType,
+			})
+			continue
+		}
 
 		block.Add(&configEntry{
 			kind:         "field",
@@ -243,6 +253,8 @@ func getFieldType(t reflect.Type) (string, error) {
 		return "string", nil
 	case "flagext.StringSliceCSV":
 		return "string", nil
+	case "[]*relabel.Config":
+		return "relabel_config...", nil
 	}
 
 	// Fallback to auto-detection of built-in data types
@@ -297,6 +309,9 @@ func getFieldType(t reflect.Type) (string, error) {
 }
 
 func getFieldFlag(field reflect.StructField, fieldValue reflect.Value, flags map[uintptr]*flag.Flag) (*flag.Flag, error) {
+	if isAbsentInCLI(field) {
+		return nil, nil
+	}
 	fieldPtr := fieldValue.Addr().Pointer()
 	fieldFlag, ok := flags[fieldPtr]
 	if !ok {
@@ -395,6 +410,10 @@ func isFieldHidden(f reflect.StructField) bool {
 	return getDocTagFlag(f, "hidden")
 }
 
+func isAbsentInCLI(f reflect.StructField) bool {
+	return getDocTagFlag(f, "nocli")
+}
+
 func isFieldRequired(f reflect.StructField) bool {
 	return getDocTagFlag(f, "required")
 }