From c0ccea9a3bbcea1f4b187c7b69eb493bb31baeea Mon Sep 17 00:00:00 2001
From: Julien Pivotto <roidelapluie@inuits.eu>
Date: Tue, 13 Oct 2020 01:05:28 +0200
Subject: [PATCH 1/5] Support metric_relabel_configs in distributor

Signed-off-by: Julien Pivotto <roidelapluie@inuits.eu>
---
 CHANGELOG.md                                  |  1 +
 docs/configuration/config-file-reference.md   |  6 ++
 .../config-file-reference.template            |  1 +
 pkg/distributor/distributor.go                |  6 ++
 pkg/distributor/distributor_test.go           | 75 +++++++++++++++++++
 pkg/util/validation/limits.go                 |  8 ++
 tools/doc-generator/parser.go                 | 19 +++++
 7 files changed, 116 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 45d0f96f51e..d8039251a4e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -51,6 +51,7 @@
 * [FEATURE] Shuffle sharding: added support for shuffle-sharding queriers in the query-frontend. When configured (`-frontend.max-queriers-per-tenant` globally, or using per-tenant limit `max_queriers_per_tenant`), each tenants's requests will be handled by different set of queriers. #3113 #3257
 * [FEATURE] Shuffle sharding: added support for shuffle-sharding ingesters on the read path. When ingesters shuffle-sharding is enabled and `-querier.shuffle-sharding-ingesters-lookback-period` is set, queriers will fetch in-memory series from the minimum set of required ingesters, selecting only ingesters which may have received series since 'now - lookback period'. #3252
 * [FEATURE] Query-frontend: added `compression` config to support results cache with compression. #3217
+* [FEATURE] Added support for applying Prometheus relabel configs on series received by the distributor. A `metric_relabel_configs` field has been added to the per-tenant limits configuration. #3329
 * [ENHANCEMENT] Allow to specify multiple comma-separated Cortex services to `-target` CLI option (or its respective YAML config option). For example, `-target=all,compactor` can be used to start Cortex single-binary with compactor as well. #3275
 * [ENHANCEMENT] Expose additional HTTP configs for the S3 backend client. New flag are listed below: #3244
   - `-blocks-storage.s3.http.idle-conn-timeout`
diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md
index f9f419a74e4..801a14f67ea 100644
--- a/docs/configuration/config-file-reference.md
+++ b/docs/configuration/config-file-reference.md
@@ -24,6 +24,7 @@ To specify which configuration file to load, pass the `-config.file` flag at the
 * `<string>`: a regular string
 * `<url>`: an URL
 * `<prefix>`: a CLI flag prefix based on the context (look at the parent configuration block to see which CLI flags prefix should be used)
+* `<relabel_config>`: a [prometheus relabeling configuration](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config).
 * `<time>`: a timestamp, with available formats: `2006-01-20` (midnight, local timezone), `2006-01-20T15:04` (local timezone), and RFC 3339 formats: `2006-01-20T15:04:05Z` (UTC) or `2006-01-20T15:04:05+07:00` (explicit timezone)
 
 ### Use environment variables in the configuration
@@ -2799,6 +2800,11 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s
 # CLI flag: -distributor.ingestion-tenant-shard-size
 [ingestion_tenant_shard_size: <int> | default = 0]
 
+# List of metric relabel configurations. Note that in most situations, it is
+# more effective to use metrics relabeling directly in the Prometheus server,
+# e.g. remote_write.write_relabel_configs.
+[metric_relabel_configs: <relabel_config...> | default = ]
+
 # The maximum number of series for which a query can fetch samples from each
 # ingester. This limit is enforced only in the ingesters (when querying samples
 # not flushed to the storage yet) and it's a per-instance limit. This limit is
diff --git a/docs/configuration/config-file-reference.template b/docs/configuration/config-file-reference.template
index 07abacea35a..5fab2eade04 100644
--- a/docs/configuration/config-file-reference.template
+++ b/docs/configuration/config-file-reference.template
@@ -24,6 +24,7 @@ To specify which configuration file to load, pass the `-config.file` flag at the
 * `<string>`: a regular string
 * `<url>`: an URL
 * `<prefix>`: a CLI flag prefix based on the context (look at the parent configuration block to see which CLI flags prefix should be used)
+* `<relabel_config>`: a [prometheus relabeling configuration](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config).
 * `<time>`: a timestamp, with available formats: `2006-01-20` (midnight, local timezone), `2006-01-20T15:04` (local timezone), and RFC 3339 formats: `2006-01-20T15:04:05Z` (UTC) or `2006-01-20T15:04:05+07:00` (explicit timezone)
 
 ### Use environment variables in the configuration
diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go
index 3567b795e5e..b2c626aab8f 100644
--- a/pkg/distributor/distributor.go
+++ b/pkg/distributor/distributor.go
@@ -15,6 +15,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/pkg/labels"
+	"github.com/prometheus/prometheus/pkg/relabel"
 	"github.com/prometheus/prometheus/scrape"
 	"github.com/weaveworks/common/httpgrpc"
 	"github.com/weaveworks/common/instrument"
@@ -447,6 +448,11 @@ func (d *Distributor) Push(ctx context.Context, req *client.WriteRequest) (*clie
 			latestSampleTimestampMs = util.Max64(latestSampleTimestampMs, ts.Samples[len(ts.Samples)-1].TimestampMs)
 		}
 
+		if mrc := d.limits.MetricRelabelConfigs(userID); len(mrc) > 0 {
+			l := relabel.Process(client.FromLabelAdaptersToLabels(ts.Labels), mrc...)
+			ts.Labels = client.FromLabelsToLabelAdapters(l)
+		}
+
 		// If we found both the cluster and replica labels, we only want to include the cluster label when
 		// storing series in Cortex. If we kept the replica label we would end up with another series for the same
 		// series we're trying to dedupe when HA tracking moves over to a different replica.
diff --git a/pkg/distributor/distributor_test.go b/pkg/distributor/distributor_test.go
index 3a6bac9da8c..ee56c7bb44b 100644
--- a/pkg/distributor/distributor_test.go
+++ b/pkg/distributor/distributor_test.go
@@ -18,6 +18,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/pkg/labels"
+	"github.com/prometheus/prometheus/pkg/relabel"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/weaveworks/common/httpgrpc"
@@ -1720,6 +1721,80 @@ func TestSortLabels(t *testing.T) {
 	})
 }
 
+func TestDistributor_Push_Relabel(t *testing.T) {
+	ctx = user.InjectOrgID(context.Background(), "user")
+
+	type testcase struct {
+		inputSeries          labels.Labels
+		expectedSeries       labels.Labels
+		metricRelabelConfigs []*relabel.Config
+	}
+
+	cases := []testcase{
+		// No relabel config.
+		{
+			inputSeries: labels.Labels{
+				{Name: "__name__", Value: "foo"},
+				{Name: "cluster", Value: "one"},
+			},
+			expectedSeries: labels.Labels{
+				{Name: "__name__", Value: "foo"},
+				{Name: "cluster", Value: "one"},
+			},
+		},
+		{
+			inputSeries: labels.Labels{
+				{Name: "__name__", Value: "foo"},
+				{Name: "cluster", Value: "one"},
+			},
+			expectedSeries: labels.Labels{
+				{Name: "__name__", Value: "foo"},
+				{Name: "cluster", Value: "two"},
+			},
+			metricRelabelConfigs: []*relabel.Config{
+				{
+					SourceLabels: []model.LabelName{"cluster"},
+					Action:       relabel.DefaultRelabelConfig.Action,
+					Regex:        relabel.DefaultRelabelConfig.Regex,
+					TargetLabel:  "cluster",
+					Replacement:  "two",
+				},
+			},
+		},
+	}
+
+	for _, tc := range cases {
+		var err error
+		var limits validation.Limits
+		flagext.DefaultValues(&limits)
+		limits.MetricRelabelConfigs = tc.metricRelabelConfigs
+
+		ds, ingesters, r := prepare(t, prepConfig{
+			numIngesters:     2,
+			happyIngesters:   2,
+			numDistributors:  1,
+			shardByAllLabels: true,
+			limits:           &limits,
+		})
+		defer stopAll(ds, r)
+
+		// Push the series to the distributor
+		req := mockWriteRequest(tc.inputSeries, 1, 1)
+		_, err = ds[0].Push(ctx, req)
+		require.NoError(t, err)
+
+		// Since each test pushes only 1 series, we do expect the ingester
+		// to have received exactly 1 series
+		for i := range ingesters {
+			timeseries := ingesters[i].series()
+			assert.Equal(t, 1, len(timeseries))
+			for _, v := range timeseries {
+				assert.Equal(t, tc.expectedSeries, client.FromLabelAdaptersToLabels(v.Labels))
+			}
+		}
+	}
+}
+
 func countMockIngestersCalls(ingesters []mockIngester, name string) int {
 	count := 0
 	for i := 0; i < len(ingesters); i++ {
diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go
index c546d1d1148..bea0fa6cecb 100644
--- a/pkg/util/validation/limits.go
+++ b/pkg/util/validation/limits.go
@@ -5,6 +5,8 @@ import (
 	"flag"
 	"time"
 
+	"github.com/prometheus/prometheus/pkg/relabel"
+
 	"github.com/cortexproject/cortex/pkg/util/flagext"
 )
 
@@ -46,6 +48,7 @@ type Limits struct {
 	EnforceMetadataMetricName bool                `yaml:"enforce_metadata_metric_name"`
 	EnforceMetricName         bool                `yaml:"enforce_metric_name"`
 	IngestionTenantShardSize  int                 `yaml:"ingestion_tenant_shard_size"`
+	MetricRelabelConfigs      []*relabel.Config   `yaml:"metric_relabel_configs,omitempty" doc:"nocli|description=List of metric relabel configurations. Note that in most situations, it is more effective to use metrics relabeling directly in the Prometheus server, e.g. remote_write.write_relabel_configs."`
 
 	// Ingester enforced limits.
 	// Series
@@ -372,6 +375,11 @@ func (o *Overrides) EvaluationDelay(userID string) time.Duration {
 	return o.getOverridesForUser(userID).RulerEvaluationDelay
 }
 
+// MetricRelabelConfigs returns the metric relabel configs for a given user.
+func (o *Overrides) MetricRelabelConfigs(userID string) []*relabel.Config {
+	return o.getOverridesForUser(userID).MetricRelabelConfigs
+}
+
 // RulerTenantShardSize returns shard size (number of rulers) used by this tenant when using shuffle-sharding strategy.
 func (o *Overrides) RulerTenantShardSize(userID string) int {
 	return o.getOverridesForUser(userID).RulerTenantShardSize
diff --git a/tools/doc-generator/parser.go b/tools/doc-generator/parser.go
index 2beadb2627e..0d1ef2d442a 100644
--- a/tools/doc-generator/parser.go
+++ b/tools/doc-generator/parser.go
@@ -193,6 +193,16 @@ func parseConfig(block *configBlock, cfg interface{}, flags map[uintptr]*flag.Fl
 		if err != nil {
 			return nil, errors.Wrapf(err, "config=%s.%s", t.PkgPath(), t.Name())
 		}
+		if fieldFlag == nil {
+			block.Add(&configEntry{
+				kind:      "field",
+				name:      fieldName,
+				required:  isFieldRequired(field),
+				fieldDesc: getFieldDescription(field, ""),
+				fieldType: fieldType,
+			})
+			continue
+		}
 
 		block.Add(&configEntry{
 			kind:         "field",
@@ -243,6 +253,8 @@ func getFieldType(t reflect.Type) (string, error) {
 		return "string", nil
 	case "flagext.StringSliceCSV":
 		return "string", nil
+	case "[]*relabel.Config":
+		return "relabel_config...", nil
 	}
 
 	// Fallback to auto-detection of built-in data types
@@ -297,6 +309,9 @@ func getFieldType(t reflect.Type) (string, error) {
 }
 
 func getFieldFlag(field reflect.StructField, fieldValue reflect.Value, flags map[uintptr]*flag.Flag) (*flag.Flag, error) {
+	if isAbsentInCLI(field) {
+		return nil, nil
+	}
 	fieldPtr := fieldValue.Addr().Pointer()
 	fieldFlag, ok := flags[fieldPtr]
 	if !ok {
@@ -395,6 +410,10 @@ func isFieldHidden(f reflect.StructField) bool {
 	return getDocTagFlag(f, "hidden")
 }
 
+func isAbsentInCLI(f reflect.StructField) bool {
+	return getDocTagFlag(f, "nocli")
+}
+
 func isFieldRequired(f reflect.StructField) bool {
 	return getDocTagFlag(f, "required")
 }

From 8b657af02ba65548be3980e62b6cfc35e5a99ddf Mon Sep 17 00:00:00 2001
From: Julien Pivotto <roidelapluie@inuits.eu>
Date: Tue, 20 Oct 2020 20:49:20 +0200
Subject: [PATCH 2/5] Add unmarshal test

Signed-off-by: Julien Pivotto <roidelapluie@inuits.eu>
---
 pkg/util/validation/limits_test.go | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/pkg/util/validation/limits_test.go b/pkg/util/validation/limits_test.go
index eb4bbdcf9e0..8dcc615a832 100644
--- a/pkg/util/validation/limits_test.go
+++ b/pkg/util/validation/limits_test.go
@@ -3,6 +3,8 @@ package validation
 import (
 	"testing"
 
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/prometheus/pkg/relabel"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v2"
@@ -82,9 +84,32 @@ func TestLimitsLoadingFromYaml(t *testing.T) {
 	inp := `ingestion_rate: 0.5`
 
 	l := Limits{}
-	err := yaml.Unmarshal([]byte(inp), &l)
+	err := yaml.UnmarshalStrict([]byte(inp), &l)
 	require.NoError(t, err)
 
 	assert.Equal(t, 0.5, l.IngestionRate, "from yaml")
 	assert.Equal(t, 100, l.MaxLabelNameLength, "from defaults")
 }
+
+func TestMetricRelabelConfigLimitsLoadingFromYaml(t *testing.T) {
+	SetDefaultLimitsForYAMLUnmarshalling(Limits{})
+
+	inp := `
+metric_relabel_configs:
+- action: drop
+  source_labels: [le]
+  regex: .+
+`
+	exp := relabel.DefaultRelabelConfig
+	exp.Action = relabel.Drop
+	regex, err := relabel.NewRegexp(".+")
+	require.NoError(t, err)
+	exp.Regex = regex
+	exp.SourceLabels = model.LabelNames([]model.LabelName{"le"})
+
+	l := Limits{}
+	err = yaml.UnmarshalStrict([]byte(inp), &l)
+	require.NoError(t, err)
+
+	assert.Equal(t, []*relabel.Config{&exp}, l.MetricRelabelConfigs)
+}

From f8e05bd08287443746fc17a3d706796e916e5b3c Mon Sep 17 00:00:00 2001
From: Julien Pivotto <roidelapluie@inuits.eu>
Date: Mon, 26 Oct 2020 18:31:39 +0100
Subject: [PATCH 3/5] Mark as experimental

Signed-off-by: Julien Pivotto <roidelapluie@inuits.eu>
---
 docs/configuration/v1-guarantees.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/configuration/v1-guarantees.md b/docs/configuration/v1-guarantees.md
index 86a768f82fb..9f2d6e0df04 100644
--- a/docs/configuration/v1-guarantees.md
+++ b/docs/configuration/v1-guarantees.md
@@ -51,3 +51,4 @@ Currently experimental features are:
 - TLS configuration in gRPC and HTTP clients.
 - TLS configuration in Etcd client.
 - Blocksconvert tools
+- Metric relabeling in the distributor.

From 199e07f622e6e9adc1bdba577411b4f7f38667f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Peter=20=C5=A0tibran=C3=BD?= <pstibrany@gmail.com>
Date: Mon, 26 Oct 2020 21:54:57 +0100
Subject: [PATCH 4/5] Update docs/configuration/config-file-reference.template
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Marco Pracucci <marco@pracucci.com>
Signed-off-by: Peter Štibraný <peter.stibrany@grafana.com>
---
 docs/configuration/config-file-reference.template | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/configuration/config-file-reference.template b/docs/configuration/config-file-reference.template
index 5fab2eade04..d7ea7c0a2d9 100644
--- a/docs/configuration/config-file-reference.template
+++ b/docs/configuration/config-file-reference.template
@@ -24,7 +24,7 @@ To specify which configuration file to load, pass the `-config.file` flag at the
 * `<string>`: a regular string
 * `<url>`: an URL
 * `<prefix>`: a CLI flag prefix based on the context (look at the parent configuration block to see which CLI flags prefix should be used)
-* `<relabel_config>`: a [prometheus relabeling configuration](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config).
+* `<relabel_config>`: a [Prometheus relabeling configuration](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config).
 * `<time>`: a timestamp, with available formats: `2006-01-20` (midnight, local timezone), `2006-01-20T15:04` (local timezone), and RFC 3339 formats: `2006-01-20T15:04:05Z` (UTC) or `2006-01-20T15:04:05+07:00` (explicit timezone)
 
 ### Use environment variables in the configuration

From 8a1e3f8109160f1d073dcfb4b5ea3699096cef05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= <peter.stibrany@grafana.com>
Date: Mon, 26 Oct 2020 22:16:17 +0100
Subject: [PATCH 5/5] Fix docs.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Peter Štibraný <peter.stibrany@grafana.com>
---
 docs/configuration/config-file-reference.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md
index 2dd24b75b82..316abb37a15 100644
--- a/docs/configuration/config-file-reference.md
+++ b/docs/configuration/config-file-reference.md
@@ -24,7 +24,7 @@ To specify which configuration file to load, pass the `-config.file` flag at the
 * `<string>`: a regular string
 * `<url>`: an URL
 * `<prefix>`: a CLI flag prefix based on the context (look at the parent configuration block to see which CLI flags prefix should be used)
-* `<relabel_config>`: a [prometheus relabeling configuration](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config).
+* `<relabel_config>`: a [Prometheus relabeling configuration](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config).
 * `<time>`: a timestamp, with available formats: `2006-01-20` (midnight, local timezone), `2006-01-20T15:04` (local timezone), and RFC 3339 formats: `2006-01-20T15:04:05Z` (UTC) or `2006-01-20T15:04:05+07:00` (explicit timezone)
 
 ### Use environment variables in the configuration