Merge branch 'master' into partitioning-compaction

Signed-off-by: Alex Le <[email protected]>

Author: alexqyle

.github/workflows/scripts/install-docker.sh

@@ -4,4 +4,8 @@ set -x
 VER="20.10.19"
 curl -L -o /tmp/docker-$VER.tgz https://download.docker.com/linux/static/stable/x86_64/docker-$VER.tgz
 tar -xz -C /tmp -f /tmp/docker-$VER.tgz
+mkdir -vp ~/.docker/cli-plugins/
+curl --silent -L "https://github.com/docker/buildx/releases/download/v0.3.0/buildx-v0.3.0.linux-amd64" > ~/.docker/cli-plugins/docker-buildx
+chmod a+x ~/.docker/cli-plugins/docker-buildx
 mv /tmp/docker/* /usr/bin
+docker run --privileged --rm tonistiigi/binfmt --install all

.github/workflows/test-build-deploy.yml

@@ -88,6 +88,16 @@ jobs:
   integration:
     needs: build
     runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        tags:
+        - requires_docker
+        - integration_alertmanager
+        - integration_backward_compatibility
+        - integration_memberlist
+        - integration_querier
+        - integration_ruler
     steps:
       - name: Upgrade golang
         uses: actions/setup-go@v2
@@ -116,35 +126,30 @@ jobs:
         # locally and the download time doesn't account in the test execution time, which is subject
         # to a timeout
         run: |
-          docker pull minio/minio:RELEASE.2019-12-30T05-45-39Z
-          docker pull amazon/dynamodb-local:1.11.477
+          docker pull minio/minio:RELEASE.2021-10-13T00-23-17Z
           docker pull consul:1.8.4
           docker pull gcr.io/etcd-development/etcd:v3.4.7
-          docker pull quay.io/cortexproject/cortex:v1.0.0
-          docker pull quay.io/cortexproject/cortex:v1.1.0
-          docker pull quay.io/cortexproject/cortex:v1.2.0
-          docker pull quay.io/cortexproject/cortex:v1.3.0
-          docker pull quay.io/cortexproject/cortex:v1.4.0
-          docker pull quay.io/cortexproject/cortex:v1.5.0
-          docker pull quay.io/cortexproject/cortex:v1.6.0
-          docker pull quay.io/cortexproject/cortex:v1.7.0
-          docker pull quay.io/cortexproject/cortex:v1.8.0
-          docker pull quay.io/cortexproject/cortex:v1.9.0
-          docker pull quay.io/cortexproject/cortex:v1.10.0
-          docker pull quay.io/cortexproject/cortex:v1.11.1
-          docker pull quay.io/cortexproject/cortex:v1.13.1
-          docker pull quay.io/cortexproject/cortex:v1.14.0
-          docker pull shopify/bigtable-emulator:0.1.0
+          if [ "$TEST_TAGS" = "integration_backward_compatibility" ]; then
+            docker pull quay.io/cortexproject/cortex:v1.6.0
+            docker pull quay.io/cortexproject/cortex:v1.7.0
+            docker pull quay.io/cortexproject/cortex:v1.8.0
+            docker pull quay.io/cortexproject/cortex:v1.9.0
+            docker pull quay.io/cortexproject/cortex:v1.10.0
+            docker pull quay.io/cortexproject/cortex:v1.11.1
+            docker pull quay.io/cortexproject/cortex:v1.13.1
+            docker pull quay.io/cortexproject/cortex:v1.14.0
+          fi
           docker pull memcached:1.6.1
-          docker pull bouncestorage/swift-aio:55ba4331
+        env:
+          TEST_TAGS: ${{ matrix.tags }}
       - name: Integration Tests
         run: |
           export CORTEX_IMAGE_PREFIX="${IMAGE_PREFIX:-quay.io/cortexproject/}"
           export IMAGE_TAG=$(make image-tag)
-          export CORTEX_IMAGE="${CORTEX_IMAGE_PREFIX}cortex:$IMAGE_TAG"
+          export CORTEX_IMAGE="${CORTEX_IMAGE_PREFIX}cortex:$IMAGE_TAG-amd64"
           export CORTEX_CHECKOUT_DIR="/go/src/github.com/cortexproject/cortex"
           echo "Running integration tests with image: $CORTEX_IMAGE"
-          go test -tags=requires_docker -timeout 2400s -v -count=1 ./integration/...
+          go test -tags=integration,${{ matrix.tags }} -timeout 2400s -v -count=1 ./integration/...
         env:
           IMAGE_PREFIX: ${{ secrets.IMAGE_PREFIX }}
 

.golangci.yml

@@ -33,3 +33,9 @@ run:
   build-tags:
     - netgo
     - requires_docker
+    - integration
+    - integration_alertmanager
+    - integration_backward_compatibility
+    - integration_memberlist
+    - integration_querier
+    - integration_ruler

CHANGELOG.md

@@ -1,16 +1,28 @@
 # Changelog
 
 ## master / unreleased
+* [CHANGE] Alertmanager: Local file disclosure vulnerability in OpsGenie configuration has been fixed. #5045
 * [ENHANCEMENT] Update Go version to 1.19.3. #4988
 * [ENHANCEMENT] Querier: limit series query to only ingesters if `start` param is not specified. #4976
 * [ENHANCEMENT] Query-frontend/scheduler: add a new limit `frontend.max-outstanding-requests-per-tenant` for configuring queue size per tenant. Started deprecating two flags `-query-scheduler.max-outstanding-requests-per-tenant` and `-querier.max-outstanding-requests-per-tenant`, and change their value default to 0. Now if both the old flag and new flag are specified, the old flag's queue size will be picked. #5005
 * [ENHANCEMENT] Query-tee: Add `/api/v1/query_exemplars` API endpoint support. #5010
+* [ENHANCEMENT] Let blocks_cleaner delete blocks concurrently(default 16 goroutines). #5028
+* [ENHANCEMENT] Query Frontend/Query Scheduler: Increase upper bound to 60s for queue duration histogram metric. #5029
+* [ENHANCEMENT] Query Frontend: Log Vertical sharding information when `query_stats_enabled` is enabled. #5037
+* [ENHANCEMENT] Ingester: The metadata APIs should honour `querier.query-ingesters-within` when `querier.query-store-for-labels-enabled` is true. #5027
+* [ENHANCEMENT] Query Frontend: Skip instant query roundtripper if sharding is not applicable. #5062
 * [FEATURE] Querier/Query Frontend: support Prometheus /api/v1/status/buildinfo API. #4978
 * [FEATURE] Ingester: Add active series to all_user_stats page. #4972
 * [FEATURE] Ingester: Added `-blocks-storage.tsdb.head-chunks-write-queue-size` allowing to configure the size of the in-memory queue used before flushing chunks to the disk . #5000
 * [FEATURE] Query Frontend: Log query params in query frontend even if error happens. #5005
 * [FEATURE] Ingester: Enable snapshotting of In-memory TSDB on disk during shutdown via `-blocks-storage.tsdb.memory-snapshot-on-shutdown`. #5011
+* [FEATURE] Query Frontend/Scheduler: Add a new counter metric `cortex_request_queue_requests_total` for total requests going to queue. #5030
+* [FEATURE] Build ARM docker images. #5041
+* [FEATURE] Query-frontend/Querier: Create spans to measure time to merge promql responses. #5041
 * [BUGFIX] Updated `golang.org/x/net` dependency to fix CVE-2022-27664. #5008
+* [BUGFIX] Fix panic when otel and xray tracing is enabled. #5044
+* [BUGFIX] Fixed no compact block got grouped in shuffle sharding grouper. #5055
+* [BUGFIX] Tracing: Fix missing object storage span instrumentation. #5074
 
 ## 1.14.0 2022-12-02
 

Makefile

@@ -9,6 +9,9 @@
 VERSION=$(shell cat "./VERSION" 2> /dev/null)
 GOPROXY_VALUE=$(shell go env GOPROXY)
 
+# ARCHS
+ARCHS       = amd64 arm64
+
 # Boiler plate for building Docker containers.
 # All this must go at top of file I'm afraid.
 IMAGE_PREFIX ?= quay.io/cortexproject/
@@ -37,8 +40,9 @@ SED ?= $(shell which gsed 2>/dev/null || which sed)
 # Dependencies (i.e. things that go in the image) still need to be explicitly
 # declared.
 %/$(UPTODATE): %/Dockerfile
-	@echo
-	$(SUDO) docker build --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) -t $(IMAGE_PREFIX)$(shell basename $(@D)) -t $(IMAGE_PREFIX)$(shell basename $(@D)):$(IMAGE_TAG) $(@D)/
+	for arch in $(ARCHS); do \
+		$(SUDO) docker buildx build --platform linux/$$arch --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) -t $(IMAGE_PREFIX)$(shell basename $(@D)) -t $(IMAGE_PREFIX)$(shell basename $(@D)):$(IMAGE_TAG)-$$arch $(@D)/ ; \
+	done
 	@echo
 	@echo Please use push-multiarch-build-image to build and push build image for all supported architectures.
 	touch $@
@@ -160,7 +164,11 @@ else
 exes: $(EXES)
 
 $(EXES):
-	CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D)
+	@for arch in $(ARCHS); do \
+  		echo "Building $@ for $$arch";\
+  		CGO_ENABLED=0 GOARCH=$$arch GOOS=linux go build $(GO_FLAGS) -o $@-$$arch ./$(@D); \
+  	done
+
 
 protos: $(PROTO_GOS)
 
@@ -176,7 +184,7 @@ lint:
 	golangci-lint run
 
 	# Ensure no blocklisted package is imported.
-	GOFLAGS="-tags=requires_docker" faillint -paths "github.com/bmizerany/assert=github.com/stretchr/testify/assert,\
+	GOFLAGS="-tags=requires_docker,integration,integration_alertmanager,integration_backward_compatibility,integration_memberlist,integration_querier,integration_ruler" faillint -paths "github.com/bmizerany/assert=github.com/stretchr/testify/assert,\
 		golang.org/x/net/context=context,\
 		sync/atomic=go.uber.org/atomic,\
 		github.com/prometheus/client_golang/prometheus.{MultiError}=github.com/prometheus/prometheus/tsdb/errors.{NewMulti},\
@@ -272,16 +280,20 @@ clean-protos:
 
 save-images:
 	@mkdir -p docker-images
-	for image_name in $(IMAGE_NAMES); do \
+	@for image_name in $(IMAGE_NAMES); do \
 		if ! echo $$image_name | grep build; then \
-			docker save $$image_name:$(IMAGE_TAG) -o docker-images/$$(echo $$image_name | tr "/" _):$(IMAGE_TAG); \
+		  	for arch in $(ARCHS); do \
+				docker save $$image_name:$(IMAGE_TAG)-$$arch -o docker-images/$$(echo $$image_name | tr "/" _):$(IMAGE_TAG)-$$arch; \
+			done;\
 		fi \
 	done
 
 load-images:
-	for image_name in $(IMAGE_NAMES); do \
+	@for image_name in $(IMAGE_NAMES); do \
 		if ! echo $$image_name | grep build; then \
-			docker load -i docker-images/$$(echo $$image_name | tr "/" _):$(IMAGE_TAG); \
+		  	for arch in $(ARCHS); do \
+				docker load -i docker-images/$$(echo $$image_name | tr "/" _):$(IMAGE_TAG)-$$arch; \
+			done;\
 		fi \
 	done
 

cmd/cortex/Dockerfile

@@ -1,7 +1,9 @@
 FROM       alpine:3.14
+ARG TARGETARCH
+
 RUN        apk add --no-cache ca-certificates
 COPY       migrations /migrations/
-COPY       cortex /bin/cortex
+COPY       cortex-$TARGETARCH /bin/cortex
 EXPOSE     80
 ENTRYPOINT [ "/bin/cortex" ]
 

cmd/query-tee/Dockerfile

@@ -1,6 +1,8 @@
 FROM       alpine:3.14
+ARG TARGETARCH
+
 RUN        apk add --no-cache ca-certificates
-COPY       query-tee /
+COPY       query-tee-$TARGETARCH /query-tee
 ENTRYPOINT ["/query-tee"]
 
 ARG revision

cmd/test-exporter/Dockerfile

@@ -1,6 +1,7 @@
 FROM       alpine:3.14
+ARG TARGETARCH
 RUN        apk add --no-cache ca-certificates
-COPY       test-exporter /
+COPY       test-exporter-$TARGETARCH /test-exporter
 ENTRYPOINT ["/test-exporter"]
 
 ARG revision

cmd/thanosconvert/Dockerfile

@@ -1,6 +1,7 @@
 FROM       alpine:3.14
+ARG TARGETARCH
 RUN        apk add --no-cache ca-certificates
-COPY       thanosconvert /
+COPY       thanosconvert-$TARGETARCH /thanosconvert
 ENTRYPOINT ["/thanosconvert"]
 
 ARG revision

docs/blocks-storage/querier.md

@@ -131,8 +131,9 @@ querier:
   # CLI flag: -querier.query-ingesters-within
   [query_ingesters_within: <duration> | default = 0s]
 
-  # Query long-term store for series, label values and label names APIs. Works
-  # only with blocks engine.
+  # Deprecated (Querying long-term store for labels will be always enabled in
+  # the future.): Query long-term store for series, label values and label names
+  # APIs.
   # CLI flag: -querier.query-store-for-labels-enabled
   [query_store_for_labels_enabled: <boolean> | default = false]