fix query frontend incorrect error response format

Signed-off-by: Ben Ye <[email protected]>

Author: yeya24

integration/query_frontend_test.go

@@ -6,6 +6,7 @@ package integration
 import (
 	"crypto/x509"
 	"crypto/x509/pkix"
+	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -35,6 +36,14 @@ type queryFrontendTestConfig struct {
 	setup                 func(t *testing.T, s *e2e.Scenario) (configFile string, flags map[string]string)
 }
 
+type response struct {
+	Status    status      `json:"status"`
+	Data      interface{} `json:"data,omitempty"`
+	ErrorType errorType   `json:"errorType,omitempty"`
+	Error     string      `json:"error,omitempty"`
+	Warnings  []string    `json:"warnings,omitempty"`
+}
+
 func TestQueryFrontendWithBlocksStorageViaFlags(t *testing.T) {
 	runQueryFrontendTest(t, queryFrontendTestConfig{
 		testMissingMetricName: false,
@@ -345,6 +354,26 @@ func runQueryFrontendTest(t *testing.T, cfg queryFrontendTestConfig) {
 			assert.Equal(t, model.LabelSet{labels.MetricName: "series_1"}, result[0])
 		}
 
+		// No need to repeat the query 400 test for each user.
+		if userID == 0 {
+			start := time.Unix(1595846748, 806*1e6)
+			end := time.Unix(1595846750, 806*1e6)
+
+			_, err := c.QueryRange("up)", start, end, time.Second)
+			require.Error(t, err)
+
+			// Expect the error response format to be correct.
+			type response struct {
+				Status    string `json:"status"`
+				ErrorType string `json:"errorType,omitempty"`
+				Error     string `json:"error,omitempty"`
+			}
+			var res response
+			err = json.Unmarshal([]byte(err.Error()), &res)
+			require.NoError(t, err)
+			require.Equal(t, response.ErrorType, "bad_data")
+		}
+
 		for q := 0; q < numQueriesPerUser; q++ {
 			go func() {
 				defer wg.Done()
@@ -359,7 +388,7 @@ func runQueryFrontendTest(t *testing.T, cfg queryFrontendTestConfig) {
 
 	wg.Wait()
 
-	extra := float64(2)
+	extra := float64(3)
 	if cfg.testMissingMetricName {
 		extra++
 	}

pkg/frontend/transport/handler.go

@@ -203,12 +203,12 @@ func (f *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 }
 
 func formatGrafanaStatsFields(r *http.Request) []interface{} {
-	fields := make([]interface{}, 0, 2)
+	fields := make([]interface{}, 0, 4)
 	if dashboardUID := r.Header.Get("X-Dashboard-Uid"); dashboardUID != "" {
-		fields = append(fields, dashboardUID)
+		fields = append(fields, "X-Dashboard-Uid", dashboardUID)
 	}
 	if panelID := r.Header.Get("X-Panel-Id"); panelID != "" {
-		fields = append(fields, panelID)
+		fields = append(fields, "X-Panel-Id", panelID)
 	}
 	return fields
 }

pkg/querier/tripperware/queryrange/split_by_interval.go

@@ -47,7 +47,12 @@ func (s splitByInterval) Do(ctx context.Context, r tripperware.Request) (tripper
 	// to line up the boundaries with step.
 	reqs, err := splitQuery(r, s.interval(r))
 	if err != nil {
-		return nil, err
+		// If the query itself is bad, we don't return error but send the query
+		// to querier to return the expected error message. This is not very efficient
+		// but should be okay for now.
+		// TODO(yeya24): query frontend can reuse the Prometheus API handler and return
+		// expected error message locally without passing it to the querier through network.
+		return s.next.Do(ctx, r)
 	}
 	s.splitByCounter.Add(float64(len(reqs)))