Acceptable error rate for scale-down is a fraction

Increase test error rate to match

Author: bboreham

pkg/chunk/aws/metrics_autoscaling.go

@@ -22,7 +22,7 @@ const (
 	queueLengthScaledown   = 10000   // consider scaling down if queue smaller than this
 	queueLengthAcceptable  = 100000  // we don't mind queues smaller than this
 	queueLengthMax         = 1000000 // always scale up if queue bigger than this
-	errorRateScaledown     = 1
+	errorFractionScaledown = 0.1
 	scaledown              = 0.9
 	scaleup                = 1.2
 )
@@ -45,7 +45,7 @@ func (d dynamoTableClient) metricsAutoScale(ctx context.Context, current, expect
 	level.Info(util.Logger).Log("msg", "checking metrics", "table", current.Name, "queueLengths", fmt.Sprint(m.queueLengths), "errorRate", errorRate)
 
 	switch {
-	case m.queueLengths[2] < queueLengthScaledown && errorRate < errorRateScaledown:
+	case m.queueLengths[2] < queueLengthScaledown && errorRate < errorFractionScaledown*float64(current.ProvisionedWrite):
 		// No big queue, low errors -> scale down
 		expected.ProvisionedWrite = int64(float64(current.ProvisionedWrite) * scaledown)
 		level.Info(util.Logger).Log("msg", "metrics scale-down", "table", current.Name, "write", expected.ProvisionedWrite)

pkg/chunk/aws/metrics_autoscaling_test.go

@@ -92,7 +92,7 @@ func TestTableManagerMetricsAutoScaling(t *testing.T) {
 			staticTable(0, read, 233, read, 162)...), // - scale down both again
 	)
 
-	mockProm.SetResponse(0, 0, 0, 1, 1, 1, 1)
+	mockProm.SetResponse(0, 0, 0, 30, 30, 30, 30)
 	test(t, client, tableManager, "Next week",
 		startTime.Add(tablePeriod),
 		// Nothing much happening - expect table 0 write rates to stay as-is and table 1 to be created with defaults
@@ -102,7 +102,7 @@ func TestTableManagerMetricsAutoScaling(t *testing.T) {
 	)
 
 	// No errors on last week's index table, still some on chunk table
-	mockProm.SetResponse(0, 0, 0, 0, 1, 1, 1)
+	mockProm.SetResponse(0, 0, 0, 0, 30, 30, 30)
 	test(t, client, tableManager, "Next week plus a bit",
 		startTime.Add(tablePeriod).Add(time.Minute*10),
 		append(append(baseTable("a", inactiveRead, inactiveWrite),