Catch context error in the s3 bucket client (cortexproject#5240)

Signed-off-by: Xiaochao Dong (@damnever) <[email protected]>
Signed-off-by: Ben Ye <[email protected]>

Author: damnever

CHANGELOG.md

@@ -59,6 +59,7 @@
 * [BUGFIX] Compactor: Fix issue that shuffle sharding planner return error if block is under visit by other compactor. #5188
 * [BUGFIX] Fix S3 BucketWithRetries upload empty content issue #5217
 * [BUGFIX] Query Frontend: Disable `absent`, `absent_over_time` and `scalar` for vertical sharding. #5221
+* [BUGFIX] Catch context error in the s3 bucket client. #5240
 
 ## 1.14.0 2022-12-02
 

pkg/storage/bucket/s3/bucket_client.go

@@ -124,7 +124,7 @@ func (b *BucketWithRetries) retry(ctx context.Context, f func() error, operation
 		level.Error(b.logger).Log("msg", "bucket operation fail after retries", "err", lastErr, "operation", operationInfo)
 		return lastErr
 	}
-	return nil
+	return retries.Err()
 }
 
 func (b *BucketWithRetries) Name() string {

pkg/storage/bucket/s3/bucket_client_test.go

@@ -73,6 +73,25 @@ func TestBucketWithRetries_UploadFailed(t *testing.T) {
 	require.ErrorContains(t, err, "failed upload: ")
 }
 
+func TestBucketWithRetries_ContextCanceled(t *testing.T) {
+	t.Parallel()
+
+	m := mockBucket{}
+	b := BucketWithRetries{
+		logger:           log.NewNopLogger(),
+		bucket:           &m,
+		operationRetries: 5,
+		retryMinBackoff:  10 * time.Millisecond,
+		retryMaxBackoff:  time.Second,
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	obj, err := b.GetRange(ctx, "dummy", 0, 10)
+	require.ErrorIs(t, err, context.Canceled)
+	require.Nil(t, obj)
+}
+
 type fakeReader struct {
 }
 
@@ -121,7 +140,7 @@ func (m *mockBucket) Get(ctx context.Context, name string) (io.ReadCloser, error
 
 // GetRange mocks objstore.Bucket.GetRange()
 func (m *mockBucket) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error) {
-	return nil, nil
+	return io.NopCloser(bytes.NewBuffer(bytes.Repeat([]byte{1}, int(length)))), nil
 }
 
 // Exists mocks objstore.Bucket.Exists()