Add retry functionality on 429 & 5xx errors

optimizely/client/client.go

@@ -8,8 +8,9 @@ import (
 )
 
 type OptimizelyClient struct {
-	Address string
-	Token   string
+	Address    string
+	Token      string
+	HttpClient http.Client
 }
 
 func (c OptimizelyClient) sendHttpRequest(method, url string, body io.Reader) ([]byte, error) {
@@ -24,8 +25,7 @@ func (c OptimizelyClient) sendHttpRequest(method, url string, body io.Reader) ([
 		req.Header.Set("Content-type", "application/json")
 	}
 
-	httpClient := http.Client{}
-	resp, err := httpClient.Do(req)
+	resp, err := c.HttpClient.Do(req)
 	if err != nil {
 		return nil, err
 	}

optimizely/provider.go

@@ -2,12 +2,14 @@ package optimizely
 
 import (
 	"context"
+	"net/http"
 
 	"github.com/dusan-dragon/terraform-provider-optimizely/optimizely/audience"
 	"github.com/dusan-dragon/terraform-provider-optimizely/optimizely/client"
 	"github.com/dusan-dragon/terraform-provider-optimizely/optimizely/environment"
 	"github.com/dusan-dragon/terraform-provider-optimizely/optimizely/flag"
 	"github.com/dusan-dragon/terraform-provider-optimizely/optimizely/project"
+	"github.com/dusan-dragon/terraform-provider-optimizely/optimizely/transport"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/diag"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 )
@@ -23,6 +25,12 @@ func Provider() *schema.Provider {
 				Type:     schema.TypeString,
 				Required: true,
 			},
+			"http_client_retry_enabled": {
+				Type:        schema.TypeBool,
+				Optional:    true,
+				DefaultFunc: schema.EnvDefaultFunc("OPTIMIZELY_HTTP_CLIENT_RETRY_ENABLED", true),
+				Description: "Enables request retries on HTTP status codes 429 and 5xx. Defaults to `true`.",
+			},
 		},
 		ResourcesMap: map[string]*schema.Resource{
 			"optimizely_feature":  flag.ResourceFeature(),
@@ -41,10 +49,18 @@ func providerConfigure(ctx context.Context, d *schema.ResourceData) (interface{}
 
 	address := d.Get("host").(string)
 	token := d.Get("token").(string)
+	httpRetryEnabled := d.Get("http_client_retry_enabled").(bool)
+
+	httpClient := http.DefaultClient
+	if httpRetryEnabled {
+		customTransport := transport.NewCustomTransport(httpClient.Transport)
+		httpClient.Transport = customTransport
+	}
 
 	optimizelyClient := client.OptimizelyClient{
-		Address: address,
-		Token:   token,
+		Address:    address,
+		Token:      token,
+		HttpClient: *httpClient,
 	}
 
 	return optimizelyClient, diags

optimizely/transport/custom_transport.go

@@ -0,0 +1,122 @@
+package transport
+
+import (
+	"bytes"
+	"context"
+	"io/ioutil"
+	"math"
+	"net/http"
+	"strconv"
+	"time"
+)
+
+var (
+	maxRetries                       = 5
+	defaultBackOffMultiplier float64 = 2
+	defaultBackOffBase       float64 = 2
+	defaultHTTPRetryTimeout          = 5 * 60 * time.Second
+	rateLimitResetHeader             = "X-Ratelimit-Reset"
+)
+
+// CustomTransport holds DefaultTransport configuration and is used to for custom http error handling
+type CustomTransport struct {
+	defaultTransport http.RoundTripper
+}
+
+// RoundTrip method used to retry http errors
+func (customTransport *CustomTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	var ccancel context.CancelFunc
+	ctx := req.Context()
+	if _, set := ctx.Deadline(); !set {
+		ctx, ccancel = context.WithTimeout(ctx, defaultHTTPRetryTimeout)
+		defer ccancel()
+	}
+
+	var rawBody []byte
+	if req.Body != nil && req.Body != http.NoBody {
+		rawBody, _ = ioutil.ReadAll(req.Body)
+		req.Body.Close()
+	}
+	var resp *http.Response
+	var respErr error
+	retryCount := 0
+	for {
+		if retryCount == maxRetries {
+			ccancel()
+		}
+
+		newRequest := customTransport.cloneRequest(req, &rawBody)
+		resp, respErr = customTransport.defaultTransport.RoundTrip(newRequest)
+		// Close the body so connection can be re-used
+		if resp != nil {
+			localVarBody, _ := ioutil.ReadAll(resp.Body)
+			resp.Body.Close()
+			resp.Body = ioutil.NopCloser(bytes.NewBuffer(localVarBody))
+		}
+		if respErr != nil {
+			return resp, respErr
+		}
+
+		// Check if request should be retried and get retry time
+		retryDuration, retry := customTransport.retryRequest(resp, retryCount)
+		if !retry {
+			return resp, respErr
+		}
+
+		select {
+		case <-ctx.Done():
+			return resp, respErr
+		case <-time.After(*retryDuration):
+			retryCount++
+			continue
+		}
+	}
+}
+
+func (t *CustomTransport) cloneRequest(r *http.Request, rawBody *[]byte) *http.Request {
+	newRequest := *r
+
+	if r.Body == nil || r.Body == http.NoBody {
+		return &newRequest
+	}
+	newRequest.Body = ioutil.NopCloser(bytes.NewBuffer(*rawBody))
+
+	return &newRequest
+}
+
+func (t *CustomTransport) retryRequest(response *http.Response, retryCount int) (*time.Duration, bool) {
+	var err error
+	if v := response.Header.Get(rateLimitResetHeader); v != "" && response.StatusCode == 429 {
+		vInt, err := strconv.ParseInt(v, 10, 64)
+		if err == nil {
+			retryDuration := time.Duration(vInt) * time.Second
+			return &retryDuration, true
+		}
+	}
+
+	// Calculate retry for 5xx errors or if unable to parse value of rateLimitResetHeader
+	if response.StatusCode >= 500 || err != nil {
+		// Calculate the retry val (base * multiplier^retryCount)
+		retryVal := defaultBackOffBase * math.Pow(defaultBackOffMultiplier, float64(retryCount))
+		// retry duration shouldn't exceed default timeout period
+		retryVal = math.Min(float64(defaultHTTPRetryTimeout/time.Second), retryVal)
+		retryDuration := time.Duration(retryVal) * time.Second
+		return &retryDuration, true
+	}
+
+	return nil, false
+}
+
+// NewCustomTransport returns new CustomTransport struct
+func NewCustomTransport(roundTripper http.RoundTripper) *CustomTransport {
+	// Use default transport if one provided is nil
+	if roundTripper == nil {
+		roundTripper = http.DefaultTransport
+	}
+
+	customTransport := CustomTransport{
+		defaultTransport: roundTripper,
+	}
+
+	return &customTransport
+}