Merge pull request #33 from netlify/fix-rate-limit-again

Retry algorithm tweaks

Author: bcomnes

src/deploy/upload-files.js

@@ -1,6 +1,7 @@
 const pMap = require('p-map')
 const fs = require('fs')
 const backoff = require('backoff')
+const debug = require('debug')('netlify:deploy')
 
 module.exports = uploadFiles
 async function uploadFiles(api, deployId, uploadList, { concurrentUpload, statusCb, maxRetry }) {
@@ -71,8 +72,8 @@ function retryUpload(uploadFn, maxRetry) {
     let lastError
     const fibonacciBackoff = backoff.fibonacci({
       randomisationFactor: 0.5,
-      initialDelay: 100,
-      maxDelay: 10000
+      initialDelay: 5000,
+      maxDelay: 90000
     })
     fibonacciBackoff.failAfter(maxRetry)
 
@@ -93,8 +94,11 @@ function retryUpload(uploadFn, maxRetry) {
         .catch(e => {
           lastError = e
           switch (true) {
-            case e.status === 408: // request timeout
+            case e.status >= 400: // observed errors: 408, 401 (4** swallowed), 502
             case e.name === 'FetchError': {
+              debug(`Upload failed... retrying`)
+              const msg = e.json || e.data
+              if (msg) debug('O%', msg)
               return fibonacciBackoff.backoff()
             }
             default: {

src/open-api/index.js

@@ -119,32 +119,31 @@ exports.generateMethod = method => {
       // Adapted from:
       // https://github.com/netlify/open-api/blob/master/go/porcelain/http/http.go
 
-      const MAX_RETRY = 5
+      const MAX_RETRY = 10
       const DEFAULT_RETRY_DELAY = 5000 //ms
 
+      let last_retry_delay = DEFAULT_RETRY_DELAY
       for (let index = 0; index <= MAX_RETRY; index++) {
+        debug('Rate limit attempt ' + index + ' for ' + path)
         const response = await makeRequest()
         if (http.STATUS_CODES[response.status] !== 'Too Many Requests' || index === MAX_RETRY) {
+          if (index === MAX_RETRY) debug(`Rate limit retry exhausted, aborting request...`)
           return response
         } else {
-          debug(`Rate limited, retrying`)
           try {
             const rateLimitReset = response.headers.get('X-RateLimit-Reset')
-            debug('rateLimitReset: %O', rateLimitReset)
             const resetTime = Number.parseInt(rateLimitReset)
             if (!existy(resetTime)) {
               debug('Issue getting resetTime: %O', resetTime)
               throw new Error('Header missing reset time')
             }
-            debug(`resetTime: ${resetTime}`)
             const now = unixNow()
-            debug(`unixNow(): ${now}`)
-            const sleepTime = (resetTime - now < 0 ? 0 : resetTime - now) * 1000
-            debug(`sleeping for ${sleepTime}ms`)
-            await sleep(sleepTime)
+            last_retry_delay = ((resetTime - now < 0 ? 0 : resetTime - now) + 1) * 1000 // minimum 1 second
+            debug(`sleeping for ${last_retry_delay}ms`)
+            await sleep(last_retry_delay)
           } catch (e) {
-            debug(`sleeping for ${DEFAULT_RETRY_DELAY}ms`)
-            await sleep(DEFAULT_RETRY_DELAY) // Default to 5 seconds if the header is borked
+            debug(`sleeping for ${last_retry_delay}ms`)
+            await sleep(last_retry_delay)
           }
         }
       }