From 307db7e592838e95bd4c7916ed572714f9d632dc Mon Sep 17 00:00:00 2001 From: Bartek Ogryczak Date: Tue, 9 May 2017 16:43:35 -0700 Subject: [PATCH 1/4] Add `--retry-status ` to allow specifying custom HTTP staus codes to retry on. This is useful for AWS S3 or Cloudflare, which at times return codes other than 503 for itermitent failures. --- news/4473.feature | 3 +++ pip/basecommand.py | 1 + pip/cmdoptions.py | 11 +++++++++++ pip/download.py | 4 +++- 4 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 news/4473.feature diff --git a/news/4473.feature b/news/4473.feature new file mode 100644 index 00000000000..2ff1abcdc42 --- /dev/null +++ b/news/4473.feature @@ -0,0 +1,3 @@ +Add `--retry-status ` to allow specifying custom HTTP status codes to retry on. +This is useful for AWS S3 or Cloudflare, which at times return codes other than 503 for +intermittent failures. diff --git a/pip/basecommand.py b/pip/basecommand.py index e88c7c57a9d..c5177f2bcf4 100644 --- a/pip/basecommand.py +++ b/pip/basecommand.py @@ -70,6 +70,7 @@ def _build_session(self, options, retries=None, timeout=None): ), retries=retries if retries is not None else options.retries, insecure_hosts=options.trusted_hosts, + retry_status=options.retry_status, ) # Handle custom ca-bundles from the user diff --git a/pip/cmdoptions.py b/pip/cmdoptions.py index 494be4db467..f76454e37e6 100644 --- a/pip/cmdoptions.py +++ b/pip/cmdoptions.py @@ -158,6 +158,16 @@ def getname(n): help="Maximum number of retries each connection should attempt " "(default %default times).") +retry_status = partial( + Option, + '--retry-status', + dest='retry_status', + type='int', + action='append', + help="HTTP status on which to attempt retry " + "(default 503). " + "Can be used multiple times for multiple statuses") + timeout = partial( Option, '--timeout', '--default-timeout', @@ -521,6 +531,7 @@ def _merge_hash(option, opt_str, value, parser): no_input, proxy, retries, + retry_status, timeout, skip_requirements_regex, exists_action, diff --git a/pip/download.py b/pip/download.py index 894d0ff9a58..330aba094a1 100644 --- a/pip/download.py +++ b/pip/download.py @@ -328,6 +328,8 @@ def __init__(self, *args, **kwargs): retries = kwargs.pop("retries", 0) cache = kwargs.pop("cache", None) insecure_hosts = kwargs.pop("insecure_hosts", []) + # if retry_status not set, fall back to the default 503 + retry_status = kwargs.pop("retry_status", [503]) super(PipSession, self).__init__(*args, **kwargs) @@ -348,7 +350,7 @@ def __init__(self, *args, **kwargs): # connection got interrupted in some way. A 503 error in general # is typically considered a transient error so we'll go ahead and # retry it. - status_forcelist=[503], + status_forcelist=retry_status, # Add a small amount of back off between failed requests in # order to prevent hammering the service. From 864b66965aab84f05860a824d7e0b1df7dabe0a4 Mon Sep 17 00:00:00 2001 From: Bartek Ogryczak Date: Thu, 19 Oct 2017 14:03:20 -0700 Subject: [PATCH 2/4] stripping option related code, hardcoding the statuses --- news/4473.feature | 5 ++--- src/pip/_internal/basecommand.py | 1 - src/pip/_internal/cmdoptions.py | 11 ----------- src/pip/_internal/download.py | 6 +++--- 4 files changed, 5 insertions(+), 18 deletions(-) diff --git a/news/4473.feature b/news/4473.feature index 2ff1abcdc42..ef0640bd43f 100644 --- a/news/4473.feature +++ b/news/4473.feature @@ -1,3 +1,2 @@ -Add `--retry-status ` to allow specifying custom HTTP status codes to retry on. -This is useful for AWS S3 or Cloudflare, which at times return codes other than 503 for -intermittent failures. +More HTTP status codes for intermittent failures. Currently it's only the standard 503. +Some popular CDNs use non-standard ones, for example AWS S3 (500) or Cloudflare (520, 527) diff --git a/src/pip/_internal/basecommand.py b/src/pip/_internal/basecommand.py index ba2b3999598..21aa8d950f9 100644 --- a/src/pip/_internal/basecommand.py +++ b/src/pip/_internal/basecommand.py @@ -79,7 +79,6 @@ def _build_session(self, options, retries=None, timeout=None): ), retries=retries if retries is not None else options.retries, insecure_hosts=options.trusted_hosts, - retry_status=options.retry_status, ) # Handle custom ca-bundles from the user diff --git a/src/pip/_internal/cmdoptions.py b/src/pip/_internal/cmdoptions.py index ca7d0c196a6..acf73ed66ed 100644 --- a/src/pip/_internal/cmdoptions.py +++ b/src/pip/_internal/cmdoptions.py @@ -173,16 +173,6 @@ def getname(n): "(default %default times).", ) # type: Any -retry_status = partial( - Option, - '--retry-status', - dest='retry_status', - type='int', - action='append', - help="HTTP status on which to attempt retry " - "(default 503). " - "Can be used multiple times for multiple statuses") - timeout = partial( Option, '--timeout', '--default-timeout', @@ -567,7 +557,6 @@ def _merge_hash(option, opt_str, value, parser): no_input, proxy, retries, - retry_status, timeout, skip_requirements_regex, exists_action, diff --git a/src/pip/_internal/download.py b/src/pip/_internal/download.py index 6042bd44a7a..f392ebabce1 100644 --- a/src/pip/_internal/download.py +++ b/src/pip/_internal/download.py @@ -328,8 +328,6 @@ def __init__(self, *args, **kwargs): retries = kwargs.pop("retries", 0) cache = kwargs.pop("cache", None) insecure_hosts = kwargs.pop("insecure_hosts", []) - # if retry_status not set, fall back to the default 503 - retry_status = kwargs.pop("retry_status", [503]) super(PipSession, self).__init__(*args, **kwargs) @@ -350,7 +348,9 @@ def __init__(self, *args, **kwargs): # connection got interrupted in some way. A 503 error in general # is typically considered a transient error so we'll go ahead and # retry it. - status_forcelist=retry_status, + # A 500 may indicate transient errror in Amazon S3 + # A 520 or 527 – may indicate transient errror in CloudFlare + status_forcelist=[500, 503, 520, 527], # Add a small amount of back off between failed requests in # order to prevent hammering the service. From b597e98b6ba410780e7a4fab4a7db9034ffd6d11 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 20 Oct 2017 03:12:17 +0530 Subject: [PATCH 3/4] Reword the news file --- news/4473.feature | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/news/4473.feature b/news/4473.feature index ef0640bd43f..5e93d322496 100644 --- a/news/4473.feature +++ b/news/4473.feature @@ -1,2 +1,4 @@ -More HTTP status codes for intermittent failures. Currently it's only the standard 503. -Some popular CDNs use non-standard ones, for example AWS S3 (500) or Cloudflare (520, 527) +pip now retries on more HTTP status codes, for intermittent failures. + +Previously, it only retried on the standard 503. Now, it also retries on 500 +(transient failures on AWS S3), 520 and 527 (transient failures on Cloudflare). From a425eb9dbd0546ae72dfe3c20e6e854e551c3bde Mon Sep 17 00:00:00 2001 From: Bartek Ogryczak Date: Mon, 23 Oct 2017 16:05:43 -0700 Subject: [PATCH 4/4] changing ndash to regular ascii dash --- src/pip/_internal/download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/download.py b/src/pip/_internal/download.py index f392ebabce1..180ea77bfaf 100644 --- a/src/pip/_internal/download.py +++ b/src/pip/_internal/download.py @@ -349,7 +349,7 @@ def __init__(self, *args, **kwargs): # is typically considered a transient error so we'll go ahead and # retry it. # A 500 may indicate transient errror in Amazon S3 - # A 520 or 527 – may indicate transient errror in CloudFlare + # A 520 or 527 - may indicate transient errror in CloudFlare status_forcelist=[500, 503, 520, 527], # Add a small amount of back off between failed requests in