From 999dc68123a148b92af914b2f9acaed4789572f2 Mon Sep 17 00:00:00 2001 From: Kjell Wooding Date: Wed, 9 Jan 2019 15:48:41 -0500 Subject: [PATCH 1/3] Downloading was broken in certain cases on python 2. FancyUrlopener.urlretrieve was failing on redirect, but in an insidious way, because of a bug in the python-future module: https://github.com/PythonCharmers/python-future/issues/425 Replace with the requests module for the time being. --- requirements.txt | 1 + toolchain.py | 29 +++++++++++------------------ 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/requirements.txt b/requirements.txt index 946b8cd8e..0e7f0252d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ pbxproj==2.5.1 +requests>=2.13 diff --git a/toolchain.py b/toolchain.py index 7b514459c..ac52fbfe3 100755 --- a/toolchain.py +++ b/toolchain.py @@ -20,14 +20,11 @@ import tempfile from datetime import datetime try: - from urllib.request import FancyURLopener, urlcleanup -except ImportError: - from urllib import FancyURLopener, urlcleanup -try: + import requests from pbxproj import XcodeProject from pbxproj.pbxextensions.ProjectFiles import FileOptions except ImportError: - print("ERROR: pbxproj requirements is missing") + print("ERROR: Python requirements are missing") print("To install: pip install -r requirements.txt") sys.exit(0) curdir = dirname(__file__) @@ -65,15 +62,6 @@ def _cache_execution(self, *args, **kwargs): state[key_time] = str(datetime.utcnow()) return _cache_execution - -class ChromeDownloader(FancyURLopener): - version = ( - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' - '(KHTML, like Gecko) Chrome/28.0.1500.71 Safari/537.36') - -urlretrieve = ChromeDownloader().retrieve - - class JsonStore(object): """Replacement of shelve using json, needed for support python 2 and 3. """ @@ -468,11 +456,16 @@ def report_hook(index, blksize, size): if exists(filename): unlink(filename) - # Clean up temporary files just in case before downloading. - urlcleanup() - print('Downloading {0}'.format(url)) - urlretrieve(url, filename, report_hook) + headers = {'User-agent': 'Mozilla/5.0 (X11; Linux x86_64) ' + 'AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Chrome/28.0.1500.71 Safari/537.36'} + + r = requests.get(url, headers=headers) + + with open(filename, "wb") as fw: + fw.write(r.content) + return filename def extract_file(self, filename, cwd): From 328a7428497c7908052df048769312c041b7f704 Mon Sep 17 00:00:00 2001 From: Kjell Wooding Date: Wed, 9 Jan 2019 18:01:00 -0500 Subject: [PATCH 2/3] requests can't handle ftp. Only use it if urlretrieve fails --- toolchain.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/toolchain.py b/toolchain.py index ac52fbfe3..521de6e5e 100755 --- a/toolchain.py +++ b/toolchain.py @@ -19,6 +19,11 @@ import fnmatch import tempfile from datetime import datetime +try: + from urllib.request import FancyURLopener, urlcleanup +except ImportError: + from urllib import FancyURLopener, urlcleanup + try: import requests from pbxproj import XcodeProject @@ -62,6 +67,14 @@ def _cache_execution(self, *args, **kwargs): state[key_time] = str(datetime.utcnow()) return _cache_execution +class ChromeDownloader(FancyURLopener): + version = ( + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' + '(KHTML, like Gecko) Chrome/28.0.1500.71 Safari/537.36') + +urlretrieve = ChromeDownloader().retrieve + + class JsonStore(object): """Replacement of shelve using json, needed for support python 2 and 3. """ @@ -456,15 +469,20 @@ def report_hook(index, blksize, size): if exists(filename): unlink(filename) + # Clean up temporary files just in case before downloading. + urlcleanup() + print('Downloading {0}'.format(url)) headers = {'User-agent': 'Mozilla/5.0 (X11; Linux x86_64) ' 'AppleWebKit/537.36 (KHTML, like Gecko) ' 'Chrome/28.0.1500.71 Safari/537.36'} + try: + urlretrieve(url, filename, report_hook) + except: + r = requests.get(url, headers=headers) - r = requests.get(url, headers=headers) - - with open(filename, "wb") as fw: - fw.write(r.content) + with open(filename, "wb") as fw: + fw.write(r.content) return filename From e7ec5d35b8ffedaed0fb259bb00f1557783fe353 Mon Sep 17 00:00:00 2001 From: Kjell Wooding Date: Thu, 10 Jan 2019 13:55:31 -0500 Subject: [PATCH 3/3] make workaround specific to the PY2 issue and incorporate @jonast's retry PR --- toolchain.py | 47 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/toolchain.py b/toolchain.py index 521de6e5e..a9f22476e 100755 --- a/toolchain.py +++ b/toolchain.py @@ -18,6 +18,7 @@ import shutil import fnmatch import tempfile +import time from datetime import datetime try: from urllib.request import FancyURLopener, urlcleanup @@ -25,7 +26,6 @@ from urllib import FancyURLopener, urlcleanup try: - import requests from pbxproj import XcodeProject from pbxproj.pbxextensions.ProjectFiles import FileOptions except ImportError: @@ -39,6 +39,7 @@ IS_PY3 = sys.version_info[0] >= 3 +IS_PY2 = sys.version_info[0] == 2 def shprint(command, *args, **kwargs): @@ -473,16 +474,40 @@ def report_hook(index, blksize, size): urlcleanup() print('Downloading {0}'.format(url)) - headers = {'User-agent': 'Mozilla/5.0 (X11; Linux x86_64) ' - 'AppleWebKit/537.36 (KHTML, like Gecko) ' - 'Chrome/28.0.1500.71 Safari/537.36'} - try: - urlretrieve(url, filename, report_hook) - except: - r = requests.get(url, headers=headers) - - with open(filename, "wb") as fw: - fw.write(r.content) + attempts = 0 + while True: + try: + urlretrieve(url, filename, report_hook) + except AttributeError: + if IS_PY2: + # This is caused by bug in python-future, causing occasional + # AttributeError: '_fileobject' object has no attribute 'readinto' + # It can be removed if the upstream fix is accepted. See also: + # * https://github.com/kivy/kivy-ios/issues/322 + # * https://github.com/PythonCharmers/python-future/pull/423 + import requests + + print("Warning: urlretrieve failed. Falling back to request") + + headers = {'User-agent': 'Mozilla/5.0 (X11; Linux x86_64) ' + 'AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Chrome/28.0.1500.71 Safari/537.36'} + r = requests.get(url, headers=headers) + + with open(filename, "wb") as fw: + fw.write(r.content) + break + else: + raise + except OSError as e: + attempts += 1 + if attempts >= 5: + print('Max download attempts reached: {}'.format(attempts)) + raise e + print('Download failed. Retrying in 1 second...') + time.sleep(1) + continue + break return filename