From 153cfac191f0dccb07416a4379c2a35d62387331 Mon Sep 17 00:00:00 2001 From: hyxbiao Date: Mon, 24 Jul 2017 18:42:20 +0800 Subject: [PATCH 1/9] Add connection/read timeout for requests adapter --- hyper/contrib.py | 8 +++++--- hyper/http11/connection.py | 18 ++++++++++++++++-- hyper/http20/connection.py | 18 ++++++++++++++++-- 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/hyper/contrib.py b/hyper/contrib.py index c269ab90..748892ef 100644 --- a/hyper/contrib.py +++ b/hyper/contrib.py @@ -33,7 +33,7 @@ def __init__(self, *args, **kwargs): self.connections = {} def get_connection(self, host, port, scheme, cert=None, verify=True, - proxy=None): + proxy=None, **kwargs): """ Gets an appropriate HTTP/2 connection object based on host/port/scheme/cert tuples. @@ -77,7 +77,8 @@ def get_connection(self, host, port, scheme, cert=None, verify=True, secure=secure, ssl_context=ssl_context, proxy_host=proxy_netloc, - proxy_headers=proxy_headers) + proxy_headers=proxy_headers, + **kwargs) self.connections[connection_key] = conn return conn @@ -98,7 +99,8 @@ def send(self, request, stream=False, cert=None, verify=True, proxies=None, parsed.scheme, cert=cert, verify=verify, - proxy=proxy) + proxy=proxy, + **kwargs) # Build the selector. selector = parsed.path diff --git a/hyper/http11/connection.py b/hyper/http11/connection.py index 225ef98e..86305ff9 100644 --- a/hyper/http11/connection.py +++ b/hyper/http11/connection.py @@ -150,6 +150,16 @@ def __init__(self, host, port=None, secure=None, ssl_context=None, #: the standard hyper parsing interface. self.parser = Parser() + # timeout + timeout = kwargs.get('timeout') + if isinstance(timeout, tuple): + self._connect_timeout = timeout[0] + self._read_timeout = timeout[1] + else: + self._connect_timeout = timeout + self._read_timeout = timeout + + def connect(self): """ Connect to the server specified when the object was created. This is a @@ -172,10 +182,11 @@ def connect(self): # Simple http proxy sock = socket.create_connection( (self.proxy_host, self.proxy_port), - 5 + timeout=self._connect_timeout ) else: - sock = socket.create_connection((self.host, self.port), 5) + sock = socket.create_connection((self.host, self.port), + timeout=self._connect_timeout) proto = None if self.secure: @@ -184,6 +195,9 @@ def connect(self): log.debug("Selected protocol: %s", proto) sock = BufferedSocket(sock, self.network_buffer_size) + # Set read timeout + sock.settimeout(self._read_timeout) + if proto not in ('http/1.1', None): raise TLSUpgrade(proto, sock) diff --git a/hyper/http20/connection.py b/hyper/http20/connection.py index 2451c3fe..2056f1ee 100644 --- a/hyper/http20/connection.py +++ b/hyper/http20/connection.py @@ -151,6 +151,15 @@ def __init__(self, host, port=None, secure=None, window_manager=None, self.__wm_class = window_manager or FlowControlManager self.__init_state() + # timeout + timeout = kwargs.get('timeout') + if isinstance(timeout, tuple): + self._connect_timeout = timeout[0] + self._read_timeout = timeout[1] + else: + self._connect_timeout = timeout + self._read_timeout = timeout + return def __init_state(self): @@ -355,10 +364,12 @@ def connect(self): elif self.proxy_host: # Simple http proxy sock = socket.create_connection( - (self.proxy_host, self.proxy_port) + (self.proxy_host, self.proxy_port), + timeout=self._connect_timeout ) else: - sock = socket.create_connection((self.host, self.port)) + sock = socket.create_connection((self.host, self.port), + timeout=self._connect_timeout) if self.secure: sock, proto = wrap_socket(sock, self.host, self.ssl_context, @@ -374,6 +385,9 @@ def connect(self): self._sock = BufferedSocket(sock, self.network_buffer_size) + # Set read timeout + self._sock.settimeout(self._read_timeout) + self._send_preamble() def _connect_upgrade(self, sock): From 1e7739a4b8c6eaf1a1f11711d49f4d53bf2b1e9e Mon Sep 17 00:00:00 2001 From: hyxbiao Date: Mon, 24 Jul 2017 19:12:42 +0800 Subject: [PATCH 2/9] Add connection/read timeout for requests adapter --- hyper/http11/connection.py | 3 +-- hyper/http20/connection.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/hyper/http11/connection.py b/hyper/http11/connection.py index 86305ff9..d25806ab 100644 --- a/hyper/http11/connection.py +++ b/hyper/http11/connection.py @@ -159,7 +159,6 @@ def __init__(self, host, port=None, secure=None, ssl_context=None, self._connect_timeout = timeout self._read_timeout = timeout - def connect(self): """ Connect to the server specified when the object was created. This is a @@ -185,7 +184,7 @@ def connect(self): timeout=self._connect_timeout ) else: - sock = socket.create_connection((self.host, self.port), + sock = socket.create_connection((self.host, self.port), timeout=self._connect_timeout) proto = None diff --git a/hyper/http20/connection.py b/hyper/http20/connection.py index 2056f1ee..f0dfcda7 100644 --- a/hyper/http20/connection.py +++ b/hyper/http20/connection.py @@ -368,7 +368,7 @@ def connect(self): timeout=self._connect_timeout ) else: - sock = socket.create_connection((self.host, self.port), + sock = socket.create_connection((self.host, self.port), timeout=self._connect_timeout) if self.secure: From 7be26b6f3a325632f1a0890aee6a3c08590af355 Mon Sep 17 00:00:00 2001 From: hyxbiao Date: Mon, 24 Jul 2017 19:44:33 +0800 Subject: [PATCH 3/9] update requests integration test --- test/test_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_integration.py b/test/test_integration.py index e1c87673..625ac6f8 100644 --- a/test/test_integration.py +++ b/test/test_integration.py @@ -1290,7 +1290,7 @@ def socket_handler(listener): s = requests.Session() s.mount('https://%s' % self.host, HTTP20Adapter()) - r = s.get('https://%s:%s/some/path' % (self.host, self.port)) + r = s.get('https://%s:%s/some/path' % (self.host, self.port), timeout=(10, 30)) # Assert about the received values. assert r.status_code == 200 From 8c310c79807b7c4622dbea97323216003b378367 Mon Sep 17 00:00:00 2001 From: hyxbiao Date: Tue, 25 Jul 2017 10:08:59 +0800 Subject: [PATCH 4/9] explicit timeout and more timeout tests --- hyper/contrib.py | 7 +-- hyper/http11/connection.py | 3 +- hyper/http20/connection.py | 3 +- test/server.py | 15 ++++-- test/test_http11.py | 12 +++++ test/test_integration.py | 103 ++++++++++++++++++++++++++++++++++++- 6 files changed, 130 insertions(+), 13 deletions(-) diff --git a/hyper/contrib.py b/hyper/contrib.py index 748892ef..2e781783 100644 --- a/hyper/contrib.py +++ b/hyper/contrib.py @@ -33,7 +33,7 @@ def __init__(self, *args, **kwargs): self.connections = {} def get_connection(self, host, port, scheme, cert=None, verify=True, - proxy=None, **kwargs): + proxy=None, timeout=None): """ Gets an appropriate HTTP/2 connection object based on host/port/scheme/cert tuples. @@ -78,7 +78,7 @@ def get_connection(self, host, port, scheme, cert=None, verify=True, ssl_context=ssl_context, proxy_host=proxy_netloc, proxy_headers=proxy_headers, - **kwargs) + timeout=timeout) self.connections[connection_key] = conn return conn @@ -93,6 +93,7 @@ def send(self, request, stream=False, cert=None, verify=True, proxies=None, proxy = prepend_scheme_if_needed(proxy, 'http') parsed = urlparse(request.url) + timeout = kwargs.get('timeout') conn = self.get_connection( parsed.hostname, parsed.port, @@ -100,7 +101,7 @@ def send(self, request, stream=False, cert=None, verify=True, proxies=None, cert=cert, verify=verify, proxy=proxy, - **kwargs) + timeout=timeout) # Build the selector. selector = parsed.path diff --git a/hyper/http11/connection.py b/hyper/http11/connection.py index d25806ab..56cfc154 100644 --- a/hyper/http11/connection.py +++ b/hyper/http11/connection.py @@ -101,7 +101,7 @@ class HTTP11Connection(object): def __init__(self, host, port=None, secure=None, ssl_context=None, proxy_host=None, proxy_port=None, proxy_headers=None, - **kwargs): + timeout=None, **kwargs): if port is None: self.host, self.port = to_host_port_tuple(host, default_port=80) else: @@ -151,7 +151,6 @@ def __init__(self, host, port=None, secure=None, ssl_context=None, self.parser = Parser() # timeout - timeout = kwargs.get('timeout') if isinstance(timeout, tuple): self._connect_timeout = timeout[0] self._read_timeout = timeout[1] diff --git a/hyper/http20/connection.py b/hyper/http20/connection.py index f0dfcda7..c6928c7e 100644 --- a/hyper/http20/connection.py +++ b/hyper/http20/connection.py @@ -102,7 +102,7 @@ class HTTP20Connection(object): def __init__(self, host, port=None, secure=None, window_manager=None, enable_push=False, ssl_context=None, proxy_host=None, proxy_port=None, force_proto=None, proxy_headers=None, - **kwargs): + timeout=None, **kwargs): """ Creates an HTTP/2 connection to a specific server. """ @@ -152,7 +152,6 @@ def __init__(self, host, port=None, secure=None, window_manager=None, self.__init_state() # timeout - timeout = kwargs.get('timeout') if isinstance(timeout, tuple): self._connect_timeout = timeout[0] self._read_timeout = timeout[1] diff --git a/test/server.py b/test/server.py index 482bf734..edc28755 100644 --- a/test/server.py +++ b/test/server.py @@ -108,12 +108,13 @@ class SocketLevelTest(object): A test-class that defines a few helper methods for running socket-level tests. """ - def set_up(self, secure=True, proxy=False): + def set_up(self, secure=True, proxy=False, timeout=None): self.host = None self.port = None self.socket_security = SocketSecuritySetting(secure) self.proxy = proxy self.server_thread = None + self.timeout = timeout def _start_server(self, socket_handler): """ @@ -146,18 +147,22 @@ def secure(self, value): def get_connection(self): if self.h2: if not self.proxy: - return HTTP20Connection(self.host, self.port, self.secure) + return HTTP20Connection(self.host, self.port, self.secure, + timeout=self.timeout) else: return HTTP20Connection('http2bin.org', secure=self.secure, proxy_host=self.host, - proxy_port=self.port) + proxy_port=self.port, + timeout=self.timeout) else: if not self.proxy: - return HTTP11Connection(self.host, self.port, self.secure) + return HTTP11Connection(self.host, self.port, self.secure, + timeout=self.timeout) else: return HTTP11Connection('httpbin.org', secure=self.secure, proxy_host=self.host, - proxy_port=self.port) + proxy_port=self.port, + timeout=self.timeout) def get_encoder(self): """ diff --git a/test/test_http11.py b/test/test_http11.py index 40fea8a9..f71289d8 100644 --- a/test/test_http11.py +++ b/test/test_http11.py @@ -110,6 +110,18 @@ def test_initialization_with_ipv6_addresses_proxy_inline_port(self): assert c.proxy_host == 'ffff:aaaa::1' assert c.proxy_port == 8443 + def test_initialization_timeout(self): + c = HTTP11Connection('httpbin.org', timeout=30) + + assert c._connect_timeout == 30 + assert c._read_timeout == 30 + + def test_initialization_tuple_timeout(self): + c = HTTP11Connection('httpbin.org', timeout=(5, 60)) + + assert c._connect_timeout == 5 + assert c._read_timeout == 60 + def test_basic_request(self): c = HTTP11Connection('httpbin.org') c._sock = sock = DummySocket() diff --git a/test/test_integration.py b/test/test_integration.py index 625ac6f8..31a1a30d 100644 --- a/test/test_integration.py +++ b/test/test_integration.py @@ -1230,6 +1230,106 @@ def do_connect(conn): self.tear_down() + def test_connection_timeout(self): + self.set_up(timeout=0.5) + + def socket_handler(listener): + time.sleep(1) + sock = listener.accept()[0] + sock.close() + + self._start_server(socket_handler) + conn = self.get_connection() + try: + conn.connect() + assert False + except ssl.SSLError as e: + assert 'timed out' in e.message + + self.tear_down() + + def test_read_timeout(self): + self.set_up(timeout=0.5) + + req_event = threading.Event() + recv_event = threading.Event() + + def socket_handler(listener): + sock = listener.accept()[0] + + # We get two messages for the connection open and then a HEADERS + # frame. + receive_preamble(sock) + sock.recv(65535) + + # Wait for request + req_event.wait(5) + # Now, send the headers for the response. This response has no body + time.sleep(1) + + # Now, send the headers for the response. This response has no body + f = build_headers_frame( + [(':status', '204'), ('content-length', '0')] + ) + f.flags.add('END_STREAM') + f.stream_id = 1 + sock.send(f.serialize()) + + # Wait for the message from the main thread. + recv_event.wait(5) + sock.close() + + self._start_server(socket_handler) + conn = self.get_connection() + conn.request('GET', '/') + req_event.set() + + try: + conn.get_response() + assert False + except ssl.SSLError as e: + assert 'timed out' in e.message + + # Awesome, we're done now. + recv_event.set() + self.tear_down() + + def test_default_connection_timeout(self): + self.set_up(timeout=None) + + # Confirm that we send the connection upgrade string and the initial + # SettingsFrame. + data = [] + send_event = threading.Event() + + def socket_handler(listener): + time.sleep(1) + sock = listener.accept()[0] + + # We should get one big chunk. + first = sock.recv(65535) + data.append(first) + + # We need to send back a SettingsFrame. + f = SettingsFrame(0) + sock.send(f.serialize()) + + send_event.set() + sock.close() + + self._start_server(socket_handler) + conn = self.get_connection() + try: + conn.connect() + except ssl.SSLError: + assert False + + send_event.wait(5) + + assert data[0].startswith(b'PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n') + + self.tear_down() + @patch('hyper.http20.connection.H2_NPN_PROTOCOLS', PROTOCOLS) class TestRequestsAdapter(SocketLevelTest): @@ -1290,7 +1390,8 @@ def socket_handler(listener): s = requests.Session() s.mount('https://%s' % self.host, HTTP20Adapter()) - r = s.get('https://%s:%s/some/path' % (self.host, self.port), timeout=(10, 30)) + r = s.get('https://%s:%s/some/path' % (self.host, self.port), + timeout=(10, 60)) # Assert about the received values. assert r.status_code == 200 From 28c569957c69ddca143d26911637f13b6ecb8f74 Mon Sep 17 00:00:00 2001 From: hyxbiao Date: Tue, 25 Jul 2017 10:30:16 +0800 Subject: [PATCH 5/9] update timeout tests for py3 --- test/test_integration.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/test/test_integration.py b/test/test_integration.py index 31a1a30d..df6b1dcb 100644 --- a/test/test_integration.py +++ b/test/test_integration.py @@ -13,6 +13,7 @@ import hyper import hyper.http11.connection import pytest +from socket import timeout as SocketTimeout from contextlib import contextmanager from mock import patch from concurrent.futures import ThreadPoolExecutor, TimeoutError @@ -1243,7 +1244,9 @@ def socket_handler(listener): try: conn.connect() assert False - except ssl.SSLError as e: + except (SocketTimeout, ssl.SSLError) as e: + # Py2 raises this as a BaseSSLError, + # Py3 raises it as socket timeout. assert 'timed out' in e.message self.tear_down() @@ -1287,7 +1290,9 @@ def socket_handler(listener): try: conn.get_response() assert False - except ssl.SSLError as e: + except (SocketTimeout, ssl.SSLError) as e: + # Py2 raises this as a BaseSSLError, + # Py3 raises it as socket timeout. assert 'timed out' in e.message # Awesome, we're done now. @@ -1321,7 +1326,9 @@ def socket_handler(listener): conn = self.get_connection() try: conn.connect() - except ssl.SSLError: + except (SocketTimeout, ssl.SSLError): + # Py2 raises this as a BaseSSLError, + # Py3 raises it as socket timeout. assert False send_event.wait(5) From 1417c51d79a5bbcb11d9d839b8a9c45f210ee60f Mon Sep 17 00:00:00 2001 From: hyxbiao Date: Tue, 25 Jul 2017 10:50:38 +0800 Subject: [PATCH 6/9] update timeout tests for py3 --- test/test_integration.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/test_integration.py b/test/test_integration.py index df6b1dcb..2ba1f317 100644 --- a/test/test_integration.py +++ b/test/test_integration.py @@ -1244,10 +1244,11 @@ def socket_handler(listener): try: conn.connect() assert False - except (SocketTimeout, ssl.SSLError) as e: + except (SocketTimeout, ssl.SSLError): # Py2 raises this as a BaseSSLError, # Py3 raises it as socket timeout. - assert 'timed out' in e.message + # assert 'timed out' in e.message + pass self.tear_down() @@ -1290,10 +1291,11 @@ def socket_handler(listener): try: conn.get_response() assert False - except (SocketTimeout, ssl.SSLError) as e: + except (SocketTimeout, ssl.SSLError): # Py2 raises this as a BaseSSLError, # Py3 raises it as socket timeout. - assert 'timed out' in e.message + # assert 'timed out' in e.message + pass # Awesome, we're done now. recv_event.set() From 09dfa864f8c8b9a0caeeb4b42653a01dbae77160 Mon Sep 17 00:00:00 2001 From: hyxbiao Date: Tue, 25 Jul 2017 15:27:31 +0800 Subject: [PATCH 7/9] explicit timeout in send method and update timeout tests --- hyper/contrib.py | 3 +-- test/test_integration.py | 22 ++++++---------------- 2 files changed, 7 insertions(+), 18 deletions(-) diff --git a/hyper/contrib.py b/hyper/contrib.py index 2e781783..5a580f29 100644 --- a/hyper/contrib.py +++ b/hyper/contrib.py @@ -84,7 +84,7 @@ def get_connection(self, host, port, scheme, cert=None, verify=True, return conn def send(self, request, stream=False, cert=None, verify=True, proxies=None, - **kwargs): + timeout=None, **kwargs): """ Sends a HTTP message to the server. """ @@ -93,7 +93,6 @@ def send(self, request, stream=False, cert=None, verify=True, proxies=None, proxy = prepend_scheme_if_needed(proxy, 'http') parsed = urlparse(request.url) - timeout = kwargs.get('timeout') conn = self.get_connection( parsed.hostname, parsed.port, diff --git a/test/test_integration.py b/test/test_integration.py index 2ba1f317..983250ff 100644 --- a/test/test_integration.py +++ b/test/test_integration.py @@ -1243,12 +1243,13 @@ def socket_handler(listener): conn = self.get_connection() try: conn.connect() - assert False except (SocketTimeout, ssl.SSLError): # Py2 raises this as a BaseSSLError, # Py3 raises it as socket timeout. # assert 'timed out' in e.message pass + else: + assert False self.tear_down() @@ -1256,7 +1257,6 @@ def test_read_timeout(self): self.set_up(timeout=0.5) req_event = threading.Event() - recv_event = threading.Event() def socket_handler(listener): sock = listener.accept()[0] @@ -1268,19 +1268,10 @@ def socket_handler(listener): # Wait for request req_event.wait(5) - # Now, send the headers for the response. This response has no body - time.sleep(1) - # Now, send the headers for the response. This response has no body - f = build_headers_frame( - [(':status', '204'), ('content-length', '0')] - ) - f.flags.add('END_STREAM') - f.stream_id = 1 - sock.send(f.serialize()) + # Sleep wait for read timeout + time.sleep(1) - # Wait for the message from the main thread. - recv_event.wait(5) sock.close() self._start_server(socket_handler) @@ -1290,15 +1281,14 @@ def socket_handler(listener): try: conn.get_response() - assert False except (SocketTimeout, ssl.SSLError): # Py2 raises this as a BaseSSLError, # Py3 raises it as socket timeout. # assert 'timed out' in e.message pass + else: + assert False - # Awesome, we're done now. - recv_event.set() self.tear_down() def test_default_connection_timeout(self): From f0bc05c3268d558b4be2c8570d4b8e4261745e68 Mon Sep 17 00:00:00 2001 From: hyxbiao Date: Wed, 26 Jul 2017 10:17:55 +0800 Subject: [PATCH 8/9] add timeout in common/HTTPConnection and update timeout tests --- hyper/common/connection.py | 11 ++++++++--- test/test_hyper.py | 12 ++++++++++++ test/test_integration.py | 23 ++++++----------------- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/hyper/common/connection.py b/hyper/common/connection.py index e225852e..67c4f8db 100644 --- a/hyper/common/connection.py +++ b/hyper/common/connection.py @@ -58,6 +58,7 @@ def __init__(self, proxy_host=None, proxy_port=None, proxy_headers=None, + timeout=None, **kwargs): self._host = host @@ -78,8 +79,10 @@ def __init__(self, self._h1_kwargs.update(kwargs) self._h2_kwargs.update(kwargs) + self._timeout = timeout + self._conn = HTTP11Connection( - self._host, self._port, **self._h1_kwargs + self._host, self._port, timeout=self._timeout, **self._h1_kwargs ) def request(self, method, url, body=None, headers=None): @@ -113,7 +116,8 @@ def request(self, method, url, body=None, headers=None): assert e.negotiated in H2_NPN_PROTOCOLS self._conn = HTTP20Connection( - self._host, self._port, **self._h2_kwargs + self._host, self._port, + timeout=self._timeout, **self._h2_kwargs ) self._conn._sock = e.sock @@ -138,7 +142,8 @@ def get_response(self, *args, **kwargs): assert e.negotiated == H2C_PROTOCOL self._conn = HTTP20Connection( - self._host, self._port, **self._h2_kwargs + self._host, self._port, + timeout=self._timeout, **self._h2_kwargs ) self._conn._connect_upgrade(e.sock) diff --git a/test/test_hyper.py b/test/test_hyper.py index 76a68cfe..a8513a9f 100644 --- a/test/test_hyper.py +++ b/test/test_hyper.py @@ -98,6 +98,18 @@ def test_connection_version(self): c = HTTP20Connection('www.google.com') assert c.version is HTTPVersion.http20 + def test_connection_timeout(self): + c = HTTP20Connection('httpbin.org', timeout=30) + + assert c._connect_timeout == 30 + assert c._read_timeout == 30 + + def test_connection_tuple_timeout(self): + c = HTTP20Connection('httpbin.org', timeout=(5, 60)) + + assert c._connect_timeout == 5 + assert c._read_timeout == 60 + def test_ping(self, frame_buffer): def data_callback(chunk, **kwargs): frame_buffer.add_data(chunk) diff --git a/test/test_integration.py b/test/test_integration.py index 983250ff..34e9e5a4 100644 --- a/test/test_integration.py +++ b/test/test_integration.py @@ -1236,20 +1236,14 @@ def test_connection_timeout(self): def socket_handler(listener): time.sleep(1) - sock = listener.accept()[0] - sock.close() self._start_server(socket_handler) conn = self.get_connection() - try: - conn.connect() - except (SocketTimeout, ssl.SSLError): + + with pytest.raises((SocketTimeout, ssl.SSLError)): # Py2 raises this as a BaseSSLError, # Py3 raises it as socket timeout. - # assert 'timed out' in e.message - pass - else: - assert False + conn.connect() self.tear_down() @@ -1279,15 +1273,10 @@ def socket_handler(listener): conn.request('GET', '/') req_event.set() - try: - conn.get_response() - except (SocketTimeout, ssl.SSLError): + with pytest.raises((SocketTimeout, ssl.SSLError)): # Py2 raises this as a BaseSSLError, # Py3 raises it as socket timeout. - # assert 'timed out' in e.message - pass - else: - assert False + conn.get_response() self.tear_down() @@ -1321,7 +1310,7 @@ def socket_handler(listener): except (SocketTimeout, ssl.SSLError): # Py2 raises this as a BaseSSLError, # Py3 raises it as socket timeout. - assert False + pytest.fail() send_event.wait(5) From ead832bbc9786893e1d901d4ca91783e09454406 Mon Sep 17 00:00:00 2001 From: hyxbiao Date: Wed, 26 Jul 2017 20:10:53 +0800 Subject: [PATCH 9/9] move timeout in _h1/2_kwargs; add timeout for _create_tunnel funciton; add more tests --- hyper/common/connection.py | 16 +++--- hyper/http11/connection.py | 27 +++++---- hyper/http20/connection.py | 23 ++++---- test/test_abstraction.py | 6 +- test/test_http11.py | 6 +- test/test_hyper.py | 6 +- test/test_integration.py | 97 ++++++++++++++++++++++++++++++++- test/test_integration_http11.py | 67 +++++++++++++++++++++++ 8 files changed, 204 insertions(+), 44 deletions(-) diff --git a/hyper/common/connection.py b/hyper/common/connection.py index 67c4f8db..855994f8 100644 --- a/hyper/common/connection.py +++ b/hyper/common/connection.py @@ -66,23 +66,23 @@ def __init__(self, self._h1_kwargs = { 'secure': secure, 'ssl_context': ssl_context, 'proxy_host': proxy_host, 'proxy_port': proxy_port, - 'proxy_headers': proxy_headers, 'enable_push': enable_push + 'proxy_headers': proxy_headers, 'enable_push': enable_push, + 'timeout': timeout } self._h2_kwargs = { 'window_manager': window_manager, 'enable_push': enable_push, 'secure': secure, 'ssl_context': ssl_context, 'proxy_host': proxy_host, 'proxy_port': proxy_port, - 'proxy_headers': proxy_headers + 'proxy_headers': proxy_headers, + 'timeout': timeout } # Add any unexpected kwargs to both dictionaries. self._h1_kwargs.update(kwargs) self._h2_kwargs.update(kwargs) - self._timeout = timeout - self._conn = HTTP11Connection( - self._host, self._port, timeout=self._timeout, **self._h1_kwargs + self._host, self._port, **self._h1_kwargs ) def request(self, method, url, body=None, headers=None): @@ -116,8 +116,7 @@ def request(self, method, url, body=None, headers=None): assert e.negotiated in H2_NPN_PROTOCOLS self._conn = HTTP20Connection( - self._host, self._port, - timeout=self._timeout, **self._h2_kwargs + self._host, self._port, **self._h2_kwargs ) self._conn._sock = e.sock @@ -142,8 +141,7 @@ def get_response(self, *args, **kwargs): assert e.negotiated == H2C_PROTOCOL self._conn = HTTP20Connection( - self._host, self._port, - timeout=self._timeout, **self._h2_kwargs + self._host, self._port, **self._h2_kwargs ) self._conn._connect_upgrade(e.sock) diff --git a/hyper/http11/connection.py b/hyper/http11/connection.py index 56cfc154..4311d307 100644 --- a/hyper/http11/connection.py +++ b/hyper/http11/connection.py @@ -39,14 +39,14 @@ def _create_tunnel(proxy_host, proxy_port, target_host, target_port, - proxy_headers=None): + proxy_headers=None, timeout=None): """ Sends CONNECT method to a proxy and returns a socket with established connection to the target. :returns: socket """ - conn = HTTP11Connection(proxy_host, proxy_port) + conn = HTTP11Connection(proxy_host, proxy_port, timeout=timeout) conn.request('CONNECT', '%s:%d' % (target_host, target_port), headers=proxy_headers) @@ -151,12 +151,7 @@ def __init__(self, host, port=None, secure=None, ssl_context=None, self.parser = Parser() # timeout - if isinstance(timeout, tuple): - self._connect_timeout = timeout[0] - self._read_timeout = timeout[1] - else: - self._connect_timeout = timeout - self._read_timeout = timeout + self._timeout = timeout def connect(self): """ @@ -167,6 +162,13 @@ def connect(self): """ if self._sock is None: + if isinstance(self._timeout, tuple): + connect_timeout = self._timeout[0] + read_timeout = self._timeout[1] + else: + connect_timeout = self._timeout + read_timeout = self._timeout + if self.proxy_host and self.secure: # Send http CONNECT method to a proxy and acquire the socket sock = _create_tunnel( @@ -174,17 +176,18 @@ def connect(self): self.proxy_port, self.host, self.port, - proxy_headers=self.proxy_headers + proxy_headers=self.proxy_headers, + timeout=self._timeout ) elif self.proxy_host: # Simple http proxy sock = socket.create_connection( (self.proxy_host, self.proxy_port), - timeout=self._connect_timeout + timeout=connect_timeout ) else: sock = socket.create_connection((self.host, self.port), - timeout=self._connect_timeout) + timeout=connect_timeout) proto = None if self.secure: @@ -194,7 +197,7 @@ def connect(self): sock = BufferedSocket(sock, self.network_buffer_size) # Set read timeout - sock.settimeout(self._read_timeout) + sock.settimeout(read_timeout) if proto not in ('http/1.1', None): raise TLSUpgrade(proto, sock) diff --git a/hyper/http20/connection.py b/hyper/http20/connection.py index c6928c7e..b8be292b 100644 --- a/hyper/http20/connection.py +++ b/hyper/http20/connection.py @@ -152,12 +152,7 @@ def __init__(self, host, port=None, secure=None, window_manager=None, self.__init_state() # timeout - if isinstance(timeout, tuple): - self._connect_timeout = timeout[0] - self._read_timeout = timeout[1] - else: - self._connect_timeout = timeout - self._read_timeout = timeout + self._timeout = timeout return @@ -351,6 +346,13 @@ def connect(self): if self._sock is not None: return + if isinstance(self._timeout, tuple): + connect_timeout = self._timeout[0] + read_timeout = self._timeout[1] + else: + connect_timeout = self._timeout + read_timeout = self._timeout + if self.proxy_host and self.secure: # Send http CONNECT method to a proxy and acquire the socket sock = _create_tunnel( @@ -358,17 +360,18 @@ def connect(self): self.proxy_port, self.host, self.port, - proxy_headers=self.proxy_headers + proxy_headers=self.proxy_headers, + timeout=self._timeout ) elif self.proxy_host: # Simple http proxy sock = socket.create_connection( (self.proxy_host, self.proxy_port), - timeout=self._connect_timeout + timeout=connect_timeout ) else: sock = socket.create_connection((self.host, self.port), - timeout=self._connect_timeout) + timeout=connect_timeout) if self.secure: sock, proto = wrap_socket(sock, self.host, self.ssl_context, @@ -385,7 +388,7 @@ def connect(self): self._sock = BufferedSocket(sock, self.network_buffer_size) # Set read timeout - self._sock.settimeout(self._read_timeout) + self._sock.settimeout(read_timeout) self._send_preamble() diff --git a/test/test_abstraction.py b/test/test_abstraction.py index d48b3954..00ee16ec 100644 --- a/test/test_abstraction.py +++ b/test/test_abstraction.py @@ -10,7 +10,7 @@ def test_h1_kwargs(self): c = HTTPConnection( 'test', 443, secure=False, window_manager=True, enable_push=True, ssl_context=False, proxy_host=False, proxy_port=False, - proxy_headers=False, other_kwarg=True + proxy_headers=False, other_kwarg=True, timeout=5 ) assert c._h1_kwargs == { @@ -21,13 +21,14 @@ def test_h1_kwargs(self): 'proxy_headers': False, 'other_kwarg': True, 'enable_push': True, + 'timeout': 5, } def test_h2_kwargs(self): c = HTTPConnection( 'test', 443, secure=False, window_manager=True, enable_push=True, ssl_context=True, proxy_host=False, proxy_port=False, - proxy_headers=False, other_kwarg=True + proxy_headers=False, other_kwarg=True, timeout=(10, 30) ) assert c._h2_kwargs == { @@ -39,6 +40,7 @@ def test_h2_kwargs(self): 'proxy_port': False, 'proxy_headers': False, 'other_kwarg': True, + 'timeout': (10, 30), } def test_tls_upgrade(self, monkeypatch): diff --git a/test/test_http11.py b/test/test_http11.py index f71289d8..21dd7f70 100644 --- a/test/test_http11.py +++ b/test/test_http11.py @@ -113,14 +113,12 @@ def test_initialization_with_ipv6_addresses_proxy_inline_port(self): def test_initialization_timeout(self): c = HTTP11Connection('httpbin.org', timeout=30) - assert c._connect_timeout == 30 - assert c._read_timeout == 30 + assert c._timeout == 30 def test_initialization_tuple_timeout(self): c = HTTP11Connection('httpbin.org', timeout=(5, 60)) - assert c._connect_timeout == 5 - assert c._read_timeout == 60 + assert c._timeout == (5, 60) def test_basic_request(self): c = HTTP11Connection('httpbin.org') diff --git a/test/test_hyper.py b/test/test_hyper.py index a8513a9f..f4a5994d 100644 --- a/test/test_hyper.py +++ b/test/test_hyper.py @@ -101,14 +101,12 @@ def test_connection_version(self): def test_connection_timeout(self): c = HTTP20Connection('httpbin.org', timeout=30) - assert c._connect_timeout == 30 - assert c._read_timeout == 30 + assert c._timeout == 30 def test_connection_tuple_timeout(self): c = HTTP20Connection('httpbin.org', timeout=(5, 60)) - assert c._connect_timeout == 5 - assert c._read_timeout == 60 + assert c._timeout == (5, 60) def test_ping(self, frame_buffer): def data_callback(chunk, **kwargs): diff --git a/test/test_integration.py b/test/test_integration.py index 34e9e5a4..bde7d393 100644 --- a/test/test_integration.py +++ b/test/test_integration.py @@ -1247,9 +1247,26 @@ def socket_handler(listener): self.tear_down() - def test_read_timeout(self): + def test_hyper_connection_timeout(self): self.set_up(timeout=0.5) + def socket_handler(listener): + time.sleep(1) + + self._start_server(socket_handler) + conn = hyper.HTTPConnection(self.host, self.port, self.secure, + timeout=self.timeout) + + with pytest.raises((SocketTimeout, ssl.SSLError)): + # Py2 raises this as a BaseSSLError, + # Py3 raises it as socket timeout. + conn.request('GET', '/') + + self.tear_down() + + def test_read_timeout(self): + self.set_up(timeout=(10, 0.5)) + req_event = threading.Event() def socket_handler(listener): @@ -1378,8 +1395,7 @@ def socket_handler(listener): s = requests.Session() s.mount('https://%s' % self.host, HTTP20Adapter()) - r = s.get('https://%s:%s/some/path' % (self.host, self.port), - timeout=(10, 60)) + r = s.get('https://%s:%s/some/path' % (self.host, self.port)) # Assert about the received values. assert r.status_code == 200 @@ -1626,3 +1642,78 @@ def socket_handler(listener): assert r.content == b'' self.tear_down() + + def test_adapter_connection_timeout(self, monkeypatch, frame_buffer): + self.set_up() + + # We need to patch the ssl_wrap_socket method to ensure that we + # forcefully upgrade. + old_wrap_socket = hyper.http11.connection.wrap_socket + + def wrap(*args): + sock, _ = old_wrap_socket(*args) + return sock, 'h2' + + monkeypatch.setattr(hyper.http11.connection, 'wrap_socket', wrap) + + def socket_handler(listener): + time.sleep(1) + + self._start_server(socket_handler) + + s = requests.Session() + s.mount('https://%s' % self.host, HTTP20Adapter()) + + with pytest.raises((SocketTimeout, ssl.SSLError)): + # Py2 raises this as a BaseSSLError, + # Py3 raises it as socket timeout. + s.get('https://%s:%s/some/path' % (self.host, self.port), + timeout=0.5) + + self.tear_down() + + def test_adapter_read_timeout(self, monkeypatch, frame_buffer): + self.set_up() + + # We need to patch the ssl_wrap_socket method to ensure that we + # forcefully upgrade. + old_wrap_socket = hyper.http11.connection.wrap_socket + + def wrap(*args): + sock, _ = old_wrap_socket(*args) + return sock, 'h2' + + monkeypatch.setattr(hyper.http11.connection, 'wrap_socket', wrap) + + def socket_handler(listener): + sock = listener.accept()[0] + + # Do the handshake: conn header, settings, send settings, recv ack. + frame_buffer.add_data(receive_preamble(sock)) + + # Now expect some data. One headers frame. + req_wait = True + while req_wait: + frame_buffer.add_data(sock.recv(65535)) + with reusable_frame_buffer(frame_buffer) as fr: + for f in fr: + if isinstance(f, HeadersFrame): + req_wait = False + + # Sleep wait for read timeout + time.sleep(1) + + sock.close() + + self._start_server(socket_handler) + + s = requests.Session() + s.mount('https://%s' % self.host, HTTP20Adapter()) + + with pytest.raises((SocketTimeout, ssl.SSLError)): + # Py2 raises this as a BaseSSLError, + # Py3 raises it as socket timeout. + s.get('https://%s:%s/some/path' % (self.host, self.port), + timeout=(10, 0.5)) + + self.tear_down() diff --git a/test/test_integration_http11.py b/test/test_integration_http11.py index ee318797..7ec3846a 100644 --- a/test/test_integration_http11.py +++ b/test/test_integration_http11.py @@ -9,6 +9,8 @@ import hyper import threading import pytest +import time +from socket import timeout as SocketTimeout from hyper.compat import ssl from server import SocketLevelTest, SocketSecuritySetting @@ -442,3 +444,68 @@ def socket_handler(listener): with pytest.raises(HTTPUpgrade): c.get_response() + + def test_connection_timeout(self): + self.set_up(timeout=0.5) + + def socket_handler(listener): + time.sleep(1) + + self._start_server(socket_handler) + conn = self.get_connection() + + with pytest.raises((SocketTimeout, ssl.SSLError)): + # Py2 raises this as a BaseSSLError, + # Py3 raises it as socket timeout. + conn.connect() + + self.tear_down() + + def test_hyper_connection_timeout(self): + self.set_up(timeout=0.5) + + def socket_handler(listener): + time.sleep(1) + + self._start_server(socket_handler) + conn = hyper.HTTPConnection(self.host, self.port, self.secure, + timeout=self.timeout) + + with pytest.raises((SocketTimeout, ssl.SSLError)): + # Py2 raises this as a BaseSSLError, + # Py3 raises it as socket timeout. + conn.request('GET', '/') + + self.tear_down() + + def test_read_timeout(self): + self.set_up(timeout=(10, 0.5)) + + send_event = threading.Event() + + def socket_handler(listener): + sock = listener.accept()[0] + + # We should get the initial request. + data = b'' + while not data.endswith(b'\r\n\r\n'): + data += sock.recv(65535) + + send_event.wait() + + # Sleep wait for read timeout + time.sleep(1) + + sock.close() + + self._start_server(socket_handler) + conn = self.get_connection() + conn.request('GET', '/') + send_event.set() + + with pytest.raises((SocketTimeout, ssl.SSLError)): + # Py2 raises this as a BaseSSLError, + # Py3 raises it as socket timeout. + conn.get_response() + + self.tear_down()