From f289cab3e70595d6e79fb797069e029403d991d6 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sun, 23 Apr 2017 13:51:28 +0200 Subject: [PATCH 1/7] bpo-31399: Let OpenSSL verify hostname and IP The ssl module now uses OpenSSL's X509_VERIFY_PARAM_set1_host() and X509_VERIFY_PARAM_set1_ip() API to verify hostname and IP addresses. Signed-off-by: Christian Heimes --- Lib/ssl.py | 16 +- Lib/test/test_asyncio/test_events.py | 6 +- Lib/test/test_ftplib.py | 3 + Lib/test/test_imaplib.py | 6 +- Lib/test/test_poplib.py | 3 + Lib/test/test_ssl.py | 5 +- Lib/test/test_urllib2_localnet.py | 2 +- .../2017-09-08-14-05-33.bpo-31399.FtBrrt.rst | 1 + Modules/_ssl.c | 150 +++++++++++++++--- 9 files changed, 157 insertions(+), 35 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2017-09-08-14-05-33.bpo-31399.FtBrrt.rst diff --git a/Lib/ssl.py b/Lib/ssl.py index fa83606e7cd5a5..08bb57a580a223 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -148,6 +148,10 @@ lambda name: name.startswith('CERT_'), source=_ssl) +_IntFlag._convert( + 'HostFlags', __name__, + lambda name: name.startswith('HOSTFLAG_'), + source=_ssl) PROTOCOL_SSLv23 = _SSLMethod.PROTOCOL_SSLv23 = _SSLMethod.PROTOCOL_TLS _PROTOCOL_NAMES = {value: name for name, value in _SSLMethod.__members__.items()} @@ -216,9 +220,7 @@ '!aNULL:!eNULL:!MD5:!DSS:!RC4:!3DES' ) - -class CertificateError(ValueError): - pass +CertificateError = SSLCertVerificationError def _dnsname_match(dn, hostname): @@ -473,6 +475,14 @@ def options(self): def options(self, value): super(SSLContext, SSLContext).options.__set__(self, value) + @property + def host_flags(self): + return HostFlags(super().host_flags) + + @host_flags.setter + def host_flags(self, value): + super(SSLContext, SSLContext).host_flags.__set__(self, value) + @property def verify_flags(self): return VerifyFlags(super().verify_flags) diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index e4b053681b4e91..cf217538a06f21 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -1148,11 +1148,13 @@ def test_create_server_ssl_match_failed(self): with test_utils.disable_logger(): with self.assertRaisesRegex( ssl.CertificateError, - "hostname '127.0.0.1' doesn't match 'localhost'"): + "IP address mismatch, certificate is not valid for " + "'127.0.0.1'"): self.loop.run_until_complete(f_c) # close connection - proto.transport.close() + # transport is None because TLS ALERT aborted the handshake + self.assertIsNone(proto.transport) server.close() @support.skip_unless_bind_unix_socket diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index f1b0185b2bfa37..bb5a67f27e31f6 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -330,6 +330,9 @@ def _do_ssl_handshake(self): return elif err.args[0] == ssl.SSL_ERROR_EOF: return self.handle_close() + # TODO: SSLError does not expose alert information + elif "SSLV3_ALERT_BAD_CERTIFICATE" in err.args[1]: + return self.handle_close() raise except OSError as err: if err.args[0] == errno.ECONNABORTED: diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 4a45be6572475a..f16bacd0006c87 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -485,7 +485,8 @@ def test_ssl_raises(self): ssl_context.load_verify_locations(CAFILE) with self.assertRaisesRegex(ssl.CertificateError, - "hostname '127.0.0.1' doesn't match 'localhost'"): + "IP address mismatch, certificate is not valid for " + "'127.0.0.1'"): _, server = self._setup(SimpleIMAPHandler) client = self.imap_class(*server.server_address, ssl_context=ssl_context) @@ -874,7 +875,8 @@ def test_ssl_verified(self): with self.assertRaisesRegex( ssl.CertificateError, - "hostname '127.0.0.1' doesn't match 'localhost'"): + "IP address mismatch, certificate is not valid for " + "'127.0.0.1'"): with self.reaped_server(SimpleIMAPHandler) as server: client = self.imap_class(*server.server_address, ssl_context=ssl_context) diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py index 9ba678f2039d66..4d7a4394086ae5 100644 --- a/Lib/test/test_poplib.py +++ b/Lib/test/test_poplib.py @@ -176,6 +176,9 @@ def _do_tls_handshake(self): return elif err.args[0] == ssl.SSL_ERROR_EOF: return self.handle_close() + # TODO: SSLError does not expose alert information + elif "SSLV3_ALERT_BAD_CERTIFICATE" in err.args[1]: + return self.handle_close() raise except OSError as err: if err.args[0] == errno.ECONNABORTED: diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index e3fa4233ed6860..68989f19b70be6 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -2536,8 +2536,9 @@ def test_check_hostname(self): with server: with client_context.wrap_socket(socket.socket(), server_hostname="invalid") as s: - with self.assertRaisesRegex(ssl.CertificateError, - "hostname 'invalid' doesn't match 'localhost'"): + with self.assertRaisesRegex( + ssl.CertificateError, + "Hostname mismatch, certificate is not valid for 'invalid'."): s.connect((HOST, server.port)) # missing server_hostname arg should cause an exception, too diff --git a/Lib/test/test_urllib2_localnet.py b/Lib/test/test_urllib2_localnet.py index b2d1e5f980498a..52c897af433cce 100644 --- a/Lib/test/test_urllib2_localnet.py +++ b/Lib/test/test_urllib2_localnet.py @@ -573,7 +573,7 @@ def test_https_with_cafile(self): cafile=CERT_fakehostname) # Good cert, but mismatching hostname handler = self.start_https_server(certfile=CERT_fakehostname) - with self.assertRaises(ssl.CertificateError) as cm: + with self.assertRaises(urllib.error.URLError) as cm: self.urlopen("https://localhost:%s/bizarre" % handler.port, cafile=CERT_fakehostname) diff --git a/Misc/NEWS.d/next/Library/2017-09-08-14-05-33.bpo-31399.FtBrrt.rst b/Misc/NEWS.d/next/Library/2017-09-08-14-05-33.bpo-31399.FtBrrt.rst new file mode 100644 index 00000000000000..b58b9c0a0c3cfa --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-09-08-14-05-33.bpo-31399.FtBrrt.rst @@ -0,0 +1 @@ +Let OpenSSL verify hostname and IP addressw diff --git a/Modules/_ssl.c b/Modules/_ssl.c index c5eec7eded6292..30ee5fdf596f14 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -317,6 +317,10 @@ typedef struct { PyObject *set_hostname; #endif int check_hostname; + /* OpenSSL has no API to get hostflags from X509_VERIFY_PARAM* struct. + * We have to maintain our own copy. OpenSSL's hostflags default to 0. + */ + unsigned int hostflags; } PySSLContext; typedef struct { @@ -701,6 +705,61 @@ _setSSLError (const char *errstr, int errcode, const char *filename, int lineno) return NULL; } +/* + * SSL objects + */ + +static int +_ssl_configure_hostname(PySSLSocket *self, const char* server_hostname) +{ + int retval = -1; + ASN1_OCTET_STRING *ip; + PyObject *hostname; + + assert(server_hostname); + + /* inet_pton is not available on all platforms. */ + ip = a2i_IPADDRESS(server_hostname); + if (ip == NULL) { + ERR_clear_error(); + } + + hostname = PyUnicode_Decode(server_hostname, strlen(server_hostname), + "idna", "strict"); + if (hostname == NULL) { + goto error; + } + self->server_hostname = hostname; + + /* Only send SNI extension for non-IP hostnames */ + if (ip == NULL) { + if (!SSL_set_tlsext_host_name(self->ssl, server_hostname)) { + _setSSLError(NULL, 0, __FILE__, __LINE__); + } + } + if (self->ctx->check_hostname) { + X509_VERIFY_PARAM *param = SSL_get0_param(self->ssl); + if (ip == NULL) { + if (!X509_VERIFY_PARAM_set1_host(param, server_hostname, 0)) { + _setSSLError(NULL, 0, __FILE__, __LINE__); + goto error; + } + } else { + if (!X509_VERIFY_PARAM_set1_ip(param, ASN1_STRING_data(ip), + ASN1_STRING_length(ip))) { + _setSSLError(NULL, 0, __FILE__, __LINE__); + goto error; + } + } + } + retval = 0; + error: + if (ip != NULL) { + ASN1_OCTET_STRING_free(ip); + } + return retval; +} + static PySSLSocket * newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, enum py_ssl_server_or_client socket_type, @@ -722,15 +781,6 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, self->shutdown_seen_zero = 0; self->owner = NULL; self->server_hostname = NULL; - if (server_hostname != NULL) { - PyObject *hostname = PyUnicode_Decode(server_hostname, strlen(server_hostname), - "idna", "strict"); - if (hostname == NULL) { - Py_DECREF(self); - return NULL; - } - self->server_hostname = hostname; - } self->ssl_errno = 0; self->c_errno = 0; #ifdef MS_WINDOWS @@ -761,10 +811,12 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, #endif SSL_set_mode(self->ssl, mode); -#if HAVE_SNI - if (server_hostname != NULL) - SSL_set_tlsext_host_name(self->ssl, server_hostname); -#endif + if (server_hostname != NULL) { + if (_ssl_configure_hostname(self, server_hostname) < 0) { + Py_DECREF(self); + return NULL; + } + } /* If the socket is in non-blocking mode or timeout mode, set the BIO * to non-blocking mode (blocking is the default) */ @@ -2711,6 +2763,7 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) PySSLContext *self; long options; SSL_CTX *ctx = NULL; + X509_VERIFY_PARAM *params; int result; #if defined(SSL_MODE_RELEASE_BUFFERS) unsigned long libver; @@ -2760,6 +2813,10 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) return NULL; } self->ctx = ctx; + self->hostflags = ( + X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS | + X509_CHECK_FLAG_SINGLE_LABEL_SUBDOMAINS + ); #if defined(OPENSSL_NPN_NEGOTIATED) && !defined(OPENSSL_NO_NEXTPROTONEG) self->npn_protocols = NULL; #endif @@ -2858,14 +2915,13 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) sizeof(SID_CTX)); #undef SID_CTX + params = SSL_CTX_get0_param(self->ctx); #ifdef X509_V_FLAG_TRUSTED_FIRST - { - /* Improve trust chain building when cross-signed intermediate - certificates are present. See https://bugs.python.org/issue23476. */ - X509_STORE *store = SSL_CTX_get_cert_store(self->ctx); - X509_STORE_set_flags(store, X509_V_FLAG_TRUSTED_FIRST); - } + /* Improve trust chain building when cross-signed intermediate + certificates are present. See https://bugs.python.org/issue23476. */ + X509_VERIFY_PARAM_set_flags(params, X509_V_FLAG_TRUSTED_FIRST); #endif + X509_VERIFY_PARAM_set_hostflags(params, self->hostflags); return (PyObject *)self; } @@ -3152,12 +3208,10 @@ set_verify_mode(PySSLContext *self, PyObject *arg, void *c) static PyObject * get_verify_flags(PySSLContext *self, void *c) { - X509_STORE *store; X509_VERIFY_PARAM *param; unsigned long flags; - store = SSL_CTX_get_cert_store(self->ctx); - param = X509_STORE_get0_param(store); + param = SSL_CTX_get0_param(self->ctx); flags = X509_VERIFY_PARAM_get_flags(param); return PyLong_FromUnsignedLong(flags); } @@ -3165,14 +3219,12 @@ get_verify_flags(PySSLContext *self, void *c) static int set_verify_flags(PySSLContext *self, PyObject *arg, void *c) { - X509_STORE *store; X509_VERIFY_PARAM *param; unsigned long new_flags, flags, set, clear; if (!PyArg_Parse(arg, "k", &new_flags)) return -1; - store = SSL_CTX_get_cert_store(self->ctx); - param = X509_STORE_get0_param(store); + param = SSL_CTX_get0_param(self->ctx); flags = X509_VERIFY_PARAM_get_flags(param); clear = flags & ~new_flags; set = ~flags & new_flags; @@ -3220,6 +3272,27 @@ set_options(PySSLContext *self, PyObject *arg, void *c) return 0; } +static PyObject * +get_host_flags(PySSLContext *self, void *c) +{ + return PyLong_FromUnsignedLong(self->hostflags); +} + +static int +set_host_flags(PySSLContext *self, PyObject *arg, void *c) +{ + X509_VERIFY_PARAM *param; + unsigned int new_flags = 0; + + if (!PyArg_Parse(arg, "I", &new_flags)) + return -1; + + param = SSL_CTX_get0_param(self->ctx); + self->hostflags = new_flags; + X509_VERIFY_PARAM_set_hostflags(param, new_flags); + return 0; +} + static PyObject * get_check_hostname(PySSLContext *self, void *c) { @@ -4104,6 +4177,8 @@ _ssl__SSLContext_get_ca_certs_impl(PySSLContext *self, int binary_form) static PyGetSetDef context_getsetlist[] = { {"check_hostname", (getter) get_check_hostname, (setter) set_check_hostname, NULL}, + {"host_flags", (getter) get_host_flags, + (setter) set_host_flags, NULL}, {"options", (getter) get_options, (setter) set_options, NULL}, {"verify_flags", (getter) get_verify_flags, @@ -5491,6 +5566,31 @@ PyInit__ssl(void) SSL_OP_NO_COMPRESSION); #endif +#ifdef X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT + PyModule_AddIntConstant(m, "HOSTFLAG_ALWAYS_CHECK_SUBJECT", + X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT); +#endif +#ifdef X509_CHECK_FLAG_NEVER_CHECK_SUBJECT + PyModule_AddIntConstant(m, "HOSTFLAG_NEVER_CHECK_SUBJECT", + X509_CHECK_FLAG_NEVER_CHECK_SUBJECT); +#endif +#ifdef X509_CHECK_FLAG_NO_WILDCARDS + PyModule_AddIntConstant(m, "HOSTFLAG_NO_WILDCARDS", + X509_CHECK_FLAG_NO_WILDCARDS); +#endif +#ifdef X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS + PyModule_AddIntConstant(m, "HOSTFLAG_NO_PARTIAL_WILDCARDS", + X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS); +#endif +#ifdef X509_CHECK_FLAG_MULTI_LABEL_WILDCARDS + PyModule_AddIntConstant(m, "HOSTFLAG_MULTI_LABEL_WILDCARDS", + X509_CHECK_FLAG_MULTI_LABEL_WILDCARDS); +#endif +#ifdef X509_CHECK_FLAG_SINGLE_LABEL_SUBDOMAINS + PyModule_AddIntConstant(m, "HOSTFLAG_SINGLE_LABEL_SUBDOMAINS", + X509_CHECK_FLAG_SINGLE_LABEL_SUBDOMAINS); +#endif + #if HAVE_SNI r = Py_True; #else From 9dbf5f775d88e14667d7ab3ab2894b503a895863 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Tue, 16 Jan 2018 22:06:28 +0100 Subject: [PATCH 2/7] Remove match_hostname calls Signed-off-by: Christian Heimes --- Lib/asyncio/sslproto.py | 7 +------ Lib/http/client.py | 10 ++-------- Lib/ssl.py | 5 ----- 3 files changed, 3 insertions(+), 19 deletions(-) diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index 2d377c4ae39b5d..5e96390487b820 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -590,12 +590,7 @@ def _on_handshake_complete(self, handshake_exc): raise handshake_exc peercert = sslobj.getpeercert() - if not hasattr(self._sslcontext, 'check_hostname'): - # Verify hostname if requested, Python 3.4+ uses check_hostname - # and checks the hostname in do_handshake() - if (self._server_hostname and - self._sslcontext.verify_mode != ssl.CERT_NONE): - ssl.match_hostname(peercert, self._server_hostname) + # Since 3.7, the hostname is matched by OpenSSL during handshake. except BaseException as exc: if self._loop.get_debug(): if isinstance(exc, ssl.CertificateError): diff --git a/Lib/http/client.py b/Lib/http/client.py index 1a852cd76e4920..1292db74784ccc 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -1375,7 +1375,8 @@ def __init__(self, host, port=None, key_file=None, cert_file=None, if key_file or cert_file: context.load_cert_chain(cert_file, key_file) self._context = context - self._check_hostname = check_hostname + if check_hostname is not None: + self._context.check_hostname = check_hostname def connect(self): "Connect to a host on a given (SSL) port." @@ -1389,13 +1390,6 @@ def connect(self): self.sock = self._context.wrap_socket(self.sock, server_hostname=server_hostname) - if not self._context.check_hostname and self._check_hostname: - try: - ssl.match_hostname(self.sock.getpeercert(), server_hostname) - except Exception: - self.sock.shutdown(socket.SHUT_RDWR) - self.sock.close() - raise __all__.append("HTTPSConnection") diff --git a/Lib/ssl.py b/Lib/ssl.py index 08bb57a580a223..c9f5a4bf93aebd 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -709,11 +709,6 @@ def pending(self): def do_handshake(self): """Start the SSL/TLS handshake.""" self._sslobj.do_handshake() - if self.context.check_hostname: - if not self.server_hostname: - raise ValueError("check_hostname needs server_hostname " - "argument") - match_hostname(self.getpeercert(), self.server_hostname) def unwrap(self): """Start the SSL shutdown handshake.""" From 802807d23805ebf043e44066b7f2f90ac1453b78 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Wed, 17 Jan 2018 11:18:54 +0100 Subject: [PATCH 3/7] Document host flags Signed-off-by: Christian Heimes --- Doc/library/ssl.rst | 59 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 4c44ffa23ac8d6..e4c1da9b637242 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -430,9 +430,14 @@ Certificate handling of the certificate, is now supported. .. versionchanged:: 3.7 + The function is no longer used. Hostname matching is now performed + by OpenSSL. + Allow wildcard when it is the leftmost and the only character in that segment. + .. deprecated:: 3.7 + .. function:: cert_time_to_seconds(cert_time) Return the time in seconds since the Epoch, given the ``cert_time`` @@ -586,6 +591,42 @@ Constants be passed, either to :meth:`SSLContext.load_verify_locations` or as a value of the ``ca_certs`` parameter to :func:`wrap_socket`. +.. class:: HostFlags + + :class:`enum.IntFlag` collection of all hostname verification flags for + :attr:`SSLContext.host_flags` + + .. versionadded:: 3.7 + +.. attribute:: HostFlags.HOSTFLAG_ALWAYS_CHECK_SUBJECT + + Consider subject CN even if the certificate contains at least one subject + alternative name. This flag violates :rfc:`6125`. + +.. attribute:: HostFlags.HOSTFLAG_NO_WILDCARDS + + Don't support wildcard certificate, e.g. ``*.example.org``. + +.. attribute:: HostFlags.HOSTFLAG_NO_PARTIAL_WILDCARDS + + Dont' support wildcard certificate with partial matches, e.g. + ``www*.example.org``. + +.. attribute:: HostFlags.HOSTFLAG_MULTI_LABEL_WILDCARDS + + Wildcards match multiple labels, e.g. ``www.subdomain.example.org`` + matches ``*.example.org`` This flag violates :rfc:`6125`. + +.. attribute:: HostFlags.HOSTFLAG_SINGLE_LABEL_SUBDOMAINS + +.. attribute:: HostFlags.HOSTFLAG_NEVER_CHECK_SUBJECT + + Ignore subject CN even if the certificate has no subject alternative + names. + + .. note:: The flag is not available when the ssl module is compiled + with OpenSSL 1.0.2 or LibreSSL. + .. class:: VerifyMode :class:`enum.IntEnum` collection of CERT_* constants. @@ -1075,6 +1116,12 @@ SSL sockets also have the following additional methods and attributes: The socket timeout is no more reset each time bytes are received or sent. The socket timeout is now to maximum total duration of the handshake. + .. versionchanged:: 3.7 + Hostname or IP address is matched by OpenSSL during handshake. The + function :func:`match_hostname` is no longer used. In case OpenSSL + refuses a hostname or IP address, the handshake is aborted early and + a TLS alert message is send to the peer. + .. method:: SSLSocket.getpeercert(binary_form=False) If there is no certificate for the peer on the other end of the connection, @@ -1730,6 +1777,14 @@ to speed up repeated connections from the same clients. The protocol version chosen when constructing the context. This attribute is read-only. +.. attribute:: SSLContext.host_flags + + The flags for validating host names. By default + :data:`HostFlags.HOSTFLAG_SINGLE_LABEL_SUBDOMAINS` and + :data:`HostFlags.HOSTFLAG_NO_PARTIAL_WILDCARDS` are set. + + .. versionadded:: 3.7 + .. attribute:: SSLContext.verify_flags The flags for certificate verification operations. You can set flags like @@ -2324,6 +2379,10 @@ in this case, the :func:`match_hostname` function can be used. This common check is automatically performed when :attr:`SSLContext.check_hostname` is enabled. +.. versionchanged:: 3.7 + Hostname matchings is now performed by OpenSSL. Python no longer uses + :func:`match_hostname`. + In server mode, if you want to authenticate your clients using the SSL layer (rather than using a higher-level authentication mechanism), you'll also have to specify :const:`CERT_REQUIRED` and similarly check the client certificate. From bb1f7aacfa1cfe10ddaf635be9f9deb4141d5199 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sat, 20 Jan 2018 10:05:38 +0100 Subject: [PATCH 4/7] Check for libssl with set1_host libssl must provide X509_VERIFY_PARAM_set1_host() Signed-off-by: Christian Heimes --- Modules/_ssl.c | 12 +++++------- PC/pyconfig.h | 3 +++ setup.py | 17 ++++++++++------- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 30ee5fdf596f14..cda2d4e5d54efe 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -64,10 +64,13 @@ static PySocketModule_APIObject PySocketModule; #include "openssl/rand.h" #include "openssl/bio.h" -/* Set HAVE_X509_VERIFY_PARAM_SET1_HOST for non-autoconf builds */ #ifndef HAVE_X509_VERIFY_PARAM_SET1_HOST -# if !defined(LIBRESSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER > 0x1000200fL +# ifdef LIBRESSL_VERSION_NUMBER +# error "LibreSSL is missing X509_VERIFY_PARAM_set1_host(), see https://github.com/libressl-portable/portable/issues/381" +# elif OPENSSL_VERSION_NUMBER > 0x1000200fL # define HAVE_X509_VERIFY_PARAM_SET1_HOST +# else +# error "libssl is too old and does not support X509_VERIFY_PARAM_set1_host()" # endif #endif @@ -217,11 +220,6 @@ static STACK_OF(X509_OBJECT) *X509_STORE_get0_objects(X509_STORE *store) { return store->objs; } -static X509_VERIFY_PARAM *X509_STORE_get0_param(X509_STORE *store) -{ - return store->param; -} - static int SSL_SESSION_has_ticket(const SSL_SESSION *s) { diff --git a/PC/pyconfig.h b/PC/pyconfig.h index db745dee761777..d2a3f5dd39bb7f 100644 --- a/PC/pyconfig.h +++ b/PC/pyconfig.h @@ -687,4 +687,7 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ /* framework name */ #define _PYTHONFRAMEWORK "" +/* Define if libssl has X509_VERIFY_PARAM_set1_host and related function */ +#define HAVE_X509_VERIFY_PARAM_SET1_HOST 1 + #endif /* !Py_CONFIG_H */ diff --git a/setup.py b/setup.py index 258094e3ada29a..47a8148ca62af4 100644 --- a/setup.py +++ b/setup.py @@ -2157,13 +2157,16 @@ def split_var(name, sep): if krb5_h: ssl_incs.extend(krb5_h) - ssl_ext = Extension( - '_ssl', ['_ssl.c'], - include_dirs=openssl_includes, - library_dirs=openssl_libdirs, - libraries=openssl_libs, - depends=['socketmodule.h'] - ) + if config_vars.get("HAVE_X509_VERIFY_PARAM_SET1_HOST"): + ssl_ext = Extension( + '_ssl', ['_ssl.c'], + include_dirs=openssl_includes, + library_dirs=openssl_libdirs, + libraries=openssl_libs, + depends=['socketmodule.h'] + ) + else: + ssl_ext = None hashlib_ext = Extension( '_hashlib', ['_hashopenssl.c'], From a48ae161a654548f6e9af03b1bcac05ebb4be707 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sat, 20 Jan 2018 10:43:07 +0100 Subject: [PATCH 5/7] bpo-31399: Add documentation for OpenSSL 1.0.2 requirement Signed-off-by: Christian Heimes --- Doc/library/ssl.rst | 7 ++++--- Doc/whatsnew/3.7.rst | 32 ++++++++++++++++++++++++++++++++ setup.py | 10 ++++++++++ 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index e4c1da9b637242..e6f7e93531809e 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -146,9 +146,10 @@ Functions, Constants, and Exceptions .. exception:: CertificateError - Raised to signal an error with a certificate (such as mismatching - hostname). Certificate errors detected by OpenSSL, though, raise - an :exc:`SSLCertVerificationError`. + An alias for :exc:`SSLCertVerificationError`. + + .. versionchanged:: 3.7 + The exception is now an alias for :exc:`SSLCertVerificationError`. Socket creation diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 0e1714ea17e732..cff00b6a64aad0 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -510,6 +510,32 @@ can be set within the scope of a group. ``'^$'`` or ``(?=-)`` that matches an empty string. (Contributed by Serhiy Storchaka in :issue:`25054`.) +ssl +--- + +The ssl module now uses OpenSSL's builtin API instead of +:func:`~ssl.match_hostname` to check host name or IP address. Values +are validated during TLS handshake. Any cert validation error including +a failing host name match now raises :exc:`~ssl.SSLCertVerificationError` and +aborts the handshake with a proper TLS Alert message. The new exception +contains additional information. Host name validation can be customized +with :attr:`~ssl.SSLContext.host_flags`. +(Contributed by Christian Heimes in :issue:`31399`.) + +.. note:: + The improved host name check requires an OpenSSL 1.0.2 or 1.1 compatible + libssl. OpenSSL 0.9.8 and 1.0.1 are no longer supported. LibreSSL is + temporarily not supported until it gains the necessary OpenSSL 1.0.2 APIs. + +The ssl module no longer sends IP addresses in SNI TLS extension. +(Contributed by Christian Heimes in :issue:`32185`.) + +:func:`~ssl.match_hostname` no longer supports partial wildcards like +``www*.example.org``. :attr:`~ssl.SSLContext.host_flags` has partial +wildcard matching disabled by default. +(Contributed by Mandeep Singh in :issue:`23033` and Christian Heimes in +:issue:`31399`.) + string ------ @@ -1056,6 +1082,12 @@ Other CPython implementation changes emitted in the first place), and an explicit ``error::BytesWarning`` warnings filter added to convert them to exceptions. +* CPython' :mod:`ssl` module requires OpenSSL 1.0.2 or 1.1 compatible libssl. + OpenSSL 1.0.1 has reached end of lifetime on 2016-12-31 and is no longer + supported. LibreSSL is temporarily not supported as well. LibreSSL releases + up to version 2.6.4 are missing required OpenSSL 1.0.2 APIs. + + Documentation ============= diff --git a/setup.py b/setup.py index 47a8148ca62af4..0ac090b9d01a98 100644 --- a/setup.py +++ b/setup.py @@ -363,6 +363,16 @@ def print_three_column(lst): print_three_column(failed) print() + if any('_ssl' in l + for l in (missing, self.failed, self.failed_on_import)): + print() + print("Could not build the ssl module!") + print("Python requires an OpenSSL 1.0.2 or 1.1 compatible " + "libssl with X509_VERIFY_PARAM_set1_host().") + print("LibreSSL 2.6.4 and earlier do not provide the necessary " + "APIs, https://github.com/libressl-portable/portable/issues/381") + print() + def build_extension(self, ext): if ext.name == '_ctypes': From 1e8b4ab2bcd9e98c66ed2624a989268c2b9c59da Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Tue, 23 Jan 2018 12:13:38 +0100 Subject: [PATCH 6/7] bpo-31399: Only expose useful hostflags Remove all hostflags except for NO_PARTIAL_WILDCARDS and NEVER_CHECK_SUBJECT. The other flags aren't that useful at the moment. Don't support OpenSSL special mode with a leading dot, e.g. ".example.org" matches "www.example.org". It's not standard conform. Signed-off-by: Christian Heimes --- Doc/library/ssl.rst | 28 ++++++---------------------- Lib/asyncio/sslproto.py | 1 - Lib/test/test_ssl.py | 10 ++++++++++ Modules/_ssl.c | 40 ++++++++++++++++------------------------ 4 files changed, 32 insertions(+), 47 deletions(-) diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index e6f7e93531809e..7b816387b15cc5 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -431,11 +431,12 @@ Certificate handling of the certificate, is now supported. .. versionchanged:: 3.7 - The function is no longer used. Hostname matching is now performed - by OpenSSL. + The function is no longer used to TLS connections. Hostname matching + is now performed by OpenSSL. Allow wildcard when it is the leftmost and the only character - in that segment. + in that segment. Partial wildcards like ``www*.example.com`` are no + longer supported. .. deprecated:: 3.7 @@ -599,31 +600,15 @@ Constants .. versionadded:: 3.7 -.. attribute:: HostFlags.HOSTFLAG_ALWAYS_CHECK_SUBJECT - - Consider subject CN even if the certificate contains at least one subject - alternative name. This flag violates :rfc:`6125`. - -.. attribute:: HostFlags.HOSTFLAG_NO_WILDCARDS - - Don't support wildcard certificate, e.g. ``*.example.org``. - .. attribute:: HostFlags.HOSTFLAG_NO_PARTIAL_WILDCARDS Dont' support wildcard certificate with partial matches, e.g. ``www*.example.org``. -.. attribute:: HostFlags.HOSTFLAG_MULTI_LABEL_WILDCARDS - - Wildcards match multiple labels, e.g. ``www.subdomain.example.org`` - matches ``*.example.org`` This flag violates :rfc:`6125`. - -.. attribute:: HostFlags.HOSTFLAG_SINGLE_LABEL_SUBDOMAINS - .. attribute:: HostFlags.HOSTFLAG_NEVER_CHECK_SUBJECT Ignore subject CN even if the certificate has no subject alternative - names. + name extension. .. note:: The flag is not available when the ssl module is compiled with OpenSSL 1.0.2 or LibreSSL. @@ -1781,8 +1766,7 @@ to speed up repeated connections from the same clients. .. attribute:: SSLContext.host_flags The flags for validating host names. By default - :data:`HostFlags.HOSTFLAG_SINGLE_LABEL_SUBDOMAINS` and - :data:`HostFlags.HOSTFLAG_NO_PARTIAL_WILDCARDS` are set. + :data:`HostFlags.HOSTFLAG_NO_PARTIAL_WILDCARDS` is set. .. versionadded:: 3.7 diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index 5e96390487b820..1130bced8ae0a4 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -590,7 +590,6 @@ def _on_handshake_complete(self, handshake_exc): raise handshake_exc peercert = sslobj.getpeercert() - # Since 3.7, the hostname is matched by OpenSSL during handshake. except BaseException as exc: if self._loop.get_debug(): if isinstance(exc, ssl.CertificateError): diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 68989f19b70be6..ef993f64cbea05 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -1511,6 +1511,16 @@ def test_bad_idna_in_server_hostname(self): ctx.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO(), server_hostname="xn--.com") + def test_bad_server_hostname(self): + ctx = ssl.create_default_context() + with self.assertRaises(ValueError): + ctx.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO(), + server_hostname="") + with self.assertRaises(ValueError): + ctx.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO(), + server_hostname=".example.org") + + class MemoryBIOTests(unittest.TestCase): def test_read_write(self): diff --git a/Modules/_ssl.c b/Modules/_ssl.c index cda2d4e5d54efe..786c2b91ae275b 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -713,17 +713,30 @@ _ssl_configure_hostname(PySSLSocket *self, const char* server_hostname) int retval = -1; ASN1_OCTET_STRING *ip; PyObject *hostname; + size_t len; assert(server_hostname); + /* Disable OpenSSL's special mode with leading dot in hostname: + * When name starts with a dot (e.g ".example.com"), it will be + * matched by a certificate valid for any sub-domain of name. + */ + len = strlen(server_hostname); + if (len == 0 || *server_hostname == '.') { + PyErr_SetString( + PyExc_ValueError, + "server_hostname cannot be an empty string or start with a " + "leading dot."); + return retval; + } + /* inet_pton is not available on all platforms. */ ip = a2i_IPADDRESS(server_hostname); if (ip == NULL) { ERR_clear_error(); } - hostname = PyUnicode_Decode(server_hostname, strlen(server_hostname), - "idna", "strict"); + hostname = PyUnicode_Decode(server_hostname, len, "idna", "strict"); if (hostname == NULL) { goto error; } @@ -2811,10 +2824,7 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) return NULL; } self->ctx = ctx; - self->hostflags = ( - X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS | - X509_CHECK_FLAG_SINGLE_LABEL_SUBDOMAINS - ); + self->hostflags = X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS; #if defined(OPENSSL_NPN_NEGOTIATED) && !defined(OPENSSL_NO_NEXTPROTONEG) self->npn_protocols = NULL; #endif @@ -5564,30 +5574,12 @@ PyInit__ssl(void) SSL_OP_NO_COMPRESSION); #endif -#ifdef X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT - PyModule_AddIntConstant(m, "HOSTFLAG_ALWAYS_CHECK_SUBJECT", - X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT); -#endif #ifdef X509_CHECK_FLAG_NEVER_CHECK_SUBJECT PyModule_AddIntConstant(m, "HOSTFLAG_NEVER_CHECK_SUBJECT", X509_CHECK_FLAG_NEVER_CHECK_SUBJECT); #endif -#ifdef X509_CHECK_FLAG_NO_WILDCARDS - PyModule_AddIntConstant(m, "HOSTFLAG_NO_WILDCARDS", - X509_CHECK_FLAG_NO_WILDCARDS); -#endif -#ifdef X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS PyModule_AddIntConstant(m, "HOSTFLAG_NO_PARTIAL_WILDCARDS", X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS); -#endif -#ifdef X509_CHECK_FLAG_MULTI_LABEL_WILDCARDS - PyModule_AddIntConstant(m, "HOSTFLAG_MULTI_LABEL_WILDCARDS", - X509_CHECK_FLAG_MULTI_LABEL_WILDCARDS); -#endif -#ifdef X509_CHECK_FLAG_SINGLE_LABEL_SUBDOMAINS - PyModule_AddIntConstant(m, "HOSTFLAG_SINGLE_LABEL_SUBDOMAINS", - X509_CHECK_FLAG_SINGLE_LABEL_SUBDOMAINS); -#endif #if HAVE_SNI r = Py_True; From 845c149b455b35de648ed2f71dc108685c306095 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Fri, 26 Jan 2018 16:26:12 +0100 Subject: [PATCH 7/7] Replace SSLContext.host_flags Host flags are now in internal API. Public API is a new attribute hostname_checks_common_name. Signed-off-by: Christian Heimes --- Doc/library/ssl.rst | 38 ++++++++----------- Lib/ssl.py | 28 ++++++++------ Lib/test/test_ssl.py | 13 +++++++ .../2017-09-08-14-05-33.bpo-31399.FtBrrt.rst | 5 ++- Modules/_ssl.c | 22 ++++++++++- 5 files changed, 69 insertions(+), 37 deletions(-) diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 7b816387b15cc5..aa1075d4b02b09 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -593,26 +593,6 @@ Constants be passed, either to :meth:`SSLContext.load_verify_locations` or as a value of the ``ca_certs`` parameter to :func:`wrap_socket`. -.. class:: HostFlags - - :class:`enum.IntFlag` collection of all hostname verification flags for - :attr:`SSLContext.host_flags` - - .. versionadded:: 3.7 - -.. attribute:: HostFlags.HOSTFLAG_NO_PARTIAL_WILDCARDS - - Dont' support wildcard certificate with partial matches, e.g. - ``www*.example.org``. - -.. attribute:: HostFlags.HOSTFLAG_NEVER_CHECK_SUBJECT - - Ignore subject CN even if the certificate has no subject alternative - name extension. - - .. note:: The flag is not available when the ssl module is compiled - with OpenSSL 1.0.2 or LibreSSL. - .. class:: VerifyMode :class:`enum.IntEnum` collection of CERT_* constants. @@ -877,6 +857,14 @@ Constants .. versionadded:: 3.5 +.. data:: HAS_NEVER_CHECK_COMMON_NAME + + Whether the OpenSSL library has built-in support not checking subject + common name and :attr:`SSLContext.hostname_checks_common_name` is + writeable. + + .. versionadded:: 3.7 + .. data:: HAS_ECDH Whether the OpenSSL library has built-in support for Elliptic Curve-based @@ -1763,13 +1751,17 @@ to speed up repeated connections from the same clients. The protocol version chosen when constructing the context. This attribute is read-only. -.. attribute:: SSLContext.host_flags +.. attribute:: SSLContext.hostname_checks_common_name - The flags for validating host names. By default - :data:`HostFlags.HOSTFLAG_NO_PARTIAL_WILDCARDS` is set. + Whether :attr:`~SSLContext.check_hostname` falls back to verify the cert's + subject common name in the absence of a subject alternative name + extension (default: true). .. versionadded:: 3.7 + .. note:: + Only writeable with OpenSSL 1.1.0 or higher. + .. attribute:: SSLContext.verify_flags The flags for certificate verification operations. You can set flags like diff --git a/Lib/ssl.py b/Lib/ssl.py index c9f5a4bf93aebd..bd82367bae192c 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -148,11 +148,6 @@ lambda name: name.startswith('CERT_'), source=_ssl) -_IntFlag._convert( - 'HostFlags', __name__, - lambda name: name.startswith('HOSTFLAG_'), - source=_ssl) - PROTOCOL_SSLv23 = _SSLMethod.PROTOCOL_SSLv23 = _SSLMethod.PROTOCOL_TLS _PROTOCOL_NAMES = {value: name for name, value in _SSLMethod.__members__.items()} @@ -176,6 +171,8 @@ else: CHANNEL_BINDING_TYPES = [] +HAS_NEVER_CHECK_COMMON_NAME = hasattr(_ssl, 'HOSTFLAG_NEVER_CHECK_SUBJECT') + # Disable weak or insecure ciphers by default # (OpenSSL's default setting is 'DEFAULT:!aNULL:!eNULL') @@ -475,13 +472,22 @@ def options(self): def options(self, value): super(SSLContext, SSLContext).options.__set__(self, value) - @property - def host_flags(self): - return HostFlags(super().host_flags) + if hasattr(_ssl, 'HOSTFLAG_NEVER_CHECK_SUBJECT'): + @property + def hostname_checks_common_name(self): + ncs = self._host_flags & _ssl.HOSTFLAG_NEVER_CHECK_SUBJECT + return ncs != _ssl.HOSTFLAG_NEVER_CHECK_SUBJECT - @host_flags.setter - def host_flags(self, value): - super(SSLContext, SSLContext).host_flags.__set__(self, value) + @hostname_checks_common_name.setter + def hostname_checks_common_name(self, value): + if value: + self._host_flags &= ~_ssl.HOSTFLAG_NEVER_CHECK_SUBJECT + else: + self._host_flags |= _ssl.HOSTFLAG_NEVER_CHECK_SUBJECT + else: + @property + def hostname_checks_common_name(self): + return True @property def verify_flags(self): diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index ef993f64cbea05..fdf727f788ca0d 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -988,6 +988,19 @@ def test_verify_mode_protocol(self): self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) self.assertTrue(ctx.check_hostname) + def test_hostname_checks_common_name(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertTrue(ctx.hostname_checks_common_name) + if ssl.HAS_NEVER_CHECK_COMMON_NAME: + ctx.hostname_checks_common_name = True + self.assertTrue(ctx.hostname_checks_common_name) + ctx.hostname_checks_common_name = False + self.assertFalse(ctx.hostname_checks_common_name) + ctx.hostname_checks_common_name = True + self.assertTrue(ctx.hostname_checks_common_name) + else: + with self.assertRaises(AttributeError): + ctx.hostname_checks_common_name = True @unittest.skipUnless(have_verify_flags(), "verify_flags need OpenSSL > 0.9.8") diff --git a/Misc/NEWS.d/next/Library/2017-09-08-14-05-33.bpo-31399.FtBrrt.rst b/Misc/NEWS.d/next/Library/2017-09-08-14-05-33.bpo-31399.FtBrrt.rst index b58b9c0a0c3cfa..e50ce2a1a93a01 100644 --- a/Misc/NEWS.d/next/Library/2017-09-08-14-05-33.bpo-31399.FtBrrt.rst +++ b/Misc/NEWS.d/next/Library/2017-09-08-14-05-33.bpo-31399.FtBrrt.rst @@ -1 +1,4 @@ -Let OpenSSL verify hostname and IP addressw +The ssl module now uses OpenSSL's X509_VERIFY_PARAM_set1_host() and +X509_VERIFY_PARAM_set1_ip() API to verify hostname and IP addresses. Subject +common name fallback can be disabled with +SSLContext.hostname_checks_common_name. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 786c2b91ae275b..ec8c8af54bdd10 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -4185,8 +4185,8 @@ _ssl__SSLContext_get_ca_certs_impl(PySSLContext *self, int binary_form) static PyGetSetDef context_getsetlist[] = { {"check_hostname", (getter) get_check_hostname, (setter) set_check_hostname, NULL}, - {"host_flags", (getter) get_host_flags, - (setter) set_host_flags, NULL}, + {"_host_flags", (getter) get_host_flags, + (setter) set_host_flags, NULL}, {"options", (getter) get_options, (setter) set_options, NULL}, {"verify_flags", (getter) get_verify_flags, @@ -5574,12 +5574,30 @@ PyInit__ssl(void) SSL_OP_NO_COMPRESSION); #endif +#ifdef X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT + PyModule_AddIntConstant(m, "HOSTFLAG_ALWAYS_CHECK_SUBJECT", + X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT); +#endif #ifdef X509_CHECK_FLAG_NEVER_CHECK_SUBJECT PyModule_AddIntConstant(m, "HOSTFLAG_NEVER_CHECK_SUBJECT", X509_CHECK_FLAG_NEVER_CHECK_SUBJECT); #endif +#ifdef X509_CHECK_FLAG_NO_WILDCARDS + PyModule_AddIntConstant(m, "HOSTFLAG_NO_WILDCARDS", + X509_CHECK_FLAG_NO_WILDCARDS); +#endif +#ifdef X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS PyModule_AddIntConstant(m, "HOSTFLAG_NO_PARTIAL_WILDCARDS", X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS); +#endif +#ifdef X509_CHECK_FLAG_MULTI_LABEL_WILDCARDS + PyModule_AddIntConstant(m, "HOSTFLAG_MULTI_LABEL_WILDCARDS", + X509_CHECK_FLAG_MULTI_LABEL_WILDCARDS); +#endif +#ifdef X509_CHECK_FLAG_SINGLE_LABEL_SUBDOMAINS + PyModule_AddIntConstant(m, "HOSTFLAG_SINGLE_LABEL_SUBDOMAINS", + X509_CHECK_FLAG_SINGLE_LABEL_SUBDOMAINS); +#endif #if HAVE_SNI r = Py_True;