From 4198f77855563f851e715208b0cb25cb0ff5bebd Mon Sep 17 00:00:00 2001 From: Arthur Hanson Date: Fri, 27 Sep 2024 16:05:26 -0700 Subject: [PATCH 01/12] Add support for socks connection to Git backend --- netbox/core/data_backends.py | 84 ++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/netbox/core/data_backends.py b/netbox/core/data_backends.py index 1b64f5f5c89..ea630eb2f3d 100644 --- a/netbox/core/data_backends.py +++ b/netbox/core/data_backends.py @@ -4,6 +4,8 @@ import tempfile from contextlib import contextmanager from pathlib import Path +from urllib3 import PoolManager, HTTPConnectionPool, HTTPSConnectionPool +from urllib3.connection import HTTPConnection, HTTPSConnection from urllib.parse import urlparse from django import forms @@ -23,6 +25,81 @@ logger = logging.getLogger('netbox.data_backends') +# These Proxy Methods are for handling socks connection proxy +class ProxyHTTPConnection(HTTPConnection): + use_rdns = False + + def __init__(self, *args, **kwargs): + socks_options = kwargs.pop('_socks_options') + self._proxy_url = socks_options['proxy_url'] + super().__init__(*args, **kwargs) + + def _new_conn(self): + from python_socks.sync import Proxy + proxy = Proxy.from_url(self._proxy_url, rdns=self.use_rdns) + return proxy.connect( + dest_host=self.host, + dest_port=self.port, + timeout=self.timeout + ) + + +class ProxyHTTPSConnection(ProxyHTTPConnection, HTTPSConnection): + pass + + +class rdnsProxyHTTPConnection(ProxyHTTPConnection): + use_rdns = True + + +class rdnsProxyHTTPSConnection(ProxyHTTPSConnection): + use_rdns = True + + +class ProxyHTTPConnectionPool(HTTPConnectionPool): + ConnectionCls = ProxyHTTPConnection + + +class ProxyHTTPSConnectionPool(HTTPSConnectionPool): + ConnectionCls = ProxyHTTPSConnection + + +class rdnsProxyHTTPConnectionPool(HTTPConnectionPool): + ConnectionCls = rdnsProxyHTTPConnection + + +class rdnsProxyHTTPSConnectionPool(HTTPSConnectionPool): + ConnectionCls = rdnsProxyHTTPSConnection + + +class ProxyPoolManager(PoolManager): + def __init__(self, proxy_url, timeout=5, num_pools=10, headers=None, + **connection_pool_kw): + + # python_socks uses rdns param to denote remote DNS parsing and + # doesn't accept the 'h' or 'a' in the proxy URL + cleaned_proxy_url = proxy_url + if use_rdns := cleaned_proxy_url.startswith(('socks5h:', 'socks5a:')): + cleaned_proxy_url = cleaned_proxy_url.replace('socks5h:', 'socks5:') + cleaned_proxy_url = cleaned_proxy_url.replace('socks5a:', 'socks5:') + + connection_pool_kw['_socks_options'] = {'proxy_url': cleaned_proxy_url} + connection_pool_kw['timeout'] = timeout + + super().__init__(num_pools, headers, **connection_pool_kw) + + if use_rdns: + self.pool_classes_by_scheme = { + 'http': rdnsProxyHTTPConnectionPool, + 'https': rdnsProxyHTTPSConnectionPool, + } + else: + self.pool_classes_by_scheme = { + 'http': ProxyHTTPConnectionPool, + 'https': ProxyHTTPSConnectionPool, + } + + @register_data_backend() class LocalBackend(DataBackend): name = 'local' @@ -67,11 +144,14 @@ def init_config(self): # Initialize backend config config = ConfigDict() + self.use_socks = False # Apply HTTP proxy (if configured) if settings.HTTP_PROXIES and self.url_scheme in ('http', 'https'): if proxy := settings.HTTP_PROXIES.get(self.url_scheme): config.set("http", "proxy", proxy) + if proxy.startswith('socks'): + self.use_socks = True return config @@ -87,6 +167,10 @@ def fetch(self): "errstream": porcelain.NoneStream(), } + # check if using socks for proxy - if so need to use custom pool_manager + if self.use_socks: + clone_args['pool_manager'] = ProxyPoolManager(settings.HTTP_PROXIES.get(self.url_scheme)) + if self.url_scheme in ('http', 'https'): if self.params.get('username'): clone_args.update( From 2d46bcdd18d7d310a1e7ec7d386baf90c4b5839a Mon Sep 17 00:00:00 2001 From: Arthur Hanson Date: Fri, 27 Sep 2024 16:47:14 -0700 Subject: [PATCH 02/12] cleanup socks detection --- netbox/core/data_backends.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/netbox/core/data_backends.py b/netbox/core/data_backends.py index ea630eb2f3d..f3012fe0f9d 100644 --- a/netbox/core/data_backends.py +++ b/netbox/core/data_backends.py @@ -79,9 +79,9 @@ def __init__(self, proxy_url, timeout=5, num_pools=10, headers=None, # python_socks uses rdns param to denote remote DNS parsing and # doesn't accept the 'h' or 'a' in the proxy URL cleaned_proxy_url = proxy_url - if use_rdns := cleaned_proxy_url.startswith(('socks5h:', 'socks5a:')): - cleaned_proxy_url = cleaned_proxy_url.replace('socks5h:', 'socks5:') - cleaned_proxy_url = cleaned_proxy_url.replace('socks5a:', 'socks5:') + if use_rdns := urlparse(cleaned_proxy_url).scheme.lower() in ['socks4h', 'socks4a' 'socks5h', 'socks5a']: + cleaned_proxy_url = cleaned_proxy_url.replace('socks5h:', 'socks5:').replace('socks5a:', 'socks5:') + cleaned_proxy_url = cleaned_proxy_url.replace('socks4h:', 'socks4:').replace('socks4a:', 'socks4:') connection_pool_kw['_socks_options'] = {'proxy_url': cleaned_proxy_url} connection_pool_kw['timeout'] = timeout @@ -150,7 +150,7 @@ def init_config(self): if settings.HTTP_PROXIES and self.url_scheme in ('http', 'https'): if proxy := settings.HTTP_PROXIES.get(self.url_scheme): config.set("http", "proxy", proxy) - if proxy.startswith('socks'): + if urlparse(proxy).scheme.lower() in ['socks4', 'socks4a', 'socks4h', 'socks5', 'socks5a', 'socks5h']: self.use_socks = True return config From 01ec68d80a2186c1e23b82114f5e2fe3dcd50bed Mon Sep 17 00:00:00 2001 From: Arthur Hanson Date: Fri, 27 Sep 2024 17:05:39 -0700 Subject: [PATCH 03/12] add documentation for installing python_socks --- docs/features/synchronized-data.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/features/synchronized-data.md b/docs/features/synchronized-data.md index 8c95c87791f..95656f6d13b 100644 --- a/docs/features/synchronized-data.md +++ b/docs/features/synchronized-data.md @@ -13,6 +13,9 @@ To enable remote data synchronization, the NetBox administrator first designates !!! info Data backends which connect to external sources typically require the installation of one or more supporting Python libraries. The Git backend requires the [`dulwich`](https://www.dulwich.io/) package, and the S3 backend requires the [`boto3`](https://boto3.amazonaws.com/v1/documentation/api/latest/index.html) package. These must be installed within NetBox's environment to enable these backends. +!!! info + If you are configuring Git and have HTTP_PROXIES set to use the SOCKS protocol, you will also need to install the [`python_socks`](https://pypi.org/project/python-socks/) package. + Each type of remote source has its own configuration parameters. For instance, a git source will ask the user to specify a branch and authentication credentials. Once the source has been created, a synchronization job is run to automatically replicate remote files in the local database. The following NetBox models can be associated with replicated data files: From c287e868a3b31ccc07bf86b9f3c3ad38223ef58b Mon Sep 17 00:00:00 2001 From: Arthur Hanson Date: Fri, 27 Sep 2024 17:13:17 -0700 Subject: [PATCH 04/12] dont need lower() --- netbox/core/data_backends.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/netbox/core/data_backends.py b/netbox/core/data_backends.py index f3012fe0f9d..488a5e61ccc 100644 --- a/netbox/core/data_backends.py +++ b/netbox/core/data_backends.py @@ -79,7 +79,7 @@ def __init__(self, proxy_url, timeout=5, num_pools=10, headers=None, # python_socks uses rdns param to denote remote DNS parsing and # doesn't accept the 'h' or 'a' in the proxy URL cleaned_proxy_url = proxy_url - if use_rdns := urlparse(cleaned_proxy_url).scheme.lower() in ['socks4h', 'socks4a' 'socks5h', 'socks5a']: + if use_rdns := urlparse(cleaned_proxy_url).scheme in ['socks4h', 'socks4a' 'socks5h', 'socks5a']: cleaned_proxy_url = cleaned_proxy_url.replace('socks5h:', 'socks5:').replace('socks5a:', 'socks5:') cleaned_proxy_url = cleaned_proxy_url.replace('socks4h:', 'socks4:').replace('socks4a:', 'socks4:') @@ -150,7 +150,7 @@ def init_config(self): if settings.HTTP_PROXIES and self.url_scheme in ('http', 'https'): if proxy := settings.HTTP_PROXIES.get(self.url_scheme): config.set("http", "proxy", proxy) - if urlparse(proxy).scheme.lower() in ['socks4', 'socks4a', 'socks4h', 'socks5', 'socks5a', 'socks5h']: + if urlparse(proxy).scheme in ['socks4', 'socks4a', 'socks4h', 'socks5', 'socks5a', 'socks5h']: self.use_socks = True return config From ed64af0c59a8dd4b7e46968a61655e67008a14db Mon Sep 17 00:00:00 2001 From: Arthur Hanson Date: Fri, 27 Sep 2024 17:18:31 -0700 Subject: [PATCH 05/12] cleanup --- netbox/core/data_backends.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/netbox/core/data_backends.py b/netbox/core/data_backends.py index 488a5e61ccc..935cb3aa9b7 100644 --- a/netbox/core/data_backends.py +++ b/netbox/core/data_backends.py @@ -25,7 +25,7 @@ logger = logging.getLogger('netbox.data_backends') -# These Proxy Methods are for handling socks connection proxy +# These Proxy Methods are for handling SOCKS connection proxy class ProxyHTTPConnection(HTTPConnection): use_rdns = False @@ -73,9 +73,7 @@ class rdnsProxyHTTPSConnectionPool(HTTPSConnectionPool): class ProxyPoolManager(PoolManager): - def __init__(self, proxy_url, timeout=5, num_pools=10, headers=None, - **connection_pool_kw): - + def __init__(self, proxy_url, timeout=5, num_pools=10, headers=None, **connection_pool_kw): # python_socks uses rdns param to denote remote DNS parsing and # doesn't accept the 'h' or 'a' in the proxy URL cleaned_proxy_url = proxy_url From c4915ae521c7a167f6c23a0c2338df938af2335f Mon Sep 17 00:00:00 2001 From: Arthur Hanson Date: Mon, 30 Sep 2024 07:28:31 -0700 Subject: [PATCH 06/12] refactor Socks to utilities --- netbox/core/data_backends.py | 76 +----------------------------------- netbox/utilities/socks.py | 75 +++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 75 deletions(-) create mode 100644 netbox/utilities/socks.py diff --git a/netbox/core/data_backends.py b/netbox/core/data_backends.py index 935cb3aa9b7..72c7bae5386 100644 --- a/netbox/core/data_backends.py +++ b/netbox/core/data_backends.py @@ -4,8 +4,6 @@ import tempfile from contextlib import contextmanager from pathlib import Path -from urllib3 import PoolManager, HTTPConnectionPool, HTTPSConnectionPool -from urllib3.connection import HTTPConnection, HTTPSConnection from urllib.parse import urlparse from django import forms @@ -14,6 +12,7 @@ from netbox.data_backends import DataBackend from netbox.utils import register_data_backend +from utilities.socks import ProxyPoolManager from .exceptions import SyncError __all__ = ( @@ -25,79 +24,6 @@ logger = logging.getLogger('netbox.data_backends') -# These Proxy Methods are for handling SOCKS connection proxy -class ProxyHTTPConnection(HTTPConnection): - use_rdns = False - - def __init__(self, *args, **kwargs): - socks_options = kwargs.pop('_socks_options') - self._proxy_url = socks_options['proxy_url'] - super().__init__(*args, **kwargs) - - def _new_conn(self): - from python_socks.sync import Proxy - proxy = Proxy.from_url(self._proxy_url, rdns=self.use_rdns) - return proxy.connect( - dest_host=self.host, - dest_port=self.port, - timeout=self.timeout - ) - - -class ProxyHTTPSConnection(ProxyHTTPConnection, HTTPSConnection): - pass - - -class rdnsProxyHTTPConnection(ProxyHTTPConnection): - use_rdns = True - - -class rdnsProxyHTTPSConnection(ProxyHTTPSConnection): - use_rdns = True - - -class ProxyHTTPConnectionPool(HTTPConnectionPool): - ConnectionCls = ProxyHTTPConnection - - -class ProxyHTTPSConnectionPool(HTTPSConnectionPool): - ConnectionCls = ProxyHTTPSConnection - - -class rdnsProxyHTTPConnectionPool(HTTPConnectionPool): - ConnectionCls = rdnsProxyHTTPConnection - - -class rdnsProxyHTTPSConnectionPool(HTTPSConnectionPool): - ConnectionCls = rdnsProxyHTTPSConnection - - -class ProxyPoolManager(PoolManager): - def __init__(self, proxy_url, timeout=5, num_pools=10, headers=None, **connection_pool_kw): - # python_socks uses rdns param to denote remote DNS parsing and - # doesn't accept the 'h' or 'a' in the proxy URL - cleaned_proxy_url = proxy_url - if use_rdns := urlparse(cleaned_proxy_url).scheme in ['socks4h', 'socks4a' 'socks5h', 'socks5a']: - cleaned_proxy_url = cleaned_proxy_url.replace('socks5h:', 'socks5:').replace('socks5a:', 'socks5:') - cleaned_proxy_url = cleaned_proxy_url.replace('socks4h:', 'socks4:').replace('socks4a:', 'socks4:') - - connection_pool_kw['_socks_options'] = {'proxy_url': cleaned_proxy_url} - connection_pool_kw['timeout'] = timeout - - super().__init__(num_pools, headers, **connection_pool_kw) - - if use_rdns: - self.pool_classes_by_scheme = { - 'http': rdnsProxyHTTPConnectionPool, - 'https': rdnsProxyHTTPSConnectionPool, - } - else: - self.pool_classes_by_scheme = { - 'http': ProxyHTTPConnectionPool, - 'https': ProxyHTTPSConnectionPool, - } - - @register_data_backend() class LocalBackend(DataBackend): name = 'local' diff --git a/netbox/utilities/socks.py b/netbox/utilities/socks.py new file mode 100644 index 00000000000..30cc782bc6e --- /dev/null +++ b/netbox/utilities/socks.py @@ -0,0 +1,75 @@ +from urllib3 import PoolManager, HTTPConnectionPool, HTTPSConnectionPool +from urllib3.connection import HTTPConnection, HTTPSConnection + + +# These Proxy Methods are for handling SOCKS connection proxy +class ProxyHTTPConnection(HTTPConnection): + use_rdns = False + + def __init__(self, *args, **kwargs): + socks_options = kwargs.pop('_socks_options') + self._proxy_url = socks_options['proxy_url'] + super().__init__(*args, **kwargs) + + def _new_conn(self): + from python_socks.sync import Proxy + proxy = Proxy.from_url(self._proxy_url, rdns=self.use_rdns) + return proxy.connect( + dest_host=self.host, + dest_port=self.port, + timeout=self.timeout + ) + + +class ProxyHTTPSConnection(ProxyHTTPConnection, HTTPSConnection): + pass + + +class RdnsProxyHTTPConnection(ProxyHTTPConnection): + use_rdns = True + + +class RdnsProxyHTTPSConnection(ProxyHTTPSConnection): + use_rdns = True + + +class ProxyHTTPConnectionPool(HTTPConnectionPool): + ConnectionCls = ProxyHTTPConnection + + +class ProxyHTTPSConnectionPool(HTTPSConnectionPool): + ConnectionCls = ProxyHTTPSConnection + + +class RdnsProxyHTTPConnectionPool(HTTPConnectionPool): + ConnectionCls = RdnsProxyHTTPConnection + + +class RdnsProxyHTTPSConnectionPool(HTTPSConnectionPool): + ConnectionCls = RdnsProxyHTTPSConnection + + +class ProxyPoolManager(PoolManager): + def __init__(self, proxy_url, timeout=5, num_pools=10, headers=None, **connection_pool_kw): + # python_socks uses rdns param to denote remote DNS parsing and + # doesn't accept the 'h' or 'a' in the proxy URL + cleaned_proxy_url = proxy_url + if use_rdns := urlparse(cleaned_proxy_url).scheme in ['socks4h', 'socks4a' 'socks5h', 'socks5a']: + cleaned_proxy_url = cleaned_proxy_url.replace('socks5h:', 'socks5:').replace('socks5a:', 'socks5:') + cleaned_proxy_url = cleaned_proxy_url.replace('socks4h:', 'socks4:').replace('socks4a:', 'socks4:') + + connection_pool_kw['_socks_options'] = {'proxy_url': cleaned_proxy_url} + connection_pool_kw['timeout'] = timeout + + super().__init__(num_pools, headers, **connection_pool_kw) + + if use_rdns: + self.pool_classes_by_scheme = { + 'http': RdnsProxyHTTPConnectionPool, + 'https': RdnsProxyHTTPSConnectionPool, + } + else: + self.pool_classes_by_scheme = { + 'http': ProxyHTTPConnectionPool, + 'https': ProxyHTTPSConnectionPool, + } From 76a0403d7e8366daf0fc81b34d0591a74b032b01 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Date: Mon, 30 Sep 2024 08:43:42 -0700 Subject: [PATCH 07/12] fix imports --- netbox/utilities/socks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/netbox/utilities/socks.py b/netbox/utilities/socks.py index 30cc782bc6e..5d7367a662c 100644 --- a/netbox/utilities/socks.py +++ b/netbox/utilities/socks.py @@ -1,3 +1,4 @@ +from urllib.parse import urlparse from urllib3 import PoolManager, HTTPConnectionPool, HTTPSConnectionPool from urllib3.connection import HTTPConnection, HTTPSConnection From dadbfac207e7c6e5ae73f1fa2a2aef2eba392f2e Mon Sep 17 00:00:00 2001 From: Arthur Hanson Date: Mon, 30 Sep 2024 09:54:29 -0700 Subject: [PATCH 08/12] fix missing comma --- netbox/utilities/socks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/utilities/socks.py b/netbox/utilities/socks.py index 5d7367a662c..0b5864e449f 100644 --- a/netbox/utilities/socks.py +++ b/netbox/utilities/socks.py @@ -55,7 +55,7 @@ def __init__(self, proxy_url, timeout=5, num_pools=10, headers=None, **connectio # python_socks uses rdns param to denote remote DNS parsing and # doesn't accept the 'h' or 'a' in the proxy URL cleaned_proxy_url = proxy_url - if use_rdns := urlparse(cleaned_proxy_url).scheme in ['socks4h', 'socks4a' 'socks5h', 'socks5a']: + if use_rdns := urlparse(cleaned_proxy_url).scheme in ['socks4h', 'socks4a', 'socks5h', 'socks5a']: cleaned_proxy_url = cleaned_proxy_url.replace('socks5h:', 'socks5:').replace('socks5a:', 'socks5:') cleaned_proxy_url = cleaned_proxy_url.replace('socks4h:', 'socks4:').replace('socks4a:', 'socks4:') From d40b9283942a4b9f66cd04072c394f61c69173d8 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Date: Mon, 30 Sep 2024 10:54:03 -0700 Subject: [PATCH 09/12] Update docs/features/synchronized-data.md Co-authored-by: Jeremy Stretch --- docs/features/synchronized-data.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/features/synchronized-data.md b/docs/features/synchronized-data.md index 95656f6d13b..23c79feed88 100644 --- a/docs/features/synchronized-data.md +++ b/docs/features/synchronized-data.md @@ -14,7 +14,7 @@ To enable remote data synchronization, the NetBox administrator first designates Data backends which connect to external sources typically require the installation of one or more supporting Python libraries. The Git backend requires the [`dulwich`](https://www.dulwich.io/) package, and the S3 backend requires the [`boto3`](https://boto3.amazonaws.com/v1/documentation/api/latest/index.html) package. These must be installed within NetBox's environment to enable these backends. !!! info - If you are configuring Git and have HTTP_PROXIES set to use the SOCKS protocol, you will also need to install the [`python_socks`](https://pypi.org/project/python-socks/) package. + If you are configuring Git and have `HTTP_PROXIES` configured to use the SOCKS protocol, you will also need to install the [`python_socks`](https://pypi.org/project/python-socks/) Python library. Each type of remote source has its own configuration parameters. For instance, a git source will ask the user to specify a branch and authentication credentials. Once the source has been created, a synchronization job is run to automatically replicate remote files in the local database. From 0a5de27335d8a39ec438ca2b24fd7e131f1eb431 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Date: Mon, 30 Sep 2024 11:46:35 -0700 Subject: [PATCH 10/12] review feedback --- netbox/core/data_backends.py | 16 +++++++++++----- netbox/utilities/constants.py | 4 ++++ netbox/utilities/socks.py | 27 +++++++++++++++++++++------ 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/netbox/core/data_backends.py b/netbox/core/data_backends.py index 72c7bae5386..f20ac017c1f 100644 --- a/netbox/core/data_backends.py +++ b/netbox/core/data_backends.py @@ -8,10 +8,12 @@ from django import forms from django.conf import settings +from django.core.exceptions import ImproperlyConfigured from django.utils.translation import gettext as _ from netbox.data_backends import DataBackend from netbox.utils import register_data_backend +from utilities.constants import DATA_SOURCE_SUPPORTED_SCHEMAS, DATA_SOURCE_SUPPORTED_SOCK_SCHEMAS from utilities.socks import ProxyPoolManager from .exceptions import SyncError @@ -71,11 +73,15 @@ def init_config(self): self.use_socks = False # Apply HTTP proxy (if configured) - if settings.HTTP_PROXIES and self.url_scheme in ('http', 'https'): - if proxy := settings.HTTP_PROXIES.get(self.url_scheme): - config.set("http", "proxy", proxy) - if urlparse(proxy).scheme in ['socks4', 'socks4a', 'socks4h', 'socks5', 'socks5a', 'socks5h']: - self.use_socks = True + if settings.HTTP_PROXIES: + if proxy := settings.HTTP_PROXIES.get(self.url_scheme, None): + if urlparse(proxy).scheme not in DATA_SOURCE_SUPPORTED_SCHEMAS: + raise ImproperlyConfigured(f"Unsupported Git DataSource proxy scheme: {urlparse(proxy).scheme}") + + if self.url_scheme in ('http', 'https'): + config.set("http", "proxy", proxy) + if urlparse(proxy).scheme in DATA_SOURCE_SUPPORTED_SOCK_SCHEMAS: + self.use_socks = True return config diff --git a/netbox/utilities/constants.py b/netbox/utilities/constants.py index c7c26f6b3ab..3ec6fd2dde1 100644 --- a/netbox/utilities/constants.py +++ b/netbox/utilities/constants.py @@ -93,3 +93,7 @@ "td": {"align"}, "th": {"align"}, } + +DATA_SOURCE_SUPPORTED_SOCK_SCHEMAS = ['socks4', 'socks4a', 'socks4h', 'socks5', 'socks5a', 'socks5h'] +DATA_SOURCE_SOCK_RDNS_SCHEMAS = ['socks4h', 'socks4a', 'socks5h', 'socks5a'] +DATA_SOURCE_SUPPORTED_SCHEMAS = ['http', 'https', 'socks4', 'socks4a', 'socks4h', 'socks5', 'socks5a', 'socks5h'] diff --git a/netbox/utilities/socks.py b/netbox/utilities/socks.py index 0b5864e449f..8df82fb7a2d 100644 --- a/netbox/utilities/socks.py +++ b/netbox/utilities/socks.py @@ -1,10 +1,15 @@ from urllib.parse import urlparse from urllib3 import PoolManager, HTTPConnectionPool, HTTPSConnectionPool from urllib3.connection import HTTPConnection, HTTPSConnection +from .constants import DATA_SOURCE_SOCK_RDNS_SCHEMAS -# These Proxy Methods are for handling SOCKS connection proxy class ProxyHTTPConnection(HTTPConnection): + """ + A Proxy connection class that uses a SOCK proxy - used to create + a urllib3 PoolManager that routes connections via the proxy. + This is for an HTTP (not HTTPS) connection + """ use_rdns = False def __init__(self, *args, **kwargs): @@ -23,14 +28,25 @@ def _new_conn(self): class ProxyHTTPSConnection(ProxyHTTPConnection, HTTPSConnection): + """ + A Proxy connection class for an HTTPS (not HTTP) connection. + """ pass class RdnsProxyHTTPConnection(ProxyHTTPConnection): + """ + A Proxy connection class for an HTTP remote-dns connection. + I.E. socks4a, socks4h, socks5a, socks5h + """ use_rdns = True class RdnsProxyHTTPSConnection(ProxyHTTPSConnection): + """ + A Proxy connection class for an HTTPS remote-dns connection. + I.E. socks4a, socks4h, socks5a, socks5h + """ use_rdns = True @@ -54,12 +70,11 @@ class ProxyPoolManager(PoolManager): def __init__(self, proxy_url, timeout=5, num_pools=10, headers=None, **connection_pool_kw): # python_socks uses rdns param to denote remote DNS parsing and # doesn't accept the 'h' or 'a' in the proxy URL - cleaned_proxy_url = proxy_url - if use_rdns := urlparse(cleaned_proxy_url).scheme in ['socks4h', 'socks4a', 'socks5h', 'socks5a']: - cleaned_proxy_url = cleaned_proxy_url.replace('socks5h:', 'socks5:').replace('socks5a:', 'socks5:') - cleaned_proxy_url = cleaned_proxy_url.replace('socks4h:', 'socks4:').replace('socks4a:', 'socks4:') + if use_rdns := urlparse(proxy_url).scheme in DATA_SOURCE_SOCK_RDNS_SCHEMAS: + proxy_url = proxy_url.replace('socks5h:', 'socks5:').replace('socks5a:', 'socks5:') + proxy_url = proxy_url.replace('socks4h:', 'socks4:').replace('socks4a:', 'socks4:') - connection_pool_kw['_socks_options'] = {'proxy_url': cleaned_proxy_url} + connection_pool_kw['_socks_options'] = {'proxy_url': proxy_url} connection_pool_kw['timeout'] = timeout super().__init__(num_pools, headers, **connection_pool_kw) From 921091d57fed249c1db6ef84c9e7a0f863c2e474 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Date: Mon, 30 Sep 2024 10:54:03 -0700 Subject: [PATCH 11/12] Update docs/features/synchronized-data.md Co-authored-by: Jeremy Stretch --- docs/features/synchronized-data.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/features/synchronized-data.md b/docs/features/synchronized-data.md index 95656f6d13b..23c79feed88 100644 --- a/docs/features/synchronized-data.md +++ b/docs/features/synchronized-data.md @@ -14,7 +14,7 @@ To enable remote data synchronization, the NetBox administrator first designates Data backends which connect to external sources typically require the installation of one or more supporting Python libraries. The Git backend requires the [`dulwich`](https://www.dulwich.io/) package, and the S3 backend requires the [`boto3`](https://boto3.amazonaws.com/v1/documentation/api/latest/index.html) package. These must be installed within NetBox's environment to enable these backends. !!! info - If you are configuring Git and have HTTP_PROXIES set to use the SOCKS protocol, you will also need to install the [`python_socks`](https://pypi.org/project/python-socks/) package. + If you are configuring Git and have `HTTP_PROXIES` configured to use the SOCKS protocol, you will also need to install the [`python_socks`](https://pypi.org/project/python-socks/) Python library. Each type of remote source has its own configuration parameters. For instance, a git source will ask the user to specify a branch and authentication credentials. Once the source has been created, a synchronization job is run to automatically replicate remote files in the local database. From fe991661140fc9e830a7aea4d684f0e515070cb0 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Date: Mon, 30 Sep 2024 16:51:02 -0700 Subject: [PATCH 12/12] review changes --- netbox/core/data_backends.py | 6 +++--- netbox/utilities/constants.py | 6 +++--- netbox/utilities/socks.py | 16 +++++++++++++--- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/netbox/core/data_backends.py b/netbox/core/data_backends.py index f20ac017c1f..8b36c699588 100644 --- a/netbox/core/data_backends.py +++ b/netbox/core/data_backends.py @@ -13,7 +13,7 @@ from netbox.data_backends import DataBackend from netbox.utils import register_data_backend -from utilities.constants import DATA_SOURCE_SUPPORTED_SCHEMAS, DATA_SOURCE_SUPPORTED_SOCK_SCHEMAS +from utilities.constants import HTTP_PROXY_SUPPORTED_SCHEMAS, HTTP_PROXY_SUPPORTED_SOCK_SCHEMAS from utilities.socks import ProxyPoolManager from .exceptions import SyncError @@ -75,12 +75,12 @@ def init_config(self): # Apply HTTP proxy (if configured) if settings.HTTP_PROXIES: if proxy := settings.HTTP_PROXIES.get(self.url_scheme, None): - if urlparse(proxy).scheme not in DATA_SOURCE_SUPPORTED_SCHEMAS: + if urlparse(proxy).scheme not in HTTP_PROXY_SUPPORTED_SCHEMAS: raise ImproperlyConfigured(f"Unsupported Git DataSource proxy scheme: {urlparse(proxy).scheme}") if self.url_scheme in ('http', 'https'): config.set("http", "proxy", proxy) - if urlparse(proxy).scheme in DATA_SOURCE_SUPPORTED_SOCK_SCHEMAS: + if urlparse(proxy).scheme in HTTP_PROXY_SUPPORTED_SOCK_SCHEMAS: self.use_socks = True return config diff --git a/netbox/utilities/constants.py b/netbox/utilities/constants.py index 3ec6fd2dde1..2b93f2b963c 100644 --- a/netbox/utilities/constants.py +++ b/netbox/utilities/constants.py @@ -94,6 +94,6 @@ "th": {"align"}, } -DATA_SOURCE_SUPPORTED_SOCK_SCHEMAS = ['socks4', 'socks4a', 'socks4h', 'socks5', 'socks5a', 'socks5h'] -DATA_SOURCE_SOCK_RDNS_SCHEMAS = ['socks4h', 'socks4a', 'socks5h', 'socks5a'] -DATA_SOURCE_SUPPORTED_SCHEMAS = ['http', 'https', 'socks4', 'socks4a', 'socks4h', 'socks5', 'socks5a', 'socks5h'] +HTTP_PROXY_SUPPORTED_SOCK_SCHEMAS = ['socks4', 'socks4a', 'socks4h', 'socks5', 'socks5a', 'socks5h'] +HTTP_PROXY_SOCK_RDNS_SCHEMAS = ['socks4h', 'socks4a', 'socks5h', 'socks5a'] +HTTP_PROXY_SUPPORTED_SCHEMAS = ['http', 'https', 'socks4', 'socks4a', 'socks4h', 'socks5', 'socks5a', 'socks5h'] diff --git a/netbox/utilities/socks.py b/netbox/utilities/socks.py index 8df82fb7a2d..bb0b6b250c7 100644 --- a/netbox/utilities/socks.py +++ b/netbox/utilities/socks.py @@ -1,7 +1,12 @@ +import logging + from urllib.parse import urlparse from urllib3 import PoolManager, HTTPConnectionPool, HTTPSConnectionPool from urllib3.connection import HTTPConnection, HTTPSConnection -from .constants import DATA_SOURCE_SOCK_RDNS_SCHEMAS +from .constants import HTTP_PROXY_SOCK_RDNS_SCHEMAS + + +logger = logging.getLogger('netbox.utilities') class ProxyHTTPConnection(HTTPConnection): @@ -18,7 +23,12 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) def _new_conn(self): - from python_socks.sync import Proxy + try: + from python_socks.sync import Proxy + except ModuleNotFoundError as e: + logger.info("Configuring an HTTP proxy using SOCKS requires the python_socks library. Check that it has been installed.") + raise e + proxy = Proxy.from_url(self._proxy_url, rdns=self.use_rdns) return proxy.connect( dest_host=self.host, @@ -70,7 +80,7 @@ class ProxyPoolManager(PoolManager): def __init__(self, proxy_url, timeout=5, num_pools=10, headers=None, **connection_pool_kw): # python_socks uses rdns param to denote remote DNS parsing and # doesn't accept the 'h' or 'a' in the proxy URL - if use_rdns := urlparse(proxy_url).scheme in DATA_SOURCE_SOCK_RDNS_SCHEMAS: + if use_rdns := urlparse(proxy_url).scheme in HTTP_PROXY_SOCK_RDNS_SCHEMAS: proxy_url = proxy_url.replace('socks5h:', 'socks5:').replace('socks5a:', 'socks5:') proxy_url = proxy_url.replace('socks4h:', 'socks4:').replace('socks4a:', 'socks4:')