From d9bd543653896a318f3bbbc8938845a6a9c858df Mon Sep 17 00:00:00 2001 From: Ben Stockermans Date: Tue, 8 Jul 2025 16:24:22 +0100 Subject: [PATCH 1/7] Added PROXIES setting. --- linkcheck/linkcheck_settings.py | 1 + linkcheck/models.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/linkcheck/linkcheck_settings.py b/linkcheck/linkcheck_settings.py index 566be34..bedbc2f 100644 --- a/linkcheck/linkcheck_settings.py +++ b/linkcheck/linkcheck_settings.py @@ -59,3 +59,4 @@ SITE_DOMAINS = getattr(settings, 'LINKCHECK_SITE_DOMAINS', []) DISABLE_LISTENERS = getattr(settings, 'LINKCHECK_DISABLE_LISTENERS', False) TOLERATE_BROKEN_ANCHOR = getattr(settings, 'LINKCHECK_TOLERATE_BROKEN_ANCHOR', True) +PROXIES = getattr(settings, 'LINKCHECK_PROXIES', {}) diff --git a/linkcheck/models.py b/linkcheck/models.py index 536fb50..d01f80f 100644 --- a/linkcheck/models.py +++ b/linkcheck/models.py @@ -31,6 +31,7 @@ LINKCHECK_CONNECTION_ATTEMPT_TIMEOUT, MAX_URL_LENGTH, MEDIA_PREFIX, + PROXIES, SITE_DOMAINS, TOLERATE_BROKEN_ANCHOR, ) @@ -386,6 +387,9 @@ def check_external(self, external_recheck_interval=EXTERNAL_RECHECK_INTERVAL): "timeout": LINKCHECK_CONNECTION_ATTEMPT_TIMEOUT, "verify": True, } + if PROXIES: + request_params["proxies"] = PROXIES + try: try: # At first try a HEAD request From e811d827586c6dd39f8f51b452dee4292872d361 Mon Sep 17 00:00:00 2001 From: Ben Stockermans Date: Wed, 9 Jul 2025 16:22:12 +0100 Subject: [PATCH 2/7] Added TRUST_PROXY_SSL setting. --- linkcheck/linkcheck_settings.py | 1 + linkcheck/models.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/linkcheck/linkcheck_settings.py b/linkcheck/linkcheck_settings.py index bedbc2f..617aef4 100644 --- a/linkcheck/linkcheck_settings.py +++ b/linkcheck/linkcheck_settings.py @@ -60,3 +60,4 @@ DISABLE_LISTENERS = getattr(settings, 'LINKCHECK_DISABLE_LISTENERS', False) TOLERATE_BROKEN_ANCHOR = getattr(settings, 'LINKCHECK_TOLERATE_BROKEN_ANCHOR', True) PROXIES = getattr(settings, 'LINKCHECK_PROXIES', {}) +TRUST_PROXY_SSL = getattr(settings, 'LINKCHECK_TRUST_PROXY_SSL', False) diff --git a/linkcheck/models.py b/linkcheck/models.py index d01f80f..d7c189f 100644 --- a/linkcheck/models.py +++ b/linkcheck/models.py @@ -34,6 +34,7 @@ PROXIES, SITE_DOMAINS, TOLERATE_BROKEN_ANCHOR, + TRUST_PROXY_SSL, ) logger = logging.getLogger(__name__) @@ -388,6 +389,7 @@ def check_external(self, external_recheck_interval=EXTERNAL_RECHECK_INTERVAL): "verify": True, } if PROXIES: + request_params["verify"] = not TRUST_PROXY_SSL request_params["proxies"] = PROXIES try: From 8df4c592c2fae8adead96fade22a014ecdcc42d3 Mon Sep 17 00:00:00 2001 From: Ben Stockermans Date: Fri, 11 Jul 2025 14:19:54 +0100 Subject: [PATCH 3/7] Added test_external_proxy_request. --- linkcheck/tests/test_linkcheck.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/linkcheck/tests/test_linkcheck.py b/linkcheck/tests/test_linkcheck.py index 8a3f1e1..1c36704 100644 --- a/linkcheck/tests/test_linkcheck.py +++ b/linkcheck/tests/test_linkcheck.py @@ -1,7 +1,7 @@ import os from datetime import datetime, timedelta from io import StringIO -from unittest.mock import patch +from unittest.mock import Mock, patch import requests_mock import urllib3 @@ -672,6 +672,25 @@ def test_external_check_blocked_user_agent_blocked_head(self): self.assertEqual(uv.redirect_to, '') self.assertEqual(uv.type, 'external') + @patch('linkcheck.models.PROXIES', {'http': 'http://proxy.example.com:8080'}) + @patch('requests.head') + def test_external_proxy_request(self, mock_head): + mock_response = Mock() + mock_response.status_code = 200 + mock_response.reason = 'OK' + mock_response.history = [] + mock_head.return_value = mock_response + request_url = 'http://test.com' + uv = Url(url=request_url) + uv.check_url() + self.assertEqual(uv.status, True) + self.assertEqual(uv.message, '200 OK') + self.assertEqual(uv.type, 'external') + mock_head.assert_called_once() + (call_url,), call_kwargs = mock_head.call_args + self.assertEqual(call_url, request_url) + self.assertEqual(call_kwargs.get('proxies'), {'http': 'http://proxy.example.com:8080'}) + def test_external_check_timedout(self): uv = Url(url=f"{self.live_server_url}/timeout/") uv.check_url() From c35ab1b09c93b80ef06d2160450efd3686d72167 Mon Sep 17 00:00:00 2001 From: Ben Stockermans Date: Fri, 11 Jul 2025 16:38:23 +0100 Subject: [PATCH 4/7] Linting. --- linkcheck/tests/test_linkcheck.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/linkcheck/tests/test_linkcheck.py b/linkcheck/tests/test_linkcheck.py index 1c36704..c67f46f 100644 --- a/linkcheck/tests/test_linkcheck.py +++ b/linkcheck/tests/test_linkcheck.py @@ -672,7 +672,10 @@ def test_external_check_blocked_user_agent_blocked_head(self): self.assertEqual(uv.redirect_to, '') self.assertEqual(uv.type, 'external') - @patch('linkcheck.models.PROXIES', {'http': 'http://proxy.example.com:8080'}) + @patch( + 'linkcheck.models.PROXIES', + {'http': 'http://proxy.example.com:8080'}, + ) @patch('requests.head') def test_external_proxy_request(self, mock_head): mock_response = Mock() @@ -689,7 +692,10 @@ def test_external_proxy_request(self, mock_head): mock_head.assert_called_once() (call_url,), call_kwargs = mock_head.call_args self.assertEqual(call_url, request_url) - self.assertEqual(call_kwargs.get('proxies'), {'http': 'http://proxy.example.com:8080'}) + self.assertEqual( + call_kwargs.get('proxies'), + {'http': 'http://proxy.example.com:8080'}, + ) def test_external_check_timedout(self): uv = Url(url=f"{self.live_server_url}/timeout/") From dff72ff841e80a716590518f2e216c72e02ed2f5 Mon Sep 17 00:00:00 2001 From: Ben Stockermans Date: Thu, 18 Sep 2025 15:52:28 +0100 Subject: [PATCH 5/7] Added an entry for both of the new settings in the README. --- README.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README.rst b/README.rst index 4b9163f..45cb5c2 100644 --- a/README.rst +++ b/README.rst @@ -217,6 +217,29 @@ Whether links with broken hash anchors should be marked as valid. Disable this if you want that links to anchors which are not contained in the link target's HTML source are marked as invalid. +LINKCHECK_PROXIES +~~~~~~~~~~~~~~~~~ + +Default: `{}` + +Allows you to make your `check_external` requests via a proxy. Expects a dictionary, e.g.: + +``` +LINKCHECK_PROXIES = { + "http": "http://...", + "https": "https://...", +} +``` + + +LINKCHECK_TRUST_PROXY_SSL +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Default: `False` + +If you are making your requests via a proxy, you can use this setting to turn off SSL verification for the proxy. + + django-filebrowser integration ------------------------------ From 9c60bd671edd2fcfb786e354bf9e6267ca60b765 Mon Sep 17 00:00:00 2001 From: Ben Stockermans Date: Thu, 18 Sep 2025 16:19:33 +0100 Subject: [PATCH 6/7] Replaced @patch('requests.head') with @requests_mock.Mocker(), bringing it in line with the other tests. --- linkcheck/tests/test_linkcheck.py | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/linkcheck/tests/test_linkcheck.py b/linkcheck/tests/test_linkcheck.py index c67f46f..d263af4 100644 --- a/linkcheck/tests/test_linkcheck.py +++ b/linkcheck/tests/test_linkcheck.py @@ -676,26 +676,20 @@ def test_external_check_blocked_user_agent_blocked_head(self): 'linkcheck.models.PROXIES', {'http': 'http://proxy.example.com:8080'}, ) - @patch('requests.head') - def test_external_proxy_request(self, mock_head): - mock_response = Mock() - mock_response.status_code = 200 - mock_response.reason = 'OK' - mock_response.history = [] - mock_head.return_value = mock_response - request_url = 'http://test.com' - uv = Url(url=request_url) + @requests_mock.Mocker() + def test_external_proxy_request(self, mocker): + mocker.register_uri('HEAD', 'http://test.com', reason='OK'), + uv = Url(url='http://test.com') + self.assertEqual(mocker.called, False) uv.check_url() + self.assertEqual(mocker.called, True) self.assertEqual(uv.status, True) self.assertEqual(uv.message, '200 OK') self.assertEqual(uv.type, 'external') - mock_head.assert_called_once() - (call_url,), call_kwargs = mock_head.call_args - self.assertEqual(call_url, request_url) - self.assertEqual( - call_kwargs.get('proxies'), - {'http': 'http://proxy.example.com:8080'}, - ) + last_request = mocker.last_request + self.assertEqual(last_request.hostname, 'test.com') + self.assertEqual(last_request.scheme, 'http') + self.assertEqual(last_request.proxies, {'http': 'http://proxy.example.com:8080'}) def test_external_check_timedout(self): uv = Url(url=f"{self.live_server_url}/timeout/") From eee6c8abd8afb8d37cc59065d1faecf6be765ef0 Mon Sep 17 00:00:00 2001 From: Ben Stockermans Date: Thu, 18 Sep 2025 16:23:08 +0100 Subject: [PATCH 7/7] Removed Mock import. --- linkcheck/tests/test_linkcheck.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkcheck/tests/test_linkcheck.py b/linkcheck/tests/test_linkcheck.py index d263af4..323669f 100644 --- a/linkcheck/tests/test_linkcheck.py +++ b/linkcheck/tests/test_linkcheck.py @@ -1,7 +1,7 @@ import os from datetime import datetime, timedelta from io import StringIO -from unittest.mock import Mock, patch +from unittest.mock import patch import requests_mock import urllib3