From f8e110c6ea6288ec6e018cfbfb5baa887183c464 Mon Sep 17 00:00:00 2001 From: 1yam <40899431+1yam@users.noreply.github.com> Date: Fri, 25 Aug 2023 11:11:43 +0200 Subject: [PATCH 01/25] Feature : AlephDNS (#47) * Feature : AlephDNS add instances support add ipfs support add program support --------- Co-authored-by: aliel --- setup.cfg | 1 + src/aleph/sdk/conf.py | 7 ++ src/aleph/sdk/domain.py | 135 ++++++++++++++++++++++++++++++++++++ src/aleph/sdk/exceptions.py | 5 ++ tests/unit/test_domains.py | 50 +++++++++++++ 5 files changed, 198 insertions(+) create mode 100644 src/aleph/sdk/domain.py create mode 100644 tests/unit/test_domains.py diff --git a/setup.cfg b/setup.cfg index b17bf7f5..013a67a5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -82,6 +82,7 @@ testing = substrate-interface py-sr25519-bindings ledgereth==0.9.0 + aiodns mqtt = aiomqtt<=0.1.3 certifi diff --git a/src/aleph/sdk/conf.py b/src/aleph/sdk/conf.py index cf60470d..1e78f52f 100644 --- a/src/aleph/sdk/conf.py +++ b/src/aleph/sdk/conf.py @@ -36,6 +36,13 @@ class Settings(BaseSettings): CODE_USES_SQUASHFS: bool = which("mksquashfs") is not None # True if command exists + # Dns resolver + DNS_IPFS_DOMAIN = "ipfs.public.aleph.sh" + DNS_PROGRAM_DOMAIN = "program.public.aleph.sh" + DNS_INSTANCE_DOMAIN = "instance.public.aleph.sh" + DNS_ROOT_DOMAIN = "static.public.aleph.sh" + DNS_RESOLVERS = ["1.1.1.1", "1.0.0.1"] + class Config: env_prefix = "ALEPH_" case_sensitive = False diff --git a/src/aleph/sdk/domain.py b/src/aleph/sdk/domain.py new file mode 100644 index 00000000..1d708797 --- /dev/null +++ b/src/aleph/sdk/domain.py @@ -0,0 +1,135 @@ +import aiodns +import re +from .conf import settings +from typing import Optional +from aleph.sdk.exceptions import DomainConfigurationError + + +class AlephDNS: + def __init__(self): + self.resolver = aiodns.DNSResolver(servers=settings.DNS_RESOLVERS) + self.fqdn_matcher = re.compile(r"https?://?") + + async def query(self, name: str, query_type: str): + try: + return await self.resolver.query(name, query_type) + except Exception as e: + print(e) + return None + + def url_to_domain(self, url): + return self.fqdn_matcher.sub("", url).strip().strip("/") + + async def get_ipv6_address(self, url: str): + domain = self.url_to_domain(url) + ipv6 = [] + query = await self.query(domain, "AAAA") + if query: + for entry in query: + ipv6.append(entry.host) + return ipv6 + + async def get_dnslink(self, url: str): + domain = self.url_to_domain(url) + query = await self.query(f"_dnslink.{domain}", "TXT") + if query is not None and len(query) > 0: + return query[0].text + + async def check_domain_configured(self, domain, target, owner): + try: + print("Check...", target) + return await self.check_domain(domain, target, owner) + except Exception as error: + raise DomainConfigurationError(error) + + async def check_domain( + self, url: str, target: str, owner: Optional[str] = None + ): + status = {"cname": False, "owner_proof": False} + + target = target.lower() + domain = self.url_to_domain(url) + + dns_rules = self.get_required_dns_rules(url, target, owner) + + for dns_rule in dns_rules: + status[dns_rule["rule_name"]] = False + + record_name = dns_rule["dns"]["name"] + record_type = dns_rule["dns"]["type"] + record_value = dns_rule["dns"]["value"] + + res = await self.query(record_name, record_type.upper()) + + if record_type == "txt": + found = False + + for _res in res: + if hasattr(_res, "text") and _res.text == record_value: + found = True + + if found == False: + raise DomainConfigurationError( + (dns_rule["info"], dns_rule["on_error"], status) + ) + + elif res is None or not hasattr(res, record_type) or getattr(res, record_type) != record_value: + raise DomainConfigurationError( + (dns_rule["info"], dns_rule["on_error"], status) + ) + + status[dns_rule["rule_name"]] = True + + return status + + def get_required_dns_rules(self, url, target, owner: Optional[str] = None): + domain = self.url_to_domain(url) + target = target.lower() + dns_rules = [] + + if target == "ipfs": + cname_value = settings.DNS_IPFS_DOMAIN + elif target == "program": + cname_value = settings.DNS_PROGRAM_DOMAIN + elif target == "instance": + cname_value = f"{domain}.{settings.DNS_INSTANCE_DOMAIN}" + + # cname rule + dns_rules.append({ + "rule_name": "cname", + "dns": { + "type": "cname", + "name": domain, + "value": cname_value + }, + "info": f"Create a CNAME record for {domain} with value {cname_value}", + "on_error": f"CNAME record not found: {domain}" + }) + + if target == "ipfs": + # ipfs rule + dns_rules.append({ + "rule_name": "delegation", + "dns": { + "type": "cname", + "name": f"_dnslink.{domain}", + "value": f"_dnslink.{domain}.{settings.DNS_ROOT_DOMAIN}" + }, + "info": f"Create a CNAME record for _dnslink.{domain} with value _dnslink.{domain}.{settings.DNS_ROOT_DOMAIN}", + "on_error": f"CNAME record not found: _dnslink.{domain}" + }) + + if owner: + # ownership rule + dns_rules.append({ + "rule_name": "owner_proof", + "dns": { + "type": "txt", + "name": f"_control.{domain}", + "value": owner + }, + "info": f"Create a TXT record for _control.{domain} with value = owner address", + "on_error": f"Owner address mismatch" + }) + + return dns_rules diff --git a/src/aleph/sdk/exceptions.py b/src/aleph/sdk/exceptions.py index 5f09e1bc..8e10eee1 100644 --- a/src/aleph/sdk/exceptions.py +++ b/src/aleph/sdk/exceptions.py @@ -50,3 +50,8 @@ class FileTooLarge(Exception): """ pass + + +class DomainConfigurationError(Exception): + "Raised when the domain checks are not satisfied" + pass diff --git a/tests/unit/test_domains.py b/tests/unit/test_domains.py new file mode 100644 index 00000000..ce78abaa --- /dev/null +++ b/tests/unit/test_domains.py @@ -0,0 +1,50 @@ +import pytest +import asyncio + +from aleph.sdk.domain import AlephDNS +from aleph.sdk.exceptions import DomainConfigurationError + + +@pytest.mark.asyncio +async def test_url_to_domain(): + alephdns = AlephDNS() + domain = alephdns.url_to_domain("https://aleph.im") + query = await alephdns.query(domain, "A") + assert query is not None + assert len(query) > 0 + assert hasattr(query[0], "host") + + +@pytest.mark.asyncio +async def test_get_ipv6_address(): + alephdns = AlephDNS() + url = "https://aleph.im" + ipv6_address = await alephdns.get_ipv6_address(url) + assert ipv6_address is not None + assert len(ipv6_address) > 0 + assert ":" in ipv6_address[0] + + +@pytest.mark.asyncio +async def test_dnslink(): + alephdns = AlephDNS() + url = "https://aleph.im" + dnslink = await alephdns.get_dnslink(url) + assert dnslink is not None + + +@pytest.mark.asyncio +async def test_configured_domain(): + alephdns = AlephDNS() + url = 'https://custom-domain-unit-test.aleph.sh' + status = await alephdns.check_domain(url, "ipfs", "0xfakeaddress") + assert type(status) is dict + + +@pytest.mark.asyncio +async def test_not_configured_domain(): + alephdns = AlephDNS() + url = 'https://not-configured-domain.aleph.sh' + with pytest.raises(DomainConfigurationError): + status = await alephdns.check_domain(url, "ipfs", "0xfakeaddress") + From c27647086c0c83b46dd571834579ad13dce0608d Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 28 Aug 2023 15:27:26 +0200 Subject: [PATCH 02/25] Fix: Reformat with `black` --- src/aleph/sdk/domain.py | 79 +++++++++++++++++++++----------------- tests/unit/test_domains.py | 8 ++-- 2 files changed, 47 insertions(+), 40 deletions(-) diff --git a/src/aleph/sdk/domain.py b/src/aleph/sdk/domain.py index 1d708797..20fa2225 100644 --- a/src/aleph/sdk/domain.py +++ b/src/aleph/sdk/domain.py @@ -1,9 +1,12 @@ -import aiodns import re -from .conf import settings from typing import Optional + +import aiodns + from aleph.sdk.exceptions import DomainConfigurationError +from .conf import settings + class AlephDNS: def __init__(self): @@ -42,9 +45,7 @@ async def check_domain_configured(self, domain, target, owner): except Exception as error: raise DomainConfigurationError(error) - async def check_domain( - self, url: str, target: str, owner: Optional[str] = None - ): + async def check_domain(self, url: str, target: str, owner: Optional[str] = None): status = {"cname": False, "owner_proof": False} target = target.lower() @@ -73,7 +74,11 @@ async def check_domain( (dns_rule["info"], dns_rule["on_error"], status) ) - elif res is None or not hasattr(res, record_type) or getattr(res, record_type) != record_value: + elif ( + res is None + or not hasattr(res, record_type) + or getattr(res, record_type) != record_value + ): raise DomainConfigurationError( (dns_rule["info"], dns_rule["on_error"], status) ) @@ -95,41 +100,43 @@ def get_required_dns_rules(self, url, target, owner: Optional[str] = None): cname_value = f"{domain}.{settings.DNS_INSTANCE_DOMAIN}" # cname rule - dns_rules.append({ - "rule_name": "cname", - "dns": { - "type": "cname", - "name": domain, - "value": cname_value - }, - "info": f"Create a CNAME record for {domain} with value {cname_value}", - "on_error": f"CNAME record not found: {domain}" - }) + dns_rules.append( + { + "rule_name": "cname", + "dns": {"type": "cname", "name": domain, "value": cname_value}, + "info": f"Create a CNAME record for {domain} with value {cname_value}", + "on_error": f"CNAME record not found: {domain}", + } + ) if target == "ipfs": # ipfs rule - dns_rules.append({ - "rule_name": "delegation", - "dns": { - "type": "cname", - "name": f"_dnslink.{domain}", - "value": f"_dnslink.{domain}.{settings.DNS_ROOT_DOMAIN}" - }, - "info": f"Create a CNAME record for _dnslink.{domain} with value _dnslink.{domain}.{settings.DNS_ROOT_DOMAIN}", - "on_error": f"CNAME record not found: _dnslink.{domain}" - }) + dns_rules.append( + { + "rule_name": "delegation", + "dns": { + "type": "cname", + "name": f"_dnslink.{domain}", + "value": f"_dnslink.{domain}.{settings.DNS_ROOT_DOMAIN}", + }, + "info": f"Create a CNAME record for _dnslink.{domain} with value _dnslink.{domain}.{settings.DNS_ROOT_DOMAIN}", + "on_error": f"CNAME record not found: _dnslink.{domain}", + } + ) if owner: # ownership rule - dns_rules.append({ - "rule_name": "owner_proof", - "dns": { - "type": "txt", - "name": f"_control.{domain}", - "value": owner - }, - "info": f"Create a TXT record for _control.{domain} with value = owner address", - "on_error": f"Owner address mismatch" - }) + dns_rules.append( + { + "rule_name": "owner_proof", + "dns": { + "type": "txt", + "name": f"_control.{domain}", + "value": owner, + }, + "info": f"Create a TXT record for _control.{domain} with value = owner address", + "on_error": f"Owner address mismatch", + } + ) return dns_rules diff --git a/tests/unit/test_domains.py b/tests/unit/test_domains.py index ce78abaa..3a8e88ab 100644 --- a/tests/unit/test_domains.py +++ b/tests/unit/test_domains.py @@ -1,6 +1,7 @@ -import pytest import asyncio +import pytest + from aleph.sdk.domain import AlephDNS from aleph.sdk.exceptions import DomainConfigurationError @@ -36,7 +37,7 @@ async def test_dnslink(): @pytest.mark.asyncio async def test_configured_domain(): alephdns = AlephDNS() - url = 'https://custom-domain-unit-test.aleph.sh' + url = "https://custom-domain-unit-test.aleph.sh" status = await alephdns.check_domain(url, "ipfs", "0xfakeaddress") assert type(status) is dict @@ -44,7 +45,6 @@ async def test_configured_domain(): @pytest.mark.asyncio async def test_not_configured_domain(): alephdns = AlephDNS() - url = 'https://not-configured-domain.aleph.sh' + url = "https://not-configured-domain.aleph.sh" with pytest.raises(DomainConfigurationError): status = await alephdns.check_domain(url, "ipfs", "0xfakeaddress") - From 24900ffc07a1355ba30494119327577603f1f2be Mon Sep 17 00:00:00 2001 From: aliel Date: Thu, 31 Aug 2023 14:20:44 +0200 Subject: [PATCH 03/25] minor change on dns settings --- src/aleph/sdk/conf.py | 2 +- src/aleph/sdk/domain.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/aleph/sdk/conf.py b/src/aleph/sdk/conf.py index 1e78f52f..12c9c0e2 100644 --- a/src/aleph/sdk/conf.py +++ b/src/aleph/sdk/conf.py @@ -40,7 +40,7 @@ class Settings(BaseSettings): DNS_IPFS_DOMAIN = "ipfs.public.aleph.sh" DNS_PROGRAM_DOMAIN = "program.public.aleph.sh" DNS_INSTANCE_DOMAIN = "instance.public.aleph.sh" - DNS_ROOT_DOMAIN = "static.public.aleph.sh" + DNS_STATIC_DOMAIN = "static.public.aleph.sh" DNS_RESOLVERS = ["1.1.1.1", "1.0.0.1"] class Config: diff --git a/src/aleph/sdk/domain.py b/src/aleph/sdk/domain.py index 20fa2225..b0f028d9 100644 --- a/src/aleph/sdk/domain.py +++ b/src/aleph/sdk/domain.py @@ -117,9 +117,9 @@ def get_required_dns_rules(self, url, target, owner: Optional[str] = None): "dns": { "type": "cname", "name": f"_dnslink.{domain}", - "value": f"_dnslink.{domain}.{settings.DNS_ROOT_DOMAIN}", + "value": f"_dnslink.{domain}.{settings.DNS_STATIC_DOMAIN}", }, - "info": f"Create a CNAME record for _dnslink.{domain} with value _dnslink.{domain}.{settings.DNS_ROOT_DOMAIN}", + "info": f"Create a CNAME record for _dnslink.{domain} with value _dnslink.{domain}.{settings.DNS_STATIC_DOMAIN}", "on_error": f"CNAME record not found: _dnslink.{domain}", } ) From 525366fe51c97861a8a763d0c72c88d43b74e23c Mon Sep 17 00:00:00 2001 From: aliel Date: Fri, 1 Sep 2023 17:46:02 +0200 Subject: [PATCH 04/25] avoid not valid local variable in certain conditions --- src/aleph/sdk/domain.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/aleph/sdk/domain.py b/src/aleph/sdk/domain.py index b0f028d9..74f508e8 100644 --- a/src/aleph/sdk/domain.py +++ b/src/aleph/sdk/domain.py @@ -92,6 +92,7 @@ def get_required_dns_rules(self, url, target, owner: Optional[str] = None): target = target.lower() dns_rules = [] + cname_value = None if target == "ipfs": cname_value = settings.DNS_IPFS_DOMAIN elif target == "program": From 967b7b4d5d0743e06d617bbbbe51e8c90f89f5f2 Mon Sep 17 00:00:00 2001 From: aliel Date: Tue, 5 Sep 2023 10:36:46 +0200 Subject: [PATCH 05/25] add method to retreive txt values --- src/aleph/sdk/domain.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/aleph/sdk/domain.py b/src/aleph/sdk/domain.py index 74f508e8..56bdb093 100644 --- a/src/aleph/sdk/domain.py +++ b/src/aleph/sdk/domain.py @@ -38,6 +38,19 @@ async def get_dnslink(self, url: str): if query is not None and len(query) > 0: return query[0].text + async def get_txt_values(self, url: str, delimiter: Optional[str] = None): + domain = self.url_to_domain(url) + res = await alephdns.query(domain, "TXT") + values = [] + if res is not None: + for _res in res: + if hasattr(_res, "text") and _res.text.startswith("0x"): + if delimiter is not None and delimiter in _res.text: + values = values + _res.text.split(delimiter) + else: + values.append(_res.text) + return values + async def check_domain_configured(self, domain, target, owner): try: print("Check...", target) From f27e00220af040fffd4e57b6336efcabee820e0d Mon Sep 17 00:00:00 2001 From: aliel Date: Wed, 6 Sep 2023 14:15:17 +0200 Subject: [PATCH 06/25] fix dns record check --- src/aleph/sdk/domain.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/aleph/sdk/domain.py b/src/aleph/sdk/domain.py index 56bdb093..ce5e55f3 100644 --- a/src/aleph/sdk/domain.py +++ b/src/aleph/sdk/domain.py @@ -77,16 +77,14 @@ async def check_domain(self, url: str, target: str, owner: Optional[str] = None) if record_type == "txt": found = False - - for _res in res: - if hasattr(_res, "text") and _res.text == record_value: - found = True - + if res is not None: + for _res in res: + if hasattr(_res, "text") and _res.text == record_value: + found = True if found == False: raise DomainConfigurationError( (dns_rule["info"], dns_rule["on_error"], status) ) - elif ( res is None or not hasattr(res, record_type) From 73c6b6ba24d575c75d2ce162e97dc4a8b0ba4413 Mon Sep 17 00:00:00 2001 From: aliel Date: Wed, 6 Sep 2023 16:34:53 +0200 Subject: [PATCH 07/25] fix program cname value --- src/aleph/sdk/domain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aleph/sdk/domain.py b/src/aleph/sdk/domain.py index ce5e55f3..76ab288d 100644 --- a/src/aleph/sdk/domain.py +++ b/src/aleph/sdk/domain.py @@ -107,7 +107,7 @@ def get_required_dns_rules(self, url, target, owner: Optional[str] = None): if target == "ipfs": cname_value = settings.DNS_IPFS_DOMAIN elif target == "program": - cname_value = settings.DNS_PROGRAM_DOMAIN + cname_value = f"{domain}.{settings.DNS_PROGRAM_DOMAIN}" elif target == "instance": cname_value = f"{domain}.{settings.DNS_INSTANCE_DOMAIN}" From 9e3a65145a15dc045520d0641a28513e7f7ff4b1 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 18 Sep 2023 17:35:17 +0200 Subject: [PATCH 08/25] Refactor: str -> Enum for DNS target --- src/aleph/sdk/domain.py | 20 +++++++++++++------- tests/unit/test_domains.py | 6 ++++-- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/aleph/sdk/domain.py b/src/aleph/sdk/domain.py index 76ab288d..786834b0 100644 --- a/src/aleph/sdk/domain.py +++ b/src/aleph/sdk/domain.py @@ -1,4 +1,5 @@ import re +from enum import Enum from typing import Optional import aiodns @@ -8,6 +9,12 @@ from .conf import settings +class Target(str, Enum): + IPFS = "ipfs" + PROGRAM = "program" + INSTANCE = "instance" + + class AlephDNS: def __init__(self): self.resolver = aiodns.DNSResolver(servers=settings.DNS_RESOLVERS) @@ -51,7 +58,7 @@ async def get_txt_values(self, url: str, delimiter: Optional[str] = None): values.append(_res.text) return values - async def check_domain_configured(self, domain, target, owner): + async def check_domain_configured(self, domain, target: Target, owner): try: print("Check...", target) return await self.check_domain(domain, target, owner) @@ -61,7 +68,6 @@ async def check_domain_configured(self, domain, target, owner): async def check_domain(self, url: str, target: str, owner: Optional[str] = None): status = {"cname": False, "owner_proof": False} - target = target.lower() domain = self.url_to_domain(url) dns_rules = self.get_required_dns_rules(url, target, owner) @@ -98,17 +104,17 @@ async def check_domain(self, url: str, target: str, owner: Optional[str] = None) return status - def get_required_dns_rules(self, url, target, owner: Optional[str] = None): + def get_required_dns_rules(self, url, target: Target, owner: Optional[str] = None): domain = self.url_to_domain(url) target = target.lower() dns_rules = [] cname_value = None - if target == "ipfs": + if target == Target.IPFS: cname_value = settings.DNS_IPFS_DOMAIN - elif target == "program": + elif target == Target.PROGRAM: cname_value = f"{domain}.{settings.DNS_PROGRAM_DOMAIN}" - elif target == "instance": + elif target == Target.INSTANCE: cname_value = f"{domain}.{settings.DNS_INSTANCE_DOMAIN}" # cname rule @@ -121,7 +127,7 @@ def get_required_dns_rules(self, url, target, owner: Optional[str] = None): } ) - if target == "ipfs": + if target == Target.IPFS: # ipfs rule dns_rules.append( { diff --git a/tests/unit/test_domains.py b/tests/unit/test_domains.py index 3a8e88ab..a2ddf37b 100644 --- a/tests/unit/test_domains.py +++ b/tests/unit/test_domains.py @@ -5,6 +5,8 @@ from aleph.sdk.domain import AlephDNS from aleph.sdk.exceptions import DomainConfigurationError +from src.aleph.sdk.domain import Target + @pytest.mark.asyncio async def test_url_to_domain(): @@ -38,7 +40,7 @@ async def test_dnslink(): async def test_configured_domain(): alephdns = AlephDNS() url = "https://custom-domain-unit-test.aleph.sh" - status = await alephdns.check_domain(url, "ipfs", "0xfakeaddress") + status = await alephdns.check_domain(url, Target.IPFS, "0xfakeaddress") assert type(status) is dict @@ -47,4 +49,4 @@ async def test_not_configured_domain(): alephdns = AlephDNS() url = "https://not-configured-domain.aleph.sh" with pytest.raises(DomainConfigurationError): - status = await alephdns.check_domain(url, "ipfs", "0xfakeaddress") + status = await alephdns.check_domain(url, Target.IPFS, "0xfakeaddress") From f20590dca34543d9abbecbe9340f059166975139 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 18 Sep 2023 17:43:59 +0200 Subject: [PATCH 09/25] Refactor: Add types to arguments --- src/aleph/sdk/domain.py | 30 +++++++++++++++--------------- tests/unit/test_domains.py | 5 +++-- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/src/aleph/sdk/domain.py b/src/aleph/sdk/domain.py index 786834b0..1370c7da 100644 --- a/src/aleph/sdk/domain.py +++ b/src/aleph/sdk/domain.py @@ -1,10 +1,12 @@ import re from enum import Enum from typing import Optional +from urllib.parse import urlparse import aiodns from aleph.sdk.exceptions import DomainConfigurationError +from pydantic import HttpUrl from .conf import settings @@ -18,7 +20,6 @@ class Target(str, Enum): class AlephDNS: def __init__(self): self.resolver = aiodns.DNSResolver(servers=settings.DNS_RESOLVERS) - self.fqdn_matcher = re.compile(r"https?://?") async def query(self, name: str, query_type: str): try: @@ -27,11 +28,8 @@ async def query(self, name: str, query_type: str): print(e) return None - def url_to_domain(self, url): - return self.fqdn_matcher.sub("", url).strip().strip("/") - - async def get_ipv6_address(self, url: str): - domain = self.url_to_domain(url) + async def get_ipv6_address(self, url: HttpUrl) -> str: + domain = urlparse(url).netloc ipv6 = [] query = await self.query(domain, "AAAA") if query: @@ -39,14 +37,14 @@ async def get_ipv6_address(self, url: str): ipv6.append(entry.host) return ipv6 - async def get_dnslink(self, url: str): - domain = self.url_to_domain(url) + async def get_dnslink(self, url: HttpUrl) -> str: + domain = urlparse(url).netloc query = await self.query(f"_dnslink.{domain}", "TXT") if query is not None and len(query) > 0: return query[0].text - async def get_txt_values(self, url: str, delimiter: Optional[str] = None): - domain = self.url_to_domain(url) + async def get_txt_values(self, url: HttpUrl, delimiter: Optional[str] = None): + domain = urlparse(url).netloc res = await alephdns.query(domain, "TXT") values = [] if res is not None: @@ -58,17 +56,19 @@ async def get_txt_values(self, url: str, delimiter: Optional[str] = None): values.append(_res.text) return values - async def check_domain_configured(self, domain, target: Target, owner): + async def check_domain_configured(self, domain: HttpUrl, target: Target, owner): try: print("Check...", target) return await self.check_domain(domain, target, owner) except Exception as error: raise DomainConfigurationError(error) - async def check_domain(self, url: str, target: str, owner: Optional[str] = None): + async def check_domain(self, url: HttpUrl, target: Target, owner: Optional[str] = None): + """Check that the domain points towards the target. + """ status = {"cname": False, "owner_proof": False} - domain = self.url_to_domain(url) + domain = urlparse(url).netloc dns_rules = self.get_required_dns_rules(url, target, owner) @@ -104,8 +104,8 @@ async def check_domain(self, url: str, target: str, owner: Optional[str] = None) return status - def get_required_dns_rules(self, url, target: Target, owner: Optional[str] = None): - domain = self.url_to_domain(url) + def get_required_dns_rules(self, url: HttpUrl, target: Target, owner: Optional[str] = None): + domain = urlparse(url).netloc target = target.lower() dns_rules = [] diff --git a/tests/unit/test_domains.py b/tests/unit/test_domains.py index a2ddf37b..7555388c 100644 --- a/tests/unit/test_domains.py +++ b/tests/unit/test_domains.py @@ -1,4 +1,5 @@ import asyncio +from urllib.parse import urlparse import pytest @@ -9,9 +10,9 @@ @pytest.mark.asyncio -async def test_url_to_domain(): +async def test_query(): alephdns = AlephDNS() - domain = alephdns.url_to_domain("https://aleph.im") + domain = urlparse("https://aleph.im").netloc query = await alephdns.query(domain, "A") assert query is not None assert len(query) > 0 From 50cd4f202c7bcc15d3a023f45529321d88f8d844 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 18 Sep 2023 17:44:28 +0200 Subject: [PATCH 10/25] Fix: variable did not exist --- src/aleph/sdk/domain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aleph/sdk/domain.py b/src/aleph/sdk/domain.py index 1370c7da..9f42f9af 100644 --- a/src/aleph/sdk/domain.py +++ b/src/aleph/sdk/domain.py @@ -45,7 +45,7 @@ async def get_dnslink(self, url: HttpUrl) -> str: async def get_txt_values(self, url: HttpUrl, delimiter: Optional[str] = None): domain = urlparse(url).netloc - res = await alephdns.query(domain, "TXT") + res = await self.query(domain, "TXT") values = [] if res is not None: for _res in res: From 828576c5fc6c37efaf5d956cde04b9c0d455a917 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 18 Sep 2023 17:48:10 +0200 Subject: [PATCH 11/25] Add typing --- src/aleph/sdk/domain.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/aleph/sdk/domain.py b/src/aleph/sdk/domain.py index 9f42f9af..6fd77956 100644 --- a/src/aleph/sdk/domain.py +++ b/src/aleph/sdk/domain.py @@ -1,6 +1,6 @@ import re from enum import Enum -from typing import Optional +from typing import Optional, List from urllib.parse import urlparse import aiodns @@ -37,16 +37,18 @@ async def get_ipv6_address(self, url: HttpUrl) -> str: ipv6.append(entry.host) return ipv6 - async def get_dnslink(self, url: HttpUrl) -> str: + async def get_dnslink(self, url: HttpUrl) -> Optional[str]: domain = urlparse(url).netloc query = await self.query(f"_dnslink.{domain}", "TXT") if query is not None and len(query) > 0: return query[0].text + else: + return None - async def get_txt_values(self, url: HttpUrl, delimiter: Optional[str] = None): + async def get_txt_values(self, url: HttpUrl, delimiter: Optional[str] = None) -> List[str]: domain = urlparse(url).netloc res = await self.query(domain, "TXT") - values = [] + values: List[str] = [] if res is not None: for _res in res: if hasattr(_res, "text") and _res.text.startswith("0x"): From 8d8862557de0fcacc67b78a4efcf765a7f9015a3 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 18 Sep 2023 17:48:50 +0200 Subject: [PATCH 12/25] Cleanup: imports with isort --- src/aleph/sdk/domain.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/aleph/sdk/domain.py b/src/aleph/sdk/domain.py index 6fd77956..80620b43 100644 --- a/src/aleph/sdk/domain.py +++ b/src/aleph/sdk/domain.py @@ -1,12 +1,11 @@ -import re from enum import Enum -from typing import Optional, List +from typing import List, Optional from urllib.parse import urlparse import aiodns +from pydantic import HttpUrl from aleph.sdk.exceptions import DomainConfigurationError -from pydantic import HttpUrl from .conf import settings From 30e842ebb6fd654a62dceb020317db09b824f556 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 18 Sep 2023 17:49:03 +0200 Subject: [PATCH 13/25] Cleanup: typing --- src/aleph/sdk/domain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aleph/sdk/domain.py b/src/aleph/sdk/domain.py index 80620b43..2b3d7dc7 100644 --- a/src/aleph/sdk/domain.py +++ b/src/aleph/sdk/domain.py @@ -27,7 +27,7 @@ async def query(self, name: str, query_type: str): print(e) return None - async def get_ipv6_address(self, url: HttpUrl) -> str: + async def get_ipv6_address(self, url: HttpUrl) -> List[str]: domain = urlparse(url).netloc ipv6 = [] query = await self.query(domain, "AAAA") From 1fa6f5c902194442d3cb2f59be661b07515c2fc0 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 18 Sep 2023 17:56:33 +0200 Subject: [PATCH 14/25] Refactor: domain_from_url --- src/aleph/sdk/domain.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/aleph/sdk/domain.py b/src/aleph/sdk/domain.py index 2b3d7dc7..bf2eac25 100644 --- a/src/aleph/sdk/domain.py +++ b/src/aleph/sdk/domain.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import List, Optional +from typing import List, Optional, Dict from urllib.parse import urlparse import aiodns @@ -16,6 +16,9 @@ class Target(str, Enum): INSTANCE = "instance" +def domain_from_url(url: HttpUrl) -> str: + return domain_from_url(url) + class AlephDNS: def __init__(self): self.resolver = aiodns.DNSResolver(servers=settings.DNS_RESOLVERS) @@ -28,7 +31,7 @@ async def query(self, name: str, query_type: str): return None async def get_ipv6_address(self, url: HttpUrl) -> List[str]: - domain = urlparse(url).netloc + domain = domain_from_url(url) ipv6 = [] query = await self.query(domain, "AAAA") if query: @@ -37,7 +40,7 @@ async def get_ipv6_address(self, url: HttpUrl) -> List[str]: return ipv6 async def get_dnslink(self, url: HttpUrl) -> Optional[str]: - domain = urlparse(url).netloc + domain = domain_from_url(url) query = await self.query(f"_dnslink.{domain}", "TXT") if query is not None and len(query) > 0: return query[0].text @@ -45,7 +48,7 @@ async def get_dnslink(self, url: HttpUrl) -> Optional[str]: return None async def get_txt_values(self, url: HttpUrl, delimiter: Optional[str] = None) -> List[str]: - domain = urlparse(url).netloc + domain = domain_from_url(url) res = await self.query(domain, "TXT") values: List[str] = [] if res is not None: @@ -69,7 +72,7 @@ async def check_domain(self, url: HttpUrl, target: Target, owner: Optional[str] """ status = {"cname": False, "owner_proof": False} - domain = urlparse(url).netloc + domain = domain_from_url(url) dns_rules = self.get_required_dns_rules(url, target, owner) @@ -105,8 +108,8 @@ async def check_domain(self, url: HttpUrl, target: Target, owner: Optional[str] return status - def get_required_dns_rules(self, url: HttpUrl, target: Target, owner: Optional[str] = None): - domain = urlparse(url).netloc + def get_required_dns_rules(self, url: HttpUrl, target: Target, owner: Optional[str] = None) -> List[Dict]: + domain = domain_from_url(url) target = target.lower() dns_rules = [] From 7642ea49da3dfcf24a026a8507a0063c4bcdf86d Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 18 Sep 2023 17:57:00 +0200 Subject: [PATCH 15/25] Refactor: use a generator --- src/aleph/sdk/domain.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/aleph/sdk/domain.py b/src/aleph/sdk/domain.py index bf2eac25..dc07834f 100644 --- a/src/aleph/sdk/domain.py +++ b/src/aleph/sdk/domain.py @@ -1,5 +1,6 @@ from enum import Enum -from typing import List, Optional, Dict +from ipaddress import IPv6Address +from typing import List, Optional, Dict, Iterable from urllib.parse import urlparse import aiodns @@ -24,20 +25,14 @@ def __init__(self): self.resolver = aiodns.DNSResolver(servers=settings.DNS_RESOLVERS) async def query(self, name: str, query_type: str): - try: - return await self.resolver.query(name, query_type) - except Exception as e: - print(e) - return None + return await self.resolver.query(name, query_type) - async def get_ipv6_address(self, url: HttpUrl) -> List[str]: + async def get_ipv6_address(self, url: HttpUrl) -> Iterable[IPv6Address]: domain = domain_from_url(url) - ipv6 = [] query = await self.query(domain, "AAAA") if query: for entry in query: - ipv6.append(entry.host) - return ipv6 + yield entry.host async def get_dnslink(self, url: HttpUrl) -> Optional[str]: domain = domain_from_url(url) From 720c0bdb19ffa55f15d00bd49d130a459fb1012f Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Mon, 18 Sep 2023 17:57:14 +0200 Subject: [PATCH 16/25] Cleanup: Add docstring and property --- src/aleph/sdk/domain.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/aleph/sdk/domain.py b/src/aleph/sdk/domain.py index dc07834f..03f344ad 100644 --- a/src/aleph/sdk/domain.py +++ b/src/aleph/sdk/domain.py @@ -21,6 +21,11 @@ def domain_from_url(url: HttpUrl) -> str: return domain_from_url(url) class AlephDNS: + """ + Tools used to analyze domain names used on the aleph.im network. + """ + resolver: aiodns.DNSResolver + def __init__(self): self.resolver = aiodns.DNSResolver(servers=settings.DNS_RESOLVERS) From 3c747f0cd99d6c47c8584734f429cb92474cdbd1 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 19 Sep 2023 16:50:29 +0200 Subject: [PATCH 17/25] WIP: Refactor and cleanup domain related code --- setup.cfg | 2 + src/aleph/sdk/conf.py | 2 +- src/aleph/sdk/domain.py | 170 +++++++++++++++++++++---------------- tests/unit/test_domains.py | 36 ++++---- 4 files changed, 118 insertions(+), 92 deletions(-) diff --git a/setup.cfg b/setup.cfg index 013a67a5..00caba2c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -83,6 +83,8 @@ testing = py-sr25519-bindings ledgereth==0.9.0 aiodns +dns = + aiodns mqtt = aiomqtt<=0.1.3 certifi diff --git a/src/aleph/sdk/conf.py b/src/aleph/sdk/conf.py index 12c9c0e2..f8d798c6 100644 --- a/src/aleph/sdk/conf.py +++ b/src/aleph/sdk/conf.py @@ -41,7 +41,7 @@ class Settings(BaseSettings): DNS_PROGRAM_DOMAIN = "program.public.aleph.sh" DNS_INSTANCE_DOMAIN = "instance.public.aleph.sh" DNS_STATIC_DOMAIN = "static.public.aleph.sh" - DNS_RESOLVERS = ["1.1.1.1", "1.0.0.1"] + DNS_RESOLVERS = ["9.9.9.9", "1.1.1.1"] class Config: env_prefix = "ALEPH_" diff --git a/src/aleph/sdk/domain.py b/src/aleph/sdk/domain.py index 03f344ad..a97f3b9c 100644 --- a/src/aleph/sdk/domain.py +++ b/src/aleph/sdk/domain.py @@ -1,80 +1,101 @@ +import logging from enum import Enum from ipaddress import IPv6Address -from typing import List, Optional, Dict, Iterable +from typing import Dict, Iterable, List, Optional, NewType, Union from urllib.parse import urlparse import aiodns from pydantic import HttpUrl -from aleph.sdk.exceptions import DomainConfigurationError - from .conf import settings +from .exceptions import DomainConfigurationError + +logger = logging.getLogger(__name__) + +Hostname = NewType("Hostname", str) -class Target(str, Enum): +class TargetType(str, Enum): IPFS = "ipfs" PROGRAM = "program" INSTANCE = "instance" -def domain_from_url(url: HttpUrl) -> str: - return domain_from_url(url) +def hostname_from_url(url: Union[HttpUrl, str]) -> Hostname: + return Hostname(urlparse(url).netloc) -class AlephDNS: + +class DomainValidator: """ Tools used to analyze domain names used on the aleph.im network. """ + resolver: aiodns.DNSResolver def __init__(self): self.resolver = aiodns.DNSResolver(servers=settings.DNS_RESOLVERS) - async def query(self, name: str, query_type: str): - return await self.resolver.query(name, query_type) - - async def get_ipv6_address(self, url: HttpUrl) -> Iterable[IPv6Address]: - domain = domain_from_url(url) - query = await self.query(domain, "AAAA") - if query: - for entry in query: - yield entry.host - - async def get_dnslink(self, url: HttpUrl) -> Optional[str]: - domain = domain_from_url(url) - query = await self.query(f"_dnslink.{domain}", "TXT") - if query is not None and len(query) > 0: - return query[0].text - else: - return None - - async def get_txt_values(self, url: HttpUrl, delimiter: Optional[str] = None) -> List[str]: - domain = domain_from_url(url) - res = await self.query(domain, "TXT") - values: List[str] = [] - if res is not None: - for _res in res: - if hasattr(_res, "text") and _res.text.startswith("0x"): - if delimiter is not None and delimiter in _res.text: - values = values + _res.text.split(delimiter) - else: - values.append(_res.text) - return values - - async def check_domain_configured(self, domain: HttpUrl, target: Target, owner): + async def get_ipv6_addresses(self, hostname: Hostname) -> List[IPv6Address]: + """Returns all IPv6 addresses for a domain""" + entries: Iterable = await self.resolver.query(hostname, "AAAA") or [] + return [entry.host for entry in entries] + + async def get_dnslinks(self, hostname: Hostname) -> List[str]: + """Returns all DNSLink values for a domain.""" + entries = await self.resolver.query(f"_dnslink.{hostname}", "TXT") + return [entry.text for entry in entries] + + async def get_dnslink(self, hostname: Hostname) -> Optional[str]: + """Returns the DNSLink corresponding to a domain. + + Since it is possible to add multiple TXT records containing a DNSLink to + the same domain, a behaviour has to be defined. + + - Some IPFS implementations might use the first valid dnslink= record they find. + - Others might throw an error indicating that the DNSLink resolution is ambiguous due to multiple records. + - Still, others might try to fetch content from all provided DNSLinks, + though this behavior would be less common and may introduce overhead. + """ + dnslinks = await self.get_dnslinks(hostname) + return dnslinks[0] if dnslinks else None + + async def get_txt_values( + self, hostname: Hostname, delimiter: Optional[str] = None + ) -> Iterable[str]: + """Returns all TXT values for a domain""" + entries: Iterable = await self.resolver.query(hostname, "TXT") or [] + for entry in entries: + if not hasattr(entry, "text"): + logger.debug("An entry does not have any text") + continue + if not entry.text.startswith("0x"): + logger.debug("Does not look like an Ethereum address") + continue + + if delimiter: + for part in entry.text.split(delimiter): + yield part + else: + yield entry.text + + async def check_domain_configured( + self, hostname: Hostname, target: TargetType, owner + ): + """Check if a domain is configured... for what ?""" try: - print("Check...", target) - return await self.check_domain(domain, target, owner) + logger.debug(f"Checking {target}") + return await self.check_domain(hostname, target, owner) except Exception as error: + # FIXME: Do not catch any exception raise DomainConfigurationError(error) - async def check_domain(self, url: HttpUrl, target: Target, owner: Optional[str] = None): - """Check that the domain points towards the target. - """ + async def check_domain( + self, hostname: Hostname, target: TargetType, owner: Optional[str] = None + ) -> Dict: + """Check that the domain points towards the target.""" status = {"cname": False, "owner_proof": False} - domain = domain_from_url(url) - - dns_rules = self.get_required_dns_rules(url, target, owner) + dns_rules = self.get_required_dns_rules(hostname, target, owner) for dns_rule in dns_rules: status[dns_rule["rule_name"]] = False @@ -83,22 +104,20 @@ async def check_domain(self, url: HttpUrl, target: Target, owner: Optional[str] record_type = dns_rule["dns"]["type"] record_value = dns_rule["dns"]["value"] - res = await self.query(record_name, record_type.upper()) + entries = await self.resolver.query(record_name, record_type.upper()) - if record_type == "txt": - found = False - if res is not None: - for _res in res: - if hasattr(_res, "text") and _res.text == record_value: - found = True - if found == False: + if entries and record_type == "txt": + for entry in entries: + if hasattr(entry, "text") and entry.text == record_value: + break + else: raise DomainConfigurationError( (dns_rule["info"], dns_rule["on_error"], status) ) elif ( - res is None - or not hasattr(res, record_type) - or getattr(res, record_type) != record_value + entries is None + or not hasattr(entries, record_type) + or getattr(entries, record_type) != record_value ): raise DomainConfigurationError( (dns_rule["info"], dns_rule["on_error"], status) @@ -108,41 +127,42 @@ async def check_domain(self, url: HttpUrl, target: Target, owner: Optional[str] return status - def get_required_dns_rules(self, url: HttpUrl, target: Target, owner: Optional[str] = None) -> List[Dict]: - domain = domain_from_url(url) + def get_required_dns_rules( + self, hostname: Hostname, target: TargetType, owner: Optional[str] = None + ) -> List[Dict]: target = target.lower() dns_rules = [] cname_value = None - if target == Target.IPFS: + if target == TargetType.IPFS: cname_value = settings.DNS_IPFS_DOMAIN - elif target == Target.PROGRAM: - cname_value = f"{domain}.{settings.DNS_PROGRAM_DOMAIN}" - elif target == Target.INSTANCE: - cname_value = f"{domain}.{settings.DNS_INSTANCE_DOMAIN}" + elif target == TargetType.PROGRAM: + cname_value = f"{hostname}.{settings.DNS_PROGRAM_DOMAIN}" + elif target == TargetType.INSTANCE: + cname_value = f"{hostname}.{settings.DNS_INSTANCE_DOMAIN}" # cname rule dns_rules.append( { "rule_name": "cname", - "dns": {"type": "cname", "name": domain, "value": cname_value}, - "info": f"Create a CNAME record for {domain} with value {cname_value}", - "on_error": f"CNAME record not found: {domain}", + "dns": {"type": "cname", "name": hostname, "value": cname_value}, + "info": f"Create a CNAME record for {hostname} with value {cname_value}", + "on_error": f"CNAME record not found: {hostname}", } ) - if target == Target.IPFS: + if target == TargetType.IPFS: # ipfs rule dns_rules.append( { "rule_name": "delegation", "dns": { "type": "cname", - "name": f"_dnslink.{domain}", - "value": f"_dnslink.{domain}.{settings.DNS_STATIC_DOMAIN}", + "name": f"_dnslink.{hostname}", + "value": f"_dnslink.{hostname}.{settings.DNS_STATIC_DOMAIN}", }, - "info": f"Create a CNAME record for _dnslink.{domain} with value _dnslink.{domain}.{settings.DNS_STATIC_DOMAIN}", - "on_error": f"CNAME record not found: _dnslink.{domain}", + "info": f"Create a CNAME record for _dnslink.{hostname} with value _dnslink.{hostname}.{settings.DNS_STATIC_DOMAIN}", + "on_error": f"CNAME record not found: _dnslink.{hostname}", } ) @@ -153,10 +173,10 @@ def get_required_dns_rules(self, url: HttpUrl, target: Target, owner: Optional[s "rule_name": "owner_proof", "dns": { "type": "txt", - "name": f"_control.{domain}", + "name": f"_control.{hostname}", "value": owner, }, - "info": f"Create a TXT record for _control.{domain} with value = owner address", + "info": f"Create a TXT record for _control.{hostname} with value = owner address", "on_error": f"Owner address mismatch", } ) diff --git a/tests/unit/test_domains.py b/tests/unit/test_domains.py index 7555388c..221bd517 100644 --- a/tests/unit/test_domains.py +++ b/tests/unit/test_domains.py @@ -3,17 +3,17 @@ import pytest -from aleph.sdk.domain import AlephDNS +from aleph.sdk.domain import DomainValidator from aleph.sdk.exceptions import DomainConfigurationError -from src.aleph.sdk.domain import Target +from src.aleph.sdk.domain import TargetType, hostname_from_url @pytest.mark.asyncio async def test_query(): - alephdns = AlephDNS() - domain = urlparse("https://aleph.im").netloc - query = await alephdns.query(domain, "A") + alephdns = DomainValidator() + hostname = hostname_from_url("https://aleph.im") + query = await alephdns.resolver.query(hostname, "A") assert query is not None assert len(query) > 0 assert hasattr(query[0], "host") @@ -21,33 +21,37 @@ async def test_query(): @pytest.mark.asyncio async def test_get_ipv6_address(): - alephdns = AlephDNS() + alephdns = DomainValidator() url = "https://aleph.im" - ipv6_address = await alephdns.get_ipv6_address(url) - assert ipv6_address is not None - assert len(ipv6_address) > 0 - assert ":" in ipv6_address[0] + hostname = hostname_from_url(url) + ipv6_addresses = await alephdns.get_ipv6_addresses(hostname) + assert ipv6_addresses is not None + assert len(ipv6_addresses) > 0 + assert ":" in ipv6_addresses[0] @pytest.mark.asyncio async def test_dnslink(): - alephdns = AlephDNS() + alephdns = DomainValidator() url = "https://aleph.im" - dnslink = await alephdns.get_dnslink(url) + hostname = hostname_from_url(url) + dnslink = await alephdns.get_dnslink(hostname) assert dnslink is not None @pytest.mark.asyncio async def test_configured_domain(): - alephdns = AlephDNS() + alephdns = DomainValidator() url = "https://custom-domain-unit-test.aleph.sh" - status = await alephdns.check_domain(url, Target.IPFS, "0xfakeaddress") + hostname = hostname_from_url(url) + status = await alephdns.check_domain(hostname, TargetType.IPFS, "0xfakeaddress") assert type(status) is dict @pytest.mark.asyncio async def test_not_configured_domain(): - alephdns = AlephDNS() + alephdns = DomainValidator() url = "https://not-configured-domain.aleph.sh" + hostname = hostname_from_url(url) with pytest.raises(DomainConfigurationError): - status = await alephdns.check_domain(url, Target.IPFS, "0xfakeaddress") + status = await alephdns.check_domain(hostname, TargetType.IPFS, "0xfakeaddress") From 2409cd0a292712df9813207e72f2dcacf7ebe0d0 Mon Sep 17 00:00:00 2001 From: aliel Date: Thu, 5 Oct 2023 09:48:40 +0200 Subject: [PATCH 18/25] [dns] let checks continue --- src/aleph/sdk/domain.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/aleph/sdk/domain.py b/src/aleph/sdk/domain.py index a97f3b9c..60cf7408 100644 --- a/src/aleph/sdk/domain.py +++ b/src/aleph/sdk/domain.py @@ -1,7 +1,7 @@ import logging from enum import Enum from ipaddress import IPv6Address -from typing import Dict, Iterable, List, Optional, NewType, Union +from typing import Dict, Iterable, List, NewType, Optional, Union from urllib.parse import urlparse import aiodns @@ -104,7 +104,11 @@ async def check_domain( record_type = dns_rule["dns"]["type"] record_value = dns_rule["dns"]["value"] - entries = await self.resolver.query(record_name, record_type.upper()) + try: + entries = await self.resolver.query(record_name, record_type.upper()) + except aiodns.error.DNSError: + """Continue checks""" + entries = None if entries and record_type == "txt": for entry in entries: @@ -176,8 +180,8 @@ def get_required_dns_rules( "name": f"_control.{hostname}", "value": owner, }, - "info": f"Create a TXT record for _control.{hostname} with value = owner address", - "on_error": f"Owner address mismatch", + "info": f"Create a TXT record for _control.{hostname} with value {owner}", + "on_error": "Owner address mismatch", } ) From 779e01a7c022318d717493f311e250115fc3812e Mon Sep 17 00:00:00 2001 From: aliel Date: Thu, 5 Oct 2023 14:57:12 +0200 Subject: [PATCH 19/25] fix hostname_from_url when it's already a hostname --- src/aleph/sdk/domain.py | 6 +++++- tests/unit/test_domains.py | 8 +++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/aleph/sdk/domain.py b/src/aleph/sdk/domain.py index 60cf7408..b014f353 100644 --- a/src/aleph/sdk/domain.py +++ b/src/aleph/sdk/domain.py @@ -22,7 +22,11 @@ class TargetType(str, Enum): def hostname_from_url(url: Union[HttpUrl, str]) -> Hostname: - return Hostname(urlparse(url).netloc) + parsed = urlparse(url) + if all([parsed.scheme, parsed.netloc]) is True: + url = parsed.netloc + + return Hostname(url) class DomainValidator: diff --git a/tests/unit/test_domains.py b/tests/unit/test_domains.py index 221bd517..ffb1f7ae 100644 --- a/tests/unit/test_domains.py +++ b/tests/unit/test_domains.py @@ -5,10 +5,15 @@ from aleph.sdk.domain import DomainValidator from aleph.sdk.exceptions import DomainConfigurationError - from src.aleph.sdk.domain import TargetType, hostname_from_url +def test_hostname(): + hostname = hostname_from_url("https://aleph.im") + assert hostname == "aleph.im" + hostname = hostname_from_url("aleph.im") + assert hostname == "aleph.im" + @pytest.mark.asyncio async def test_query(): alephdns = DomainValidator() @@ -55,3 +60,4 @@ async def test_not_configured_domain(): hostname = hostname_from_url(url) with pytest.raises(DomainConfigurationError): status = await alephdns.check_domain(hostname, TargetType.IPFS, "0xfakeaddress") + assert type(status) is None From ec263bf97a467dca5d92a23a842a4253175b917d Mon Sep 17 00:00:00 2001 From: aliel Date: Thu, 5 Oct 2023 15:02:36 +0200 Subject: [PATCH 20/25] remove unused import --- tests/unit/test_domains.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/unit/test_domains.py b/tests/unit/test_domains.py index ffb1f7ae..f5fc3b89 100644 --- a/tests/unit/test_domains.py +++ b/tests/unit/test_domains.py @@ -1,6 +1,3 @@ -import asyncio -from urllib.parse import urlparse - import pytest from aleph.sdk.domain import DomainValidator @@ -14,6 +11,7 @@ def test_hostname(): hostname = hostname_from_url("aleph.im") assert hostname == "aleph.im" + @pytest.mark.asyncio async def test_query(): alephdns = DomainValidator() From d79d4f1071ee3622e89168f76aa85dfe7ae0e97e Mon Sep 17 00:00:00 2001 From: aliel Date: Mon, 9 Oct 2023 11:34:28 +0200 Subject: [PATCH 21/25] speedup dns detection using authoritative ns server --- src/aleph/sdk/domain.py | 61 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 59 insertions(+), 2 deletions(-) diff --git a/src/aleph/sdk/domain.py b/src/aleph/sdk/domain.py index b014f353..d00c010f 100644 --- a/src/aleph/sdk/domain.py +++ b/src/aleph/sdk/domain.py @@ -1,6 +1,6 @@ import logging from enum import Enum -from ipaddress import IPv6Address +from ipaddress import IPv4Address, IPv6Address from typing import Dict, Iterable, List, NewType, Optional, Union from urllib.parse import urlparse @@ -22,6 +22,8 @@ class TargetType(str, Enum): def hostname_from_url(url: Union[HttpUrl, str]) -> Hostname: + """Extract FQDN from url""" + parsed = urlparse(url) if all([parsed.scheme, parsed.netloc]) is True: url = parsed.netloc @@ -39,6 +41,60 @@ class DomainValidator: def __init__(self): self.resolver = aiodns.DNSResolver(servers=settings.DNS_RESOLVERS) + async def get_ns_servers(self, hostname: Hostname): + """Get ns servers of a domain""" + dns_servers = settings.DNS_RESOLVERS + fqdn = hostname + + stop = False + while stop == False: + """**Detect and get authoritative NS server of subdomains if delegated**""" + try: + entries = await self.resolver.query(fqdn, "NS") + servers = [] + for entry in entries: + servers += await self.get_ipv6_addresses(entry.host) + servers += await self.get_ipv4_addresses(entry.host) + + dns_servers = servers + stop = True + except aiodns.error.DNSError: + sub_domains = fqdn.split(".") + if len(sub_domains) > 2: + fqdn = ".".join(sub_domains[1:]) + continue + + if len(sub_domains) == 2: + stop = True + + return dns_servers + + async def get_resolver_for(self, hostname: Hostname): + dns_servers = await self.get_ns_servers(hostname) + return aiodns.DNSResolver(servers=dns_servers) + + async def get_target_type(self, fqdn: Hostname) -> Optional[TargetType]: + domain_validator = DomainValidator() + resolver = await domain_validator.get_resolver_for(fqdn) + try: + entry = await resolver.query(fqdn, "CNAME") + cname = getattr(entry, "cname") + if cname == settings.DNS_IPFS_DOMAIN: + return TargetType.IPFS + elif cname == settings.DNS_PROGRAM_DOMAIN: + return TargetType.PROGRAM + elif cname == settings.DNS_INSTANCE_DOMAIN: + return TargetType.INSTANCE + + return None + except aiodns.error.DNSError: + return None + + async def get_ipv4_addresses(self, hostname: Hostname) -> List[IPv4Address]: + """Returns all IPv4 addresses for a domain""" + entries: Iterable = await self.resolver.query(hostname, "A") or [] + return [entry.host for entry in entries] + async def get_ipv6_addresses(self, hostname: Hostname) -> List[IPv6Address]: """Returns all IPv6 addresses for a domain""" entries: Iterable = await self.resolver.query(hostname, "AAAA") or [] @@ -109,7 +165,8 @@ async def check_domain( record_value = dns_rule["dns"]["value"] try: - entries = await self.resolver.query(record_name, record_type.upper()) + resolver = await self.get_resolver_for(hostname) + entries = await resolver.query(record_name, record_type.upper()) except aiodns.error.DNSError: """Continue checks""" entries = None From a75a7aebac472b3561311afa12d9b1feabdd6216 Mon Sep 17 00:00:00 2001 From: aliel Date: Mon, 9 Oct 2023 15:21:31 +0200 Subject: [PATCH 22/25] fix cname to target --- src/aleph/sdk/domain.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/aleph/sdk/domain.py b/src/aleph/sdk/domain.py index d00c010f..671e7f24 100644 --- a/src/aleph/sdk/domain.py +++ b/src/aleph/sdk/domain.py @@ -79,11 +79,11 @@ async def get_target_type(self, fqdn: Hostname) -> Optional[TargetType]: try: entry = await resolver.query(fqdn, "CNAME") cname = getattr(entry, "cname") - if cname == settings.DNS_IPFS_DOMAIN: + if settings.DNS_IPFS_DOMAIN in cname: return TargetType.IPFS - elif cname == settings.DNS_PROGRAM_DOMAIN: + elif settings.DNS_PROGRAM_DOMAIN in cname: return TargetType.PROGRAM - elif cname == settings.DNS_INSTANCE_DOMAIN: + elif settings.DNS_INSTANCE_DOMAIN in cname: return TargetType.INSTANCE return None From 282a28712362321a4d19ac125bfbd3327376928f Mon Sep 17 00:00:00 2001 From: aliel Date: Thu, 12 Oct 2023 12:09:24 +0200 Subject: [PATCH 23/25] fix cond --- src/aleph/sdk/domain.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/aleph/sdk/domain.py b/src/aleph/sdk/domain.py index 671e7f24..ff8e3b09 100644 --- a/src/aleph/sdk/domain.py +++ b/src/aleph/sdk/domain.py @@ -1,7 +1,7 @@ import logging from enum import Enum from ipaddress import IPv4Address, IPv6Address -from typing import Dict, Iterable, List, NewType, Optional, Union +from typing import Any, AsyncIterable, Dict, Iterable, List, NewType, Optional, Union from urllib.parse import urlparse import aiodns @@ -47,11 +47,11 @@ async def get_ns_servers(self, hostname: Hostname): fqdn = hostname stop = False - while stop == False: + while stop is False: """**Detect and get authoritative NS server of subdomains if delegated**""" try: entries = await self.resolver.query(fqdn, "NS") - servers = [] + servers: List[Any] = [] for entry in entries: servers += await self.get_ipv6_addresses(entry.host) servers += await self.get_ipv4_addresses(entry.host) @@ -61,7 +61,7 @@ async def get_ns_servers(self, hostname: Hostname): except aiodns.error.DNSError: sub_domains = fqdn.split(".") if len(sub_domains) > 2: - fqdn = ".".join(sub_domains[1:]) + fqdn = Hostname(".".join(sub_domains[1:])) continue if len(sub_domains) == 2: @@ -121,7 +121,7 @@ async def get_dnslink(self, hostname: Hostname) -> Optional[str]: async def get_txt_values( self, hostname: Hostname, delimiter: Optional[str] = None - ) -> Iterable[str]: + ) -> AsyncIterable[str]: """Returns all TXT values for a domain""" entries: Iterable = await self.resolver.query(hostname, "TXT") or [] for entry in entries: From 3e0c98ffd4a9942972de9aa29faada0e272262d6 Mon Sep 17 00:00:00 2001 From: aliel Date: Thu, 9 Nov 2023 09:47:13 +0100 Subject: [PATCH 24/25] fix while loop --- src/aleph/sdk/domain.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/aleph/sdk/domain.py b/src/aleph/sdk/domain.py index ff8e3b09..32e92004 100644 --- a/src/aleph/sdk/domain.py +++ b/src/aleph/sdk/domain.py @@ -46,8 +46,7 @@ async def get_ns_servers(self, hostname: Hostname): dns_servers = settings.DNS_RESOLVERS fqdn = hostname - stop = False - while stop is False: + while True: """**Detect and get authoritative NS server of subdomains if delegated**""" try: entries = await self.resolver.query(fqdn, "NS") @@ -57,7 +56,7 @@ async def get_ns_servers(self, hostname: Hostname): servers += await self.get_ipv4_addresses(entry.host) dns_servers = servers - stop = True + break except aiodns.error.DNSError: sub_domains = fqdn.split(".") if len(sub_domains) > 2: @@ -65,7 +64,10 @@ async def get_ns_servers(self, hostname: Hostname): continue if len(sub_domains) == 2: - stop = True + break + except Exception as err: + logger.debug(f"Unexpected {err=}, {type(err)=}") + break return dns_servers From f94e30c389b64baed4adefee6ce98726ac4cf022 Mon Sep 17 00:00:00 2001 From: mhh Date: Tue, 21 Nov 2023 10:00:47 +0100 Subject: [PATCH 25/25] Fix typing; add docs; refactor for clarity --- src/aleph/sdk/domain.py | 173 ++++++++++++++++++++++-------------- src/aleph/sdk/exceptions.py | 3 +- tests/unit/test_domains.py | 5 +- 3 files changed, 112 insertions(+), 69 deletions(-) diff --git a/src/aleph/sdk/domain.py b/src/aleph/sdk/domain.py index 32e92004..fc654230 100644 --- a/src/aleph/sdk/domain.py +++ b/src/aleph/sdk/domain.py @@ -5,7 +5,7 @@ from urllib.parse import urlparse import aiodns -from pydantic import HttpUrl +from pydantic import BaseModel, HttpUrl from .conf import settings from .exceptions import DomainConfigurationError @@ -16,11 +16,39 @@ class TargetType(str, Enum): + """ + The type of target that a domain points to. + + - IPFS: The domain points to an IPFS hash. + - PROGRAM: The domain points to an aleph.im program. + - INSTANCE: The domain points to an aleph.im instance. + """ + IPFS = "ipfs" PROGRAM = "program" INSTANCE = "instance" +class DNSRule(BaseModel): + """ + A DNS rule is a DNS record that is required for a domain to be configured. + + Args: + name: The name of the rule. + dns: The DNS record. + info: Instructions to configure the DNS record. + on_error: Error message when the rule is not found. + """ + + name: str + dns: Dict[str, Any] + info: str + on_error: str + + def raise_error(self, status: Dict[str, bool]): + raise DomainConfigurationError((self.info, self.on_error, status)) + + def hostname_from_url(url: Union[HttpUrl, str]) -> Hostname: """Extract FQDN from url""" @@ -31,6 +59,25 @@ def hostname_from_url(url: Union[HttpUrl, str]) -> Hostname: return Hostname(url) +async def get_target_type(fqdn: Hostname) -> Optional[TargetType]: + """Returns the aleph.im target type of the domain""" + domain_validator = DomainValidator() + resolver = await domain_validator.get_resolver_for(fqdn) + try: + entry = await resolver.query(fqdn, "CNAME") + cname = getattr(entry, "cname") + if settings.DNS_IPFS_DOMAIN in cname: + return TargetType.IPFS + elif settings.DNS_PROGRAM_DOMAIN in cname: + return TargetType.PROGRAM + elif settings.DNS_INSTANCE_DOMAIN in cname: + return TargetType.INSTANCE + + return None + except aiodns.error.DNSError: + return None + + class DomainValidator: """ Tools used to analyze domain names used on the aleph.im network. @@ -41,13 +88,13 @@ class DomainValidator: def __init__(self): self.resolver = aiodns.DNSResolver(servers=settings.DNS_RESOLVERS) - async def get_ns_servers(self, hostname: Hostname): - """Get ns servers of a domain""" + async def get_name_servers(self, hostname: Hostname): + """Get DNS name servers (NS) of a domain""" dns_servers = settings.DNS_RESOLVERS fqdn = hostname while True: - """**Detect and get authoritative NS server of subdomains if delegated**""" + # Detect and get authoritative NS of subdomains if delegated try: entries = await self.resolver.query(fqdn, "NS") servers: List[Any] = [] @@ -72,26 +119,9 @@ async def get_ns_servers(self, hostname: Hostname): return dns_servers async def get_resolver_for(self, hostname: Hostname): - dns_servers = await self.get_ns_servers(hostname) + dns_servers = await self.get_name_servers(hostname) return aiodns.DNSResolver(servers=dns_servers) - async def get_target_type(self, fqdn: Hostname) -> Optional[TargetType]: - domain_validator = DomainValidator() - resolver = await domain_validator.get_resolver_for(fqdn) - try: - entry = await resolver.query(fqdn, "CNAME") - cname = getattr(entry, "cname") - if settings.DNS_IPFS_DOMAIN in cname: - return TargetType.IPFS - elif settings.DNS_PROGRAM_DOMAIN in cname: - return TargetType.PROGRAM - elif settings.DNS_INSTANCE_DOMAIN in cname: - return TargetType.INSTANCE - - return None - except aiodns.error.DNSError: - return None - async def get_ipv4_addresses(self, hostname: Hostname) -> List[IPv4Address]: """Returns all IPv4 addresses for a domain""" entries: Iterable = await self.resolver.query(hostname, "A") or [] @@ -140,37 +170,39 @@ async def get_txt_values( else: yield entry.text - async def check_domain_configured( - self, hostname: Hostname, target: TargetType, owner - ): - """Check if a domain is configured... for what ?""" - try: - logger.debug(f"Checking {target}") - return await self.check_domain(hostname, target, owner) - except Exception as error: - # FIXME: Do not catch any exception - raise DomainConfigurationError(error) - async def check_domain( self, hostname: Hostname, target: TargetType, owner: Optional[str] = None ) -> Dict: - """Check that the domain points towards the target.""" + """ + Checks that the domain points towards the given aleph.im target. + + Args: + hostname: The hostname of the domain. + target: The aleph.im target type. + owner: The owner wallet address of the domain for ownership proof. + + Raises: + DomainConfigurationError: If the domain is not configured. + + Returns: + A dictionary containing the status of the domain configuration. + """ status = {"cname": False, "owner_proof": False} dns_rules = self.get_required_dns_rules(hostname, target, owner) for dns_rule in dns_rules: - status[dns_rule["rule_name"]] = False + status[dns_rule.name] = False - record_name = dns_rule["dns"]["name"] - record_type = dns_rule["dns"]["type"] - record_value = dns_rule["dns"]["value"] + record_name = dns_rule.dns["name"] + record_type = dns_rule.dns["type"] + record_value = dns_rule.dns["value"] try: resolver = await self.get_resolver_for(hostname) entries = await resolver.query(record_name, record_type.upper()) except aiodns.error.DNSError: - """Continue checks""" + # Continue checks entries = None if entries and record_type == "txt": @@ -178,25 +210,32 @@ async def check_domain( if hasattr(entry, "text") and entry.text == record_value: break else: - raise DomainConfigurationError( - (dns_rule["info"], dns_rule["on_error"], status) - ) + return dns_rule.raise_error(status) elif ( entries is None or not hasattr(entries, record_type) or getattr(entries, record_type) != record_value ): - raise DomainConfigurationError( - (dns_rule["info"], dns_rule["on_error"], status) - ) + return dns_rule.raise_error(status) - status[dns_rule["rule_name"]] = True + status[dns_rule.name] = True return status def get_required_dns_rules( self, hostname: Hostname, target: TargetType, owner: Optional[str] = None - ) -> List[Dict]: + ) -> List[DNSRule]: + """ + Returns the DNS rules (CNAME, TXT) required for a domain to be configured. + + Args: + hostname: The hostname of the domain. + target: The aleph.im target type. + owner: The owner wallet address of the domain to add as an ownership proof. + + Returns: + A list of DNS rules with + """ target = target.lower() dns_rules = [] @@ -210,42 +249,46 @@ def get_required_dns_rules( # cname rule dns_rules.append( - { - "rule_name": "cname", - "dns": {"type": "cname", "name": hostname, "value": cname_value}, - "info": f"Create a CNAME record for {hostname} with value {cname_value}", - "on_error": f"CNAME record not found: {hostname}", - } + DNSRule( + name="cname", + dns={ + "type": "cname", + "name": hostname, + "value": cname_value, + }, + info=f"Create a CNAME record for {hostname} with value {cname_value}", + on_error=f"CNAME record not found: {hostname}", + ) ) if target == TargetType.IPFS: # ipfs rule dns_rules.append( - { - "rule_name": "delegation", - "dns": { + DNSRule( + name="delegation", + dns={ "type": "cname", "name": f"_dnslink.{hostname}", "value": f"_dnslink.{hostname}.{settings.DNS_STATIC_DOMAIN}", }, - "info": f"Create a CNAME record for _dnslink.{hostname} with value _dnslink.{hostname}.{settings.DNS_STATIC_DOMAIN}", - "on_error": f"CNAME record not found: _dnslink.{hostname}", - } + info=f"Create a CNAME record for _dnslink.{hostname} with value _dnslink.{hostname}.{settings.DNS_STATIC_DOMAIN}", + on_error=f"CNAME record not found: _dnslink.{hostname}", + ) ) if owner: # ownership rule dns_rules.append( - { - "rule_name": "owner_proof", - "dns": { + DNSRule( + name="owner_proof", + dns={ "type": "txt", "name": f"_control.{hostname}", "value": owner, }, - "info": f"Create a TXT record for _control.{hostname} with value {owner}", - "on_error": "Owner address mismatch", - } + info=f"Create a TXT record for _control.{hostname} with value {owner}", + on_error="Owner address mismatch", + ) ) return dns_rules diff --git a/src/aleph/sdk/exceptions.py b/src/aleph/sdk/exceptions.py index 8e10eee1..7ac7ae89 100644 --- a/src/aleph/sdk/exceptions.py +++ b/src/aleph/sdk/exceptions.py @@ -53,5 +53,6 @@ class FileTooLarge(Exception): class DomainConfigurationError(Exception): - "Raised when the domain checks are not satisfied" + """Raised when the domain checks are not satisfied""" + pass diff --git a/tests/unit/test_domains.py b/tests/unit/test_domains.py index f5fc3b89..380e4bb5 100644 --- a/tests/unit/test_domains.py +++ b/tests/unit/test_domains.py @@ -1,8 +1,7 @@ import pytest -from aleph.sdk.domain import DomainValidator +from aleph.sdk.domain import DomainValidator, TargetType, hostname_from_url from aleph.sdk.exceptions import DomainConfigurationError -from src.aleph.sdk.domain import TargetType, hostname_from_url def test_hostname(): @@ -30,7 +29,7 @@ async def test_get_ipv6_address(): ipv6_addresses = await alephdns.get_ipv6_addresses(hostname) assert ipv6_addresses is not None assert len(ipv6_addresses) > 0 - assert ":" in ipv6_addresses[0] + assert ":" in str(ipv6_addresses[0]) @pytest.mark.asyncio