diff --git a/examples/config.yml b/examples/config.yml index 84bc1ab4..9cba1248 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -193,6 +193,12 @@ # keyaliases: # com.example.another.plugin: "@com.example.another" +# It is possible to include the IP addresses of repos and mirrors in +# the index file. Enabling this requires network access and will +# cause DNS lookups to happen every time `fdroid update` is run. +# +# include_dns_lookups: true + # The full path to the root of the repository. It must be specified in # rsync/ssh format for a remote host/path. This is used for syncing a locally # generated repo to the server that is it hosted on. It must end in the diff --git a/fdroidserver/index.py b/fdroidserver/index.py index 9a3388b0..a71ac4b6 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -37,6 +37,7 @@ import logging import os import re import shutil +import socket import sys import tempfile import urllib.parse @@ -111,6 +112,15 @@ def make(apps, apks, repodir, archive): 'archive_url', common.config['repo_url'][:-4] + 'archive' ) repodict['address'] = archive_url + + # do dns lookup and store results for later use + ip4_array = get_dnsa_results(archive_url) + if len(ip4_array) > 0: + repodict['dnsA'] = ip4_array + ip6_array = get_dnsaaaa_results(archive_url) + if len(ip6_array) > 0: + repodict['dnsAAAA'] = ip6_array + if 'archive_web_base_url' in common.config: repodict["webBaseUrl"] = common.config['archive_web_base_url'] repo_section = os.path.basename(urllib.parse.urlparse(archive_url).path) @@ -118,6 +128,15 @@ def make(apps, apks, repodir, archive): repodict['name'] = common.config['repo_name'] repodict['icon'] = repo_icon repodict['address'] = common.config['repo_url'] + + # do dns lookup and store results for later use + ip4_array = get_dnsa_results(common.config['repo_url']) + if len(ip4_array) > 0: + repodict['dnsA'] = ip4_array + ip6_array = get_dnsaaaa_results(common.config['repo_url']) + if len(ip6_array) > 0: + repodict['dnsAAAA'] = ip6_array + if 'repo_web_base_url' in common.config: repodict["webBaseUrl"] = common.config['repo_web_base_url'] repodict['description'] = common.config['repo_description'] @@ -158,6 +177,30 @@ def make(apps, apks, repodir, archive): ) +def get_dnsa_results(url): + return get_dns_results(url, socket.AF_INET) + + +def get_dnsaaaa_results(url): + return get_dns_results(url, socket.AF_INET6) + + +def get_dns_results(url, inet_type): + ip_array = set() + if not common.config or not common.config.get('include_dns_lookups'): + return ip_array + try: + dns_results = socket.getaddrinfo(urllib.parse.urlparse(url).hostname, 443) + for result in dns_results: + socket_address = result[4] + ip_address = socket_address[0] + if result[0] == inet_type: + ip_array.add(ip_address) + except Exception as e: + logging.warning('Failed to get DNS results for ' + url + " " + str(e)) + return sorted(ip_array) + + def _should_file_be_generated(path, magic_string): if os.path.exists(path): with open(path) as f: @@ -715,10 +758,9 @@ def v2_repo(repodict, repodir, archive): repo["icon"] = localized_config["icon"] repo["address"] = repodict["address"] - if "mirrors" in repodict: - repo["mirrors"] = repodict["mirrors"] - if "webBaseUrl" in repodict: - repo["webBaseUrl"] = repodict["webBaseUrl"] + for key in 'dnsA', 'dnsAAAA', 'mirrors', 'webBaseUrl': + if key in repodict: + repo[key] = repodict[key] repo["timestamp"] = repodict["timestamp"] @@ -1685,6 +1727,15 @@ def add_mirrors_to_repodict(repo_section, repodict): found_primary = False errors = 0 for mirror in mirrors: + + # do dns lookup to store results for later use + ip4_array = get_dnsa_results(mirror['url']) + if len(ip4_array) > 0: + repodict['dnsA'] = ip4_array + ip6_array = get_dnsaaaa_results(mirror['url']) + if len(ip6_array) > 0: + repodict['dnsAAAA'] = ip6_array + if canonical_url == mirror['url']: found_primary = True mirror['isPrimary'] = True @@ -1706,7 +1757,12 @@ def add_mirrors_to_repodict(repo_section, repodict): raise FDroidException(_('"isPrimary" key should not be added to mirrors!')) if repodict['mirrors'] and not found_primary: - repodict['mirrors'].insert(0, {'isPrimary': True, 'url': repodict['address']}) + primary = {'isPrimary': True, 'url': repodict['address']} + if 'dnsA' in repodict: + primary['dnsA'] = repodict['dnsA'] + if 'dnsAAAA' in repodict: + primary['dnsAAAA'] = repodict['dnsAAAA'] + repodict['mirrors'].insert(0, primary) def get_mirror_service_urls(mirror): diff --git a/tests/test_index.py b/tests/test_index.py index dedc4226..63d186e7 100755 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -28,7 +28,20 @@ class Options: verbose = False -class IndexTest(unittest.TestCase): +class SetUpTearDownMixin: + """A mixin with no tests in it for shared setUp and tearDown.""" + + def setUp(self): + common.config = None + self._td = mkdtemp() + self.testdir = self._td.name + + def tearDown(self): + common.config = None + self._td.cleanup() + + +class IndexTest(SetUpTearDownMixin, unittest.TestCase): @classmethod def setUpClass(cls): # TODO something should remove cls.index_v1_jar, but it was @@ -38,6 +51,7 @@ class IndexTest(unittest.TestCase): cls.index_v1_jar = basedir / 'repo' / 'index-v1.jar' def setUp(self): + super().setUp() (basedir / common.CONFIG_FILE).chmod(0o600) os.chdir(basedir) # so read_config() can find config.yml @@ -49,12 +63,6 @@ class IndexTest(unittest.TestCase): signindex.config = config update.config = config - self._td = mkdtemp() - self.testdir = self._td.name - - def tearDown(self): - self._td.cleanup() - def _sign_test_index_v1_jar(self): if not self.index_v1_jar.exists(): signindex.sign_index(self.index_v1_jar.parent, 'index-v1.json') @@ -1088,3 +1096,30 @@ class AltstoreIndexTest(unittest.TestCase): }, json.load(f), ) + + +class DnsCacheTest(SetUpTearDownMixin, unittest.TestCase): + + url = 'https://f-droid.org/repo/entry.jar' + + def test_defaults_to_no_dns(self): + self.assertFalse(index.get_dnsa_results(self.url)) + self.assertFalse(index.get_dnsaaaa_results(self.url)) + + def test_f_droid_org_a(self): + common.config = {'include_dns_lookups': True} + self.assertTrue(index.get_dnsa_results(self.url)) + + def test_f_droid_org_aaaa(self): + common.config = {'include_dns_lookups': True} + self.assertTrue(index.get_dnsaaaa_results(self.url)) + + def test_no_A_duplicates(self): + common.config = {'include_dns_lookups': True} + a = index.get_dnsa_results(self.url) + self.assertEqual(a, sorted(set(a))) + + def test_no_AAAA_duplicates(self): + common.config = {'include_dns_lookups': True} + aaaa = index.get_dnsaaaa_results(self.url) + self.assertEqual(aaaa, sorted(set(aaaa))) diff --git a/tests/test_integration.py b/tests/test_integration.py index 7011d24b..456e489e 100755 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -1706,3 +1706,17 @@ class IntegrationTest(unittest.TestCase): 'fdroid/repo/index-v1.json', }, ) + + def test_dns_in_index_v2(self): + self.fdroid_init_with_prebuilt_keystore() + self.update_yaml( + common.CONFIG_FILE, + {"include_dns_lookups": True, "mirrors": ["https://f-droid.org/fdroid"]}, + ) + self.assert_run(self.fdroid_cmd + ["update", "--pretty", "--nosign"]) + with open('repo/index-v2.json') as fp: + data = json.load(fp) + self.assertIsNotNone(data['repo'].get('dnsA')) + self.assertIsNotNone(data['repo'].get('dnsAAAA')) + self.assertIsNotNone(data['repo']['mirrors'][0].get('dnsA')) + self.assertIsNotNone(data['repo']['mirrors'][0].get('dnsAAAA'))