Merge branch 'add_dns_info' into 'master'

Add DNS results when building repo index

See merge request fdroid/fdroidserver!1786

examples/config.yml

@@ -193,6 +193,12 @@
 # keyaliases:
 #   com.example.another.plugin: "@com.example.another"
 
+# It is possible to include the IP addresses of repos and mirrors in
+# the index file.  Enabling this requires network access and will
+# cause DNS lookups to happen every time `fdroid update` is run.
+#
+# include_dns_lookups: true
+
 # The full path to the root of the repository. It must be specified in
 # rsync/ssh format for a remote host/path. This is used for syncing a locally
 # generated repo to the server that is it hosted on. It must end in the

fdroidserver/index.py

@@ -37,6 +37,7 @@ import logging
 import os
 import re
 import shutil
+import socket
 import sys
 import tempfile
 import urllib.parse
@@ -111,6 +112,15 @@ def make(apps, apks, repodir, archive):
             'archive_url', common.config['repo_url'][:-4] + 'archive'
         )
         repodict['address'] = archive_url
+
+        # do dns lookup and store results for later use
+        ip4_array = get_dnsa_results(archive_url)
+        if len(ip4_array) > 0:
+            repodict['dnsA'] = ip4_array
+        ip6_array = get_dnsaaaa_results(archive_url)
+        if len(ip6_array) > 0:
+            repodict['dnsAAAA'] = ip6_array
+
         if 'archive_web_base_url' in common.config:
             repodict["webBaseUrl"] = common.config['archive_web_base_url']
         repo_section = os.path.basename(urllib.parse.urlparse(archive_url).path)
@@ -118,6 +128,15 @@ def make(apps, apks, repodir, archive):
         repodict['name'] = common.config['repo_name']
         repodict['icon'] = repo_icon
         repodict['address'] = common.config['repo_url']
+
+        # do dns lookup and store results for later use
+        ip4_array = get_dnsa_results(common.config['repo_url'])
+        if len(ip4_array) > 0:
+            repodict['dnsA'] = ip4_array
+        ip6_array = get_dnsaaaa_results(common.config['repo_url'])
+        if len(ip6_array) > 0:
+            repodict['dnsAAAA'] = ip6_array
+
         if 'repo_web_base_url' in common.config:
             repodict["webBaseUrl"] = common.config['repo_web_base_url']
         repodict['description'] = common.config['repo_description']
@@ -158,6 +177,30 @@ def make(apps, apks, repodir, archive):
     )
 
 
+def get_dnsa_results(url):
+    return get_dns_results(url, socket.AF_INET)
+
+
+def get_dnsaaaa_results(url):
+    return get_dns_results(url, socket.AF_INET6)
+
+
+def get_dns_results(url, inet_type):
+    ip_array = set()
+    if not common.config or not common.config.get('include_dns_lookups'):
+        return ip_array
+    try:
+        dns_results = socket.getaddrinfo(urllib.parse.urlparse(url).hostname, 443)
+        for result in dns_results:
+            socket_address = result[4]
+            ip_address = socket_address[0]
+            if result[0] == inet_type:
+                ip_array.add(ip_address)
+    except Exception as e:
+        logging.warning('Failed to get DNS results for ' + url + " " + str(e))
+    return sorted(ip_array)
+
+
 def _should_file_be_generated(path, magic_string):
     if os.path.exists(path):
         with open(path) as f:
@@ -715,10 +758,9 @@ def v2_repo(repodict, repodir, archive):
             repo["icon"] = localized_config["icon"]
 
     repo["address"] = repodict["address"]
-    if "mirrors" in repodict:
-        repo["mirrors"] = repodict["mirrors"]
-    if "webBaseUrl" in repodict:
-        repo["webBaseUrl"] = repodict["webBaseUrl"]
+    for key in 'dnsA', 'dnsAAAA', 'mirrors', 'webBaseUrl':
+        if key in repodict:
+            repo[key] = repodict[key]
 
     repo["timestamp"] = repodict["timestamp"]
 
@@ -1685,6 +1727,15 @@ def add_mirrors_to_repodict(repo_section, repodict):
     found_primary = False
     errors = 0
     for mirror in mirrors:
+
+        # do dns lookup to store results for later use
+        ip4_array = get_dnsa_results(mirror['url'])
+        if len(ip4_array) > 0:
+            repodict['dnsA'] = ip4_array
+        ip6_array = get_dnsaaaa_results(mirror['url'])
+        if len(ip6_array) > 0:
+            repodict['dnsAAAA'] = ip6_array
+
         if canonical_url == mirror['url']:
             found_primary = True
             mirror['isPrimary'] = True
@@ -1706,7 +1757,12 @@ def add_mirrors_to_repodict(repo_section, repodict):
         raise FDroidException(_('"isPrimary" key should not be added to mirrors!'))
 
     if repodict['mirrors'] and not found_primary:
-        repodict['mirrors'].insert(0, {'isPrimary': True, 'url': repodict['address']})
+        primary = {'isPrimary': True, 'url': repodict['address']}
+        if 'dnsA' in repodict:
+            primary['dnsA'] = repodict['dnsA']
+        if 'dnsAAAA' in repodict:
+            primary['dnsAAAA'] = repodict['dnsAAAA']
+        repodict['mirrors'].insert(0, primary)
 
 
 def get_mirror_service_urls(mirror):

tests/test_index.py

@@ -28,7 +28,20 @@ class Options:
     verbose = False
 
 
-class IndexTest(unittest.TestCase):
+class SetUpTearDownMixin:
+    """A mixin with no tests in it for shared setUp and tearDown."""
+
+    def setUp(self):
+        common.config = None
+        self._td = mkdtemp()
+        self.testdir = self._td.name
+
+    def tearDown(self):
+        common.config = None
+        self._td.cleanup()
+
+
+class IndexTest(SetUpTearDownMixin, unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         # TODO something should remove cls.index_v1_jar, but it was
@@ -38,6 +51,7 @@ class IndexTest(unittest.TestCase):
         cls.index_v1_jar = basedir / 'repo' / 'index-v1.jar'
 
     def setUp(self):
+        super().setUp()
         (basedir / common.CONFIG_FILE).chmod(0o600)
         os.chdir(basedir)  # so read_config() can find config.yml
 
@@ -49,12 +63,6 @@ class IndexTest(unittest.TestCase):
         signindex.config = config
         update.config = config
 
-        self._td = mkdtemp()
-        self.testdir = self._td.name
-
-    def tearDown(self):
-        self._td.cleanup()
-
     def _sign_test_index_v1_jar(self):
         if not self.index_v1_jar.exists():
             signindex.sign_index(self.index_v1_jar.parent, 'index-v1.json')
@@ -1088,3 +1096,30 @@ class AltstoreIndexTest(unittest.TestCase):
                     },
                     json.load(f),
                 )
+
+
+class DnsCacheTest(SetUpTearDownMixin, unittest.TestCase):
+
+    url = 'https://f-droid.org/repo/entry.jar'
+
+    def test_defaults_to_no_dns(self):
+        self.assertFalse(index.get_dnsa_results(self.url))
+        self.assertFalse(index.get_dnsaaaa_results(self.url))
+
+    def test_f_droid_org_a(self):
+        common.config = {'include_dns_lookups': True}
+        self.assertTrue(index.get_dnsa_results(self.url))
+
+    def test_f_droid_org_aaaa(self):
+        common.config = {'include_dns_lookups': True}
+        self.assertTrue(index.get_dnsaaaa_results(self.url))
+
+    def test_no_A_duplicates(self):
+        common.config = {'include_dns_lookups': True}
+        a = index.get_dnsa_results(self.url)
+        self.assertEqual(a, sorted(set(a)))
+
+    def test_no_AAAA_duplicates(self):
+        common.config = {'include_dns_lookups': True}
+        aaaa = index.get_dnsaaaa_results(self.url)
+        self.assertEqual(aaaa, sorted(set(aaaa)))

tests/test_integration.py

@@ -1706,3 +1706,17 @@ class IntegrationTest(unittest.TestCase):
                 'fdroid/repo/index-v1.json',
             },
         )
+
+    def test_dns_in_index_v2(self):
+        self.fdroid_init_with_prebuilt_keystore()
+        self.update_yaml(
+            common.CONFIG_FILE,
+            {"include_dns_lookups": True, "mirrors": ["https://f-droid.org/fdroid"]},
+        )
+        self.assert_run(self.fdroid_cmd + ["update", "--pretty", "--nosign"])
+        with open('repo/index-v2.json') as fp:
+            data = json.load(fp)
+        self.assertIsNotNone(data['repo'].get('dnsA'))
+        self.assertIsNotNone(data['repo'].get('dnsAAAA'))
+        self.assertIsNotNone(data['repo']['mirrors'][0].get('dnsA'))
+        self.assertIsNotNone(data['repo']['mirrors'][0].get('dnsAAAA'))