move 'added' to internal index-v2 format

This ended up being a refactoring of the whole KnownApks setup.

fdroidserver/common.py

@@ -2903,21 +2903,21 @@ def check_system_clock(dt_obj, path):
                         + 'sudo date -s "' + str(dt_obj) + '"')
 
 
-class KnownApks:
-    """Permanent store of existing APKs with the date they were added.
+class PackageAddedCache:
+    """Store/fetch the added timestamp of packages.
+
+    This uses a cached value for "now" that is set once to avoid
+    preferencing apps based on any particular sort, like alpha sort of
+    file name or Application ID.  This way all new apps in a single
+    run get the same "added" time.
 
-    This is currently the only way to permanently store the "updated"
-    date of APKs.
     """
 
-    def __init__(self):
-        """Load filename/date info about previously seen APKs.
-
-        Since the appid and date strings both will never have spaces,
-        this is parsed as a list from the end to allow the filename to
-        have any combo of spaces.
-        """
-        self.apks = {}
+    def __init__(self, use_date_from_file=False):
+        """Load filename/date info about previously seen Versions."""
+        self.now = int(time.time() * 1000)
+        self.use_date_from_file = use_date_from_file
+        self.versions = {}
         for part in ('repo', 'archive'):
             path = os.path.join(part, 'index-v2.json')
             if os.path.isfile(path):
@@ -2925,24 +2925,31 @@ class KnownApks:
                     index = json.load(f)
                     for appid, data in index["packages"].items():
                         for version in data["versions"].values():
-                            filename = version["file"]["name"][1:]
-                            date = datetime.fromtimestamp(version["added"] // 1000, tz=timezone.utc)
-                            self.apks[filename] = date
+                            vpath = os.path.join(
+                                part,
+                                version["file"]["name"][1:],
+                            )
+                            self.versions[vpath] = version["added"]
 
-    def recordapk(self, apkName, default_date=None):
-        """
-        Record an APK (if it's new, otherwise does nothing).
+    def get(self, vpath, use_date_from_file=False):  # TODO do this based on sha256
+        """Get 'added' time, set current time as 'added' if file is new.
+
+        Parameters
+        ----------
+        vpath
+          path to file that is being added to the index
 
         Returns
         -------
-        datetime
-          the date it was added as a datetime instance.
+        int
+          timestamp as Java milliseconds since UNIX epoch
         """
-        if apkName not in self.apks:
-            if default_date is None:
-                default_date = datetime.now(timezone.utc)
-            self.apks[apkName] = default_date
-        return self.apks[apkName]
+        if vpath not in self.versions:
+            if use_date_from_file or self.use_date_from_file:
+                self.versions[vpath] = int(os.stat(vpath).st_mtime * 1000)
+            else:
+                self.versions[vpath] = self.now
+        return self.versions[vpath]
 
 
 def get_file_extension(filename):

fdroidserver/index.py

@@ -513,6 +513,19 @@ def dict_diff(source, target):
     return result
 
 
+def datetime_from_millis(millis):
+    """Convert epoch milliseconds to datetime instance.
+
+    This is the format returned by Java's System.currentTimeMillis().
+
+    Parameters
+    ----------
+    millis
+      Java-style integer time since UNIX epoch in milliseconds
+    """
+    return datetime.utcfromtimestamp(millis / 1000)
+
+
 def convert_datetime(obj):
     if isinstance(obj, datetime):
         # Java prefers milliseconds
@@ -598,13 +611,10 @@ def convert_version(version, app, repodir):
     metadata file.
 
     """
-    ver = {}
-    if "added" in version:
-        ver["added"] = convert_datetime(version["added"])
-    else:
-        ver["added"] = 0
-
-    ver["file"] = version["file"]
+    ver = {
+        "added": version["added"],
+        "file": version["file"],
+    }
     ver["file"]["name"] = f'/{version["file"]["name"]}'  # TODO remove for index-v3
 
     ipfsCIDv1 = version.get("ipfsCIDv1")
@@ -1269,9 +1279,11 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, signer_fingerprints):
 
         addElement('id', app.id, doc, apel)
         if app.added:
-            addElement('added', app.added.strftime('%Y-%m-%d'), doc, apel)
+            added = datetime_from_millis(app.added)
+            addElement('added', added.strftime('%Y-%m-%d'), doc, apel)
         if app.lastUpdated:
-            addElement('lastupdated', app.lastUpdated.strftime('%Y-%m-%d'), doc, apel)
+            lastUpdated = datetime_from_millis(app.lastUpdated)
+            addElement('lastupdated', lastUpdated.strftime('%Y-%m-%d'), doc, apel)
 
         addElementCheckLocalized('name', app, 'Name', doc, apel, name_from_apk)
         addElementCheckLocalized('summary', app, 'Summary', doc, apel)
@@ -1393,7 +1405,8 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, signer_fingerprints):
                 'obbPatchFileSha256', apk, 'obbPatchFileSha256', doc, apkel
             )
             if 'added' in apk:
-                addElement('added', apk['added'].strftime('%Y-%m-%d'), doc, apkel)
+                added = datetime_from_millis(apk['added'])
+                addElement('added', added.strftime('%Y-%m-%d'), doc, apkel)
 
             if file_extension == 'apk':  # sig is required for APKs, but only APKs
                 addElement('sig', apk['sig'], doc, apkel)
@@ -2053,7 +2066,7 @@ def make_altstore(apps, apks, config, repodir, pretty=False):
                 if apk['packageName'] == packageName and file_extension == 'ipa':
                     v = {
                         "version": apk["manifest"]["versionName"],
-                        "date": apk["added"].isoformat(),
+                        "date": datetime_from_millis(apk["added"]).isoformat(),
                         "downloadURL": f"{config['repo_url']}/{apk['file']['name']}",
                         "size": apk['file']['size'],
                     }

fdroidserver/update.py

@@ -36,7 +36,7 @@ import time
 import warnings
 import zipfile
 from argparse import ArgumentParser
-from datetime import datetime, timezone
+from datetime import datetime
 from pathlib import Path
 
 import asn1crypto.cms
@@ -383,8 +383,6 @@ def get_cache():
         if 'antiFeatures' in v:
             if not isinstance(v['antiFeatures'], dict):
                 v['antiFeatures'] = {k: {} for k in sorted(v['antiFeatures'])}
-        if 'added' in v:
-            v['added'] = datetime.fromtimestamp(v['added'], tz=timezone.utc)
         if v.get('srcname') and not v.get('srcnameSha256'):
             f = f'archive/{v["srcname"]}'
             if not os.path.exists(f):
@@ -684,7 +682,7 @@ def parse_ipa(ipa_path, file_size, sha256):
     return ipa
 
 
-def scan_repo_for_ipas(apkcache, repodir, knownapks):
+def scan_repo_for_ipas(apkcache, repodir, package_added_cache):
     """Scan for IPA files in a given repo directory.
 
     Parameters
@@ -693,7 +691,7 @@ def scan_repo_for_ipas(apkcache, repodir, knownapks):
       cache dictionary containting cached file infos from previous runs
     repodir
       repo directory to scan
-    knownapks
+    package_added_cache
       list of all known files, as per metadata.read_metadata
 
     Returns
@@ -720,9 +718,7 @@ def scan_repo_for_ipas(apkcache, repodir, knownapks):
             apkcache[ipa_name] = ipa
             cachechanged = True
 
-        added = knownapks.recordapk(ipa_name)
-        if added:
-            ipa['added'] = added
+        ipa['added'] = package_added_cache.get(ipa_path)
 
         ipas.append(ipa)
 
@@ -1542,7 +1538,7 @@ def insert_localized_ios_app_metadata(apps_with_packages):
             shutil.copy2(icon_src, icon_dest)
 
 
-def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
+def scan_repo_files(apkcache, repodir, package_added_cache, use_date_from_file=False):
     """Scan a repo for all files with an extension except APK/OBB/IPA.
 
     This allows putting all kinds of files into repostories. E.g. Media Files,
@@ -1554,7 +1550,7 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
       current cached info about all repo files
     repodir
       repo directory to scan
-    knownapks
+    package_added_cache
       list of all known files, as per metadata.read_metadata
     use_date_from_file
       use date from file (instead of current date) for newly added files
@@ -1633,18 +1629,7 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
             apkcache[name_utf8] = repo_file
             cachechanged = True
 
-        if use_date_from_file:
-            timestamp = stat.st_ctime
-            default_date_param = datetime.fromtimestamp(timestamp, tz=timezone.utc)
-        else:
-            default_date_param = None
-
-        # Record in knownapks, getting the added date at the same time..
-        added = knownapks.recordapk(
-            repo_file['file']['name'], default_date=default_date_param
-        )
-        if added:
-            repo_file['added'] = added
+        repo_file['added'] = package_added_cache.get(filename.decode(), use_date_from_file)
 
         repo_files.append(repo_file)
 
@@ -2011,7 +1996,7 @@ def _fill_reason(msg):
     return d
 
 
-def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
+def process_apk(apkcache, apkfilename, repodir, package_added_cache, use_date_from_apk=False,
                 allow_disabled_algorithms=False, archive_bad_sig=False, apps=None, cache_timestamp=0):
     """Process the apk with the given filename in the given repo directory.
 
@@ -2025,8 +2010,8 @@ def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=Fal
       the filename of the apk to scan
     repodir
       repo directory to scan
-    knownapks
-      known apks info
+    package_added_cache
+      list of all known files, as per metadata.read_metadata
     use_date_from_apk
       use date from APK (instead of current date) for newly added APKs
     allow_disabled_algorithms
@@ -2187,15 +2172,7 @@ def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=Fal
         # resize existing icons for densities missing in the APK
         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
 
-        if use_date_from_apk:
-            default_date_param = datetime.fromtimestamp(os.stat(apkfile).st_mtime, tz=timezone.utc)
-        else:
-            default_date_param = None
-
-        # Record in known apks, getting the added date at the same time..
-        added = knownapks.recordapk(apk['file']['name'], default_date=default_date_param)
-        if added:
-            apk['added'] = added
+        apk['added'] = package_added_cache.get(apkfile, use_date_from_apk)
 
         apkcache[apkfilename] = apk
         cachechanged = True
@@ -2203,7 +2180,7 @@ def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=Fal
     return False, apk, cachechanged
 
 
-def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False, apps=None, cache_timestamp=0):
+def process_apks(apkcache, repodir, package_added_cache, use_date_from_apk=False, apps=None, cache_timestamp=0):
     """Process the apks in the given repo directory.
 
     This also extracts the icons.
@@ -2214,8 +2191,8 @@ def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False, apps=Non
       current apk cache information
     repodir
       repo directory to scan
-    knownapks
-     b known apks info
+    package_added_cache
+      list of all known files, as per metadata.read_metadata
     use_date_from_apk
       use date from APK (instead of current date) for newly added APKs
     cache_timestamp
@@ -2240,7 +2217,7 @@ def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False, apps=Non
     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
         apkfilename = apkfile[len(repodir) + 1 :]
         ada = disabled_algorithms_allowed()
-        (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
+        (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, package_added_cache,
                                              use_date_from_apk, ada, True, apps, cache_timestamp)
         if skip:
             continue
@@ -2895,7 +2872,7 @@ def main():
     apps = metadata.read_metadata()
 
     # Read known apks data (will be updated and written back when we've finished)
-    knownapks = common.KnownApks()
+    package_added_cache = common.PackageAddedCache(options.use_date_from_apk)
 
     # Get APK cache
     apkcache = get_cache()
@@ -2910,20 +2887,18 @@ def main():
     apks, cachechanged = process_apks(
         apkcache,
         repodirs[0],
-        knownapks,
+        package_added_cache,
         options.use_date_from_apk,
         apps,
         cache_timestamp,
     )
 
     output_status_stage(status_output, 'scan_repo_files')
-    files, fcachechanged = scan_repo_files(
-        apkcache, repodirs[0], knownapks, options.use_date_from_apk
-    )
+    files, fcachechanged = scan_repo_files(apkcache, repodirs[0], package_added_cache)
     cachechanged = cachechanged or fcachechanged
     apks += files
 
-    ipas, icachechanged = scan_repo_for_ipas(apkcache, repodirs[0], knownapks)
+    ipas, icachechanged = scan_repo_for_ipas(apkcache, repodirs[0], package_added_cache)
     cachechanged = cachechanged or icachechanged
     apks += ipas
 
@@ -3003,7 +2978,7 @@ def main():
 
     # Scan the archive repo for apks as well
     if len(repodirs) > 1:
-        archapks, cc = process_apks(apkcache, repodirs[1], knownapks,
+        archapks, cc = process_apks(apkcache, repodirs[1], package_added_cache,
                                     options.use_date_from_apk, apps, cache_timestamp)
         if cc:
             cachechanged = True

tests/test_common.py

@@ -3005,36 +3005,36 @@ class CommonTest(SetUpTearDownMixin, unittest.TestCase):
             [{'url': s}], fdroidserver.common.parse_list_of_dicts(mirrors)
         )
 
-    def test_KnownApks_recordapk(self):
+    def test_PackageAddedCache_get(self):
         """Test that added dates are being fetched from the index.
 
-        There are more related tests in tests/run-tests.
+        There are more related tests in tests/test_integration.py.
 
         """
-        now = datetime.now(timezone.utc)
-        knownapks = fdroidserver.common.KnownApks()
-        for apkName in knownapks.apks:
-            knownapks.recordapk(apkName, default_date=now)
-        for added in knownapks.apks.values():
-            self.assertNotEqual(added, now)
+        package_added_cache = fdroidserver.common.PackageAddedCache()
+        package_added_cache.now = 1234567890
+        for filename in package_added_cache.versions:
+            package_added_cache.get(filename)
+        for added in package_added_cache.versions.values():
+            self.assertNotEqual(added, package_added_cache.now)
 
-    def test_KnownApks_recordapk_new(self):
+    def test_PackageAddedCache_get_new(self):
         """Test that new added dates work, and are not replaced later.
 
-        There are more related tests in tests/run-tests.
+        There are more related tests in tests/test_integration.py.
 
         """
-        now = datetime.now(timezone.utc)
-        knownapks = fdroidserver.common.KnownApks()
+        package_added_cache = fdroidserver.common.PackageAddedCache()
+        package_added_cache.now = 1234567890
         fake_apk = 'fake.apk'
-        knownapks.recordapk(fake_apk, default_date=now)
-        for apk, added in knownapks.apks.items():
+        package_added_cache.get(fake_apk)
+        for apk, added in package_added_cache.versions.items():
             if apk == fake_apk:
-                self.assertEqual(added, now)
+                self.assertEqual(added, package_added_cache.now)
             else:
-                self.assertNotEqual(added, now)
-        knownapks.recordapk(fake_apk, default_date=datetime.now(timezone.utc))
-        self.assertEqual(knownapks.apks[fake_apk], now)
+                self.assertNotEqual(added, package_added_cache.now)
+        package_added_cache.get(fake_apk)
+        self.assertEqual(package_added_cache.versions[fake_apk], package_added_cache.now)
 
     def test_get_mirrors_fdroidorg(self):
         mirrors = fdroidserver.common.get_mirrors(

tests/test_index.py

@@ -519,7 +519,7 @@ class IndexTest(unittest.TestCase):
             'sig': '03f9b2f848d22fd1d8d1331e8b1b486d',
             'signer': '51cfa5c8a743833ad89acf81cb755936876a5c8b8eca54d1ffdcec0cdca25d0e',
             'icon': 'info.zwanenburg.caffeinetile.4.xml',
-            'added': datetime.datetime.fromtimestamp(1539122400),
+            'added': 1539122400000,
         }
         requestsdict = {'install': [], 'uninstall': []}
         common.config['repo_pubkey'] = 'ffffffffffffffffffffffffffffffffff'
@@ -918,7 +918,7 @@ class AltstoreIndexTest(unittest.TestCase):
                 "manifest": {
                     "versionName": "v123",
                 },
-                "added": datetime.datetime(2000, 2, 2, 2, 2, 2),
+                "added": 949456922000,
                 "ipa_MinimumOSVersion": "10.0",
                 "ipa_DTPlatformVersion": "12.0",
                 "ipa_permissions": [

tests/test_integration.py

@@ -771,20 +771,28 @@ class IntegrationTest(unittest.TestCase):
         self.update_yaml("metadata/com.politedroid.yml", {"ArchivePolicy": 1})
 
         self.assert_run(self.fdroid_cmd + ["update", "--pretty", "--nosign"])
-        timestamp = 1498176000000
+        timestamp = 1498176000000  # value cached in repo/index-v2.json
         index_v1_json = json.loads(Path("repo/index-v1.json").read_text())
         self.assertEqual(timestamp, index_v1_json['apps'][0]['added'])
-        self.assertEqual(timestamp, index_v1_json['apps'][0]['lastUpdated'])
         self.assertEqual(
             timestamp, index_v1_json['packages']['com.politedroid'][0]['added']
         )
-        # the archive will have the added timestamp for the app and for the apk,
-        # both need to be there
+        # Since there is no archive/index-v2.json, a cached 'added'
+        # value cannot be loaded from it. Therefore, the archive APK's
+        # 'added' value is "now" which is newer than 'timestamp',
+        # which was loaded from repo/index-v2.json.
         archive_json = json.loads(Path("archive/index-v1.json").read_text())
-        self.assertEqual(timestamp, archive_json['apps'][0]['added'])
-        self.assertEqual(timestamp, archive_json['apps'][0]['lastUpdated'])
         self.assertEqual(
-            timestamp, archive_json['packages']['com.politedroid'][0]['added']
+            index_v1_json['apps'][0]['added'],
+            archive_json['apps'][0]['added'],
+        )
+        self.assertEqual(
+            archive_json['apps'][0]['lastUpdated'],
+            archive_json['packages']['com.politedroid'][0]['added'],
+        )
+        self.assertEqual(
+            archive_json['apps'][0]['lastUpdated'],
+            index_v1_json['apps'][0]['lastUpdated'],
         )
 
     def test_whatsnew_from_fastlane_without_cvc_set(self):

tests/test_update.py

@@ -17,7 +17,6 @@ import time
 import unittest
 import zipfile
 from binascii import hexlify
-from datetime import datetime
 from pathlib import Path
 from unittest import mock
 
@@ -367,8 +366,8 @@ class UpdateTest(unittest.TestCase):
 
         apps = fdroidserver.metadata.read_metadata()
         apps['info.guardianproject.urzip']['CurrentVersionCode'] = 100
-        knownapks = fdroidserver.common.KnownApks()
-        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks, False)
+        package_added_cache = fdroidserver.common.PackageAddedCache()
+        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', package_added_cache, False)
         fdroidserver.update.insert_localized_app_metadata(apps)
         fdroidserver.update.ingest_screenshots_from_repo_dir(apps)
         fdroidserver.update.apply_info_from_latest_apk(apps, apks)
@@ -454,8 +453,8 @@ class UpdateTest(unittest.TestCase):
         fdroidserver.update.options.delete_unknown = True
 
         apps = fdroidserver.metadata.read_metadata()
-        knownapks = fdroidserver.common.KnownApks()
-        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks, False)
+        package_added_cache = fdroidserver.common.PackageAddedCache()
+        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', package_added_cache, False)
 
         appid = 'info.guardianproject.checkey'
         testapps = {appid: copy.copy(apps[appid])}
@@ -721,8 +720,8 @@ class UpdateTest(unittest.TestCase):
         fdroidserver.update.options.delete_unknown = True
 
         apps = fdroidserver.metadata.read_metadata()
-        knownapks = fdroidserver.common.KnownApks()
-        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks, False)
+        package_added_cache = fdroidserver.common.PackageAddedCache()
+        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', package_added_cache, False)
         self.assertEqual(len(apks), 18)
         apk = apks[1]
         manifest = apk['manifest']
@@ -779,13 +778,13 @@ class UpdateTest(unittest.TestCase):
         fdroidserver.update.options.delete_unknown = True
 
         fdroidserver.metadata.read_metadata()
-        knownapks = fdroidserver.common.KnownApks()
+        package_added_cache = fdroidserver.common.PackageAddedCache()
         apkcache = fdroidserver.update.get_cache()
         self.assertEqual(2, len(apkcache))
         self.assertEqual(fdroidserver.update.METADATA_VERSION, apkcache["METADATA_VERSION"])
         self.assertEqual(fdroidserver.update.options.allow_disabled_algorithms,
                          apkcache['allow_disabled_algorithms'])
-        apks, cachechanged = fdroidserver.update.process_apks(apkcache, 'repo', knownapks, False)
+        apks, cachechanged = fdroidserver.update.process_apks(apkcache, 'repo', package_added_cache, False)
         fdroidserver.update.write_cache(apkcache)
 
         fdroidserver.update.options.clean = False
@@ -808,13 +807,13 @@ class UpdateTest(unittest.TestCase):
         os.mkdir('repo')
         filename = 'Norway_bouvet_europe_2.obf.zip'
         shutil.copy(basedir / filename, 'repo')
-        knownapks = fdroidserver.common.KnownApks()
-        files, fcachechanged = fdroidserver.update.scan_repo_files(dict(), 'repo', knownapks, False)
+        package_added_cache = fdroidserver.common.PackageAddedCache()
+        files, fcachechanged = fdroidserver.update.scan_repo_files(dict(), 'repo', package_added_cache, False)
         self.assertTrue(fcachechanged)
 
         info = files[0]
         self.assertEqual(filename, info['file']['name'])
-        self.assertEqual(datetime, type(info['added']))
+        self.assertEqual(int, type(info['added']))
         self.assertEqual(
             os.path.getsize(os.path.join('repo', filename)),
             info['file']['size'],
@@ -833,8 +832,8 @@ class UpdateTest(unittest.TestCase):
         fdroidserver.update.config = config
         fdroidserver.common.options = Options
         apps = fdroidserver.metadata.read_metadata()
-        knownapks = fdroidserver.common.KnownApks()
-        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks)
+        package_added_cache = fdroidserver.common.PackageAddedCache()
+        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', package_added_cache)
         fdroidserver.update.read_added_date_from_all_apks(apps, apks)
 
     def test_apply_info_from_latest_apk(self):
@@ -847,8 +846,8 @@ class UpdateTest(unittest.TestCase):
         fdroidserver.common.options = Options
         fdroidserver.update.options = fdroidserver.common.options
         apps = fdroidserver.metadata.read_metadata()
-        knownapks = fdroidserver.common.KnownApks()
-        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks)
+        package_added_cache = fdroidserver.common.PackageAddedCache()
+        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', package_added_cache)
         fdroidserver.update.apply_info_from_latest_apk(apps, apks)
 
     def test_uses_permission_empty(self):
@@ -1107,14 +1106,14 @@ class UpdateTest(unittest.TestCase):
             if not os.path.exists(icon_dir):
                 os.makedirs(icon_dir)
 
-        knownapks = fdroidserver.common.KnownApks()
+        package_added_cache = fdroidserver.common.PackageAddedCache()
         apkList = ['../urzip.apk', '../org.dyndns.fules.ck_20.apk', 'org.maxsdkversion_4.apk']
 
         for apkName in apkList:
-            _, apk, cachechanged = fdroidserver.update.process_apk({}, apkName, 'repo', knownapks,
+            _, apk, cachechanged = fdroidserver.update.process_apk({}, apkName, 'repo', package_added_cache,
                                                                    False)
             # Don't care about the date added to the repo and relative apkName
-            self.assertEqual(datetime, type(apk['added']))
+            self.assertEqual(int, type(apk['added']))
             del apk['added']
 
             # ensure that icons have been extracted properly
@@ -1157,14 +1156,14 @@ class UpdateTest(unittest.TestCase):
         fdroidserver.update.options.verbose = True
         fdroidserver.update.options.delete_unknown = True
 
-        knownapks = fdroidserver.common.KnownApks()
+        package_added_cache = fdroidserver.common.PackageAddedCache()
 
         with mkdtemp() as tmptestsdir, TmpCwd(tmptestsdir):
             os.mkdir('repo')
             os.mkdir('archive')
             # setup the repo, create icons dirs, etc.
-            fdroidserver.update.process_apks({}, 'repo', knownapks)
-            fdroidserver.update.process_apks({}, 'archive', knownapks)
+            fdroidserver.update.process_apks({}, 'repo', package_added_cache)
+            fdroidserver.update.process_apks({}, 'archive', package_added_cache)
 
             disabledsigs = ['org.bitbucket.tickytacky.mirrormirror_2.apk']
             for apkName in disabledsigs:
@@ -1172,7 +1171,7 @@ class UpdateTest(unittest.TestCase):
                             os.path.join(tmptestsdir, 'repo'))
 
                 skip, apk, cachechanged = fdroidserver.update.process_apk({}, apkName, 'repo',
-                                                                          knownapks,
+                                                                          package_added_cache,
                                                                           allow_disabled_algorithms=True,
                                                                           archive_bad_sig=False)
                 self.assertFalse(skip)
@@ -1195,7 +1194,7 @@ class UpdateTest(unittest.TestCase):
                 # that has MD5 listed in jdk.jar.disabledAlgorithms in java.security
                 # https://blogs.oracle.com/java-platform-group/oracle-jre-will-no-longer-trust-md5-signed-code-by-default
                 skip, apk, cachechanged = fdroidserver.update.process_apk({}, apkName, 'repo',
-                                                                          knownapks,
+                                                                          package_added_cache,
                                                                           allow_disabled_algorithms=False,
                                                                           archive_bad_sig=True)
                 self.assertTrue(skip)
@@ -1205,7 +1204,7 @@ class UpdateTest(unittest.TestCase):
                 self.assertFalse(os.path.exists(os.path.join('repo', apkName)))
 
                 skip, apk, cachechanged = fdroidserver.update.process_apk({}, apkName, 'archive',
-                                                                          knownapks,
+                                                                          package_added_cache,
                                                                           allow_disabled_algorithms=False,
                                                                           archive_bad_sig=False)
                 self.assertFalse(skip)
@@ -1227,7 +1226,7 @@ class UpdateTest(unittest.TestCase):
                             os.path.join(self.testdir, 'repo'))
 
                 skip, apk, cachechanged = fdroidserver.update.process_apk({}, apkName, 'repo',
-                                                                          knownapks,
+                                                                          package_added_cache,
                                                                           allow_disabled_algorithms=False,
                                                                           archive_bad_sig=False)
                 self.assertTrue(skip)
@@ -1247,10 +1246,11 @@ class UpdateTest(unittest.TestCase):
         fdroidserver.update.options = fdroidserver.common.options
         fdroidserver.update.options.delete_unknown = False
 
-        knownapks = fdroidserver.common.KnownApks()
+        package_added_cache = fdroidserver.common.PackageAddedCache()
         apk = 'fake.ota.update_1234.zip'  # this is not an APK, scanning should fail
-        (skip, apk, cachechanged) = fdroidserver.update.process_apk({}, apk, 'repo', knownapks,
-                                                                    False)
+        (skip, apk, cachechanged) = fdroidserver.update.process_apk(
+            {}, apk, 'repo', package_added_cache, False
+        )
 
         self.assertTrue(skip)
         self.assertIsNone(apk)
@@ -1302,8 +1302,8 @@ class UpdateTest(unittest.TestCase):
         fdroidserver.update.options = fdroidserver.common.options
 
         app = fdroidserver.metadata.App()
-        knownapks = fdroidserver.common.KnownApks()
-        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks)
+        package_added_cache = fdroidserver.common.PackageAddedCache()
+        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', package_added_cache)
         apkfile = 'v1.v2.sig_1020.apk'
         self.assertIn(
             apkfile,
@@ -1311,7 +1311,7 @@ class UpdateTest(unittest.TestCase):
             f'{apkfile} was archived or otherwise removed from "repo"',
         )
         (skip, apk, cachechanged) = fdroidserver.update.process_apk(
-            {}, apkfile, 'repo', knownapks, False
+            {}, apkfile, 'repo', package_added_cache, False
         )
 
         r = fdroidserver.update.get_apks_without_allowed_signatures(app, apk)
@@ -1332,11 +1332,11 @@ class UpdateTest(unittest.TestCase):
                 'AllowedAPKSigningKeys': '32a23624c201b949f085996ba5ed53d40f703aca4989476949cae891022e0ed6'
             }
         )
-        knownapks = fdroidserver.common.KnownApks()
-        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks)
+        package_added_cache = fdroidserver.common.PackageAddedCache()
+        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', package_added_cache)
         apkfile = 'v1.v2.sig_1020.apk'
         (skip, apk, cachechanged) = fdroidserver.update.process_apk(
-            {}, apkfile, 'repo', knownapks, False
+            {}, apkfile, 'repo', package_added_cache, False
         )
 
         r = fdroidserver.update.get_apks_without_allowed_signatures(app, apk)
@@ -1357,11 +1357,11 @@ class UpdateTest(unittest.TestCase):
                 'AllowedAPKSigningKeys': 'fa4edeadfa4edeadfa4edeadfa4edeadfa4edeadfa4edeadfa4edeadfa4edead'
             }
         )
-        knownapks = fdroidserver.common.KnownApks()
-        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks)
+        package_added_cache = fdroidserver.common.PackageAddedCache()
+        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', package_added_cache)
         apkfile = 'v1.v2.sig_1020.apk'
         (skip, apk, cachechanged) = fdroidserver.update.process_apk(
-            {}, apkfile, 'repo', knownapks, False
+            {}, apkfile, 'repo', package_added_cache, False
         )
 
         r = fdroidserver.update.get_apks_without_allowed_signatures(app, apk)
@@ -1428,8 +1428,8 @@ class UpdateTest(unittest.TestCase):
         fdroidserver.update.options.delete_unknown = True
 
         apps = fdroidserver.metadata.read_metadata()
-        knownapks = fdroidserver.common.KnownApks()
-        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks, False)
+        package_added_cache = fdroidserver.common.PackageAddedCache()
+        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', package_added_cache, False)
         fdroidserver.update.translate_per_build_anti_features(apps, apks)
         self.assertEqual(len(apks), 18)
         foundtest = False
@@ -1457,8 +1457,8 @@ class UpdateTest(unittest.TestCase):
         fdroidserver.update.options = fdroidserver.common.options
         fdroidserver.update.options.clean = True
 
-        knownapks = fdroidserver.common.KnownApks()
-        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks, False)
+        package_added_cache = fdroidserver.common.PackageAddedCache()
+        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', package_added_cache, False)
         self.assertEqual(1, len(apks))
         apk = apks[0]
 
@@ -1997,8 +1997,8 @@ class UpdateTest(unittest.TestCase):
         build.disable = "disabled"
         app['Builds'] = [build]
 
-        knownapks = fdroidserver.common.KnownApks()
-        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks, False, apps)
+        package_added_cache = fdroidserver.common.PackageAddedCache()
+        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', package_added_cache, False, apps)
         self.assertEqual([], apks)
 
     def test_archive_old_apks_ArchivePolicy_0(self):
@@ -2295,7 +2295,7 @@ class TestScanRepoForIpas(unittest.TestCase):
 
             apkcache = mock.MagicMock()
             repodir = "repo"
-            knownapks = mock.MagicMock()
+            package_added_cache = mock.MagicMock()
 
             def mocked_parse(p, s, c):
                 # pylint: disable=unused-argument
@@ -2303,7 +2303,7 @@ class TestScanRepoForIpas(unittest.TestCase):
 
             with mock.patch('fdroidserver.update.parse_ipa', mocked_parse):
                 ipas, checkchanged = fdroidserver.update.scan_repo_for_ipas(
-                    apkcache, repodir, knownapks
+                    apkcache, repodir, package_added_cache
                 )
 
             self.assertEqual(checkchanged, True)
@@ -2319,12 +2319,14 @@ class TestScanRepoForIpas(unittest.TestCase):
             self.assertTrue('xyz' in apkcache_setter_package_name)
             self.assertEqual(apkcache.__setitem__.call_count, 2)
 
-            knownapks.recordapk.call_count = 2
+            package_added_cache.get.call_count = 2
             self.assertTrue(
-                unittest.mock.call('abc.Def_123.ipa') in knownapks.recordapk.mock_calls
+                unittest.mock.call('repo/abc.Def_123.ipa')
+                in package_added_cache.get.mock_calls
             )
             self.assertTrue(
-                unittest.mock.call('xyz.XXX_123.ipa') in knownapks.recordapk.mock_calls
+                unittest.mock.call('repo/xyz.XXX_123.ipa')
+                in package_added_cache.get.mock_calls
             )