From a1ea7cf3257af320a1997246337e4536befe6fde Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 19 Feb 2025 08:30:53 +0100 Subject: [PATCH] move 'added' to internal index-v2 format This ended up being a refactoring of the whole KnownApks setup. --- fdroidserver/common.py | 57 ++++++++++++---------- fdroidserver/index.py | 35 ++++++++----- fdroidserver/update.py | 65 ++++++++----------------- tests/test_common.py | 36 +++++++------- tests/test_index.py | 4 +- tests/test_integration.py | 22 ++++++--- tests/test_update.py | 100 +++++++++++++++++++------------------- 7 files changed, 162 insertions(+), 157 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index ec389262..03ec1082 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -2903,21 +2903,21 @@ def check_system_clock(dt_obj, path): + 'sudo date -s "' + str(dt_obj) + '"') -class KnownApks: - """Permanent store of existing APKs with the date they were added. +class PackageAddedCache: + """Store/fetch the added timestamp of packages. + + This uses a cached value for "now" that is set once to avoid + preferencing apps based on any particular sort, like alpha sort of + file name or Application ID. This way all new apps in a single + run get the same "added" time. - This is currently the only way to permanently store the "updated" - date of APKs. """ - def __init__(self): - """Load filename/date info about previously seen APKs. - - Since the appid and date strings both will never have spaces, - this is parsed as a list from the end to allow the filename to - have any combo of spaces. - """ - self.apks = {} + def __init__(self, use_date_from_file=False): + """Load filename/date info about previously seen Versions.""" + self.now = int(time.time() * 1000) + self.use_date_from_file = use_date_from_file + self.versions = {} for part in ('repo', 'archive'): path = os.path.join(part, 'index-v2.json') if os.path.isfile(path): @@ -2925,24 +2925,31 @@ class KnownApks: index = json.load(f) for appid, data in index["packages"].items(): for version in data["versions"].values(): - filename = version["file"]["name"][1:] - date = datetime.fromtimestamp(version["added"] // 1000, tz=timezone.utc) - self.apks[filename] = date + vpath = os.path.join( + part, + version["file"]["name"][1:], + ) + self.versions[vpath] = version["added"] - def recordapk(self, apkName, default_date=None): - """ - Record an APK (if it's new, otherwise does nothing). + def get(self, vpath, use_date_from_file=False): # TODO do this based on sha256 + """Get 'added' time, set current time as 'added' if file is new. + + Parameters + ---------- + vpath + path to file that is being added to the index Returns ------- - datetime - the date it was added as a datetime instance. + int + timestamp as Java milliseconds since UNIX epoch """ - if apkName not in self.apks: - if default_date is None: - default_date = datetime.now(timezone.utc) - self.apks[apkName] = default_date - return self.apks[apkName] + if vpath not in self.versions: + if use_date_from_file or self.use_date_from_file: + self.versions[vpath] = int(os.stat(vpath).st_mtime * 1000) + else: + self.versions[vpath] = self.now + return self.versions[vpath] def get_file_extension(filename): diff --git a/fdroidserver/index.py b/fdroidserver/index.py index e9cc2e43..57fdaf8e 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -513,6 +513,19 @@ def dict_diff(source, target): return result +def datetime_from_millis(millis): + """Convert epoch milliseconds to datetime instance. + + This is the format returned by Java's System.currentTimeMillis(). + + Parameters + ---------- + millis + Java-style integer time since UNIX epoch in milliseconds + """ + return datetime.utcfromtimestamp(millis / 1000) + + def convert_datetime(obj): if isinstance(obj, datetime): # Java prefers milliseconds @@ -598,13 +611,10 @@ def convert_version(version, app, repodir): metadata file. """ - ver = {} - if "added" in version: - ver["added"] = convert_datetime(version["added"]) - else: - ver["added"] = 0 - - ver["file"] = version["file"] + ver = { + "added": version["added"], + "file": version["file"], + } ver["file"]["name"] = f'/{version["file"]["name"]}' # TODO remove for index-v3 ipfsCIDv1 = version.get("ipfsCIDv1") @@ -1269,9 +1279,11 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, signer_fingerprints): addElement('id', app.id, doc, apel) if app.added: - addElement('added', app.added.strftime('%Y-%m-%d'), doc, apel) + added = datetime_from_millis(app.added) + addElement('added', added.strftime('%Y-%m-%d'), doc, apel) if app.lastUpdated: - addElement('lastupdated', app.lastUpdated.strftime('%Y-%m-%d'), doc, apel) + lastUpdated = datetime_from_millis(app.lastUpdated) + addElement('lastupdated', lastUpdated.strftime('%Y-%m-%d'), doc, apel) addElementCheckLocalized('name', app, 'Name', doc, apel, name_from_apk) addElementCheckLocalized('summary', app, 'Summary', doc, apel) @@ -1393,7 +1405,8 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, signer_fingerprints): 'obbPatchFileSha256', apk, 'obbPatchFileSha256', doc, apkel ) if 'added' in apk: - addElement('added', apk['added'].strftime('%Y-%m-%d'), doc, apkel) + added = datetime_from_millis(apk['added']) + addElement('added', added.strftime('%Y-%m-%d'), doc, apkel) if file_extension == 'apk': # sig is required for APKs, but only APKs addElement('sig', apk['sig'], doc, apkel) @@ -2053,7 +2066,7 @@ def make_altstore(apps, apks, config, repodir, pretty=False): if apk['packageName'] == packageName and file_extension == 'ipa': v = { "version": apk["manifest"]["versionName"], - "date": apk["added"].isoformat(), + "date": datetime_from_millis(apk["added"]).isoformat(), "downloadURL": f"{config['repo_url']}/{apk['file']['name']}", "size": apk['file']['size'], } diff --git a/fdroidserver/update.py b/fdroidserver/update.py index c88bfb5b..3c655a72 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -36,7 +36,7 @@ import time import warnings import zipfile from argparse import ArgumentParser -from datetime import datetime, timezone +from datetime import datetime from pathlib import Path import asn1crypto.cms @@ -383,8 +383,6 @@ def get_cache(): if 'antiFeatures' in v: if not isinstance(v['antiFeatures'], dict): v['antiFeatures'] = {k: {} for k in sorted(v['antiFeatures'])} - if 'added' in v: - v['added'] = datetime.fromtimestamp(v['added'], tz=timezone.utc) if v.get('srcname') and not v.get('srcnameSha256'): f = f'archive/{v["srcname"]}' if not os.path.exists(f): @@ -684,7 +682,7 @@ def parse_ipa(ipa_path, file_size, sha256): return ipa -def scan_repo_for_ipas(apkcache, repodir, knownapks): +def scan_repo_for_ipas(apkcache, repodir, package_added_cache): """Scan for IPA files in a given repo directory. Parameters @@ -693,7 +691,7 @@ def scan_repo_for_ipas(apkcache, repodir, knownapks): cache dictionary containting cached file infos from previous runs repodir repo directory to scan - knownapks + package_added_cache list of all known files, as per metadata.read_metadata Returns @@ -720,9 +718,7 @@ def scan_repo_for_ipas(apkcache, repodir, knownapks): apkcache[ipa_name] = ipa cachechanged = True - added = knownapks.recordapk(ipa_name) - if added: - ipa['added'] = added + ipa['added'] = package_added_cache.get(ipa_path) ipas.append(ipa) @@ -1542,7 +1538,7 @@ def insert_localized_ios_app_metadata(apps_with_packages): shutil.copy2(icon_src, icon_dest) -def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): +def scan_repo_files(apkcache, repodir, package_added_cache, use_date_from_file=False): """Scan a repo for all files with an extension except APK/OBB/IPA. This allows putting all kinds of files into repostories. E.g. Media Files, @@ -1554,7 +1550,7 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): current cached info about all repo files repodir repo directory to scan - knownapks + package_added_cache list of all known files, as per metadata.read_metadata use_date_from_file use date from file (instead of current date) for newly added files @@ -1633,18 +1629,7 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): apkcache[name_utf8] = repo_file cachechanged = True - if use_date_from_file: - timestamp = stat.st_ctime - default_date_param = datetime.fromtimestamp(timestamp, tz=timezone.utc) - else: - default_date_param = None - - # Record in knownapks, getting the added date at the same time.. - added = knownapks.recordapk( - repo_file['file']['name'], default_date=default_date_param - ) - if added: - repo_file['added'] = added + repo_file['added'] = package_added_cache.get(filename.decode(), use_date_from_file) repo_files.append(repo_file) @@ -2011,7 +1996,7 @@ def _fill_reason(msg): return d -def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False, +def process_apk(apkcache, apkfilename, repodir, package_added_cache, use_date_from_apk=False, allow_disabled_algorithms=False, archive_bad_sig=False, apps=None, cache_timestamp=0): """Process the apk with the given filename in the given repo directory. @@ -2025,8 +2010,8 @@ def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=Fal the filename of the apk to scan repodir repo directory to scan - knownapks - known apks info + package_added_cache + list of all known files, as per metadata.read_metadata use_date_from_apk use date from APK (instead of current date) for newly added APKs allow_disabled_algorithms @@ -2187,15 +2172,7 @@ def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=Fal # resize existing icons for densities missing in the APK fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir) - if use_date_from_apk: - default_date_param = datetime.fromtimestamp(os.stat(apkfile).st_mtime, tz=timezone.utc) - else: - default_date_param = None - - # Record in known apks, getting the added date at the same time.. - added = knownapks.recordapk(apk['file']['name'], default_date=default_date_param) - if added: - apk['added'] = added + apk['added'] = package_added_cache.get(apkfile, use_date_from_apk) apkcache[apkfilename] = apk cachechanged = True @@ -2203,7 +2180,7 @@ def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=Fal return False, apk, cachechanged -def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False, apps=None, cache_timestamp=0): +def process_apks(apkcache, repodir, package_added_cache, use_date_from_apk=False, apps=None, cache_timestamp=0): """Process the apks in the given repo directory. This also extracts the icons. @@ -2214,8 +2191,8 @@ def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False, apps=Non current apk cache information repodir repo directory to scan - knownapks - b known apks info + package_added_cache + list of all known files, as per metadata.read_metadata use_date_from_apk use date from APK (instead of current date) for newly added APKs cache_timestamp @@ -2240,7 +2217,7 @@ def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False, apps=Non for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))): apkfilename = apkfile[len(repodir) + 1 :] ada = disabled_algorithms_allowed() - (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks, + (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, package_added_cache, use_date_from_apk, ada, True, apps, cache_timestamp) if skip: continue @@ -2895,7 +2872,7 @@ def main(): apps = metadata.read_metadata() # Read known apks data (will be updated and written back when we've finished) - knownapks = common.KnownApks() + package_added_cache = common.PackageAddedCache(options.use_date_from_apk) # Get APK cache apkcache = get_cache() @@ -2910,20 +2887,18 @@ def main(): apks, cachechanged = process_apks( apkcache, repodirs[0], - knownapks, + package_added_cache, options.use_date_from_apk, apps, cache_timestamp, ) output_status_stage(status_output, 'scan_repo_files') - files, fcachechanged = scan_repo_files( - apkcache, repodirs[0], knownapks, options.use_date_from_apk - ) + files, fcachechanged = scan_repo_files(apkcache, repodirs[0], package_added_cache) cachechanged = cachechanged or fcachechanged apks += files - ipas, icachechanged = scan_repo_for_ipas(apkcache, repodirs[0], knownapks) + ipas, icachechanged = scan_repo_for_ipas(apkcache, repodirs[0], package_added_cache) cachechanged = cachechanged or icachechanged apks += ipas @@ -3003,7 +2978,7 @@ def main(): # Scan the archive repo for apks as well if len(repodirs) > 1: - archapks, cc = process_apks(apkcache, repodirs[1], knownapks, + archapks, cc = process_apks(apkcache, repodirs[1], package_added_cache, options.use_date_from_apk, apps, cache_timestamp) if cc: cachechanged = True diff --git a/tests/test_common.py b/tests/test_common.py index 962faa7f..c15bcc04 100755 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -3005,36 +3005,36 @@ class CommonTest(SetUpTearDownMixin, unittest.TestCase): [{'url': s}], fdroidserver.common.parse_list_of_dicts(mirrors) ) - def test_KnownApks_recordapk(self): + def test_PackageAddedCache_get(self): """Test that added dates are being fetched from the index. - There are more related tests in tests/run-tests. + There are more related tests in tests/test_integration.py. """ - now = datetime.now(timezone.utc) - knownapks = fdroidserver.common.KnownApks() - for apkName in knownapks.apks: - knownapks.recordapk(apkName, default_date=now) - for added in knownapks.apks.values(): - self.assertNotEqual(added, now) + package_added_cache = fdroidserver.common.PackageAddedCache() + package_added_cache.now = 1234567890 + for filename in package_added_cache.versions: + package_added_cache.get(filename) + for added in package_added_cache.versions.values(): + self.assertNotEqual(added, package_added_cache.now) - def test_KnownApks_recordapk_new(self): + def test_PackageAddedCache_get_new(self): """Test that new added dates work, and are not replaced later. - There are more related tests in tests/run-tests. + There are more related tests in tests/test_integration.py. """ - now = datetime.now(timezone.utc) - knownapks = fdroidserver.common.KnownApks() + package_added_cache = fdroidserver.common.PackageAddedCache() + package_added_cache.now = 1234567890 fake_apk = 'fake.apk' - knownapks.recordapk(fake_apk, default_date=now) - for apk, added in knownapks.apks.items(): + package_added_cache.get(fake_apk) + for apk, added in package_added_cache.versions.items(): if apk == fake_apk: - self.assertEqual(added, now) + self.assertEqual(added, package_added_cache.now) else: - self.assertNotEqual(added, now) - knownapks.recordapk(fake_apk, default_date=datetime.now(timezone.utc)) - self.assertEqual(knownapks.apks[fake_apk], now) + self.assertNotEqual(added, package_added_cache.now) + package_added_cache.get(fake_apk) + self.assertEqual(package_added_cache.versions[fake_apk], package_added_cache.now) def test_get_mirrors_fdroidorg(self): mirrors = fdroidserver.common.get_mirrors( diff --git a/tests/test_index.py b/tests/test_index.py index a9a7473f..3a72d999 100755 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -519,7 +519,7 @@ class IndexTest(unittest.TestCase): 'sig': '03f9b2f848d22fd1d8d1331e8b1b486d', 'signer': '51cfa5c8a743833ad89acf81cb755936876a5c8b8eca54d1ffdcec0cdca25d0e', 'icon': 'info.zwanenburg.caffeinetile.4.xml', - 'added': datetime.datetime.fromtimestamp(1539122400), + 'added': 1539122400000, } requestsdict = {'install': [], 'uninstall': []} common.config['repo_pubkey'] = 'ffffffffffffffffffffffffffffffffff' @@ -918,7 +918,7 @@ class AltstoreIndexTest(unittest.TestCase): "manifest": { "versionName": "v123", }, - "added": datetime.datetime(2000, 2, 2, 2, 2, 2), + "added": 949456922000, "ipa_MinimumOSVersion": "10.0", "ipa_DTPlatformVersion": "12.0", "ipa_permissions": [ diff --git a/tests/test_integration.py b/tests/test_integration.py index f6b463e7..7011d24b 100755 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -771,20 +771,28 @@ class IntegrationTest(unittest.TestCase): self.update_yaml("metadata/com.politedroid.yml", {"ArchivePolicy": 1}) self.assert_run(self.fdroid_cmd + ["update", "--pretty", "--nosign"]) - timestamp = 1498176000000 + timestamp = 1498176000000 # value cached in repo/index-v2.json index_v1_json = json.loads(Path("repo/index-v1.json").read_text()) self.assertEqual(timestamp, index_v1_json['apps'][0]['added']) - self.assertEqual(timestamp, index_v1_json['apps'][0]['lastUpdated']) self.assertEqual( timestamp, index_v1_json['packages']['com.politedroid'][0]['added'] ) - # the archive will have the added timestamp for the app and for the apk, - # both need to be there + # Since there is no archive/index-v2.json, a cached 'added' + # value cannot be loaded from it. Therefore, the archive APK's + # 'added' value is "now" which is newer than 'timestamp', + # which was loaded from repo/index-v2.json. archive_json = json.loads(Path("archive/index-v1.json").read_text()) - self.assertEqual(timestamp, archive_json['apps'][0]['added']) - self.assertEqual(timestamp, archive_json['apps'][0]['lastUpdated']) self.assertEqual( - timestamp, archive_json['packages']['com.politedroid'][0]['added'] + index_v1_json['apps'][0]['added'], + archive_json['apps'][0]['added'], + ) + self.assertEqual( + archive_json['apps'][0]['lastUpdated'], + archive_json['packages']['com.politedroid'][0]['added'], + ) + self.assertEqual( + archive_json['apps'][0]['lastUpdated'], + index_v1_json['apps'][0]['lastUpdated'], ) def test_whatsnew_from_fastlane_without_cvc_set(self): diff --git a/tests/test_update.py b/tests/test_update.py index ec687e5a..0496c402 100755 --- a/tests/test_update.py +++ b/tests/test_update.py @@ -17,7 +17,6 @@ import time import unittest import zipfile from binascii import hexlify -from datetime import datetime from pathlib import Path from unittest import mock @@ -367,8 +366,8 @@ class UpdateTest(unittest.TestCase): apps = fdroidserver.metadata.read_metadata() apps['info.guardianproject.urzip']['CurrentVersionCode'] = 100 - knownapks = fdroidserver.common.KnownApks() - apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks, False) + package_added_cache = fdroidserver.common.PackageAddedCache() + apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', package_added_cache, False) fdroidserver.update.insert_localized_app_metadata(apps) fdroidserver.update.ingest_screenshots_from_repo_dir(apps) fdroidserver.update.apply_info_from_latest_apk(apps, apks) @@ -454,8 +453,8 @@ class UpdateTest(unittest.TestCase): fdroidserver.update.options.delete_unknown = True apps = fdroidserver.metadata.read_metadata() - knownapks = fdroidserver.common.KnownApks() - apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks, False) + package_added_cache = fdroidserver.common.PackageAddedCache() + apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', package_added_cache, False) appid = 'info.guardianproject.checkey' testapps = {appid: copy.copy(apps[appid])} @@ -721,8 +720,8 @@ class UpdateTest(unittest.TestCase): fdroidserver.update.options.delete_unknown = True apps = fdroidserver.metadata.read_metadata() - knownapks = fdroidserver.common.KnownApks() - apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks, False) + package_added_cache = fdroidserver.common.PackageAddedCache() + apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', package_added_cache, False) self.assertEqual(len(apks), 18) apk = apks[1] manifest = apk['manifest'] @@ -779,13 +778,13 @@ class UpdateTest(unittest.TestCase): fdroidserver.update.options.delete_unknown = True fdroidserver.metadata.read_metadata() - knownapks = fdroidserver.common.KnownApks() + package_added_cache = fdroidserver.common.PackageAddedCache() apkcache = fdroidserver.update.get_cache() self.assertEqual(2, len(apkcache)) self.assertEqual(fdroidserver.update.METADATA_VERSION, apkcache["METADATA_VERSION"]) self.assertEqual(fdroidserver.update.options.allow_disabled_algorithms, apkcache['allow_disabled_algorithms']) - apks, cachechanged = fdroidserver.update.process_apks(apkcache, 'repo', knownapks, False) + apks, cachechanged = fdroidserver.update.process_apks(apkcache, 'repo', package_added_cache, False) fdroidserver.update.write_cache(apkcache) fdroidserver.update.options.clean = False @@ -808,13 +807,13 @@ class UpdateTest(unittest.TestCase): os.mkdir('repo') filename = 'Norway_bouvet_europe_2.obf.zip' shutil.copy(basedir / filename, 'repo') - knownapks = fdroidserver.common.KnownApks() - files, fcachechanged = fdroidserver.update.scan_repo_files(dict(), 'repo', knownapks, False) + package_added_cache = fdroidserver.common.PackageAddedCache() + files, fcachechanged = fdroidserver.update.scan_repo_files(dict(), 'repo', package_added_cache, False) self.assertTrue(fcachechanged) info = files[0] self.assertEqual(filename, info['file']['name']) - self.assertEqual(datetime, type(info['added'])) + self.assertEqual(int, type(info['added'])) self.assertEqual( os.path.getsize(os.path.join('repo', filename)), info['file']['size'], @@ -833,8 +832,8 @@ class UpdateTest(unittest.TestCase): fdroidserver.update.config = config fdroidserver.common.options = Options apps = fdroidserver.metadata.read_metadata() - knownapks = fdroidserver.common.KnownApks() - apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks) + package_added_cache = fdroidserver.common.PackageAddedCache() + apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', package_added_cache) fdroidserver.update.read_added_date_from_all_apks(apps, apks) def test_apply_info_from_latest_apk(self): @@ -847,8 +846,8 @@ class UpdateTest(unittest.TestCase): fdroidserver.common.options = Options fdroidserver.update.options = fdroidserver.common.options apps = fdroidserver.metadata.read_metadata() - knownapks = fdroidserver.common.KnownApks() - apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks) + package_added_cache = fdroidserver.common.PackageAddedCache() + apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', package_added_cache) fdroidserver.update.apply_info_from_latest_apk(apps, apks) def test_uses_permission_empty(self): @@ -1107,14 +1106,14 @@ class UpdateTest(unittest.TestCase): if not os.path.exists(icon_dir): os.makedirs(icon_dir) - knownapks = fdroidserver.common.KnownApks() + package_added_cache = fdroidserver.common.PackageAddedCache() apkList = ['../urzip.apk', '../org.dyndns.fules.ck_20.apk', 'org.maxsdkversion_4.apk'] for apkName in apkList: - _, apk, cachechanged = fdroidserver.update.process_apk({}, apkName, 'repo', knownapks, + _, apk, cachechanged = fdroidserver.update.process_apk({}, apkName, 'repo', package_added_cache, False) # Don't care about the date added to the repo and relative apkName - self.assertEqual(datetime, type(apk['added'])) + self.assertEqual(int, type(apk['added'])) del apk['added'] # ensure that icons have been extracted properly @@ -1157,14 +1156,14 @@ class UpdateTest(unittest.TestCase): fdroidserver.update.options.verbose = True fdroidserver.update.options.delete_unknown = True - knownapks = fdroidserver.common.KnownApks() + package_added_cache = fdroidserver.common.PackageAddedCache() with mkdtemp() as tmptestsdir, TmpCwd(tmptestsdir): os.mkdir('repo') os.mkdir('archive') # setup the repo, create icons dirs, etc. - fdroidserver.update.process_apks({}, 'repo', knownapks) - fdroidserver.update.process_apks({}, 'archive', knownapks) + fdroidserver.update.process_apks({}, 'repo', package_added_cache) + fdroidserver.update.process_apks({}, 'archive', package_added_cache) disabledsigs = ['org.bitbucket.tickytacky.mirrormirror_2.apk'] for apkName in disabledsigs: @@ -1172,7 +1171,7 @@ class UpdateTest(unittest.TestCase): os.path.join(tmptestsdir, 'repo')) skip, apk, cachechanged = fdroidserver.update.process_apk({}, apkName, 'repo', - knownapks, + package_added_cache, allow_disabled_algorithms=True, archive_bad_sig=False) self.assertFalse(skip) @@ -1195,7 +1194,7 @@ class UpdateTest(unittest.TestCase): # that has MD5 listed in jdk.jar.disabledAlgorithms in java.security # https://blogs.oracle.com/java-platform-group/oracle-jre-will-no-longer-trust-md5-signed-code-by-default skip, apk, cachechanged = fdroidserver.update.process_apk({}, apkName, 'repo', - knownapks, + package_added_cache, allow_disabled_algorithms=False, archive_bad_sig=True) self.assertTrue(skip) @@ -1205,7 +1204,7 @@ class UpdateTest(unittest.TestCase): self.assertFalse(os.path.exists(os.path.join('repo', apkName))) skip, apk, cachechanged = fdroidserver.update.process_apk({}, apkName, 'archive', - knownapks, + package_added_cache, allow_disabled_algorithms=False, archive_bad_sig=False) self.assertFalse(skip) @@ -1227,7 +1226,7 @@ class UpdateTest(unittest.TestCase): os.path.join(self.testdir, 'repo')) skip, apk, cachechanged = fdroidserver.update.process_apk({}, apkName, 'repo', - knownapks, + package_added_cache, allow_disabled_algorithms=False, archive_bad_sig=False) self.assertTrue(skip) @@ -1247,10 +1246,11 @@ class UpdateTest(unittest.TestCase): fdroidserver.update.options = fdroidserver.common.options fdroidserver.update.options.delete_unknown = False - knownapks = fdroidserver.common.KnownApks() + package_added_cache = fdroidserver.common.PackageAddedCache() apk = 'fake.ota.update_1234.zip' # this is not an APK, scanning should fail - (skip, apk, cachechanged) = fdroidserver.update.process_apk({}, apk, 'repo', knownapks, - False) + (skip, apk, cachechanged) = fdroidserver.update.process_apk( + {}, apk, 'repo', package_added_cache, False + ) self.assertTrue(skip) self.assertIsNone(apk) @@ -1302,8 +1302,8 @@ class UpdateTest(unittest.TestCase): fdroidserver.update.options = fdroidserver.common.options app = fdroidserver.metadata.App() - knownapks = fdroidserver.common.KnownApks() - apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks) + package_added_cache = fdroidserver.common.PackageAddedCache() + apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', package_added_cache) apkfile = 'v1.v2.sig_1020.apk' self.assertIn( apkfile, @@ -1311,7 +1311,7 @@ class UpdateTest(unittest.TestCase): f'{apkfile} was archived or otherwise removed from "repo"', ) (skip, apk, cachechanged) = fdroidserver.update.process_apk( - {}, apkfile, 'repo', knownapks, False + {}, apkfile, 'repo', package_added_cache, False ) r = fdroidserver.update.get_apks_without_allowed_signatures(app, apk) @@ -1332,11 +1332,11 @@ class UpdateTest(unittest.TestCase): 'AllowedAPKSigningKeys': '32a23624c201b949f085996ba5ed53d40f703aca4989476949cae891022e0ed6' } ) - knownapks = fdroidserver.common.KnownApks() - apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks) + package_added_cache = fdroidserver.common.PackageAddedCache() + apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', package_added_cache) apkfile = 'v1.v2.sig_1020.apk' (skip, apk, cachechanged) = fdroidserver.update.process_apk( - {}, apkfile, 'repo', knownapks, False + {}, apkfile, 'repo', package_added_cache, False ) r = fdroidserver.update.get_apks_without_allowed_signatures(app, apk) @@ -1357,11 +1357,11 @@ class UpdateTest(unittest.TestCase): 'AllowedAPKSigningKeys': 'fa4edeadfa4edeadfa4edeadfa4edeadfa4edeadfa4edeadfa4edeadfa4edead' } ) - knownapks = fdroidserver.common.KnownApks() - apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks) + package_added_cache = fdroidserver.common.PackageAddedCache() + apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', package_added_cache) apkfile = 'v1.v2.sig_1020.apk' (skip, apk, cachechanged) = fdroidserver.update.process_apk( - {}, apkfile, 'repo', knownapks, False + {}, apkfile, 'repo', package_added_cache, False ) r = fdroidserver.update.get_apks_without_allowed_signatures(app, apk) @@ -1428,8 +1428,8 @@ class UpdateTest(unittest.TestCase): fdroidserver.update.options.delete_unknown = True apps = fdroidserver.metadata.read_metadata() - knownapks = fdroidserver.common.KnownApks() - apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks, False) + package_added_cache = fdroidserver.common.PackageAddedCache() + apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', package_added_cache, False) fdroidserver.update.translate_per_build_anti_features(apps, apks) self.assertEqual(len(apks), 18) foundtest = False @@ -1457,8 +1457,8 @@ class UpdateTest(unittest.TestCase): fdroidserver.update.options = fdroidserver.common.options fdroidserver.update.options.clean = True - knownapks = fdroidserver.common.KnownApks() - apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks, False) + package_added_cache = fdroidserver.common.PackageAddedCache() + apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', package_added_cache, False) self.assertEqual(1, len(apks)) apk = apks[0] @@ -1997,8 +1997,8 @@ class UpdateTest(unittest.TestCase): build.disable = "disabled" app['Builds'] = [build] - knownapks = fdroidserver.common.KnownApks() - apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks, False, apps) + package_added_cache = fdroidserver.common.PackageAddedCache() + apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', package_added_cache, False, apps) self.assertEqual([], apks) def test_archive_old_apks_ArchivePolicy_0(self): @@ -2295,7 +2295,7 @@ class TestScanRepoForIpas(unittest.TestCase): apkcache = mock.MagicMock() repodir = "repo" - knownapks = mock.MagicMock() + package_added_cache = mock.MagicMock() def mocked_parse(p, s, c): # pylint: disable=unused-argument @@ -2303,7 +2303,7 @@ class TestScanRepoForIpas(unittest.TestCase): with mock.patch('fdroidserver.update.parse_ipa', mocked_parse): ipas, checkchanged = fdroidserver.update.scan_repo_for_ipas( - apkcache, repodir, knownapks + apkcache, repodir, package_added_cache ) self.assertEqual(checkchanged, True) @@ -2319,12 +2319,14 @@ class TestScanRepoForIpas(unittest.TestCase): self.assertTrue('xyz' in apkcache_setter_package_name) self.assertEqual(apkcache.__setitem__.call_count, 2) - knownapks.recordapk.call_count = 2 + package_added_cache.get.call_count = 2 self.assertTrue( - unittest.mock.call('abc.Def_123.ipa') in knownapks.recordapk.mock_calls + unittest.mock.call('repo/abc.Def_123.ipa') + in package_added_cache.get.mock_calls ) self.assertTrue( - unittest.mock.call('xyz.XXX_123.ipa') in knownapks.recordapk.mock_calls + unittest.mock.call('repo/xyz.XXX_123.ipa') + in package_added_cache.get.mock_calls )