diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 03ec1082..7b7bfc7a 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -2903,55 +2903,6 @@ def check_system_clock(dt_obj, path): + 'sudo date -s "' + str(dt_obj) + '"') -class PackageAddedCache: - """Store/fetch the added timestamp of packages. - - This uses a cached value for "now" that is set once to avoid - preferencing apps based on any particular sort, like alpha sort of - file name or Application ID. This way all new apps in a single - run get the same "added" time. - - """ - - def __init__(self, use_date_from_file=False): - """Load filename/date info about previously seen Versions.""" - self.now = int(time.time() * 1000) - self.use_date_from_file = use_date_from_file - self.versions = {} - for part in ('repo', 'archive'): - path = os.path.join(part, 'index-v2.json') - if os.path.isfile(path): - with open(path, 'r', encoding='utf-8') as f: - index = json.load(f) - for appid, data in index["packages"].items(): - for version in data["versions"].values(): - vpath = os.path.join( - part, - version["file"]["name"][1:], - ) - self.versions[vpath] = version["added"] - - def get(self, vpath, use_date_from_file=False): # TODO do this based on sha256 - """Get 'added' time, set current time as 'added' if file is new. - - Parameters - ---------- - vpath - path to file that is being added to the index - - Returns - ------- - int - timestamp as Java milliseconds since UNIX epoch - """ - if vpath not in self.versions: - if use_date_from_file or self.use_date_from_file: - self.versions[vpath] = int(os.stat(vpath).st_mtime * 1000) - else: - self.versions[vpath] = self.now - return self.versions[vpath] - - def get_file_extension(filename): """Get the normalized file extension, can be blank string but never None.""" if isinstance(filename, bytes): diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 3c655a72..a4e2aea8 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -111,6 +111,55 @@ SCREENSHOT_DIRS = ( BLANK_PNG_INFO = PngImagePlugin.PngInfo() +class PackageAddedCache: + """Store/fetch the added timestamp of packages. + + This uses a cached value for "now" that is set once to avoid + preferencing apps based on any particular sort, like alpha sort of + file name or Application ID. This way all new apps in a single + run get the same "added" time. + + """ + + def __init__(self, use_date_from_file=False): + """Load filename/date info about previously seen Versions.""" + self.now = int(time.time() * 1000) + self.use_date_from_file = use_date_from_file + self.versions = {} + for part in ('repo', 'archive'): + path = os.path.join(part, 'index-v2.json') + if os.path.isfile(path): + with open(path, 'r', encoding='utf-8') as f: + index = json.load(f) + for appid, data in index["packages"].items(): + for version in data["versions"].values(): + vpath = os.path.join( + part, + version["file"]["name"][1:], + ) + self.versions[vpath] = version["added"] + + def get(self, vpath, use_date_from_file=False): # TODO do this based on sha256 + """Get 'added' time, set current time as 'added' if file is new. + + Parameters + ---------- + vpath + path to file that is being added to the index + + Returns + ------- + int + timestamp as Java milliseconds since UNIX epoch + """ + if vpath not in self.versions: + if use_date_from_file or self.use_date_from_file: + self.versions[vpath] = int(os.stat(vpath).st_mtime * 1000) + else: + self.versions[vpath] = self.now + return self.versions[vpath] + + def dpi_to_px(density): return (int(density) * 48) / 160 @@ -2872,7 +2921,7 @@ def main(): apps = metadata.read_metadata() # Read known apks data (will be updated and written back when we've finished) - package_added_cache = common.PackageAddedCache(options.use_date_from_apk) + package_added_cache = PackageAddedCache(options.use_date_from_apk) # Get APK cache apkcache = get_cache() diff --git a/tests/test_common.py b/tests/test_common.py index c15bcc04..8330aba5 100755 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -3005,37 +3005,6 @@ class CommonTest(SetUpTearDownMixin, unittest.TestCase): [{'url': s}], fdroidserver.common.parse_list_of_dicts(mirrors) ) - def test_PackageAddedCache_get(self): - """Test that added dates are being fetched from the index. - - There are more related tests in tests/test_integration.py. - - """ - package_added_cache = fdroidserver.common.PackageAddedCache() - package_added_cache.now = 1234567890 - for filename in package_added_cache.versions: - package_added_cache.get(filename) - for added in package_added_cache.versions.values(): - self.assertNotEqual(added, package_added_cache.now) - - def test_PackageAddedCache_get_new(self): - """Test that new added dates work, and are not replaced later. - - There are more related tests in tests/test_integration.py. - - """ - package_added_cache = fdroidserver.common.PackageAddedCache() - package_added_cache.now = 1234567890 - fake_apk = 'fake.apk' - package_added_cache.get(fake_apk) - for apk, added in package_added_cache.versions.items(): - if apk == fake_apk: - self.assertEqual(added, package_added_cache.now) - else: - self.assertNotEqual(added, package_added_cache.now) - package_added_cache.get(fake_apk) - self.assertEqual(package_added_cache.versions[fake_apk], package_added_cache.now) - def test_get_mirrors_fdroidorg(self): mirrors = fdroidserver.common.get_mirrors( 'https://f-droid.org/repo', 'entry.jar' diff --git a/tests/test_update.py b/tests/test_update.py index 0496c402..82933a7c 100755 --- a/tests/test_update.py +++ b/tests/test_update.py @@ -366,7 +366,7 @@ class UpdateTest(unittest.TestCase): apps = fdroidserver.metadata.read_metadata() apps['info.guardianproject.urzip']['CurrentVersionCode'] = 100 - package_added_cache = fdroidserver.common.PackageAddedCache() + package_added_cache = fdroidserver.update.PackageAddedCache() apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', package_added_cache, False) fdroidserver.update.insert_localized_app_metadata(apps) fdroidserver.update.ingest_screenshots_from_repo_dir(apps) @@ -453,7 +453,7 @@ class UpdateTest(unittest.TestCase): fdroidserver.update.options.delete_unknown = True apps = fdroidserver.metadata.read_metadata() - package_added_cache = fdroidserver.common.PackageAddedCache() + package_added_cache = fdroidserver.update.PackageAddedCache() apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', package_added_cache, False) appid = 'info.guardianproject.checkey' @@ -720,7 +720,7 @@ class UpdateTest(unittest.TestCase): fdroidserver.update.options.delete_unknown = True apps = fdroidserver.metadata.read_metadata() - package_added_cache = fdroidserver.common.PackageAddedCache() + package_added_cache = fdroidserver.update.PackageAddedCache() apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', package_added_cache, False) self.assertEqual(len(apks), 18) apk = apks[1] @@ -778,7 +778,7 @@ class UpdateTest(unittest.TestCase): fdroidserver.update.options.delete_unknown = True fdroidserver.metadata.read_metadata() - package_added_cache = fdroidserver.common.PackageAddedCache() + package_added_cache = fdroidserver.update.PackageAddedCache() apkcache = fdroidserver.update.get_cache() self.assertEqual(2, len(apkcache)) self.assertEqual(fdroidserver.update.METADATA_VERSION, apkcache["METADATA_VERSION"]) @@ -807,8 +807,10 @@ class UpdateTest(unittest.TestCase): os.mkdir('repo') filename = 'Norway_bouvet_europe_2.obf.zip' shutil.copy(basedir / filename, 'repo') - package_added_cache = fdroidserver.common.PackageAddedCache() - files, fcachechanged = fdroidserver.update.scan_repo_files(dict(), 'repo', package_added_cache, False) + package_added_cache = fdroidserver.update.PackageAddedCache() + files, fcachechanged = fdroidserver.update.scan_repo_files( + dict(), 'repo', package_added_cache, False + ) self.assertTrue(fcachechanged) info = files[0] @@ -832,7 +834,7 @@ class UpdateTest(unittest.TestCase): fdroidserver.update.config = config fdroidserver.common.options = Options apps = fdroidserver.metadata.read_metadata() - package_added_cache = fdroidserver.common.PackageAddedCache() + package_added_cache = fdroidserver.update.PackageAddedCache() apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', package_added_cache) fdroidserver.update.read_added_date_from_all_apks(apps, apks) @@ -846,7 +848,7 @@ class UpdateTest(unittest.TestCase): fdroidserver.common.options = Options fdroidserver.update.options = fdroidserver.common.options apps = fdroidserver.metadata.read_metadata() - package_added_cache = fdroidserver.common.PackageAddedCache() + package_added_cache = fdroidserver.update.PackageAddedCache() apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', package_added_cache) fdroidserver.update.apply_info_from_latest_apk(apps, apks) @@ -1106,7 +1108,7 @@ class UpdateTest(unittest.TestCase): if not os.path.exists(icon_dir): os.makedirs(icon_dir) - package_added_cache = fdroidserver.common.PackageAddedCache() + package_added_cache = fdroidserver.update.PackageAddedCache() apkList = ['../urzip.apk', '../org.dyndns.fules.ck_20.apk', 'org.maxsdkversion_4.apk'] for apkName in apkList: @@ -1156,7 +1158,7 @@ class UpdateTest(unittest.TestCase): fdroidserver.update.options.verbose = True fdroidserver.update.options.delete_unknown = True - package_added_cache = fdroidserver.common.PackageAddedCache() + package_added_cache = fdroidserver.update.PackageAddedCache() with mkdtemp() as tmptestsdir, TmpCwd(tmptestsdir): os.mkdir('repo') @@ -1246,7 +1248,7 @@ class UpdateTest(unittest.TestCase): fdroidserver.update.options = fdroidserver.common.options fdroidserver.update.options.delete_unknown = False - package_added_cache = fdroidserver.common.PackageAddedCache() + package_added_cache = fdroidserver.update.PackageAddedCache() apk = 'fake.ota.update_1234.zip' # this is not an APK, scanning should fail (skip, apk, cachechanged) = fdroidserver.update.process_apk( {}, apk, 'repo', package_added_cache, False @@ -1279,9 +1281,9 @@ class UpdateTest(unittest.TestCase): fdroidserver.update.options = fdroidserver.common.options fdroidserver.update.options.delete_unknown = False - knownapks = fdroidserver.common.KnownApks() + package_added_cache = fdroidserver.update.PackageAddedCache() (skip, apk, cachechanged) = fdroidserver.update.process_apk( - {}, apk, 'archive', knownapks, False + {}, apk, 'archive', package_added_cache, False ) self.assertFalse(skip) self.assertTrue(cachechanged) @@ -1291,6 +1293,37 @@ class UpdateTest(unittest.TestCase): # translations only available if compiled locally: make -C locale compile self.assertNotEqual(da['en-US'], da['ro']) + def test_PackageAddedCache_get(self): + """Test that added dates are being fetched from the index. + + There are more related tests in tests/test_integration.py. + + """ + package_added_cache = fdroidserver.update.PackageAddedCache() + package_added_cache.now = 1234567890 + for filename in package_added_cache.versions: + package_added_cache.get(filename) + for added in package_added_cache.versions.values(): + self.assertNotEqual(added, package_added_cache.now) + + def test_PackageAddedCache_get_new(self): + """Test that new added dates work, and are not replaced later. + + There are more related tests in tests/test_integration.py. + + """ + package_added_cache = fdroidserver.update.PackageAddedCache() + package_added_cache.now = 1234567890 + fake_apk = 'fake.apk' + package_added_cache.get(fake_apk) + for apk, added in package_added_cache.versions.items(): + if apk == fake_apk: + self.assertEqual(added, package_added_cache.now) + else: + self.assertNotEqual(added, package_added_cache.now) + package_added_cache.get(fake_apk) + self.assertEqual(package_added_cache.versions[fake_apk], package_added_cache.now) + def test_get_apks_without_allowed_signatures(self): """Test when no AllowedAPKSigningKeys is specified""" os.chdir(self.testdir) @@ -1302,7 +1335,7 @@ class UpdateTest(unittest.TestCase): fdroidserver.update.options = fdroidserver.common.options app = fdroidserver.metadata.App() - package_added_cache = fdroidserver.common.PackageAddedCache() + package_added_cache = fdroidserver.update.PackageAddedCache() apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', package_added_cache) apkfile = 'v1.v2.sig_1020.apk' self.assertIn( @@ -1332,7 +1365,7 @@ class UpdateTest(unittest.TestCase): 'AllowedAPKSigningKeys': '32a23624c201b949f085996ba5ed53d40f703aca4989476949cae891022e0ed6' } ) - package_added_cache = fdroidserver.common.PackageAddedCache() + package_added_cache = fdroidserver.update.PackageAddedCache() apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', package_added_cache) apkfile = 'v1.v2.sig_1020.apk' (skip, apk, cachechanged) = fdroidserver.update.process_apk( @@ -1357,7 +1390,7 @@ class UpdateTest(unittest.TestCase): 'AllowedAPKSigningKeys': 'fa4edeadfa4edeadfa4edeadfa4edeadfa4edeadfa4edeadfa4edeadfa4edead' } ) - package_added_cache = fdroidserver.common.PackageAddedCache() + package_added_cache = fdroidserver.update.PackageAddedCache() apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', package_added_cache) apkfile = 'v1.v2.sig_1020.apk' (skip, apk, cachechanged) = fdroidserver.update.process_apk( @@ -1428,7 +1461,7 @@ class UpdateTest(unittest.TestCase): fdroidserver.update.options.delete_unknown = True apps = fdroidserver.metadata.read_metadata() - package_added_cache = fdroidserver.common.PackageAddedCache() + package_added_cache = fdroidserver.update.PackageAddedCache() apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', package_added_cache, False) fdroidserver.update.translate_per_build_anti_features(apps, apks) self.assertEqual(len(apks), 18) @@ -1457,7 +1490,7 @@ class UpdateTest(unittest.TestCase): fdroidserver.update.options = fdroidserver.common.options fdroidserver.update.options.clean = True - package_added_cache = fdroidserver.common.PackageAddedCache() + package_added_cache = fdroidserver.update.PackageAddedCache() apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', package_added_cache, False) self.assertEqual(1, len(apks)) apk = apks[0] @@ -1997,7 +2030,7 @@ class UpdateTest(unittest.TestCase): build.disable = "disabled" app['Builds'] = [build] - package_added_cache = fdroidserver.common.PackageAddedCache() + package_added_cache = fdroidserver.update.PackageAddedCache() apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', package_added_cache, False, apps) self.assertEqual([], apks)