diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 17425b8a..00fd097c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,7 +20,7 @@ buildserver run-tests: image: registry.gitlab.com/fdroid/fdroidserver:buildserver script: - apt-get update - - apt-get install gnupg-agent python3-babel python3-clint python3-pycountry + - apt-get install gnupg-agent python3-babel python3-biplist python3-clint python3-pycountry - ./tests/run-tests # make sure that translations do not cause stacktraces - cd $CI_PROJECT_DIR/locale @@ -42,7 +42,7 @@ metadata_v0: image: registry.gitlab.com/fdroid/fdroidserver:buildserver variables: GIT_DEPTH: 1000 - RELEASE_COMMIT_ID: a1c4f803de8d4dc92ebd6b571a493183d14a00bf # after ArchivePolicy: 0 + RELEASE_COMMIT_ID: 50aa35772b058e76b950c01e16019c072c191b73 # after switching to `git rev-parse` script: - git fetch https://gitlab.com/fdroid/fdroidserver.git $RELEASE_COMMIT_ID - cd tests @@ -154,8 +154,8 @@ ubuntu_jammy_pip: - $pip install sdkmanager - sdkmanager 'build-tools;33.0.0' - # pycountry is only for linting config/mirrors.yml, so its not in setup.py - - $pip install pycountry + # Install extras_require.optional from setup.py + - $pip install biplist pycountry - $pip install dist/fdroidserver-*.tar.gz - tar xzf dist/fdroidserver-*.tar.gz diff --git a/MANIFEST.in b/MANIFEST.in index 1aed9975..05a022b2 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -543,6 +543,7 @@ include tests/build-tools/28.0.3/aapt-output-souch.smsbypass_9.txt include tests/build-tools/generate.sh include tests/check-fdroid-apk include tests/checkupdates.TestCase +include tests/com.fake.IpaApp_1000000000001.ipa include tests/common.TestCase include tests/config.py include tests/config/antiFeatures.yml diff --git a/fdroidserver/update.py b/fdroidserver/update.py index a1d4dbc4..26e248d5 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -49,10 +49,10 @@ from binascii import hexlify from . import _ from . import common -from . import index from . import metadata from .common import DEFAULT_LOCALE from .exception import BuildException, FDroidException, VerificationException +import fdroidserver.index from PIL import Image, PngImagePlugin @@ -524,6 +524,94 @@ def insert_obbs(repodir, apps, apks): break +VERSION_STRING_RE = re.compile(r'^([0-9]+)\.([0-9]+)\.([0-9]+)$') + + +def version_string_to_int(version): + """ + Convert sermver version designation to version code. + + Approximately convert a [Major].[Minor].[Patch] version string + consisting of numeric characters (0-9) and periods to a number. The + exponents are chosen such that it still fits in the 64bit JSON/Android range. + """ + m = VERSION_STRING_RE.match(version) + if not m: + raise ValueError(f"invalid version string '{version}'") + major = int(m.group(1)) + minor = int(m.group(2)) + patch = int(m.group(3)) + return major * 10**12 + minor * 10**6 + patch + + +def parse_ipa(ipa_path, file_size, sha256): + from biplist import readPlist + + ipa = { + "apkName": os.path.basename(ipa_path), + "hash": sha256, + "hashType": "sha256", + "size": file_size, + } + + with zipfile.ZipFile(ipa_path) as ipa_zip: + for info in ipa_zip.infolist(): + if re.match("Payload/[^/]*.app/Info.plist", info.filename): + with ipa_zip.open(info) as plist_file: + plist = readPlist(plist_file) + ipa["packageName"] = plist["CFBundleIdentifier"] + # https://developer.apple.com/documentation/bundleresources/information_property_list/cfbundleshortversionstring + ipa["versionCode"] = version_string_to_int(plist["CFBundleShortVersionString"]) + ipa["versionName"] = plist["CFBundleShortVersionString"] + return ipa + + +def scan_repo_for_ipas(apkcache, repodir, knownapks): + """Scan for IPA files in a given repo directory. + + Parameters + ---------- + apkcache + cache dictionary containting cached file infos from previous runs + repodir + repo directory to scan + knownapks + list of all known files, as per metadata.read_metadata + + Returns + ------- + ipas + list of file infos for ipa files in ./repo folder + cachechanged + ture if new ipa files were found and added to `apkcache` + """ + cachechanged = False + ipas = [] + for ipa_path in glob.glob(os.path.join(repodir, '*.ipa')): + ipa_name = os.path.basename(ipa_path) + + file_size = os.stat(ipa_path).st_size + if file_size == 0: + raise FDroidException(_('{path} is zero size!') + .format(path=ipa_path)) + + sha256 = common.sha256sum(ipa_path) + ipa = apkcache.get(ipa_name, {}) + + if ipa.get('hash') != sha256: + ipa = fdroidserver.update.parse_ipa(ipa_path, file_size, sha256) + apkcache[ipa_name] = ipa + cachechanged = True + + added = knownapks.recordapk(ipa_name, ipa['packageName']) + if added: + ipa['added'] = added + + ipas.append(ipa) + + return ipas, cachechanged + + def translate_per_build_anti_features(apps, apks): """Grab the anti-features list from the build metadata. @@ -1121,7 +1209,10 @@ def insert_localized_app_metadata(apps): def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): - """Scan a repo for all files with an extension except APK/OBB. + """Scan a repo for all files with an extension except APK/OBB/IPA. + + This allows putting all kinds of files into repostories. E.g. Media Files, + Zip archives, ... Parameters ---------- @@ -1138,22 +1229,29 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): repo_files = [] repodir = repodir.encode() for name in os.listdir(repodir): + # skip files based on file extensions, that are handled elsewhere file_extension = common.get_file_extension(name) - if file_extension in ('apk', 'obb'): + if file_extension in ('apk', 'obb', 'ipa'): continue + + # skip source tarballs generated by fdroidserver filename = os.path.join(repodir, name) name_utf8 = name.decode() if filename.endswith(b'_src.tar.gz'): logging.debug(_('skipping source tarball: {path}') .format(path=filename.decode())) continue + + # skip all other files generated by fdroidserver if not common.is_repo_file(filename): continue + stat = os.stat(filename) if stat.st_size == 0: raise FDroidException(_('{path} is zero size!') .format(path=filename)) + # load file infos from cache if not stale shasum = common.sha256sum(filename) usecache = False if name_utf8 in apkcache: @@ -1166,6 +1264,7 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): logging.debug(_("Ignoring stale cache data for {apkfilename}") .format(apkfilename=name_utf8)) + # scan file if info wasn't in cache if not usecache: logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8)) repo_file = collections.OrderedDict() @@ -2254,6 +2353,11 @@ def main(): options.use_date_from_apk) cachechanged = cachechanged or fcachechanged apks += files + + ipas, icachechanged = scan_repo_for_ipas(apkcache, repodirs[0], knownapks) + cachechanged = cachechanged or icachechanged + apks += ipas + appid_has_apks = set() appid_has_repo_files = set() remove_apks = [] @@ -2329,7 +2433,7 @@ def main(): if len(repodirs) > 1: archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older']) archived_apps = prepare_apps(apps, archapks, repodirs[1]) - index.make(archived_apps, archapks, repodirs[1], True) + fdroidserver.index.make(archived_apps, archapks, repodirs[1], True) repoapps = prepare_apps(apps, apks, repodirs[0]) @@ -2342,13 +2446,13 @@ def main(): app_dict = dict() app_dict[appid] = app if os.path.isdir(repodir): - index.make(app_dict, apks, repodir, False) + fdroidserver.index.make(app_dict, apks, repodir, False) else: logging.info(_('Skipping index generation for {appid}').format(appid=appid)) return # Make the index for the main repo... - index.make(repoapps, apks, repodirs[0], False) + fdroidserver.index.make(repoapps, apks, repodirs[0], False) git_remote = config.get('binary_transparency_remote') if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')): diff --git a/setup.py b/setup.py index 522c3377..afff96b4 100755 --- a/setup.py +++ b/setup.py @@ -108,7 +108,11 @@ setup( 'sdkmanager >= 0.6.4', 'yamllint', ], + # Some requires are only needed for very limited cases: + # * biplist is only used for parsing Apple .ipa files + # * pycountry is only for linting config/mirrors.yml extras_require={ + 'optional': ['biplist', 'pycountry'], 'test': ['pyjks', 'html5print'], 'docs': [ 'sphinx', diff --git a/tests/com.fake.IpaApp_1000000000001.ipa b/tests/com.fake.IpaApp_1000000000001.ipa new file mode 100644 index 00000000..d392cb94 Binary files /dev/null and b/tests/com.fake.IpaApp_1000000000001.ipa differ diff --git a/tests/update.TestCase b/tests/update.TestCase index abce3a30..f1c07fd9 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -1922,6 +1922,86 @@ class UpdateTest(unittest.TestCase): index['repo'][CATEGORIES_CONFIG_NAME], ) + def test_parse_ipa(self): + ipa_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'com.fake.IpaApp_1000000000001.ipa') + result = fdroidserver.update.parse_ipa(ipa_path, 'fake_size', 'fake_sha') + self.maxDiff = None + self.assertDictEqual(result, { + 'apkName': 'com.fake.IpaApp_1000000000001.ipa', + 'hash': 'fake_sha', + 'hashType': 'sha256', + 'packageName': 'org.onionshare.OnionShare', + 'size': 'fake_size', + 'versionCode': 1000000000001, + 'versionName': '1.0.1', + }) + + +class TestUpdateVersionStringToInt(unittest.TestCase): + + def test_version_string_to_int(self): + self.assertEqual(fdroidserver.update.version_string_to_int("1.2.3"), 1000002000003) + self.assertEqual(fdroidserver.update.version_string_to_int("0.0.0003"), 3) + self.assertEqual(fdroidserver.update.version_string_to_int("0.0.0"), 0) + self.assertEqual(fdroidserver.update.version_string_to_int("4321.321.21"), 4321000321000021) + self.assertEqual(fdroidserver.update.version_string_to_int("18446744.073709.551615"), 18446744073709551615) + + def test_version_string_to_int_value_errors(self): + with self.assertRaises(ValueError): + fdroidserver.update.version_string_to_int("1.2.3a") + with self.assertRaises(ValueError): + fdroidserver.update.version_string_to_int("asdfasdf") + with self.assertRaises(ValueError): + fdroidserver.update.version_string_to_int("1.2.-3") + with self.assertRaises(ValueError): + fdroidserver.update.version_string_to_int("-1.2.-3") + with self.assertRaises(ValueError): + fdroidserver.update.version_string_to_int("0.0.0x3") + + +class TestScanRepoForIpas(unittest.TestCase): + + def setUp(self): + self.maxDiff = None + + def test_scan_repo_for_ipas_no_cache(self): + self.maxDiff = None + with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): + os.mkdir("repo") + with open('repo/abc.Def_123.ipa', 'w') as f: + f.write('abc') + with open('repo/xyz.XXX_123.ipa', 'w') as f: + f.write('xyz') + + apkcache = mock.MagicMock() + # apkcache['a'] = 1 + repodir = "repo" + knownapks = mock.MagicMock() + + def mocked_parse(p, s, c): + # pylint: disable=unused-argument + return { + 'packageName': 'abc' if 'abc' in p else 'xyz' + } + + with mock.patch('fdroidserver.update.parse_ipa', mocked_parse): + ipas, checkchanged = fdroidserver.update.scan_repo_for_ipas(apkcache, repodir, knownapks) + + self.assertEqual(checkchanged, True) + self.assertEqual(len(ipas), 2) + package_names_in_ipas = [x['packageName'] for x in ipas] + self.assertTrue('abc' in package_names_in_ipas) + self.assertTrue('xyz' in package_names_in_ipas) + + apkcache_setter_package_name = [x.args[1]['packageName'] for x in apkcache.__setitem__.mock_calls] + self.assertTrue('abc' in apkcache_setter_package_name) + self.assertTrue('xyz' in apkcache_setter_package_name) + self.assertEqual(apkcache.__setitem__.call_count, 2) + + knownapks.recordapk.call_count = 2 + self.assertTrue(unittest.mock.call('abc.Def_123.ipa', 'abc') in knownapks.recordapk.mock_calls) + self.assertTrue(unittest.mock.call('xyz.XXX_123.ipa', 'xyz') in knownapks.recordapk.mock_calls) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) @@ -1938,4 +2018,6 @@ if __name__ == "__main__": newSuite = unittest.TestSuite() newSuite.addTest(unittest.makeSuite(UpdateTest)) + newSuite.addTest(unittest.makeSuite(TestUpdateVersionStringToInt)) + newSuite.addTest(unittest.makeSuite(TestScanRepoForIpas)) unittest.main(failfast=False)