diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a72447741..7d7594f4d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -148,11 +148,12 @@ pages: only: - master script: - - ./gradlew :download:dokkaHtml :index:dokkaHtml - - mkdir public - - touch public/index.html - - cp -r download/build/dokka/html public/download - - cp -r index/build/dokka/html public/index + - ./gradlew :libs:download:dokkaHtml :libs:index:dokkaHtml :libs:database:dokkaHtml + - mkdir -p public/libs + - touch public/index.html public/libs/index.html + - cp -r libs/download/build/dokka/html public/libs/download + - cp -r libs/index/build/dokka/html public/libs/index + - cp -r libs/database/build/dokka/html public/libs/database artifacts: paths: - public @@ -173,8 +174,8 @@ deploy_nightly: - echo "${CI_PROJECT_PATH}-nightly" >> app/src/main/res/values/default_repos.xml - echo "${CI_PROJECT_URL}-nightly/raw/master/fdroid/repo" >> app/src/main/res/values/default_repos.xml - cat config/nightly-repo/repo.xml >> app/src/main/res/values/default_repos.xml - - export DB=`sed -n 's,.*DB_VERSION *= *\([0-9][0-9]*\).*,\1,p' app/src/main/java/org/fdroid/fdroid/data/DBHelper.java` - - export versionCode=`printf '%d%05d' $DB $(date '+%s'| cut -b4-8)` + - export DB=`sed -n 's,.*version *= *\([0-9][0-9]*\).*,\1,p' libs/database/src/main/java/org/fdroid/database/FDroidDatabase.kt` + - export versionCode=`printf '%d%05d' $DB $(date '+%s'| cut -b1-8)` - sed -i "s,^\(\s*versionCode\) *[0-9].*,\1 $versionCode," app/build.gradle # build the APKs! - ./gradlew assembleDebug diff --git a/README.md b/README.md index 0ba489fb7..500bb87d8 100644 --- a/README.md +++ b/README.md @@ -20,10 +20,7 @@ from our site or [browse it in the repo](https://f-droid.org/app/org.fdroid.fdro Core F-Droid functionality is split into re-usable libraries to make using F-Droid technology in your own projects as easy as possible. -Note that all libraries are still in alpha stage. -While they work, their public APIs are still subject to change. - -* [download](download) library for handling (multi-platform) HTTP download of repository indexes and APKs +[More information about libraries](libs/README.md) ## Contributing diff --git a/app/build.gradle b/app/build.gradle index b9b0f6053..47eb3af07 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -142,7 +142,9 @@ android { } dependencies { - implementation project(":download") + implementation project(":libs:download") + implementation project(":libs:index") + implementation project(":libs:database") implementation 'androidx.appcompat:appcompat:1.3.0' implementation 'androidx.preference:preference:1.1.1' implementation 'androidx.gridlayout:gridlayout:1.0.0' diff --git a/app/src/main/java/org/fdroid/fdroid/net/BluetoothDownloader.java b/app/src/main/java/org/fdroid/fdroid/net/BluetoothDownloader.java index 968a3f976..511dadc75 100644 --- a/app/src/main/java/org/fdroid/fdroid/net/BluetoothDownloader.java +++ b/app/src/main/java/org/fdroid/fdroid/net/BluetoothDownloader.java @@ -91,10 +91,18 @@ public class BluetoothDownloader extends Downloader { @Override public long totalDownloadSize() { + if (getFileSize() != null) return getFileSize(); FileDetails details = getFileDetails(); return details != null ? details.getFileSize() : -1; } + @Override + public void download(long totalSize, @Nullable String sha256) throws IOException, InterruptedException { + setFileSize(totalSize); + setSha256(sha256); + download(); + } + @Override public void download() throws IOException, InterruptedException { downloadFromStream(false); diff --git a/app/src/main/java/org/fdroid/fdroid/net/LocalFileDownloader.java b/app/src/main/java/org/fdroid/fdroid/net/LocalFileDownloader.java index 44c3bc7da..351dcf275 100644 --- a/app/src/main/java/org/fdroid/fdroid/net/LocalFileDownloader.java +++ b/app/src/main/java/org/fdroid/fdroid/net/LocalFileDownloader.java @@ -3,6 +3,7 @@ package org.fdroid.fdroid.net; import android.net.Uri; import androidx.annotation.NonNull; +import androidx.annotation.Nullable; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; @@ -69,6 +70,13 @@ public class LocalFileDownloader extends Downloader { return sourceFile.length(); } + @Override + public void download(long totalSize, @Nullable String sha256) throws IOException, InterruptedException { + setFileSize(totalSize); + setSha256(sha256); + download(); + } + @Override public void download() throws IOException, InterruptedException { if (!sourceFile.exists()) { diff --git a/app/src/main/java/org/fdroid/fdroid/net/TreeUriDownloader.java b/app/src/main/java/org/fdroid/fdroid/net/TreeUriDownloader.java index f1f0bc70f..a8b417a47 100644 --- a/app/src/main/java/org/fdroid/fdroid/net/TreeUriDownloader.java +++ b/app/src/main/java/org/fdroid/fdroid/net/TreeUriDownloader.java @@ -15,6 +15,7 @@ import java.io.InputStream; import java.net.ProtocolException; import androidx.annotation.NonNull; +import androidx.annotation.Nullable; import androidx.documentfile.provider.DocumentFile; /** @@ -94,7 +95,14 @@ public class TreeUriDownloader extends Downloader { @Override protected long totalDownloadSize() { - return documentFile.length(); // TODO how should this actually be implemented? + return getFileSize() != null ? getFileSize() : documentFile.length(); + } + + @Override + public void download(long totalSize, @Nullable String sha256) throws IOException, InterruptedException { + setFileSize(totalSize); + setSha256(sha256); + downloadFromStream(false); } @Override diff --git a/download/README.md b/download/README.md deleted file mode 100644 index 37b51a47a..000000000 --- a/download/README.md +++ /dev/null @@ -1,40 +0,0 @@ -# F-Droid multi-platform download library - -Note that advanced security and privacy features are only available for Android: - - * Rejection of TLS 1.1 and older as well as rejection of weak ciphers - * No DNS requests when using Tor as a proxy - * short TLS session timeout to prevent tracking and key re-use - -Other platforms besides Android have not been tested and might need additional work. - -## How to include in your project - -Add this to your `build.gradle` file -and replace `[version]` with the [latest version](gradle.properties): - - implementation 'org.fdroid:download:[version]' - -## Development - -You can list available gradle tasks by running the following command in the project root. - - ./gradlew :download:tasks - -### Making releases - -Bump version number in [`gradle.properties`](gradle.properties), ensure you didn't break a public API and run: - - ./gradlew :download:check :download:connectedCheck - ./gradlew :download:publish - ./gradlew closeAndReleaseRepository - -See https://github.com/vanniktech/gradle-maven-publish-plugin#gradle-maven-publish-plugin for more information. - -## License - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 814b291d8..a8cccac91 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -10,12 +10,17 @@ + + + + + @@ -51,7 +56,10 @@ - + + + + @@ -243,7 +251,7 @@ - + @@ -283,10 +291,18 @@ + + + + + + + + @@ -295,6 +311,11 @@ + + + + + @@ -326,7 +347,7 @@ - + @@ -397,11 +418,26 @@ + + + + + + + + + + + + + + + @@ -564,6 +600,11 @@ + + + + + @@ -577,6 +618,11 @@ + + + + + @@ -590,7 +636,25 @@ + + + + + + + + + + + + + + + + + + @@ -705,11 +769,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -736,11 +835,21 @@ + + + + + + + + + + @@ -829,6 +938,11 @@ + + + + + @@ -865,21 +979,11 @@ - - - - - - - - - - @@ -2236,6 +2340,11 @@ + + + + + @@ -2764,6 +2873,11 @@ + + + + + @@ -2782,6 +2896,12 @@ + + + + + + @@ -2797,12 +2917,6 @@ - - - - - - @@ -2920,6 +3034,11 @@ + + + + + @@ -3937,66 +4056,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -4148,11 +4207,6 @@ - - - - - @@ -4207,11 +4261,6 @@ - - - - - @@ -5170,11 +5219,6 @@ - - - - - @@ -5185,11 +5229,6 @@ - - - - - @@ -5215,11 +5254,6 @@ - - - - - @@ -5240,11 +5274,6 @@ - - - - - @@ -5270,11 +5299,6 @@ - - - - - @@ -5335,11 +5359,6 @@ - - - - - @@ -5370,11 +5389,6 @@ - - - - - @@ -5435,11 +5449,6 @@ - - - - - @@ -5585,11 +5594,6 @@ - - - - - @@ -5605,6 +5609,11 @@ + + + + + @@ -5680,11 +5689,6 @@ - - - - - @@ -5705,16 +5709,6 @@ - - - - - - - - - - @@ -5725,11 +5719,6 @@ - - - - - @@ -5800,11 +5789,6 @@ - - - - - @@ -5825,11 +5809,6 @@ - - - - - @@ -5880,11 +5859,6 @@ - - - - - @@ -5969,16 +5943,16 @@ - - - - - + + + + + @@ -5989,6 +5963,11 @@ + + + + + @@ -6056,16 +6035,6 @@ - - - - - - - - - - @@ -6232,6 +6201,7 @@ + diff --git a/index/LICENSE b/index/LICENSE deleted file mode 100644 index d64569567..000000000 --- a/index/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/index/README.md b/index/README.md deleted file mode 100644 index 130a7817a..000000000 --- a/index/README.md +++ /dev/null @@ -1,41 +0,0 @@ -# F-Droid multi-platform index library - -Note that some features are only available for Android: - - * index signature verification (`JarFile` is JVM only) - * index stream processing (`InputStream` is JVM only) - * index V2 diffing (reflection is JVM only) - * app device compatibility checking (requires Android) - -Other platforms besides Android have not been tested and might need additional work. - -## How to include in your project - -Add this to your `build.gradle` file -and replace `[version]` with the [latest version](gradle.properties): - - implementation 'org.fdroid:index:[version]' - -## Development - -You can list available gradle tasks by running the following command in the project root. - - ./gradlew :index:tasks - -### Making releases - -Bump version number in [`gradle.properties`](gradle.properties), ensure you didn't break a public API and run: - - ./gradlew :index:check :index:connectedCheck - ./gradlew :index:publish - ./gradlew closeAndReleaseRepository - -See https://github.com/vanniktech/gradle-maven-publish-plugin#gradle-maven-publish-plugin for more information. - -## License - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 diff --git a/download/LICENSE b/libs/LICENSE similarity index 100% rename from download/LICENSE rename to libs/LICENSE diff --git a/libs/README.md b/libs/README.md new file mode 100644 index 000000000..e9759399e --- /dev/null +++ b/libs/README.md @@ -0,0 +1,98 @@ +# F-Droid libraries + +Core F-Droid functionality is split into re-usable libraries +to make using F-Droid technology in your own projects as easy as possible. + +Note that all libraries are still in alpha stage. +While they work, their public APIs are still subject to change. + +* [download](libs/download) library for handling (multi-platform) HTTP download + of repository indexes and APKs +* [index](libs/index) library for parsing/verifying/creating repository indexes +* [database](libs/database) library to store and query F-Droid related information + in a Room-based database on Android + +## F-Droid multi-platform download library + +[API docs](https://fdroid.gitlab.io/fdroidclient/libs/download/) + +Note that advanced security and privacy features are only available for Android: + +* Rejection of TLS 1.1 and older as well as rejection of weak ciphers +* No DNS requests when using Tor as a proxy +* short TLS session timeout to prevent tracking and key re-use + +Other platforms besides Android have not been tested and might need additional work. + +### How to include in your project + +Add this to your `build.gradle` file +and replace `[version]` with the [latest version](download/index/gradle.properties): + + implementation 'org.fdroid:download:[version]' + +## F-Droid multi-platform index library + +[API docs](https://fdroid.gitlab.io/fdroidclient/libs/index/) + +Note that some features are only available for Android: + + * index signature verification (`JarFile` is JVM only) + * index stream processing (`InputStream` is JVM only) + * index V2 diffing (reflection is JVM only) + * app device compatibility checking (requires Android) + +Other platforms besides Android have not been tested and might need additional work. + +### How to include in your project + +Add this to your `build.gradle` file +and replace `[version]` with the [latest version](libs/index/gradle.properties): + + implementation 'org.fdroid:index:[version]' + +## F-Droid Android database library + +[API docs](https://fdroid.gitlab.io/fdroidclient/libs/database/) + +An Android-only database library to store and query F-Droid related information +such as repositories, apps and their versions. +This library should bring everything you need to build your own F-Droid client +that persists information. + +### How to include in your project + +Add this to your `build.gradle` file +and replace `[version]` with the [latest version](libs/database/gradle.properties): + + implementation 'org.fdroid:database:[version]' + +# Development + +You can list available gradle tasks by running the following command in the project root. + + ./gradlew :download:tasks + +Replace `download` with the name of the library you want to view tasks for. + +# Making releases + +Bump version number in the library's [`gradle.properties`](gradle.properties), +ensure you didn't break a public API and run: + + ./gradlew :download:check :index:connectedCheck + ./gradlew :download:publish + ./gradlew closeAndReleaseRepository + +Replace `download` with the name of the library you want to publish. + +See https://github.com/vanniktech/gradle-maven-publish-plugin#gradle-maven-publish-plugin +for more information. + +# License + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 diff --git a/download/.gitignore b/libs/database/.gitignore similarity index 100% rename from download/.gitignore rename to libs/database/.gitignore diff --git a/libs/database/build.gradle b/libs/database/build.gradle new file mode 100644 index 000000000..05f42c5e3 --- /dev/null +++ b/libs/database/build.gradle @@ -0,0 +1,109 @@ +plugins { + id 'kotlin-android' + id 'com.android.library' + id 'kotlin-kapt' + id 'org.jetbrains.dokka' + id "org.jlleitschuh.gradle.ktlint" version "10.2.1" +} + +android { + compileSdkVersion 31 + + defaultConfig { + minSdkVersion 21 + + consumerProguardFiles "consumer-rules.pro" + javaCompileOptions { + annotationProcessorOptions { + arguments += ["room.schemaLocation": "$projectDir/schemas".toString()] + } + } + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" + testInstrumentationRunnerArguments disableAnalytics: 'true' + } + + buildTypes { + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' + } + } + sourceSets { + androidTest { + java.srcDirs += "src/dbTest/java" + } + test { + java.srcDirs += "src/dbTest/java" + } + } + compileOptions { + sourceCompatibility JavaVersion.VERSION_1_8 + targetCompatibility JavaVersion.VERSION_1_8 + } + testOptions { + unitTests { + includeAndroidResources = true + } + } + kotlinOptions { + jvmTarget = '1.8' + freeCompilerArgs += "-Xexplicit-api=strict" + freeCompilerArgs += "-opt-in=kotlin.RequiresOptIn" + } + aaptOptions { + // needed only for instrumentation tests: assets.openFd() + noCompress "json" + } + packagingOptions { + exclude 'META-INF/AL2.0' + exclude 'META-INF/LGPL2.1' + } +} + +dependencies { + implementation project(":libs:download") + implementation project(":libs:index") + + implementation 'androidx.core:core-ktx:1.8.0' + implementation 'androidx.lifecycle:lifecycle-livedata-ktx:2.4.1' + + def room_version = "2.4.2" + implementation "androidx.room:room-runtime:$room_version" + implementation "androidx.room:room-ktx:$room_version" + kapt "androidx.room:room-compiler:$room_version" + + implementation 'io.github.microutils:kotlin-logging:2.1.21' + implementation "org.slf4j:slf4j-android:1.7.36" + + implementation "org.jetbrains.kotlinx:kotlinx-serialization-json:1.3.2" + + testImplementation project(":libs:sharedTest") + testImplementation 'junit:junit:4.13.2' + testImplementation 'io.mockk:mockk:1.12.4' + testImplementation 'org.jetbrains.kotlin:kotlin-test' + testImplementation 'androidx.test:core:1.4.0' + testImplementation 'androidx.test.ext:junit:1.1.3' + testImplementation 'androidx.arch.core:core-testing:2.1.0' + testImplementation 'org.robolectric:robolectric:4.8.1' + testImplementation 'commons-io:commons-io:2.6' + + androidTestImplementation project(":libs:sharedTest") + androidTestImplementation 'io.mockk:mockk-android:1.12.3' // 1.12.4 has strange error + androidTestImplementation 'org.jetbrains.kotlin:kotlin-test' + androidTestImplementation 'androidx.test.ext:junit:1.1.3' + androidTestImplementation 'androidx.arch.core:core-testing:2.1.0' + androidTestImplementation 'androidx.test.espresso:espresso-core:3.4.0' + androidTestImplementation 'commons-io:commons-io:2.6' +} + +import org.jetbrains.dokka.gradle.DokkaTask +tasks.withType(DokkaTask).configureEach { + pluginsMapConfiguration.set( + ["org.jetbrains.dokka.base.DokkaBase": """{ + "customAssets": ["${file("${rootProject.rootDir}/logo-icon.svg")}"], + "footerMessage": "© 2010-2022 F-Droid Limited and Contributors" + }"""] + ) +} + +apply from: "${rootProject.rootDir}/gradle/ktlint.gradle" diff --git a/libs/database/consumer-rules.pro b/libs/database/consumer-rules.pro new file mode 100644 index 000000000..e69de29bb diff --git a/libs/database/proguard-rules.pro b/libs/database/proguard-rules.pro new file mode 100644 index 000000000..481bb4348 --- /dev/null +++ b/libs/database/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile \ No newline at end of file diff --git a/libs/database/schemas/org.fdroid.database.FDroidDatabaseInt/1.json b/libs/database/schemas/org.fdroid.database.FDroidDatabaseInt/1.json new file mode 100644 index 000000000..497371163 --- /dev/null +++ b/libs/database/schemas/org.fdroid.database.FDroidDatabaseInt/1.json @@ -0,0 +1,1100 @@ +{ + "formatVersion": 1, + "database": { + "version": 1, + "identityHash": "beaebd71355e07ce5f0500c19d9045fd", + "entities": [ + { + "tableName": "CoreRepository", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`repoId` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `name` TEXT NOT NULL, `icon` TEXT, `address` TEXT NOT NULL, `webBaseUrl` TEXT, `timestamp` INTEGER NOT NULL, `version` INTEGER, `formatVersion` TEXT, `maxAge` INTEGER, `description` TEXT NOT NULL, `certificate` TEXT)", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "icon", + "columnName": "icon", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "address", + "columnName": "address", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "webBaseUrl", + "columnName": "webBaseUrl", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "timestamp", + "columnName": "timestamp", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "version", + "columnName": "version", + "affinity": "INTEGER", + "notNull": false + }, + { + "fieldPath": "formatVersion", + "columnName": "formatVersion", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "maxAge", + "columnName": "maxAge", + "affinity": "INTEGER", + "notNull": false + }, + { + "fieldPath": "description", + "columnName": "description", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "certificate", + "columnName": "certificate", + "affinity": "TEXT", + "notNull": false + } + ], + "primaryKey": { + "columnNames": [ + "repoId" + ], + "autoGenerate": true + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "Mirror", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`repoId` INTEGER NOT NULL, `url` TEXT NOT NULL, `location` TEXT, PRIMARY KEY(`repoId`, `url`), FOREIGN KEY(`repoId`) REFERENCES `CoreRepository`(`repoId`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "url", + "columnName": "url", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "location", + "columnName": "location", + "affinity": "TEXT", + "notNull": false + } + ], + "primaryKey": { + "columnNames": [ + "repoId", + "url" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [ + { + "table": "CoreRepository", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "repoId" + ], + "referencedColumns": [ + "repoId" + ] + } + ] + }, + { + "tableName": "AntiFeature", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`repoId` INTEGER NOT NULL, `id` TEXT NOT NULL, `name` TEXT NOT NULL, `description` TEXT NOT NULL, `icon_name` TEXT, `icon_sha256` TEXT, `icon_size` INTEGER, PRIMARY KEY(`repoId`, `id`), FOREIGN KEY(`repoId`) REFERENCES `CoreRepository`(`repoId`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "id", + "columnName": "id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "description", + "columnName": "description", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "icon.name", + "columnName": "icon_name", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "icon.sha256", + "columnName": "icon_sha256", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "icon.size", + "columnName": "icon_size", + "affinity": "INTEGER", + "notNull": false + } + ], + "primaryKey": { + "columnNames": [ + "repoId", + "id" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [ + { + "table": "CoreRepository", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "repoId" + ], + "referencedColumns": [ + "repoId" + ] + } + ] + }, + { + "tableName": "Category", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`repoId` INTEGER NOT NULL, `id` TEXT NOT NULL, `name` TEXT NOT NULL, `description` TEXT NOT NULL, `icon_name` TEXT, `icon_sha256` TEXT, `icon_size` INTEGER, PRIMARY KEY(`repoId`, `id`), FOREIGN KEY(`repoId`) REFERENCES `CoreRepository`(`repoId`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "id", + "columnName": "id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "description", + "columnName": "description", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "icon.name", + "columnName": "icon_name", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "icon.sha256", + "columnName": "icon_sha256", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "icon.size", + "columnName": "icon_size", + "affinity": "INTEGER", + "notNull": false + } + ], + "primaryKey": { + "columnNames": [ + "repoId", + "id" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [ + { + "table": "CoreRepository", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "repoId" + ], + "referencedColumns": [ + "repoId" + ] + } + ] + }, + { + "tableName": "ReleaseChannel", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`repoId` INTEGER NOT NULL, `id` TEXT NOT NULL, `name` TEXT NOT NULL, `description` TEXT NOT NULL, `icon_name` TEXT, `icon_sha256` TEXT, `icon_size` INTEGER, PRIMARY KEY(`repoId`, `id`), FOREIGN KEY(`repoId`) REFERENCES `CoreRepository`(`repoId`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "id", + "columnName": "id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "description", + "columnName": "description", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "icon.name", + "columnName": "icon_name", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "icon.sha256", + "columnName": "icon_sha256", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "icon.size", + "columnName": "icon_size", + "affinity": "INTEGER", + "notNull": false + } + ], + "primaryKey": { + "columnNames": [ + "repoId", + "id" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [ + { + "table": "CoreRepository", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "repoId" + ], + "referencedColumns": [ + "repoId" + ] + } + ] + }, + { + "tableName": "RepositoryPreferences", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`repoId` INTEGER NOT NULL, `weight` INTEGER NOT NULL, `enabled` INTEGER NOT NULL, `lastUpdated` INTEGER, `lastETag` TEXT, `userMirrors` TEXT, `disabledMirrors` TEXT, `username` TEXT, `password` TEXT, PRIMARY KEY(`repoId`))", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "weight", + "columnName": "weight", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "enabled", + "columnName": "enabled", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "lastUpdated", + "columnName": "lastUpdated", + "affinity": "INTEGER", + "notNull": false + }, + { + "fieldPath": "lastETag", + "columnName": "lastETag", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "userMirrors", + "columnName": "userMirrors", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "disabledMirrors", + "columnName": "disabledMirrors", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "username", + "columnName": "username", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "password", + "columnName": "password", + "affinity": "TEXT", + "notNull": false + } + ], + "primaryKey": { + "columnNames": [ + "repoId" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "AppMetadata", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`repoId` INTEGER NOT NULL, `packageName` TEXT NOT NULL, `added` INTEGER NOT NULL, `lastUpdated` INTEGER NOT NULL, `name` TEXT, `summary` TEXT, `description` TEXT, `localizedName` TEXT, `localizedSummary` TEXT, `webSite` TEXT, `changelog` TEXT, `license` TEXT, `sourceCode` TEXT, `issueTracker` TEXT, `translation` TEXT, `preferredSigner` TEXT, `video` TEXT, `authorName` TEXT, `authorEmail` TEXT, `authorWebSite` TEXT, `authorPhone` TEXT, `donate` TEXT, `liberapayID` TEXT, `liberapay` TEXT, `openCollective` TEXT, `bitcoin` TEXT, `litecoin` TEXT, `flattrID` TEXT, `categories` TEXT, `isCompatible` INTEGER NOT NULL, PRIMARY KEY(`repoId`, `packageName`), FOREIGN KEY(`repoId`) REFERENCES `CoreRepository`(`repoId`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "packageName", + "columnName": "packageName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "added", + "columnName": "added", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "lastUpdated", + "columnName": "lastUpdated", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "summary", + "columnName": "summary", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "description", + "columnName": "description", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "localizedName", + "columnName": "localizedName", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "localizedSummary", + "columnName": "localizedSummary", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "webSite", + "columnName": "webSite", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "changelog", + "columnName": "changelog", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "license", + "columnName": "license", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "sourceCode", + "columnName": "sourceCode", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "issueTracker", + "columnName": "issueTracker", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "translation", + "columnName": "translation", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "preferredSigner", + "columnName": "preferredSigner", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "video", + "columnName": "video", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "authorName", + "columnName": "authorName", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "authorEmail", + "columnName": "authorEmail", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "authorWebSite", + "columnName": "authorWebSite", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "authorPhone", + "columnName": "authorPhone", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "donate", + "columnName": "donate", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "liberapayID", + "columnName": "liberapayID", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "liberapay", + "columnName": "liberapay", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "openCollective", + "columnName": "openCollective", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "bitcoin", + "columnName": "bitcoin", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "litecoin", + "columnName": "litecoin", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "flattrID", + "columnName": "flattrID", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "categories", + "columnName": "categories", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "isCompatible", + "columnName": "isCompatible", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "repoId", + "packageName" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [ + { + "table": "CoreRepository", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "repoId" + ], + "referencedColumns": [ + "repoId" + ] + } + ] + }, + { + "ftsVersion": "FTS4", + "ftsOptions": { + "tokenizer": "simple", + "tokenizerArgs": [], + "contentTable": "AppMetadata", + "languageIdColumnName": "", + "matchInfo": "FTS4", + "notIndexedColumns": [], + "prefixSizes": [], + "preferredOrder": "ASC" + }, + "contentSyncTriggers": [ + "CREATE TRIGGER IF NOT EXISTS room_fts_content_sync_AppMetadataFts_BEFORE_UPDATE BEFORE UPDATE ON `AppMetadata` BEGIN DELETE FROM `AppMetadataFts` WHERE `docid`=OLD.`rowid`; END", + "CREATE TRIGGER IF NOT EXISTS room_fts_content_sync_AppMetadataFts_BEFORE_DELETE BEFORE DELETE ON `AppMetadata` BEGIN DELETE FROM `AppMetadataFts` WHERE `docid`=OLD.`rowid`; END", + "CREATE TRIGGER IF NOT EXISTS room_fts_content_sync_AppMetadataFts_AFTER_UPDATE AFTER UPDATE ON `AppMetadata` BEGIN INSERT INTO `AppMetadataFts`(`docid`, `repoId`, `packageName`, `localizedName`, `localizedSummary`) VALUES (NEW.`rowid`, NEW.`repoId`, NEW.`packageName`, NEW.`localizedName`, NEW.`localizedSummary`); END", + "CREATE TRIGGER IF NOT EXISTS room_fts_content_sync_AppMetadataFts_AFTER_INSERT AFTER INSERT ON `AppMetadata` BEGIN INSERT INTO `AppMetadataFts`(`docid`, `repoId`, `packageName`, `localizedName`, `localizedSummary`) VALUES (NEW.`rowid`, NEW.`repoId`, NEW.`packageName`, NEW.`localizedName`, NEW.`localizedSummary`); END" + ], + "tableName": "AppMetadataFts", + "createSql": "CREATE VIRTUAL TABLE IF NOT EXISTS `${TABLE_NAME}` USING FTS4(`repoId` INTEGER NOT NULL, `packageName` TEXT NOT NULL, `localizedName` TEXT, `localizedSummary` TEXT, content=`AppMetadata`)", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "packageName", + "columnName": "packageName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "localizedName", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "summary", + "columnName": "localizedSummary", + "affinity": "TEXT", + "notNull": false + } + ], + "primaryKey": { + "columnNames": [], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "LocalizedFile", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`repoId` INTEGER NOT NULL, `packageName` TEXT NOT NULL, `type` TEXT NOT NULL, `locale` TEXT NOT NULL, `name` TEXT NOT NULL, `sha256` TEXT, `size` INTEGER, PRIMARY KEY(`repoId`, `packageName`, `type`, `locale`), FOREIGN KEY(`repoId`, `packageName`) REFERENCES `AppMetadata`(`repoId`, `packageName`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "packageName", + "columnName": "packageName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "type", + "columnName": "type", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "locale", + "columnName": "locale", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "sha256", + "columnName": "sha256", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "size", + "columnName": "size", + "affinity": "INTEGER", + "notNull": false + } + ], + "primaryKey": { + "columnNames": [ + "repoId", + "packageName", + "type", + "locale" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [ + { + "table": "AppMetadata", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "repoId", + "packageName" + ], + "referencedColumns": [ + "repoId", + "packageName" + ] + } + ] + }, + { + "tableName": "LocalizedFileList", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`repoId` INTEGER NOT NULL, `packageName` TEXT NOT NULL, `type` TEXT NOT NULL, `locale` TEXT NOT NULL, `name` TEXT NOT NULL, `sha256` TEXT, `size` INTEGER, PRIMARY KEY(`repoId`, `packageName`, `type`, `locale`, `name`), FOREIGN KEY(`repoId`, `packageName`) REFERENCES `AppMetadata`(`repoId`, `packageName`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "packageName", + "columnName": "packageName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "type", + "columnName": "type", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "locale", + "columnName": "locale", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "sha256", + "columnName": "sha256", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "size", + "columnName": "size", + "affinity": "INTEGER", + "notNull": false + } + ], + "primaryKey": { + "columnNames": [ + "repoId", + "packageName", + "type", + "locale", + "name" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [ + { + "table": "AppMetadata", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "repoId", + "packageName" + ], + "referencedColumns": [ + "repoId", + "packageName" + ] + } + ] + }, + { + "tableName": "Version", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`repoId` INTEGER NOT NULL, `packageName` TEXT NOT NULL, `versionId` TEXT NOT NULL, `added` INTEGER NOT NULL, `releaseChannels` TEXT, `antiFeatures` TEXT, `whatsNew` TEXT, `isCompatible` INTEGER NOT NULL, `file_name` TEXT NOT NULL, `file_sha256` TEXT NOT NULL, `file_size` INTEGER, `src_name` TEXT, `src_sha256` TEXT, `src_size` INTEGER, `manifest_versionName` TEXT NOT NULL, `manifest_versionCode` INTEGER NOT NULL, `manifest_maxSdkVersion` INTEGER, `manifest_nativecode` TEXT, `manifest_features` TEXT, `manifest_usesSdk_minSdkVersion` INTEGER, `manifest_usesSdk_targetSdkVersion` INTEGER, `manifest_signer_sha256` TEXT, `manifest_signer_hasMultipleSigners` INTEGER, PRIMARY KEY(`repoId`, `packageName`, `versionId`), FOREIGN KEY(`repoId`, `packageName`) REFERENCES `AppMetadata`(`repoId`, `packageName`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "packageName", + "columnName": "packageName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "versionId", + "columnName": "versionId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "added", + "columnName": "added", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "releaseChannels", + "columnName": "releaseChannels", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "antiFeatures", + "columnName": "antiFeatures", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "whatsNew", + "columnName": "whatsNew", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "isCompatible", + "columnName": "isCompatible", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "file.name", + "columnName": "file_name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "file.sha256", + "columnName": "file_sha256", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "file.size", + "columnName": "file_size", + "affinity": "INTEGER", + "notNull": false + }, + { + "fieldPath": "src.name", + "columnName": "src_name", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "src.sha256", + "columnName": "src_sha256", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "src.size", + "columnName": "src_size", + "affinity": "INTEGER", + "notNull": false + }, + { + "fieldPath": "manifest.versionName", + "columnName": "manifest_versionName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "manifest.versionCode", + "columnName": "manifest_versionCode", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "manifest.maxSdkVersion", + "columnName": "manifest_maxSdkVersion", + "affinity": "INTEGER", + "notNull": false + }, + { + "fieldPath": "manifest.nativecode", + "columnName": "manifest_nativecode", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "manifest.features", + "columnName": "manifest_features", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "manifest.usesSdk.minSdkVersion", + "columnName": "manifest_usesSdk_minSdkVersion", + "affinity": "INTEGER", + "notNull": false + }, + { + "fieldPath": "manifest.usesSdk.targetSdkVersion", + "columnName": "manifest_usesSdk_targetSdkVersion", + "affinity": "INTEGER", + "notNull": false + }, + { + "fieldPath": "manifest.signer.sha256", + "columnName": "manifest_signer_sha256", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "manifest.signer.hasMultipleSigners", + "columnName": "manifest_signer_hasMultipleSigners", + "affinity": "INTEGER", + "notNull": false + } + ], + "primaryKey": { + "columnNames": [ + "repoId", + "packageName", + "versionId" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [ + { + "table": "AppMetadata", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "repoId", + "packageName" + ], + "referencedColumns": [ + "repoId", + "packageName" + ] + } + ] + }, + { + "tableName": "VersionedString", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`repoId` INTEGER NOT NULL, `packageName` TEXT NOT NULL, `versionId` TEXT NOT NULL, `type` TEXT NOT NULL, `name` TEXT NOT NULL, `version` INTEGER, PRIMARY KEY(`repoId`, `packageName`, `versionId`, `type`, `name`), FOREIGN KEY(`repoId`, `packageName`, `versionId`) REFERENCES `Version`(`repoId`, `packageName`, `versionId`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "packageName", + "columnName": "packageName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "versionId", + "columnName": "versionId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "type", + "columnName": "type", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "version", + "columnName": "version", + "affinity": "INTEGER", + "notNull": false + } + ], + "primaryKey": { + "columnNames": [ + "repoId", + "packageName", + "versionId", + "type", + "name" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [ + { + "table": "Version", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "repoId", + "packageName", + "versionId" + ], + "referencedColumns": [ + "repoId", + "packageName", + "versionId" + ] + } + ] + }, + { + "tableName": "AppPrefs", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`packageName` TEXT NOT NULL, `ignoreVersionCodeUpdate` INTEGER NOT NULL, `appPrefReleaseChannels` TEXT, PRIMARY KEY(`packageName`))", + "fields": [ + { + "fieldPath": "packageName", + "columnName": "packageName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "ignoreVersionCodeUpdate", + "columnName": "ignoreVersionCodeUpdate", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "appPrefReleaseChannels", + "columnName": "appPrefReleaseChannels", + "affinity": "TEXT", + "notNull": false + } + ], + "primaryKey": { + "columnNames": [ + "packageName" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + } + ], + "views": [ + { + "viewName": "LocalizedIcon", + "createSql": "CREATE VIEW `${VIEW_NAME}` AS SELECT * FROM LocalizedFile WHERE type='icon'" + }, + { + "viewName": "HighestVersion", + "createSql": "CREATE VIEW `${VIEW_NAME}` AS SELECT repoId, packageName, antiFeatures FROM Version\n GROUP BY repoId, packageName HAVING MAX(manifest_versionCode)" + } + ], + "setupQueries": [ + "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", + "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, 'beaebd71355e07ce5f0500c19d9045fd')" + ] + } +} \ No newline at end of file diff --git a/libs/database/src/dbTest/java/org/fdroid/database/AppDaoTest.kt b/libs/database/src/dbTest/java/org/fdroid/database/AppDaoTest.kt new file mode 100644 index 000000000..56822bf45 --- /dev/null +++ b/libs/database/src/dbTest/java/org/fdroid/database/AppDaoTest.kt @@ -0,0 +1,145 @@ +package org.fdroid.database + +import androidx.core.os.LocaleListCompat +import androidx.test.ext.junit.runners.AndroidJUnit4 +import org.fdroid.database.TestUtils.getOrFail +import org.fdroid.database.TestUtils.toMetadataV2 +import org.fdroid.test.TestRepoUtils.getRandomRepo +import org.fdroid.test.TestUtils.sort +import org.fdroid.test.TestVersionUtils.getRandomPackageVersionV2 +import org.junit.Test +import org.junit.runner.RunWith +import kotlin.test.assertEquals +import kotlin.test.assertTrue +import kotlin.test.fail + +@RunWith(AndroidJUnit4::class) +internal class AppDaoTest : AppTest() { + + @Test + fun insertGetDeleteSingleApp() { + val repoId = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId, packageName, app1) + + assertEquals(app1, appDao.getApp(repoId, packageName)?.toMetadataV2()?.sort()) + assertEquals(app1, appDao.getApp(packageName).getOrFail()?.toMetadataV2()?.sort()) + + appDao.deleteAppMetadata(repoId, packageName) + assertEquals(0, appDao.countApps()) + assertEquals(0, appDao.countLocalizedFiles()) + assertEquals(0, appDao.countLocalizedFileLists()) + } + + @Test + fun testGetSameAppFromTwoRepos() { + // insert same app into three repos (repoId1 has highest weight) + val repoId2 = repoDao.insertOrReplace(getRandomRepo()) + val repoId3 = repoDao.insertOrReplace(getRandomRepo()) + val repoId1 = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId1, packageName, app1, locales) + appDao.insert(repoId2, packageName, app2, locales) + appDao.insert(repoId3, packageName, app3, locales) + + // ensure expected repo weights + val repoPrefs1 = repoDao.getRepositoryPreferences(repoId1) ?: fail() + val repoPrefs2 = repoDao.getRepositoryPreferences(repoId2) ?: fail() + val repoPrefs3 = repoDao.getRepositoryPreferences(repoId3) ?: fail() + assertTrue(repoPrefs2.weight < repoPrefs3.weight) + assertTrue(repoPrefs3.weight < repoPrefs1.weight) + + // each app gets returned as stored from each repo + assertEquals(app1, appDao.getApp(repoId1, packageName)?.toMetadataV2()?.sort()) + assertEquals(app2, appDao.getApp(repoId2, packageName)?.toMetadataV2()?.sort()) + assertEquals(app3, appDao.getApp(repoId3, packageName)?.toMetadataV2()?.sort()) + + // if repo is not given, app from repo with highest weight is returned + assertEquals(app1, appDao.getApp(packageName).getOrFail()?.toMetadataV2()?.sort()) + } + + @Test + fun testUpdateCompatibility() { + // insert two apps with one version each + val repoId = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId, packageName, app1, locales) + + // without versions, app isn't compatible + assertEquals(false, appDao.getApp(repoId, packageName)?.metadata?.isCompatible) + appDao.updateCompatibility(repoId) + assertEquals(false, appDao.getApp(repoId, packageName)?.metadata?.isCompatible) + + // still incompatible with incompatible version + versionDao.insert(repoId, packageName, "1", getRandomPackageVersionV2(), false) + appDao.updateCompatibility(repoId) + assertEquals(false, appDao.getApp(repoId, packageName)?.metadata?.isCompatible) + + // only with at least one compatible version, the app becomes compatible + versionDao.insert(repoId, packageName, "2", getRandomPackageVersionV2(), true) + appDao.updateCompatibility(repoId) + assertEquals(true, appDao.getApp(repoId, packageName)?.metadata?.isCompatible) + } + + @Test + fun testAfterLocalesChanged() { + // insert app with German and French locales + val localesBefore = LocaleListCompat.forLanguageTags("de-DE") + val app = app1.copy( + name = mapOf("de-DE" to "de-DE", "fr-FR" to "fr-FR"), + summary = mapOf("de-DE" to "de-DE", "fr-FR" to "fr-FR"), + ) + val repoId = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId, packageName, app, localesBefore) + + // device is set to German, so name and summary come out German + val appBefore = appDao.getApp(repoId, packageName) + assertEquals("de-DE", appBefore?.name) + assertEquals("de-DE", appBefore?.summary) + + // device gets switched to French + val localesAfter = LocaleListCompat.forLanguageTags("fr-FR") + db.afterLocalesChanged(localesAfter) + + // device is set to French now, so name and summary come out French + val appAfter = appDao.getApp(repoId, packageName) + assertEquals("fr-FR", appAfter?.name) + assertEquals("fr-FR", appAfter?.summary) + } + + @Test + fun testGetNumberOfAppsInCategory() { + val repoId = repoDao.insertOrReplace(getRandomRepo()) + + // app1 is in A and B + appDao.insert(repoId, packageName1, app1, locales) + assertEquals(1, appDao.getNumberOfAppsInCategory("A")) + assertEquals(1, appDao.getNumberOfAppsInCategory("B")) + assertEquals(0, appDao.getNumberOfAppsInCategory("C")) + + // app2 is in A + appDao.insert(repoId, packageName2, app2, locales) + assertEquals(2, appDao.getNumberOfAppsInCategory("A")) + assertEquals(1, appDao.getNumberOfAppsInCategory("B")) + assertEquals(0, appDao.getNumberOfAppsInCategory("C")) + + // app3 is in A and B + appDao.insert(repoId, packageName3, app3, locales) + assertEquals(3, appDao.getNumberOfAppsInCategory("A")) + assertEquals(2, appDao.getNumberOfAppsInCategory("B")) + assertEquals(0, appDao.getNumberOfAppsInCategory("C")) + } + + @Test + fun testGetNumberOfAppsInRepository() { + val repoId = repoDao.insertOrReplace(getRandomRepo()) + assertEquals(0, appDao.getNumberOfAppsInRepository(repoId)) + + appDao.insert(repoId, packageName1, app1, locales) + assertEquals(1, appDao.getNumberOfAppsInRepository(repoId)) + + appDao.insert(repoId, packageName2, app2, locales) + assertEquals(2, appDao.getNumberOfAppsInRepository(repoId)) + + appDao.insert(repoId, packageName3, app3, locales) + assertEquals(3, appDao.getNumberOfAppsInRepository(repoId)) + } + +} diff --git a/libs/database/src/dbTest/java/org/fdroid/database/AppListItemsTest.kt b/libs/database/src/dbTest/java/org/fdroid/database/AppListItemsTest.kt new file mode 100644 index 000000000..a41f1dbc6 --- /dev/null +++ b/libs/database/src/dbTest/java/org/fdroid/database/AppListItemsTest.kt @@ -0,0 +1,433 @@ +package org.fdroid.database + +import android.content.pm.PackageInfo +import android.content.pm.PackageManager +import androidx.test.ext.junit.runners.AndroidJUnit4 +import io.mockk.every +import io.mockk.mockk +import org.fdroid.LocaleChooser.getBestLocale +import org.fdroid.database.AppListSortOrder.LAST_UPDATED +import org.fdroid.database.AppListSortOrder.NAME +import org.fdroid.database.TestUtils.getOrFail +import org.fdroid.index.v2.MetadataV2 +import org.fdroid.test.TestRepoUtils.getRandomRepo +import org.fdroid.test.TestUtils.getRandomString +import org.fdroid.test.TestVersionUtils.getRandomPackageVersionV2 +import org.junit.Test +import org.junit.runner.RunWith +import kotlin.random.Random +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNotEquals +import kotlin.test.assertNull +import kotlin.test.assertTrue +import kotlin.test.fail + +@RunWith(AndroidJUnit4::class) +internal class AppListItemsTest : AppTest() { + + private val pm: PackageManager = mockk() + + private val appPairs = listOf( + Pair(packageName1, app1), + Pair(packageName2, app2), + Pair(packageName3, app3), + ) + + @Test + fun testSearchQuery() { + val app1 = app1.copy(name = mapOf("en-US" to "One"), summary = mapOf("en-US" to "Onearry")) + val app2 = app2.copy(name = mapOf("en-US" to "Two"), summary = mapOf("de" to "Zfassung")) + val app3 = app3.copy(name = mapOf("de-DE" to "Drei"), summary = mapOf("de" to "Zfassung")) + // insert three apps in a random order + val repoId = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId, packageName1, app1, locales) + appDao.insert(repoId, packageName3, app3, locales) + appDao.insert(repoId, packageName2, app2, locales) + + // nothing is installed + every { pm.getInstalledPackages(0) } returns emptyList() + + // get first app by search, sort order doesn't matter + appDao.getAppListItems(pm, "One", LAST_UPDATED).getOrFail().let { apps -> + assertEquals(1, apps.size) + assertEquals(app1, apps[0]) + } + + // get second app by search, sort order doesn't matter + appDao.getAppListItems(pm, "Two", NAME).getOrFail().let { apps -> + assertEquals(1, apps.size) + assertEquals(app2, apps[0]) + } + + // get second and third app by searching for summary + appDao.getAppListItems(pm, "Zfassung", LAST_UPDATED).getOrFail().let { apps -> + assertEquals(2, apps.size) + // sort-order isn't fixes, yet + if (apps[0].packageName == packageName2) { + assertEquals(app2, apps[0]) + assertEquals(app3, apps[1]) + } else { + assertEquals(app3, apps[0]) + assertEquals(app2, apps[1]) + } + } + + // empty search for unknown search term + appDao.getAppListItems(pm, "foo bar", LAST_UPDATED).getOrFail().let { apps -> + assertEquals(0, apps.size) + } + } + + @Test + fun testSearchQueryInCategory() { + val app1 = app1.copy(name = mapOf("en-US" to "One"), summary = mapOf("en-US" to "Onearry")) + val app2 = app2.copy(name = mapOf("en-US" to "Two"), summary = mapOf("de" to "Zfassung")) + val app3 = app3.copy(name = mapOf("de-DE" to "Drei"), summary = mapOf("de" to "Zfassung")) + // insert three apps in a random order + val repoId = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId, packageName1, app1, locales) + appDao.insert(repoId, packageName3, app3, locales) + appDao.insert(repoId, packageName2, app2, locales) + + // nothing is installed + every { pm.getInstalledPackages(0) } returns emptyList() + + // get first app by search, sort order doesn't matter + appDao.getAppListItems(pm, "A", "One", LAST_UPDATED).getOrFail().let { apps -> + assertEquals(1, apps.size) + assertEquals(app1, apps[0]) + } + + // get second app by search, sort order doesn't matter + appDao.getAppListItems(pm, "A", "Two", NAME).getOrFail().let { apps -> + assertEquals(1, apps.size) + assertEquals(app2, apps[0]) + } + + // get second and third app by searching for summary + appDao.getAppListItems(pm, "A", "Zfassung", LAST_UPDATED).getOrFail().let { apps -> + assertEquals(2, apps.size) + // sort-order isn't fixes, yet + if (apps[0].packageName == packageName2) { + assertEquals(app2, apps[0]) + assertEquals(app3, apps[1]) + } else { + assertEquals(app3, apps[0]) + assertEquals(app2, apps[1]) + } + } + + // get third app by searching for summary in category B only + appDao.getAppListItems(pm, "B", "Zfassung", LAST_UPDATED).getOrFail().let { apps -> + assertEquals(1, apps.size) + assertEquals(app3, apps[0]) + } + + // empty search for unknown category + appDao.getAppListItems(pm, "C", "Zfassung", LAST_UPDATED).getOrFail().let { apps -> + assertEquals(0, apps.size) + } + + // empty search for unknown search term + appDao.getAppListItems(pm, "A", "foo bar", LAST_UPDATED).getOrFail().let { apps -> + assertEquals(0, apps.size) + } + } + + @Test + fun testSortOrderByLastUpdated() { + // insert three apps in a random order + val repoId = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId, packageName1, app1, locales) + appDao.insert(repoId, packageName3, app3, locales) + appDao.insert(repoId, packageName2, app2, locales) + + // nothing is installed + every { pm.getInstalledPackages(0) } returns emptyList() + + // get apps sorted by last updated + appDao.getAppListItems(pm, "", LAST_UPDATED).getOrFail().let { apps -> + assertEquals(3, apps.size) + // we expect apps to be sorted by last updated descending + appPairs.sortedByDescending { (_, metadataV2) -> + metadataV2.lastUpdated + }.forEachIndexed { i, pair -> + assertEquals(pair.first, apps[i].packageName) + assertEquals(pair.second, apps[i]) + } + } + } + + @Test + fun testSortOrderByName() { + // insert three apps + val repoId = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId, packageName1, app1, locales) + appDao.insert(repoId, packageName2, app2, locales) + appDao.insert(repoId, packageName3, app3, locales) + + // nothing is installed + every { pm.getInstalledPackages(0) } returns emptyList() + + // get apps sorted by name ascending + appDao.getAppListItems(pm, null, NAME).getOrFail().let { apps -> + assertEquals(3, apps.size) + // we expect apps to be sorted by last updated descending + appPairs.sortedBy { (_, metadataV2) -> + metadataV2.name.getBestLocale(locales) + }.forEachIndexed { i, pair -> + assertEquals(pair.first, apps[i].packageName) + assertEquals(pair.second, apps[i]) + } + } + } + + @Test + fun testPackageManagerInfo() { + // insert two apps + val repoId = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId, packageName1, app1, locales) + appDao.insert(repoId, packageName2, app2, locales) + + // one of the apps is installed + @Suppress("DEPRECATION") + val packageInfo2 = PackageInfo().apply { + packageName = packageName2 + versionName = getRandomString() + versionCode = Random.nextInt(1, Int.MAX_VALUE) + } + every { pm.getInstalledPackages(0) } returns listOf(packageInfo2) + + // get apps sorted by name and last update, test on both lists + listOf( + appDao.getAppListItems(pm, "", NAME).getOrFail(), + appDao.getAppListItems(pm, null, LAST_UPDATED).getOrFail(), + ).forEach { apps -> + assertEquals(2, apps.size) + // the installed app should have app data + val installed = if (apps[0].packageName == packageName1) apps[1] else apps[0] + val other = if (apps[0].packageName == packageName1) apps[0] else apps[1] + assertEquals(packageInfo2.versionName, installed.installedVersionName) + assertEquals(packageInfo2.getVersionCode(), installed.installedVersionCode) + assertNull(other.installedVersionName) + assertNull(other.installedVersionCode) + } + } + + @Test + fun testCompatibility() { + // insert two apps + val repoId = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId, packageName1, app1, locales) + appDao.insert(repoId, packageName2, app2, locales) + + // both apps are not compatible + getItems { apps -> + assertEquals(2, apps.size) + assertFalse(apps[0].isCompatible) + assertFalse(apps[1].isCompatible) + } + + // each app gets a version + versionDao.insert(repoId, packageName1, "1", getRandomPackageVersionV2(), true) + versionDao.insert(repoId, packageName2, "1", getRandomPackageVersionV2(), false) + + // updating compatibility for apps + appDao.updateCompatibility(repoId) + + // now only one is not compatible + getItems { apps -> + assertEquals(2, apps.size) + if (apps[0].packageName == packageName1) { + assertTrue(apps[0].isCompatible) + assertFalse(apps[1].isCompatible) + } else { + assertFalse(apps[0].isCompatible) + assertTrue(apps[1].isCompatible) + } + } + } + + @Test + fun testAntiFeaturesFromHighestVersion() { + // insert one app with no versions + val repoId = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId, packageName1, app1, locales) + + // app has no anti-features, because no version + getItems { apps -> + assertEquals(1, apps.size) + assertNull(apps[0].antiFeatures) + assertEquals(emptyList(), apps[0].antiFeatureKeys) + } + + // app gets a version + val version1 = getRandomPackageVersionV2(42) + versionDao.insert(repoId, packageName1, "1", version1, true) + + // app has now has the anti-features of the version + // note that installed versions don't contain anti-features, so they are ignored + getItems(alsoInstalled = false) { apps -> + assertEquals(1, apps.size) + assertEquals(version1.antiFeatures.map { it.key }, apps[0].antiFeatureKeys) + } + + // app gets another version + val version2 = getRandomPackageVersionV2(23) + versionDao.insert(repoId, packageName1, "2", version2, true) + + // app has now has the anti-features of the initial version still, because 2nd is lower + // note that installed versions don't contain anti-features, so they are ignored + getItems(alsoInstalled = false) { apps -> + assertEquals(1, apps.size) + assertEquals(version1.antiFeatures.map { it.key }, apps[0].antiFeatureKeys) + } + } + + @Test + fun testOnlyFromEnabledRepos() { + // insert two apps in two different repos + val repoId = repoDao.insertOrReplace(getRandomRepo()) + val repoId2 = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId, packageName1, app1, locales) + appDao.insert(repoId2, packageName2, app2, locales) + + // initially both apps get returned + getItems { apps -> + assertEquals(2, apps.size) + } + + // disable first repo + repoDao.setRepositoryEnabled(repoId, false) + + // now only app from enabled repo gets returned + getItems { apps -> + assertEquals(1, apps.size) + assertEquals(repoId2, apps[0].repoId) + } + } + + @Test + fun testFromRepoWithHighestWeight() { + // insert same app into three repos (repoId1 has highest weight) + val repoId2 = repoDao.insertOrReplace(getRandomRepo()) + val repoId3 = repoDao.insertOrReplace(getRandomRepo()) + val repoId1 = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId2, packageName, app2, locales) + appDao.insert(repoId1, packageName, app1, locales) + appDao.insert(repoId3, packageName, app3, locales) + + // ensure expected repo weights + val repoPrefs1 = repoDao.getRepositoryPreferences(repoId1) ?: fail() + val repoPrefs2 = repoDao.getRepositoryPreferences(repoId2) ?: fail() + val repoPrefs3 = repoDao.getRepositoryPreferences(repoId3) ?: fail() + assertTrue(repoPrefs2.weight < repoPrefs3.weight) + assertTrue(repoPrefs3.weight < repoPrefs1.weight) + + // app from repo with highest weight is returned (app1) + getItems { apps -> + assertEquals(1, apps.size) + assertEquals(packageName, apps[0].packageName) + assertEquals(app1, apps[0]) + } + } + + @Test + fun testOnlyFromGivenCategories() { + // insert three apps + val repoId = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId, packageName1, app1, locales) + appDao.insert(repoId, packageName2, app2, locales) + appDao.insert(repoId, packageName3, app3, locales) + + // only two apps are in category B + listOf( + appDao.getAppListItemsByName("B").getOrFail(), + appDao.getAppListItemsByLastUpdated("B").getOrFail(), + ).forEach { apps -> + assertEquals(2, apps.size) + assertNotEquals(packageName2, apps[0].packageName) + assertNotEquals(packageName2, apps[1].packageName) + } + + // no app is in category C + listOf( + appDao.getAppListItemsByName("C").getOrFail(), + appDao.getAppListItemsByLastUpdated("C").getOrFail(), + ).forEach { apps -> + assertEquals(0, apps.size) + } + } + + @Test + fun testGetInstalledAppListItems() { + // insert three apps + val repoId = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId, packageName1, app1, locales) + appDao.insert(repoId, packageName2, app2, locales) + appDao.insert(repoId, packageName3, app3, locales) + + // define packageInfo for each test + @Suppress("DEPRECATION") + val packageInfo1 = PackageInfo().apply { + packageName = packageName1 + versionName = getRandomString() + versionCode = Random.nextInt(1, Int.MAX_VALUE) + } + val packageInfo2 = PackageInfo().apply { packageName = packageName2 } + val packageInfo3 = PackageInfo().apply { packageName = packageName3 } + + // all apps get returned, if we consider all of them installed + every { + pm.getInstalledPackages(0) + } returns listOf(packageInfo1, packageInfo2, packageInfo3) + assertEquals(3, appDao.getInstalledAppListItems(pm).getOrFail().size) + + // one apps get returned, if we consider only that one installed + every { pm.getInstalledPackages(0) } returns listOf(packageInfo1) + appDao.getInstalledAppListItems(pm).getOrFail().let { apps -> + assertEquals(1, apps.size) + assertEquals(app1, apps[0]) + // version code and version name gets taken from supplied packageInfo + assertEquals(packageInfo1.getVersionCode(), apps[0].installedVersionCode) + assertEquals(packageInfo1.versionName, apps[0].installedVersionName) + } + + // no app gets returned, if we consider none installed + every { pm.getInstalledPackages(0) } returns emptyList() + appDao.getInstalledAppListItems(pm).getOrFail().let { apps -> + assertEquals(0, apps.size) + } + } + + /** + * Runs the given block on all getAppListItems* methods. + * Uses category "A" as all apps should be in that. + */ + private fun getItems(alsoInstalled: Boolean = true, block: (List) -> Unit) { + appDao.getAppListItemsByName().getOrFail().let(block) + appDao.getAppListItemsByName("A").getOrFail().let(block) + appDao.getAppListItemsByLastUpdated().getOrFail().let(block) + appDao.getAppListItemsByLastUpdated("A").getOrFail().let(block) + if (alsoInstalled) { + // everything is always considered to be installed + val packageInfo = + PackageInfo().apply { packageName = this@AppListItemsTest.packageName } + val packageInfo1 = PackageInfo().apply { packageName = packageName1 } + val packageInfo2 = PackageInfo().apply { packageName = packageName2 } + val packageInfo3 = PackageInfo().apply { packageName = packageName3 } + every { + pm.getInstalledPackages(0) + } returns listOf(packageInfo, packageInfo1, packageInfo2, packageInfo3) + appDao.getInstalledAppListItems(pm).getOrFail().let(block) + } + } + + private fun assertEquals(expected: MetadataV2, actual: AppListItem) { + assertEquals(expected.name.getBestLocale(locales), actual.name) + assertEquals(expected.summary.getBestLocale(locales), actual.summary) + assertEquals(expected.icon.getBestLocale(locales), actual.getIcon(locales)) + } + +} diff --git a/libs/database/src/dbTest/java/org/fdroid/database/AppOverviewItemsTest.kt b/libs/database/src/dbTest/java/org/fdroid/database/AppOverviewItemsTest.kt new file mode 100644 index 000000000..a13a005d3 --- /dev/null +++ b/libs/database/src/dbTest/java/org/fdroid/database/AppOverviewItemsTest.kt @@ -0,0 +1,298 @@ +package org.fdroid.database + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import org.fdroid.LocaleChooser.getBestLocale +import org.fdroid.database.TestUtils.getOrAwaitValue +import org.fdroid.database.TestUtils.getOrFail +import org.fdroid.index.v2.MetadataV2 +import org.fdroid.test.TestRepoUtils.getRandomRepo +import org.fdroid.test.TestVersionUtils.getRandomPackageVersionV2 +import org.junit.Test +import org.junit.runner.RunWith +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertNull + +@RunWith(AndroidJUnit4::class) +internal class AppOverviewItemsTest : AppTest() { + + @Test + fun testAntiFeatures() { + // insert one apps with without version + val repoId = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId, packageName, app1, locales) + + // without version, anti-features are empty + appDao.getAppOverviewItems().getOrFail().let { apps -> + assertEquals(1, apps.size) + assertNull(apps[0].antiFeatures) + } + + // with one version, the app has those anti-features + val version = getRandomPackageVersionV2(versionCode = 42) + versionDao.insert(repoId, packageName, "1", version, true) + appDao.getAppOverviewItems().getOrFail().let { apps -> + assertEquals(1, apps.size) + assertEquals(version.antiFeatures, apps[0].antiFeatures) + } + + // with two versions, the app has the anti-features of the highest version + val version2 = getRandomPackageVersionV2(versionCode = 23) + versionDao.insert(repoId, packageName, "2", version2, true) + appDao.getAppOverviewItems().getOrFail().let { apps -> + assertEquals(1, apps.size) + assertEquals(version.antiFeatures, apps[0].antiFeatures) + } + + // with three versions, the app has the anti-features of the highest version + val version3 = getRandomPackageVersionV2(versionCode = 1337) + versionDao.insert(repoId, packageName, "3", version3, true) + appDao.getAppOverviewItems().getOrFail().let { apps -> + assertEquals(1, apps.size) + assertEquals(version3.antiFeatures, apps[0].antiFeatures) + } + } + + @Test + fun testIcons() { + // insert one app + val repoId = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId, packageName, app1, locales) + + // icon is returned correctly + appDao.getAppOverviewItems().getOrFail().let { apps -> + assertEquals(1, apps.size) + assertEquals(app1.icon.getBestLocale(locales), apps[0].getIcon(locales)) + } + + // insert same app into another repo + val repoId2 = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId2, packageName, app2, locales) + + // now icon is returned from app in second repo + appDao.getAppOverviewItems().getOrFail().let { apps -> + assertEquals(1, apps.size) + assertEquals(app2.icon.getBestLocale(locales), apps[0].getIcon(locales)) + } + } + + @Test + fun testLimit() { + // insert three apps + val repoId = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId, packageName1, app1, locales) + appDao.insert(repoId, packageName2, app2, locales) + appDao.insert(repoId, packageName3, app3, locales) + + // limit is respected + for (i in 0..3) assertEquals(i, appDao.getAppOverviewItems(i).getOrFail().size) + assertEquals(3, appDao.getAppOverviewItems(42).getOrFail().size) + } + + @Test + fun testGetByRepoWeight() { + // insert one app with one version + val repoId = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId, packageName, app1, locales) + versionDao.insert(repoId, packageName, "1", getRandomPackageVersionV2(2), true) + + // app is returned correctly + appDao.getAppOverviewItems().getOrFail().let { apps -> + assertEquals(1, apps.size) + assertEquals(app1, apps[0]) + } + + // add another app without version + val repoId2 = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId2, packageName, app2, locales) + + // now second app from second repo is returned + appDao.getAppOverviewItems().getOrFail().let { apps -> + assertEquals(1, apps.size) + assertEquals(app2, apps[0]) + } + } + + @Test + fun testSortOrder() { + // insert two apps with one version each + val repoId = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId, packageName1, app1, locales) + appDao.insert(repoId, packageName2, app2, locales) + versionDao.insert(repoId, packageName1, "1", getRandomPackageVersionV2(), true) + versionDao.insert(repoId, packageName2, "2", getRandomPackageVersionV2(), true) + + // icons of both apps are returned correctly + appDao.getAppOverviewItems().getOrFail().let { apps -> + assertEquals(2, apps.size) + // app 2 is first, because has icon and summary + assertEquals(packageName2, apps[0].packageName) + assertEquals(icons2, apps[0].localizedIcon?.toLocalizedFileV2()) + // app 1 is next, because has icon + assertEquals(packageName1, apps[1].packageName) + assertEquals(icons1, apps[1].localizedIcon?.toLocalizedFileV2()) + } + + // app without icon is returned last + appDao.insert(repoId, packageName3, app3) + versionDao.insert(repoId, packageName3, "3", getRandomPackageVersionV2(), true) + appDao.getAppOverviewItems().getOrFail().let { apps -> + assertEquals(3, apps.size) + assertEquals(packageName2, apps[0].packageName) + assertEquals(packageName1, apps[1].packageName) + assertEquals(packageName3, apps[2].packageName) + assertEquals(emptyList(), apps[2].localizedIcon) + } + + // app1b is the same as app1 (but in another repo) and thus will not be shown again + val repoId2 = repoDao.insertOrReplace(getRandomRepo()) + val app1b = app1.copy(name = name2, icon = icons2, summary = name2) + appDao.insert(repoId2, packageName1, app1b) + // note that we don't insert a version here + assertEquals(3, appDao.getAppOverviewItems().getOrFail().size) + + // app3b is the same as app3, but has an icon, so is not last anymore + val app3b = app3.copy(icon = icons2) + appDao.insert(repoId2, packageName3, app3b) + // note that we don't insert a version here + appDao.getAppOverviewItems().getOrFail().let { apps -> + assertEquals(3, apps.size) + assertEquals(packageName3, apps[0].packageName) + assertEquals(emptyList(), apps[0].antiFeatureKeys) + assertEquals(packageName2, apps[1].packageName) + assertEquals(packageName1, apps[2].packageName) + } + } + + @Test + fun testSortOrderWithCategories() { + val repoId = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId, packageName1, app1, locales) + appDao.insert(repoId, packageName2, app2, locales) + versionDao.insert(repoId, packageName1, "1", getRandomPackageVersionV2(), true) + versionDao.insert(repoId, packageName2, "2", getRandomPackageVersionV2(), true) + + // icons of both apps are returned correctly + appDao.getAppOverviewItems("A").getOrFail().let { apps -> + assertEquals(2, apps.size) + // app 2 is first, because has icon and summary + assertEquals(packageName2, apps[0].packageName) + assertEquals(icons2, apps[0].localizedIcon?.toLocalizedFileV2()) + // app 1 is next, because has icon + assertEquals(packageName1, apps[1].packageName) + assertEquals(icons1, apps[1].localizedIcon?.toLocalizedFileV2()) + } + + // only one app is returned for category B + assertEquals(1, appDao.getAppOverviewItems("B").getOrFail().size) + + // app without icon is returned last + appDao.insert(repoId, packageName3, app3) + versionDao.insert(repoId, packageName3, "3", getRandomPackageVersionV2(), true) + appDao.getAppOverviewItems("A").getOrFail().let { apps -> + assertEquals(3, apps.size) + assertEquals(packageName2, apps[0].packageName) + assertEquals(packageName1, apps[1].packageName) + assertEquals(packageName3, apps[2].packageName) + assertEquals(emptyList(), apps[2].localizedIcon) + } + + // app1b is the same as app1 (but in another repo) and thus will not be shown again + val repoId2 = repoDao.insertOrReplace(getRandomRepo()) + val app1b = app1.copy(name = name2, icon = icons2, summary = name2) + appDao.insert(repoId2, packageName1, app1b) + // note that we don't insert a version here + assertEquals(3, appDao.getAppOverviewItems("A").getOrFail().size) + + // app3b is the same as app3, but has an icon, so is not last anymore + val app3b = app3.copy(icon = icons2) + appDao.insert(repoId2, packageName3, app3b) + // note that we don't insert a version here + appDao.getAppOverviewItems("A").getOrFail().let { apps -> + assertEquals(3, apps.size) + assertEquals(packageName3, apps[0].packageName) + assertEquals(emptyList(), apps[0].antiFeatureKeys) + assertEquals(packageName2, apps[1].packageName) + assertEquals(packageName1, apps[2].packageName) + } + + // only two apps are returned for category B + assertEquals(2, appDao.getAppOverviewItems("B").getOrFail().size) + } + + @Test + fun testOnlyFromEnabledRepos() { + val repoId = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId, packageName1, app1, locales) + appDao.insert(repoId, packageName2, app2, locales) + val repoId2 = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId2, packageName3, app3, locales) + + // 3 apps from 2 repos + assertEquals(3, appDao.getAppOverviewItems().getOrAwaitValue()?.size) + assertEquals(3, appDao.getAppOverviewItems("A").getOrAwaitValue()?.size) + + // only 1 app after disabling first repo + repoDao.setRepositoryEnabled(repoId, false) + assertEquals(1, appDao.getAppOverviewItems().getOrAwaitValue()?.size) + assertEquals(1, appDao.getAppOverviewItems("A").getOrAwaitValue()?.size) + assertEquals(1, appDao.getAppOverviewItems("B").getOrAwaitValue()?.size) + + // no more apps after disabling all repos + repoDao.setRepositoryEnabled(repoId2, false) + assertEquals(0, appDao.getAppOverviewItems().getOrAwaitValue()?.size) + assertEquals(0, appDao.getAppOverviewItems("A").getOrAwaitValue()?.size) + assertEquals(0, appDao.getAppOverviewItems("B").getOrAwaitValue()?.size) + } + + @Test + fun testGetAppOverviewItem() { + // insert three apps into two repos + val repoId = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId, packageName1, app1, locales) + appDao.insert(repoId, packageName2, app2, locales) + val repoId2 = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId2, packageName3, app3, locales) + + // each app gets returned properly + assertEquals(app1, appDao.getAppOverviewItem(repoId, packageName1)) + assertEquals(app2, appDao.getAppOverviewItem(repoId, packageName2)) + assertEquals(app3, appDao.getAppOverviewItem(repoId2, packageName3)) + + // apps don't get returned from wrong repos + assertNull(appDao.getAppOverviewItem(repoId2, packageName1)) + assertNull(appDao.getAppOverviewItem(repoId2, packageName2)) + assertNull(appDao.getAppOverviewItem(repoId, packageName3)) + } + + @Test + fun testGetAppOverviewItemWithIcons() { + // insert one app (with overlapping icons) into two repos + val repoId1 = repoDao.insertOrReplace(getRandomRepo()) + val repoId2 = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId1, packageName, app1, locales) + appDao.insert(repoId2, packageName, app2, locales) + + // each app gets returned properly + assertEquals(app1, appDao.getAppOverviewItem(repoId1, packageName)) + assertEquals(app2, appDao.getAppOverviewItem(repoId2, packageName)) + + // disable second repo + repoDao.setRepositoryEnabled(repoId2, false) + + // each app still gets returned properly + assertEquals(app1, appDao.getAppOverviewItem(repoId1, packageName)) + assertEquals(app2, appDao.getAppOverviewItem(repoId2, packageName)) + } + + private fun assertEquals(expected: MetadataV2, actual: AppOverviewItem?) { + assertNotNull(actual) + assertEquals(expected.added, actual.added) + assertEquals(expected.lastUpdated, actual.lastUpdated) + assertEquals(expected.name.getBestLocale(locales), actual.name) + assertEquals(expected.summary.getBestLocale(locales), actual.summary) + assertEquals(expected.summary.getBestLocale(locales), actual.summary) + assertEquals(expected.icon.getBestLocale(locales), actual.getIcon(locales)) + } + +} diff --git a/libs/database/src/dbTest/java/org/fdroid/database/AppTest.kt b/libs/database/src/dbTest/java/org/fdroid/database/AppTest.kt new file mode 100644 index 000000000..bbaf51cda --- /dev/null +++ b/libs/database/src/dbTest/java/org/fdroid/database/AppTest.kt @@ -0,0 +1,47 @@ +package org.fdroid.database + +import androidx.arch.core.executor.testing.InstantTaskExecutorRule +import org.fdroid.test.TestAppUtils.getRandomMetadataV2 +import org.fdroid.test.TestRepoUtils.getRandomFileV2 +import org.fdroid.test.TestUtils.getRandomString +import org.fdroid.test.TestUtils.sort +import org.junit.Rule + +internal abstract class AppTest : DbTest() { + + @get:Rule + val instantTaskExecutorRule = InstantTaskExecutorRule() + + protected val packageName = getRandomString() + protected val packageName1 = getRandomString() + protected val packageName2 = getRandomString() + protected val packageName3 = getRandomString() + protected val name1 = mapOf("en-US" to "1") + protected val name2 = mapOf("en-US" to "2") + protected val name3 = mapOf("en-US" to "3") + // it is important for testing that the icons are sharing at least one locale + protected val icons1 = mapOf("en-US" to getRandomFileV2(), "bar" to getRandomFileV2()) + protected val icons2 = mapOf("en-US" to getRandomFileV2(), "42" to getRandomFileV2()) + protected val app1 = getRandomMetadataV2().copy( + name = name1, + icon = icons1, + summary = null, + lastUpdated = 10, + categories = listOf("A", "B") + ).sort() + protected val app2 = getRandomMetadataV2().copy( + name = name2, + icon = icons2, + summary = name2, + lastUpdated = 20, + categories = listOf("A") + ).sort() + protected val app3 = getRandomMetadataV2().copy( + name = name3, + icon = null, + summary = name3, + lastUpdated = 30, + categories = listOf("A", "B") + ).sort() + +} diff --git a/libs/database/src/dbTest/java/org/fdroid/database/DbTest.kt b/libs/database/src/dbTest/java/org/fdroid/database/DbTest.kt new file mode 100644 index 000000000..33e3c4344 --- /dev/null +++ b/libs/database/src/dbTest/java/org/fdroid/database/DbTest.kt @@ -0,0 +1,120 @@ +package org.fdroid.database + +import android.content.Context +import android.content.res.AssetManager +import androidx.core.os.LocaleListCompat +import androidx.room.Room +import androidx.test.core.app.ApplicationProvider.getApplicationContext +import io.mockk.every +import io.mockk.mockkObject +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.ExperimentalCoroutinesApi +import org.fdroid.database.TestUtils.assertRepoEquals +import org.fdroid.database.TestUtils.toMetadataV2 +import org.fdroid.database.TestUtils.toPackageVersionV2 +import org.fdroid.index.v1.IndexV1StreamProcessor +import org.fdroid.index.v2.IndexV2 +import org.fdroid.index.v2.IndexV2FullStreamProcessor +import org.fdroid.test.TestUtils.sort +import org.fdroid.test.TestUtils.sorted +import org.fdroid.test.VerifierConstants.CERTIFICATE +import org.junit.After +import org.junit.Before +import java.io.IOException +import java.util.Locale +import kotlin.test.assertEquals +import kotlin.test.fail + +@OptIn(ExperimentalCoroutinesApi::class) +internal abstract class DbTest { + + internal lateinit var repoDao: RepositoryDaoInt + internal lateinit var appDao: AppDaoInt + internal lateinit var appPrefsDao: AppPrefsDaoInt + internal lateinit var versionDao: VersionDaoInt + internal lateinit var db: FDroidDatabaseInt + private val testCoroutineDispatcher = Dispatchers.Unconfined + + protected val context: Context = getApplicationContext() + protected val assets: AssetManager = context.resources.assets + protected val locales = LocaleListCompat.create(Locale.US) + + @Before + open fun createDb() { + db = Room.inMemoryDatabaseBuilder(context, FDroidDatabaseInt::class.java) + .allowMainThreadQueries() + .build() + repoDao = db.getRepositoryDao() + appDao = db.getAppDao() + appPrefsDao = db.getAppPrefsDao() + versionDao = db.getVersionDao() + + mockkObject(FDroidDatabaseHolder) + every { FDroidDatabaseHolder.dispatcher } returns testCoroutineDispatcher + } + + @After + @Throws(IOException::class) + fun closeDb() { + db.close() + } + + protected fun streamIndexV1IntoDb( + indexAssetPath: String, + address: String = "https://f-droid.org/repo", + certificate: String = CERTIFICATE, + lastTimestamp: Long = -1, + ): Long { + val repoId = db.getRepositoryDao().insertEmptyRepo(address) + val streamReceiver = DbV1StreamReceiver(db, repoId) { true } + val indexProcessor = IndexV1StreamProcessor(streamReceiver, certificate, lastTimestamp) + db.runInTransaction { + assets.open(indexAssetPath).use { indexStream -> + indexProcessor.process(indexStream) + } + } + return repoId + } + + protected fun streamIndexV2IntoDb( + indexAssetPath: String, + address: String = "https://f-droid.org/repo", + version: Long = 42L, + certificate: String = CERTIFICATE, + ): Long { + val repoId = db.getRepositoryDao().insertEmptyRepo(address) + val streamReceiver = DbV2StreamReceiver(db, repoId) { true } + val indexProcessor = IndexV2FullStreamProcessor(streamReceiver, certificate) + db.runInTransaction { + assets.open(indexAssetPath).use { indexStream -> + indexProcessor.process(version, indexStream) {} + } + } + return repoId + } + + /** + * Asserts that data associated with the given [repoId] is equal to the given [index]. + */ + protected fun assertDbEquals(repoId: Long, index: IndexV2) { + val repo = repoDao.getRepository(repoId) ?: fail() + val sortedIndex = index.sorted() + assertRepoEquals(sortedIndex.repo, repo) + assertEquals(sortedIndex.packages.size, appDao.countApps(), "number of packages") + sortedIndex.packages.forEach { (packageName, packageV2) -> + assertEquals( + packageV2.metadata, + appDao.getApp(repoId, packageName)?.toMetadataV2()?.sort() + ) + val versions = versionDao.getAppVersions(repoId, packageName).map { + it.toPackageVersionV2() + }.associateBy { it.file.sha256 } + assertEquals(packageV2.versions.size, versions.size, "number of versions") + packageV2.versions.forEach { (versionId, packageVersionV2) -> + val version = versions[versionId] ?: fail() + assertEquals(packageVersionV2, version) + } + } + } + +} diff --git a/libs/database/src/dbTest/java/org/fdroid/database/DbUpdateCheckerTest.kt b/libs/database/src/dbTest/java/org/fdroid/database/DbUpdateCheckerTest.kt new file mode 100644 index 000000000..5ceb7c3b7 --- /dev/null +++ b/libs/database/src/dbTest/java/org/fdroid/database/DbUpdateCheckerTest.kt @@ -0,0 +1,82 @@ +package org.fdroid.database + +import android.content.pm.PackageInfo +import android.content.pm.PackageManager +import androidx.test.ext.junit.runners.AndroidJUnit4 +import io.mockk.every +import io.mockk.mockk +import org.fdroid.index.RELEASE_CHANNEL_BETA +import org.fdroid.test.TestDataMidV2 +import org.fdroid.test.TestDataMinV2 +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import kotlin.test.assertEquals +import kotlin.test.assertNull + +@RunWith(AndroidJUnit4::class) +internal class DbUpdateCheckerTest : DbTest() { + + private lateinit var updateChecker: DbUpdateChecker + private val packageManager: PackageManager = mockk() + + private val packageInfo = PackageInfo().apply { + packageName = TestDataMinV2.packageName + @Suppress("DEPRECATION") + versionCode = 0 + } + + @Before + override fun createDb() { + super.createDb() + every { packageManager.systemAvailableFeatures } returns emptyArray() + updateChecker = DbUpdateChecker(db, packageManager) + } + + @Test + fun testSuggestedVersion() { + val repoId = streamIndexV2IntoDb("index-min-v2.json") + every { + packageManager.getPackageInfo(packageInfo.packageName, any()) + } returns packageInfo + val appVersion = updateChecker.getSuggestedVersion(packageInfo.packageName) + val expectedVersion = TestDataMinV2.version.toVersion( + repoId = repoId, + packageName = packageInfo.packageName, + versionId = TestDataMinV2.version.file.sha256, + isCompatible = true, + ) + assertEquals(appVersion!!.version, expectedVersion) + } + + @Test + fun testSuggestedVersionRespectsReleaseChannels() { + streamIndexV2IntoDb("index-mid-v2.json") + every { packageManager.getPackageInfo(packageInfo.packageName, any()) } returns null + + // no suggestion version, because all beta + val appVersion1 = updateChecker.getSuggestedVersion(packageInfo.packageName) + assertNull(appVersion1) + + // now suggests only available version + val appVersion2 = updateChecker.getSuggestedVersion( + packageName = packageInfo.packageName, + releaseChannels = listOf(RELEASE_CHANNEL_BETA), + preferredSigner = TestDataMidV2.version1_2.signer!!.sha256[0], + ) + assertEquals(TestDataMidV2.version1_2.versionCode, appVersion2!!.version.versionCode) + } + + @Test + fun testGetUpdatableApps() { + streamIndexV2IntoDb("index-min-v2.json") + every { packageManager.getInstalledPackages(any()) } returns listOf(packageInfo) + + val appVersions = updateChecker.getUpdatableApps() + assertEquals(1, appVersions.size) + assertEquals(0, appVersions[0].installedVersionCode) + assertEquals(TestDataMinV2.packageName, appVersions[0].packageName) + assertEquals(TestDataMinV2.version.file.sha256, appVersions[0].update.version.versionId) + } + +} diff --git a/libs/database/src/dbTest/java/org/fdroid/database/IndexV1InsertTest.kt b/libs/database/src/dbTest/java/org/fdroid/database/IndexV1InsertTest.kt new file mode 100644 index 000000000..0eb217adf --- /dev/null +++ b/libs/database/src/dbTest/java/org/fdroid/database/IndexV1InsertTest.kt @@ -0,0 +1,134 @@ +package org.fdroid.database + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import kotlinx.serialization.SerializationException +import org.apache.commons.io.input.CountingInputStream +import org.fdroid.index.IndexConverter +import org.fdroid.index.v1.IndexV1StreamProcessor +import org.fdroid.index.v1.IndexV1StreamReceiver +import org.fdroid.index.v2.AntiFeatureV2 +import org.fdroid.index.v2.CategoryV2 +import org.fdroid.index.v2.MetadataV2 +import org.fdroid.index.v2.PackageVersionV2 +import org.fdroid.index.v2.ReleaseChannelV2 +import org.fdroid.index.v2.RepoV2 +import org.fdroid.test.TestDataEmptyV1 +import org.fdroid.test.TestDataMaxV1 +import org.fdroid.test.TestDataMidV1 +import org.fdroid.test.TestDataMinV1 +import org.junit.Test +import org.junit.runner.RunWith +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +@RunWith(AndroidJUnit4::class) +internal class IndexV1InsertTest : DbTest() { + + private val indexConverter = IndexConverter() + + @Test + fun testStreamEmptyIntoDb() { + val repoId = streamIndex("index-empty-v1.json") + assertEquals(1, repoDao.getRepositories().size) + val index = indexConverter.toIndexV2(TestDataEmptyV1.index) + assertDbEquals(repoId, index) + } + + @Test + fun testStreamMinIntoDb() { + val repoId = streamIndex("index-min-v1.json") + assertTrue(repoDao.getRepositories().size == 1) + val index = indexConverter.toIndexV2(TestDataMinV1.index) + assertDbEquals(repoId, index) + } + + @Test + fun testStreamMidIntoDb() { + val repoId = streamIndex("index-mid-v1.json") + assertTrue(repoDao.getRepositories().size == 1) + val index = indexConverter.toIndexV2(TestDataMidV1.index) + assertDbEquals(repoId, index) + } + + @Test + fun testStreamMaxIntoDb() { + val repoId = streamIndex("index-max-v1.json") + assertTrue(repoDao.getRepositories().size == 1) + val index = indexConverter.toIndexV2(TestDataMaxV1.index) + assertDbEquals(repoId, index) + } + + private fun streamIndex(path: String): Long { + val repoId = db.getRepositoryDao().insertEmptyRepo("https://f-droid.org/repo") + val streamReceiver = TestStreamReceiver(repoId) + val indexProcessor = IndexV1StreamProcessor(streamReceiver, null, -1) + db.runInTransaction { + assets.open(path).use { indexStream -> + indexProcessor.process(indexStream) + } + } + return repoId + } + + @Test + fun testExceptionWhileStreamingDoesNotSaveIntoDb() { + val cIn = CountingInputStream(assets.open("index-max-v1.json")) + assertFailsWith { + db.runInTransaction { + val repoId = db.getRepositoryDao().insertEmptyRepo("https://f-droid.org/repo") + val streamReceiver = TestStreamReceiver(repoId) { + if (cIn.byteCount > 0) throw SerializationException() + } + val indexProcessor = IndexV1StreamProcessor(streamReceiver, null, -1) + cIn.use { indexStream -> + indexProcessor.process(indexStream) + } + } + } + assertTrue(repoDao.getRepositories().isEmpty()) + assertTrue(appDao.countApps() == 0) + assertTrue(appDao.countLocalizedFiles() == 0) + assertTrue(appDao.countLocalizedFileLists() == 0) + assertTrue(versionDao.countAppVersions() == 0) + assertTrue(versionDao.countVersionedStrings() == 0) + } + + @Suppress("DEPRECATION") + inner class TestStreamReceiver( + repoId: Long, + private val callback: () -> Unit = {}, + ) : IndexV1StreamReceiver { + private val streamReceiver = DbV1StreamReceiver(db, repoId) { true } + override fun receive(repo: RepoV2, version: Long, certificate: String?) { + streamReceiver.receive(repo, version, certificate) + callback() + } + + override fun receive(packageName: String, m: MetadataV2) { + streamReceiver.receive(packageName, m) + callback() + } + + override fun receive(packageName: String, v: Map) { + streamReceiver.receive(packageName, v) + callback() + } + + override fun updateRepo( + antiFeatures: Map, + categories: Map, + releaseChannels: Map, + ) { + streamReceiver.updateRepo(antiFeatures, categories, releaseChannels) + callback() + } + + override fun updateAppMetadata(packageName: String, preferredSigner: String?) { + streamReceiver.updateAppMetadata(packageName, preferredSigner) + callback() + } + + } + +} diff --git a/libs/database/src/dbTest/java/org/fdroid/database/IndexV2DiffTest.kt b/libs/database/src/dbTest/java/org/fdroid/database/IndexV2DiffTest.kt new file mode 100644 index 000000000..397a0fcf5 --- /dev/null +++ b/libs/database/src/dbTest/java/org/fdroid/database/IndexV2DiffTest.kt @@ -0,0 +1,314 @@ +package org.fdroid.database + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import kotlinx.serialization.SerializationException +import org.fdroid.index.IndexParser +import org.fdroid.index.parseV2 +import org.fdroid.index.v2.IndexV2 +import org.fdroid.index.v2.IndexV2DiffStreamProcessor +import org.fdroid.test.TestDataMaxV2 +import org.fdroid.test.TestDataMidV2 +import org.fdroid.test.TestDataMinV2 +import org.junit.Ignore +import org.junit.Test +import org.junit.runner.RunWith +import java.io.ByteArrayInputStream +import java.io.InputStream +import kotlin.test.assertFailsWith + +@RunWith(AndroidJUnit4::class) +internal class IndexV2DiffTest : DbTest() { + + @Test + @Ignore("use for testing specific index on demand") + fun testBrokenIndexDiff() { + val endPath = "tmp/index-end.json" + val endIndex = IndexParser.parseV2(assets.open(endPath)) + testDiff( + startPath = "tmp/index-start.json", + diffPath = "tmp/diff.json", + endIndex = endIndex, + ) + } + + @Test + fun testEmptyToMin() = testDiff( + startPath = "index-empty-v2.json", + diffPath = "diff-empty-min/23.json", + endIndex = TestDataMinV2.index, + ) + + @Test + fun testEmptyToMid() = testDiff( + startPath = "index-empty-v2.json", + diffPath = "diff-empty-mid/23.json", + endIndex = TestDataMidV2.index, + ) + + @Test + fun testEmptyToMax() = testDiff( + startPath = "index-empty-v2.json", + diffPath = "diff-empty-max/23.json", + endIndex = TestDataMaxV2.index, + ) + + @Test + fun testMinToMid() = testDiff( + startPath = "index-min-v2.json", + diffPath = "diff-empty-mid/42.json", + endIndex = TestDataMidV2.index, + ) + + @Test + fun testMinToMax() = testDiff( + startPath = "index-min-v2.json", + diffPath = "diff-empty-max/42.json", + endIndex = TestDataMaxV2.index, + ) + + @Test + fun testMidToMax() = testDiff( + startPath = "index-mid-v2.json", + diffPath = "diff-empty-max/1337.json", + endIndex = TestDataMaxV2.index, + ) + + @Test + fun testMinRemoveApp() { + val diffJson = """{ + "packages": { + "org.fdroid.min1": null + } + }""".trimIndent() + testJsonDiff( + startPath = "index-min-v2.json", + diff = diffJson, + endIndex = TestDataMinV2.index.copy(packages = emptyMap()), + ) + } + + @Test + fun testMinNoMetadataRemoveVersion() { + val diffJson = """{ + "packages": { + "org.fdroid.min1": { + "metadata": { + "added": 0 + }, + "versions": { + "824a109b2352138c3699760e1683385d0ed50ce526fc7982f8d65757743374bf": null + } + } + } + }""".trimIndent() + testJsonDiff( + startPath = "index-min-v2.json", + diff = diffJson, + endIndex = TestDataMinV2.index.copy( + packages = TestDataMinV2.index.packages.mapValues { + it.value.copy(versions = emptyMap()) + } + ), + ) + } + + @Test + fun testMinNoVersionsUnknownKey() { + val diffJson = """{ + "packages": { + "org.fdroid.min1": { + "metadata": { + "added": 42 + }, + "unknownKey": "should get ignored" + } + } + }""".trimIndent() + testJsonDiff( + startPath = "index-min-v2.json", + diff = diffJson, + endIndex = TestDataMinV2.index.copy( + packages = TestDataMinV2.index.packages.mapValues { + it.value.copy(metadata = it.value.metadata.copy(added = 42)) + } + ), + ) + } + + @Test + fun testMinRemoveMetadata() { + val diffJson = """{ + "packages": { + "org.fdroid.min1": { + "metadata": null + } + }, + "unknownKey": "should get ignored" + }""".trimIndent() + testJsonDiff( + startPath = "index-min-v2.json", + diff = diffJson, + endIndex = TestDataMinV2.index.copy( + packages = emptyMap() + ), + ) + } + + @Test + fun testMinRemoveVersions() { + val diffJson = """{ + "packages": { + "org.fdroid.min1": { + "versions": null + } + } + }""".trimIndent() + testJsonDiff( + startPath = "index-min-v2.json", + diff = diffJson, + endIndex = TestDataMinV2.index.copy( + packages = TestDataMinV2.index.packages.mapValues { + it.value.copy(versions = emptyMap()) + } + ), + ) + } + + @Test + fun testMinNoMetadataNoVersion() { + val diffJson = """{ + "packages": { + "org.fdroid.min1": { + } + } + }""".trimIndent() + testJsonDiff( + startPath = "index-min-v2.json", + diff = diffJson, + endIndex = TestDataMinV2.index, + ) + } + + @Test + fun testAppDenyKeyList() { + val diffRepoIdJson = """{ + "packages": { + "org.fdroid.min1": { + "metadata": { + "repoId": 1 + } + } + } + }""".trimIndent() + assertFailsWith { + testJsonDiff( + startPath = "index-min-v2.json", + diff = diffRepoIdJson, + endIndex = TestDataMinV2.index, + ) + } + val diffPackageNameJson = """{ + "packages": { + "org.fdroid.min1": { + "metadata": { + "packageName": "foo" + } + } + } + }""".trimIndent() + assertFailsWith { + testJsonDiff( + startPath = "index-min-v2.json", + diff = diffPackageNameJson, + endIndex = TestDataMinV2.index, + ) + } + } + + @Test + fun testVersionsDenyKeyList() { + assertFailsWith { + testJsonDiff( + startPath = "index-min-v2.json", + diff = getMinVersionJson(""""packageName": "foo""""), + endIndex = TestDataMinV2.index, + ) + } + assertFailsWith { + testJsonDiff( + startPath = "index-min-v2.json", + diff = getMinVersionJson(""""repoId": 1"""), + endIndex = TestDataMinV2.index, + ) + } + assertFailsWith { + testJsonDiff( + startPath = "index-min-v2.json", + diff = getMinVersionJson(""""versionId": "bar""""), + endIndex = TestDataMinV2.index, + ) + } + } + + private fun getMinVersionJson(insert: String) = """{ + "packages": { + "org.fdroid.min1": { + "versions": { + "824a109b2352138c3699760e1683385d0ed50ce526fc7982f8d65757743374bf": { + $insert + } + } + } + }""".trimIndent() + + @Test + fun testMidRemoveScreenshots() { + val diffRepoIdJson = """{ + "packages": { + "org.fdroid.fdroid": { + "metadata": { + "screenshots": null + } + } + } + }""".trimIndent() + val fdroidPackage = TestDataMidV2.packages["org.fdroid.fdroid"]!!.copy( + metadata = TestDataMidV2.packages["org.fdroid.fdroid"]!!.metadata.copy( + screenshots = null, + ) + ) + testJsonDiff( + startPath = "index-mid-v2.json", + diff = diffRepoIdJson, + endIndex = TestDataMidV2.index.copy( + packages = mapOf( + TestDataMidV2.packageName1 to TestDataMidV2.app1, + TestDataMidV2.packageName2 to fdroidPackage, + ) + ), + ) + } + + private fun testJsonDiff(startPath: String, diff: String, endIndex: IndexV2) { + testDiff(startPath, ByteArrayInputStream(diff.toByteArray()), endIndex) + } + + private fun testDiff(startPath: String, diffPath: String, endIndex: IndexV2) { + testDiff(startPath, assets.open(diffPath), endIndex) + } + + private fun testDiff(startPath: String, diffStream: InputStream, endIndex: IndexV2) { + // stream start index into the DB + val repoId = streamIndexV2IntoDb(startPath) + + // apply diff stream to the DB + val streamReceiver = DbV2DiffStreamReceiver(db, repoId) { true } + val streamProcessor = IndexV2DiffStreamProcessor(streamReceiver) + db.runInTransaction { + streamProcessor.process(42, diffStream) {} + } + // assert that changed DB data is equal to given endIndex + assertDbEquals(repoId, endIndex) + } + +} diff --git a/libs/database/src/dbTest/java/org/fdroid/database/IndexV2InsertTest.kt b/libs/database/src/dbTest/java/org/fdroid/database/IndexV2InsertTest.kt new file mode 100644 index 000000000..63e1e042f --- /dev/null +++ b/libs/database/src/dbTest/java/org/fdroid/database/IndexV2InsertTest.kt @@ -0,0 +1,81 @@ +package org.fdroid.database + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import kotlinx.serialization.SerializationException +import org.apache.commons.io.input.CountingInputStream +import org.fdroid.CompatibilityChecker +import org.fdroid.index.v2.IndexV2FullStreamProcessor +import org.fdroid.test.TestDataEmptyV2 +import org.fdroid.test.TestDataMaxV2 +import org.fdroid.test.TestDataMidV2 +import org.fdroid.test.TestDataMinV2 +import org.junit.Test +import org.junit.runner.RunWith +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +@RunWith(AndroidJUnit4::class) +internal class IndexV2InsertTest : DbTest() { + + @Test + fun testStreamEmptyIntoDb() { + val repoId = streamIndexV2IntoDb("index-empty-v2.json") + assertEquals(1, repoDao.getRepositories().size) + assertDbEquals(repoId, TestDataEmptyV2.index) + } + + @Test + fun testStreamMinIntoDb() { + val repoId = streamIndexV2IntoDb("index-min-v2.json") + assertEquals(1, repoDao.getRepositories().size) + assertDbEquals(repoId, TestDataMinV2.index) + } + + @Test + fun testStreamMinReorderedIntoDb() { + val repoId = streamIndexV2IntoDb("index-min-reordered-v2.json") + assertEquals(1, repoDao.getRepositories().size) + assertDbEquals(repoId, TestDataMinV2.index) + } + + @Test + fun testStreamMidIntoDb() { + val repoId = streamIndexV2IntoDb("index-mid-v2.json") + assertEquals(1, repoDao.getRepositories().size) + assertDbEquals(repoId, TestDataMidV2.index) + } + + @Test + fun testStreamMaxIntoDb() { + val repoId = streamIndexV2IntoDb("index-max-v2.json") + assertEquals(1, repoDao.getRepositories().size) + assertDbEquals(repoId, TestDataMaxV2.index) + } + + @Test + fun testExceptionWhileStreamingDoesNotSaveIntoDb() { + val cIn = CountingInputStream(assets.open("index-max-v2.json")) + val compatibilityChecker = CompatibilityChecker { + if (cIn.byteCount > 0) throw SerializationException() + true + } + assertFailsWith { + db.runInTransaction { + val repoId = db.getRepositoryDao().insertEmptyRepo("http://example.org") + val streamReceiver = DbV2StreamReceiver(db, repoId, compatibilityChecker) + val indexProcessor = IndexV2FullStreamProcessor(streamReceiver, "") + cIn.use { indexStream -> + indexProcessor.process(42, indexStream) {} + } + } + } + assertTrue(repoDao.getRepositories().isEmpty()) + assertTrue(appDao.countApps() == 0) + assertTrue(appDao.countLocalizedFiles() == 0) + assertTrue(appDao.countLocalizedFileLists() == 0) + assertTrue(versionDao.countAppVersions() == 0) + assertTrue(versionDao.countVersionedStrings() == 0) + } + +} diff --git a/libs/database/src/dbTest/java/org/fdroid/database/RepositoryDaoTest.kt b/libs/database/src/dbTest/java/org/fdroid/database/RepositoryDaoTest.kt new file mode 100644 index 000000000..66338059c --- /dev/null +++ b/libs/database/src/dbTest/java/org/fdroid/database/RepositoryDaoTest.kt @@ -0,0 +1,269 @@ +package org.fdroid.database + +import androidx.arch.core.executor.testing.InstantTaskExecutorRule +import androidx.test.ext.junit.runners.AndroidJUnit4 +import org.fdroid.database.TestUtils.assertRepoEquals +import org.fdroid.database.TestUtils.getOrFail +import org.fdroid.test.TestAppUtils.getRandomMetadataV2 +import org.fdroid.test.TestRepoUtils.getRandomRepo +import org.fdroid.test.TestUtils.getRandomString +import org.fdroid.test.TestUtils.orNull +import org.fdroid.test.TestVersionUtils.getRandomPackageVersionV2 +import org.junit.Rule +import org.junit.Test +import org.junit.runner.RunWith +import kotlin.random.Random +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNull +import kotlin.test.assertTrue +import kotlin.test.fail + +@RunWith(AndroidJUnit4::class) +internal class RepositoryDaoTest : DbTest() { + + @get:Rule + val instantTaskExecutorRule = InstantTaskExecutorRule() + + @Test + fun testInsertInitialRepository() { + val repo = InitialRepository( + name = getRandomString(), + address = getRandomString(), + description = getRandomString(), + certificate = getRandomString(), + version = Random.nextLong(), + enabled = Random.nextBoolean(), + weight = Random.nextInt(), + ) + val repoId = repoDao.insert(repo) + + val actualRepo = repoDao.getRepository(repoId) ?: fail() + assertEquals(repo.name, actualRepo.getName(locales)) + assertEquals(repo.address, actualRepo.address) + assertEquals(repo.description, actualRepo.getDescription(locales)) + assertEquals(repo.certificate, actualRepo.certificate) + assertEquals(repo.version, actualRepo.version) + assertEquals(repo.enabled, actualRepo.enabled) + assertEquals(repo.weight, actualRepo.weight) + assertEquals(-1, actualRepo.timestamp) + assertEquals(emptyList(), actualRepo.mirrors) + assertEquals(emptyList(), actualRepo.userMirrors) + assertEquals(emptyList(), actualRepo.disabledMirrors) + assertEquals(listOf(org.fdroid.download.Mirror(repo.address)), actualRepo.getMirrors()) + assertEquals(emptyList(), actualRepo.antiFeatures) + assertEquals(emptyList(), actualRepo.categories) + assertEquals(emptyList(), actualRepo.releaseChannels) + assertNull(actualRepo.formatVersion) + assertNull(actualRepo.repository.icon) + assertNull(actualRepo.lastUpdated) + assertNull(actualRepo.webBaseUrl) + } + + @Test + fun testInsertEmptyRepo() { + // insert empty repo + val address = getRandomString() + val username = getRandomString().orNull() + val password = getRandomString().orNull() + val repoId = repoDao.insertEmptyRepo(address, username, password) + + // check that repo got inserted as expected + val actualRepo = repoDao.getRepository(repoId) ?: fail() + assertEquals(address, actualRepo.address) + assertEquals(username, actualRepo.username) + assertEquals(password, actualRepo.password) + assertEquals(-1, actualRepo.timestamp) + assertEquals(listOf(org.fdroid.download.Mirror(address)), actualRepo.getMirrors()) + assertEquals(emptyList(), actualRepo.antiFeatures) + assertEquals(emptyList(), actualRepo.categories) + assertEquals(emptyList(), actualRepo.releaseChannels) + assertNull(actualRepo.formatVersion) + assertNull(actualRepo.repository.icon) + assertNull(actualRepo.lastUpdated) + assertNull(actualRepo.webBaseUrl) + } + + @Test + fun insertAndDeleteTwoRepos() { + // insert first repo + val repo1 = getRandomRepo() + val repoId1 = repoDao.insertOrReplace(repo1) + + // check that first repo got added and retrieved as expected + repoDao.getRepositories().let { repos -> + assertEquals(1, repos.size) + assertRepoEquals(repo1, repos[0]) + } + val repositoryPreferences1 = repoDao.getRepositoryPreferences(repoId1) + assertEquals(repoId1, repositoryPreferences1?.repoId) + + // insert second repo + val repo2 = getRandomRepo() + val repoId2 = repoDao.insertOrReplace(repo2) + + // check that both repos got added and retrieved as expected + listOf( + repoDao.getRepositories().sortedBy { it.repoId }, + repoDao.getLiveRepositories().getOrFail().sortedBy { it.repoId }, + ).forEach { repos -> + assertEquals(2, repos.size) + assertRepoEquals(repo1, repos[0]) + assertRepoEquals(repo2, repos[1]) + } + val repositoryPreferences2 = repoDao.getRepositoryPreferences(repoId2) + assertEquals(repoId2, repositoryPreferences2?.repoId) + // second repo has one weight point more than first repo + assertEquals(repositoryPreferences1?.weight?.plus(1), repositoryPreferences2?.weight) + + // remove first repo and check that the database only returns one + repoDao.deleteRepository(repoId1) + listOf( + repoDao.getRepositories(), + repoDao.getLiveRepositories().getOrFail(), + ).forEach { repos -> + assertEquals(1, repos.size) + assertRepoEquals(repo2, repos[0]) + } + assertNull(repoDao.getRepositoryPreferences(repoId1)) + + // remove second repo and check that all associated data got removed as well + repoDao.deleteRepository(repoId2) + assertEquals(0, repoDao.getRepositories().size) + assertEquals(0, repoDao.countMirrors()) + assertEquals(0, repoDao.countAntiFeatures()) + assertEquals(0, repoDao.countCategories()) + assertEquals(0, repoDao.countReleaseChannels()) + assertNull(repoDao.getRepositoryPreferences(repoId2)) + } + + @Test + fun insertTwoReposAndClearAll() { + val repo1 = getRandomRepo() + val repo2 = getRandomRepo() + repoDao.insertOrReplace(repo1) + repoDao.insertOrReplace(repo2) + assertEquals(2, repoDao.getRepositories().size) + assertEquals(2, repoDao.getLiveRepositories().getOrFail().size) + + repoDao.clearAll() + assertEquals(0, repoDao.getRepositories().size) + assertEquals(0, repoDao.getLiveRepositories().getOrFail().size) + } + + @Test + fun testSetRepositoryEnabled() { + // repo is enabled by default + val repoId = repoDao.insertOrReplace(getRandomRepo()) + assertTrue(repoDao.getRepository(repoId)?.enabled ?: fail()) + + // disabled repo is disabled + repoDao.setRepositoryEnabled(repoId, false) + assertFalse(repoDao.getRepository(repoId)?.enabled ?: fail()) + + // enabling again works + repoDao.setRepositoryEnabled(repoId, true) + assertTrue(repoDao.getRepository(repoId)?.enabled ?: fail()) + } + + @Test + fun testUpdateUserMirrors() { + // repo is enabled by default + val repoId = repoDao.insertOrReplace(getRandomRepo()) + assertEquals(emptyList(), repoDao.getRepository(repoId)?.userMirrors) + + // add user mirrors + val userMirrors = listOf(getRandomString(), getRandomString(), getRandomString()) + repoDao.updateUserMirrors(repoId, userMirrors) + val repo = repoDao.getRepository(repoId) ?: fail() + assertEquals(userMirrors, repo.userMirrors) + + // user mirrors are part of all mirrors + val userDownloadMirrors = userMirrors.map { org.fdroid.download.Mirror(it) } + assertTrue(repo.getMirrors().containsAll(userDownloadMirrors)) + + // remove user mirrors + repoDao.updateUserMirrors(repoId, emptyList()) + assertEquals(emptyList(), repoDao.getRepository(repoId)?.userMirrors) + } + + @Test + fun testUpdateUsernameAndPassword() { + // repo has no username or password initially + val repoId = repoDao.insertOrReplace(getRandomRepo()) + repoDao.getRepository(repoId)?.let { repo -> + assertEquals(null, repo.username) + assertEquals(null, repo.password) + } ?: fail() + + // add user name and password + val username = getRandomString().orNull() + val password = getRandomString().orNull() + repoDao.updateUsernameAndPassword(repoId, username, password) + repoDao.getRepository(repoId)?.let { repo -> + assertEquals(username, repo.username) + assertEquals(password, repo.password) + } ?: fail() + } + + @Test + fun testUpdateDisabledMirrors() { + // repo has no username or password initially + val repoId = repoDao.insertOrReplace(getRandomRepo()) + repoDao.getRepository(repoId)?.let { repo -> + assertEquals(null, repo.username) + assertEquals(null, repo.password) + } ?: fail() + + // add user name and password + val username = getRandomString().orNull() + val password = getRandomString().orNull() + repoDao.updateUsernameAndPassword(repoId, username, password) + repoDao.getRepository(repoId)?.let { repo -> + assertEquals(username, repo.username) + assertEquals(password, repo.password) + } ?: fail() + } + + @Test + fun clearingRepoRemovesAllAssociatedData() { + // insert one repo with one app with one version + val repoId = repoDao.insertOrReplace(getRandomRepo()) + val repositoryPreferences = repoDao.getRepositoryPreferences(repoId) + val packageName = getRandomString() + val versionId = getRandomString() + appDao.insert(repoId, packageName, getRandomMetadataV2()) + val packageVersion = getRandomPackageVersionV2() + versionDao.insert(repoId, packageName, versionId, packageVersion, Random.nextBoolean()) + + // data is there as expected + assertEquals(1, repoDao.getRepositories().size) + assertEquals(1, appDao.getAppMetadata().size) + assertEquals(1, versionDao.getAppVersions(repoId, packageName).size) + assertTrue(versionDao.getVersionedStrings(repoId, packageName).isNotEmpty()) + + // clearing the repo removes apps and versions + repoDao.clear(repoId) + assertEquals(1, repoDao.getRepositories().size) + assertEquals(0, appDao.countApps()) + assertEquals(0, appDao.countLocalizedFiles()) + assertEquals(0, appDao.countLocalizedFileLists()) + assertEquals(0, versionDao.getAppVersions(repoId, packageName).size) + assertEquals(0, versionDao.getVersionedStrings(repoId, packageName).size) + // preferences are not touched by clearing + assertEquals(repositoryPreferences, repoDao.getRepositoryPreferences(repoId)) + } + + @Test + fun certGetsUpdated() { + val repoId = repoDao.insertOrReplace(getRandomRepo()) + assertEquals(1, repoDao.getRepositories().size) + assertEquals(null, repoDao.getRepositories()[0].certificate) + + val cert = getRandomString() + repoDao.updateRepository(repoId, cert) + + assertEquals(1, repoDao.getRepositories().size) + assertEquals(cert, repoDao.getRepositories()[0].certificate) + } +} diff --git a/libs/database/src/dbTest/java/org/fdroid/database/RepositoryDiffTest.kt b/libs/database/src/dbTest/java/org/fdroid/database/RepositoryDiffTest.kt new file mode 100644 index 000000000..0aa71f618 --- /dev/null +++ b/libs/database/src/dbTest/java/org/fdroid/database/RepositoryDiffTest.kt @@ -0,0 +1,241 @@ +package org.fdroid.database + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.jsonObject +import org.fdroid.database.TestUtils.assertRepoEquals +import org.fdroid.index.v2.AntiFeatureV2 +import org.fdroid.index.v2.CategoryV2 +import org.fdroid.index.v2.ReleaseChannelV2 +import org.fdroid.index.v2.RepoV2 +import org.fdroid.test.DiffUtils.applyDiff +import org.fdroid.test.DiffUtils.randomDiff +import org.fdroid.test.TestRepoUtils.getRandomFileV2 +import org.fdroid.test.TestRepoUtils.getRandomLocalizedTextV2 +import org.fdroid.test.TestRepoUtils.getRandomMirror +import org.fdroid.test.TestRepoUtils.getRandomRepo +import org.fdroid.test.TestUtils.getRandomMap +import org.fdroid.test.TestUtils.getRandomString +import org.junit.Test +import org.junit.runner.RunWith +import kotlin.random.Random +import kotlin.test.assertEquals + +/** + * Tests that repository diffs get applied to the database correctly. + */ +@RunWith(AndroidJUnit4::class) +internal class RepositoryDiffTest : DbTest() { + + private val j = Json + + @Test + fun timestampDiff() { + val repo = getRandomRepo() + val updateTimestamp = repo.timestamp + 1 + val json = """ + { + "timestamp": $updateTimestamp + }""".trimIndent() + testDiff(repo, json) { repos -> + assertEquals(updateTimestamp, repos[0].timestamp) + assertRepoEquals(repo.copy(timestamp = updateTimestamp), repos[0]) + } + } + + @Test + fun timestampDiffTwoReposInDb() { + // insert repo + val repo = getRandomRepo() + repoDao.insertOrReplace(repo) + + // insert another repo before updating + repoDao.insertOrReplace(getRandomRepo()) + + // check that the repo got added and retrieved as expected + var repos = repoDao.getRepositories().sortedBy { it.repoId } + assertEquals(2, repos.size) + val repoId = repos[0].repoId + + val updateTimestamp = Random.nextLong() + val json = """ + { + "timestamp": $updateTimestamp + }""".trimIndent() + + // decode diff from JSON and update DB with it + val diff = j.parseToJsonElement(json).jsonObject // Json.decodeFromString(json) + repoDao.updateRepository(repoId, 42, diff) + + // fetch repos again and check that the result is as expected + repos = repoDao.getRepositories().sortedBy { it.repoId } + assertEquals(2, repos.size) + assertEquals(repoId, repos[0].repoId) + assertEquals(updateTimestamp, repos[0].timestamp) + assertRepoEquals(repo.copy(timestamp = updateTimestamp), repos[0]) + } + + @Test + fun mirrorDiff() { + val repo = getRandomRepo() + val updateMirrors = repo.mirrors.toMutableList().apply { + removeLastOrNull() + add(getRandomMirror()) + add(getRandomMirror()) + } + val json = """ + { + "mirrors": ${Json.encodeToString(updateMirrors)} + }""".trimIndent() + testDiff(repo, json) { repos -> + val expectedMirrors = updateMirrors.map { mirror -> + mirror.toMirror(repos[0].repoId) + }.toSet() + assertEquals(expectedMirrors, repos[0].mirrors.toSet()) + assertRepoEquals(repo.copy(mirrors = updateMirrors), repos[0]) + } + } + + @Test + fun descriptionDiff() { + val repo = getRandomRepo().copy(description = mapOf("de" to "foo", "en" to "bar")) + val updateText = if (Random.nextBoolean()) mapOf("de" to null, "en" to "foo") else null + val json = """ + { + "description": ${Json.encodeToString(updateText)} + }""".trimIndent() + val expectedText = if (updateText == null) emptyMap() else mapOf("en" to "foo") + testDiff(repo, json) { repos -> + assertEquals(expectedText, repos[0].repository.description) + assertRepoEquals(repo.copy(description = expectedText), repos[0]) + } + } + + @Test + fun antiFeaturesDiff() { + val repo = getRandomRepo().copy(antiFeatures = getRandomMap { + getRandomString() to AntiFeatureV2( + icon = getRandomFileV2(), + name = getRandomLocalizedTextV2(), + description = getRandomLocalizedTextV2(), + ) + }) + val antiFeatures = repo.antiFeatures.randomDiff { + AntiFeatureV2(getRandomFileV2(), getRandomLocalizedTextV2(), getRandomLocalizedTextV2()) + } + val json = """ + { + "antiFeatures": ${Json.encodeToString(antiFeatures)} + }""".trimIndent() + testDiff(repo, json) { repos -> + val expectedFeatures = repo.antiFeatures.applyDiff(antiFeatures) + val expectedRepoAntiFeatures = + expectedFeatures.toRepoAntiFeatures(repos[0].repoId) + assertEquals(expectedRepoAntiFeatures.toSet(), repos[0].antiFeatures.toSet()) + assertRepoEquals(repo.copy(antiFeatures = expectedFeatures), repos[0]) + } + } + + @Test + fun antiFeatureKeyChangeDiff() { + val antiFeatureKey = getRandomString() + val antiFeature = AntiFeatureV2( + icon = getRandomFileV2(), + name = getRandomLocalizedTextV2(), + description = getRandomLocalizedTextV2(), + ) + val antiFeatures = mapOf(antiFeatureKey to antiFeature) + val repo = getRandomRepo().copy(antiFeatures = antiFeatures) + + @Suppress("UNCHECKED_CAST") + val newAntiFeatures = mapOf(antiFeatureKey to antiFeature.copy( + icon = null, + name = getRandomLocalizedTextV2(), + description = getRandomLocalizedTextV2(), + )) + val json = """ + { + "antiFeatures": { + "$antiFeatureKey": ${Json.encodeToString(newAntiFeatures)} + } + }""".trimIndent() + testDiff(repo, json) { repos -> + val expectedFeatures = repo.antiFeatures.applyDiff(antiFeatures) + val expectedRepoAntiFeatures = + expectedFeatures.toRepoAntiFeatures(repos[0].repoId) + assertEquals(expectedRepoAntiFeatures.toSet(), repos[0].antiFeatures.toSet()) + assertRepoEquals(repo.copy(antiFeatures = expectedFeatures), repos[0]) + } + } + + @Test + fun categoriesDiff() { + val repo = getRandomRepo().copy(categories = getRandomMap { + getRandomString() to CategoryV2( + icon = getRandomFileV2(), + name = getRandomLocalizedTextV2(), + description = getRandomLocalizedTextV2(), + ) + }) + val categories = repo.categories.randomDiff { + CategoryV2(getRandomFileV2(), getRandomLocalizedTextV2(), getRandomLocalizedTextV2()) + } + val json = """ + { + "categories": ${Json.encodeToString(categories)} + }""".trimIndent() + testDiff(repo, json) { repos -> + val expectedFeatures = repo.categories.applyDiff(categories) + val expectedRepoCategories = + expectedFeatures.toRepoCategories(repos[0].repoId) + assertEquals(expectedRepoCategories.toSet(), repos[0].categories.toSet()) + assertRepoEquals(repo.copy(categories = expectedFeatures), repos[0]) + } + } + + @Test + fun releaseChannelsDiff() { + val repo = getRandomRepo().copy(releaseChannels = getRandomMap { + getRandomString() to ReleaseChannelV2( + name = getRandomLocalizedTextV2(), + description = getRandomLocalizedTextV2(), + ) + }) + val releaseChannels = repo.releaseChannels.randomDiff { + ReleaseChannelV2(getRandomLocalizedTextV2(), getRandomLocalizedTextV2()) + } + val json = """ + { + "releaseChannels": ${Json.encodeToString(releaseChannels)} + }""".trimIndent() + testDiff(repo, json) { repos -> + val expectedFeatures = repo.releaseChannels.applyDiff(releaseChannels) + val expectedRepoReleaseChannels = + expectedFeatures.toRepoReleaseChannel(repos[0].repoId) + assertEquals(expectedRepoReleaseChannels.toSet(), repos[0].releaseChannels.toSet()) + assertRepoEquals(repo.copy(releaseChannels = expectedFeatures), repos[0]) + } + } + + private fun testDiff(repo: RepoV2, json: String, repoChecker: (List) -> Unit) { + // insert repo + repoDao.insertOrReplace(repo) + + // check that the repo got added and retrieved as expected + var repos = repoDao.getRepositories() + assertEquals(1, repos.size) + val repoId = repos[0].repoId + + // decode diff from JSON and update DB with it + val diff = j.parseToJsonElement(json).jsonObject + repoDao.updateRepository(repoId, 42, diff) + + // fetch repos again and check that the result is as expected + repos = repoDao.getRepositories().sortedBy { it.repoId } + assertEquals(1, repos.size) + assertEquals(repoId, repos[0].repoId) + repoChecker(repos) + } + +} diff --git a/libs/database/src/dbTest/java/org/fdroid/database/TestUtils.kt b/libs/database/src/dbTest/java/org/fdroid/database/TestUtils.kt new file mode 100644 index 000000000..b4ea0c63f --- /dev/null +++ b/libs/database/src/dbTest/java/org/fdroid/database/TestUtils.kt @@ -0,0 +1,120 @@ +package org.fdroid.database + +import androidx.lifecycle.LiveData +import androidx.lifecycle.Observer +import org.fdroid.index.v2.FeatureV2 +import org.fdroid.index.v2.ManifestV2 +import org.fdroid.index.v2.MetadataV2 +import org.fdroid.index.v2.PackageVersionV2 +import org.fdroid.index.v2.RepoV2 +import org.junit.Assert +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertTrue +import kotlin.test.fail + +internal object TestUtils { + + fun assertTimestampRecent(timestamp: Long?) { + assertNotNull(timestamp) + assertTrue(System.currentTimeMillis() - timestamp < 2000) + } + + fun assertRepoEquals(repoV2: RepoV2, repo: Repository) { + val repoId = repo.repoId + // mirrors + val expectedMirrors = repoV2.mirrors.map { it.toMirror(repoId) }.toSet() + Assert.assertEquals(expectedMirrors, repo.mirrors.toSet()) + // anti-features + val expectedAntiFeatures = repoV2.antiFeatures.toRepoAntiFeatures(repoId).toSet() + assertEquals(expectedAntiFeatures, repo.antiFeatures.toSet()) + // categories + val expectedCategories = repoV2.categories.toRepoCategories(repoId).toSet() + assertEquals(expectedCategories, repo.categories.toSet()) + // release channels + val expectedReleaseChannels = repoV2.releaseChannels.toRepoReleaseChannel(repoId).toSet() + assertEquals(expectedReleaseChannels, repo.releaseChannels.toSet()) + // core repo + val coreRepo = repoV2.toCoreRepository( + version = repo.repository.version!!.toLong(), + formatVersion = repo.repository.formatVersion, + certificate = repo.repository.certificate, + ).copy(repoId = repoId) + assertEquals(coreRepo, repo.repository) + } + + internal fun App.toMetadataV2() = MetadataV2( + added = metadata.added, + lastUpdated = metadata.lastUpdated, + name = metadata.name, + summary = metadata.summary, + description = metadata.description, + webSite = metadata.webSite, + changelog = metadata.changelog, + license = metadata.license, + sourceCode = metadata.sourceCode, + issueTracker = metadata.issueTracker, + translation = metadata.translation, + preferredSigner = metadata.preferredSigner, + video = metadata.video, + authorName = metadata.authorName, + authorEmail = metadata.authorEmail, + authorWebSite = metadata.authorWebSite, + authorPhone = metadata.authorPhone, + donate = metadata.donate ?: emptyList(), + liberapayID = metadata.liberapayID, + liberapay = metadata.liberapay, + openCollective = metadata.openCollective, + bitcoin = metadata.bitcoin, + litecoin = metadata.litecoin, + flattrID = metadata.flattrID, + categories = metadata.categories ?: emptyList(), + icon = icon, + featureGraphic = featureGraphic, + promoGraphic = promoGraphic, + tvBanner = tvBanner, + screenshots = screenshots, + ) + + fun AppVersion.toPackageVersionV2() = PackageVersionV2( + added = added, + file = file, + src = src, + manifest = ManifestV2( + versionName = manifest.versionName, + versionCode = manifest.versionCode, + usesSdk = manifest.usesSdk, + maxSdkVersion = manifest.maxSdkVersion, + signer = manifest.signer, + usesPermission = usesPermission.sortedBy { it.name }, + usesPermissionSdk23 = usesPermissionSdk23.sortedBy { it.name }, + nativecode = manifest.nativecode?.sorted() ?: emptyList(), + features = manifest.features?.map { FeatureV2(it) } ?: emptyList(), + ), + releaseChannels = releaseChannels, + antiFeatures = version.antiFeatures ?: emptyMap(), + whatsNew = version.whatsNew ?: emptyMap(), + ) + + fun LiveData.getOrAwaitValue(): T? { + val data = arrayOfNulls(1) + val latch = CountDownLatch(1) + val observer: Observer = object : Observer { + override fun onChanged(o: T?) { + data[0] = o + latch.countDown() + removeObserver(this) + } + } + observeForever(observer) + latch.await(2, TimeUnit.SECONDS) + @Suppress("UNCHECKED_CAST") + return data[0] as T? + } + + fun LiveData.getOrFail(): T { + return getOrAwaitValue() ?: fail() + } +} diff --git a/libs/database/src/dbTest/java/org/fdroid/database/VersionTest.kt b/libs/database/src/dbTest/java/org/fdroid/database/VersionTest.kt new file mode 100644 index 000000000..20c81a458 --- /dev/null +++ b/libs/database/src/dbTest/java/org/fdroid/database/VersionTest.kt @@ -0,0 +1,234 @@ +package org.fdroid.database + +import androidx.arch.core.executor.testing.InstantTaskExecutorRule +import androidx.test.ext.junit.runners.AndroidJUnit4 +import org.fdroid.database.TestUtils.getOrFail +import org.fdroid.index.v2.PackageVersionV2 +import org.fdroid.test.TestAppUtils.getRandomMetadataV2 +import org.fdroid.test.TestRepoUtils.getRandomRepo +import org.fdroid.test.TestUtils.getRandomString +import org.fdroid.test.TestVersionUtils.getRandomPackageVersionV2 +import org.junit.Rule +import org.junit.Test +import org.junit.runner.RunWith +import kotlin.random.Random +import kotlin.test.assertEquals +import kotlin.test.fail + +@RunWith(AndroidJUnit4::class) +internal class VersionTest : DbTest() { + + @get:Rule + val instantTaskExecutorRule = InstantTaskExecutorRule() + + private val packageName = getRandomString() + private val packageVersion1 = getRandomPackageVersionV2() + private val packageVersion2 = getRandomPackageVersionV2() + private val packageVersion3 = getRandomPackageVersionV2() + private val versionId1 = packageVersion1.file.sha256 + private val versionId2 = packageVersion2.file.sha256 + private val versionId3 = packageVersion3.file.sha256 + private val isCompatible1 = Random.nextBoolean() + private val isCompatible2 = Random.nextBoolean() + private val packageVersions = mapOf( + versionId1 to packageVersion1, + versionId2 to packageVersion2, + ) + + private fun getVersion1(repoId: Long) = + packageVersion1.toVersion(repoId, packageName, versionId1, isCompatible1) + + private fun getVersion2(repoId: Long) = + packageVersion2.toVersion(repoId, packageName, versionId2, isCompatible2) + + private val compatChecker: (PackageVersionV2) -> Boolean = { + when (it.file.sha256) { + versionId1 -> isCompatible1 + versionId2 -> isCompatible2 + else -> fail() + } + } + + @Test + fun insertGetDeleteSingleVersion() { + val repoId = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId, packageName, getRandomMetadataV2()) + versionDao.insert(repoId, packageName, versionId1, packageVersion1, isCompatible1) + + val appVersions = versionDao.getAppVersions(repoId, packageName) + assertEquals(1, appVersions.size) + val appVersion = appVersions[0] + assertEquals(versionId1, appVersion.version.versionId) + assertEquals(getVersion1(repoId), appVersion.version) + val manifest = packageVersion1.manifest + assertEquals(manifest.usesPermission.toSet(), appVersion.usesPermission.toSet()) + assertEquals(manifest.usesPermissionSdk23.toSet(), appVersion.usesPermissionSdk23.toSet()) + assertEquals( + manifest.features.map { it.name }.toSet(), + appVersion.version.manifest.features?.toSet() + ) + + val versionedStrings = versionDao.getVersionedStrings(repoId, packageName) + val expectedSize = manifest.usesPermission.size + manifest.usesPermissionSdk23.size + assertEquals(expectedSize, versionedStrings.size) + + versionDao.deleteAppVersion(repoId, packageName, versionId1) + assertEquals(0, versionDao.getAppVersions(repoId, packageName).size) + assertEquals(0, versionDao.getVersionedStrings(repoId, packageName).size) + } + + @Test + fun insertGetDeleteTwoVersions() { + // insert two versions along with required objects + val repoId = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId, packageName, getRandomMetadataV2()) + versionDao.insert(repoId, packageName, versionId1, packageVersion1, isCompatible1) + versionDao.insert(repoId, packageName, versionId2, packageVersion2, isCompatible2) + + // get app versions from DB and assign them correctly + val appVersions = versionDao.getAppVersions(packageName).getOrFail() + assertEquals(2, appVersions.size) + val appVersion = if (versionId1 == appVersions[0].version.versionId) { + appVersions[0] + } else appVersions[1] + val appVersion2 = if (versionId2 == appVersions[0].version.versionId) { + appVersions[0] + } else appVersions[1] + + // check first version matches + assertEquals(getVersion1(repoId), appVersion.version) + val manifest = packageVersion1.manifest + assertEquals(manifest.usesPermission.toSet(), appVersion.usesPermission.toSet()) + assertEquals(manifest.usesPermissionSdk23.toSet(), appVersion.usesPermissionSdk23.toSet()) + assertEquals( + manifest.features.map { it.name }.toSet(), + appVersion.version.manifest.features?.toSet() + ) + + // check second version matches + assertEquals(getVersion2(repoId), appVersion2.version) + val manifest2 = packageVersion2.manifest + assertEquals(manifest2.usesPermission.toSet(), appVersion2.usesPermission.toSet()) + assertEquals(manifest2.usesPermissionSdk23.toSet(), + appVersion2.usesPermissionSdk23.toSet()) + assertEquals( + manifest.features.map { it.name }.toSet(), + appVersion.version.manifest.features?.toSet() + ) + + // delete app and check that all associated data also gets deleted + appDao.deleteAppMetadata(repoId, packageName) + assertEquals(0, versionDao.getAppVersions(repoId, packageName).size) + assertEquals(0, versionDao.getVersionedStrings(repoId, packageName).size) + } + + @Test + fun versionsOnlyFromEnabledRepo() { + // insert two versions into the same repo + val repoId = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId, packageName, getRandomMetadataV2()) + versionDao.insert(repoId, packageName, packageVersions, compatChecker) + assertEquals(2, versionDao.getAppVersions(packageName).getOrFail().size) + assertEquals(2, versionDao.getVersions(listOf(packageName)).size) + + // add another version into another repo + val repoId2 = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId2, packageName, getRandomMetadataV2()) + versionDao.insert(repoId2, packageName, versionId3, packageVersion3, true) + assertEquals(3, versionDao.getAppVersions(packageName).getOrFail().size) + assertEquals(3, versionDao.getVersions(listOf(packageName)).size) + + // disable second repo + repoDao.setRepositoryEnabled(repoId2, false) + + // now only two versions get returned + assertEquals(2, versionDao.getAppVersions(packageName).getOrFail().size) + assertEquals(2, versionDao.getVersions(listOf(packageName)).size) + } + + @Test + fun versionsSortedByVersionCode() { + // insert three versions into the same repo + val repoId = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId, packageName, getRandomMetadataV2()) + versionDao.insert(repoId, packageName, packageVersions, compatChecker) + versionDao.insert(repoId, packageName, versionId3, packageVersion3, true) + val versions1 = versionDao.getAppVersions(packageName).getOrFail() + val versions2 = versionDao.getVersions(listOf(packageName)) + assertEquals(3, versions1.size) + assertEquals(3, versions2.size) + + // check that they are sorted as expected + listOf( + packageVersion1.manifest.versionCode, + packageVersion2.manifest.versionCode, + packageVersion3.manifest.versionCode, + ).sortedDescending().forEachIndexed { i, versionCode -> + assertEquals(versionCode, versions1[i].version.manifest.versionCode) + assertEquals(versionCode, versions2[i].versionCode) + } + } + + @Test + fun getVersionsRespectsAppPrefsIgnore() { + // insert one version into the repo + val repoId = repoDao.insertOrReplace(getRandomRepo()) + val versionCode = Random.nextLong(1, Long.MAX_VALUE) + val packageVersion = getRandomPackageVersionV2(versionCode) + val versionId = packageVersion.file.sha256 + appDao.insert(repoId, packageName, getRandomMetadataV2()) + versionDao.insert(repoId, packageName, versionId, packageVersion, true) + assertEquals(1, versionDao.getVersions(listOf(packageName)).size) + + // default app prefs don't change result + var appPrefs = AppPrefs(packageName) + appPrefsDao.update(appPrefs) + assertEquals(1, versionDao.getVersions(listOf(packageName)).size) + + // ignore lower version code doesn't change result + appPrefs = appPrefs.toggleIgnoreVersionCodeUpdate(versionCode - 1) + appPrefsDao.update(appPrefs) + assertEquals(1, versionDao.getVersions(listOf(packageName)).size) + + // ignoring exact version code does change result + appPrefs = appPrefs.toggleIgnoreVersionCodeUpdate(versionCode) + appPrefsDao.update(appPrefs) + assertEquals(0, versionDao.getVersions(listOf(packageName)).size) + + // ignoring higher version code does change result + appPrefs = appPrefs.toggleIgnoreVersionCodeUpdate(versionCode + 1) + appPrefsDao.update(appPrefs) + assertEquals(0, versionDao.getVersions(listOf(packageName)).size) + + // ignoring all updates does change result + appPrefs = appPrefs.toggleIgnoreAllUpdates() + appPrefsDao.update(appPrefs) + assertEquals(0, versionDao.getVersions(listOf(packageName)).size) + + // not ignoring all updates brings back version + appPrefs = appPrefs.toggleIgnoreAllUpdates() + appPrefsDao.update(appPrefs) + assertEquals(1, versionDao.getVersions(listOf(packageName)).size) + } + + @Test + fun getVersionsConsidersOnlyGivenPackages() { + // insert two versions + val repoId = repoDao.insertOrReplace(getRandomRepo()) + appDao.insert(repoId, packageName, getRandomMetadataV2()) + versionDao.insert(repoId, packageName, packageVersions, compatChecker) + assertEquals(2, versionDao.getVersions(listOf(packageName)).size) + + // insert versions for a different package + val packageName2 = getRandomString() + appDao.insert(repoId, packageName2, getRandomMetadataV2()) + versionDao.insert(repoId, packageName2, packageVersions, compatChecker) + + // still only returns above versions + assertEquals(2, versionDao.getVersions(listOf(packageName)).size) + + // all versions are returned only if all packages are asked for + assertEquals(4, versionDao.getVersions(listOf(packageName, packageName2)).size) + } + +} diff --git a/libs/database/src/dbTest/java/org/fdroid/index/v1/IndexV1UpdaterTest.kt b/libs/database/src/dbTest/java/org/fdroid/index/v1/IndexV1UpdaterTest.kt new file mode 100644 index 000000000..7dadd8031 --- /dev/null +++ b/libs/database/src/dbTest/java/org/fdroid/index/v1/IndexV1UpdaterTest.kt @@ -0,0 +1,269 @@ +package org.fdroid.index.v1 + +import android.Manifest +import android.net.Uri +import androidx.arch.core.executor.testing.InstantTaskExecutorRule +import androidx.test.ext.junit.runners.AndroidJUnit4 +import io.mockk.Runs +import io.mockk.every +import io.mockk.just +import io.mockk.mockk +import org.fdroid.CompatibilityChecker +import org.fdroid.database.DbTest +import org.fdroid.database.Repository +import org.fdroid.database.TestUtils.getOrAwaitValue +import org.fdroid.database.TestUtils.getOrFail +import org.fdroid.download.Downloader +import org.fdroid.download.DownloaderFactory +import org.fdroid.index.IndexUpdateResult +import org.fdroid.index.SigningException +import org.fdroid.index.TempFileProvider +import org.fdroid.index.v2.ANTI_FEATURE_KNOWN_VULNERABILITY +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertIs +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue +import kotlin.test.fail + +@RunWith(AndroidJUnit4::class) +internal class IndexV1UpdaterTest : DbTest() { + + @get:Rule + var tmpFolder: TemporaryFolder = TemporaryFolder() + + @get:Rule + val instantTaskExecutorRule = InstantTaskExecutorRule() + + private val tempFileProvider: TempFileProvider = mockk() + private val downloaderFactory: DownloaderFactory = mockk() + private val downloader: Downloader = mockk() + private val compatibilityChecker: CompatibilityChecker = CompatibilityChecker { true } + private lateinit var indexUpdater: IndexV1Updater + + @Before + override fun createDb() { + super.createDb() + indexUpdater = IndexV1Updater( + database = db, + tempFileProvider = tempFileProvider, + downloaderFactory = downloaderFactory, + compatibilityChecker = compatibilityChecker, + ) + } + + @Test + fun testIndexV1Processing() { + val repoId = repoDao.insertEmptyRepo(TESTY_CANONICAL_URL) + val repo = repoDao.getRepository(repoId) ?: fail() + downloadIndex(repo, TESTY_JAR) + val result = indexUpdater.updateNewRepo(repo, TESTY_FINGERPRINT).noError() + assertIs(result) + + // repo got updated + val updatedRepo = repoDao.getRepository(repoId) ?: fail() + assertEquals(TESTY_CERT, updatedRepo.certificate) + assertEquals(TESTY_FINGERPRINT, updatedRepo.fingerprint) + + // some assertions ported from old IndexV1UpdaterTest + assertEquals(1, repoDao.getRepositories().size) + assertEquals(63, appDao.countApps()) + listOf("fake.app.one", "org.adaway", "This_does_not_exist").forEach { packageName -> + assertNull(appDao.getApp(packageName).getOrAwaitValue()) + } + appDao.getAppMetadata().forEach { app -> + val numVersions = versionDao.getVersions(listOf(app.packageName)).size + assertTrue(numVersions > 0) + } + assertEquals(1497639511824, updatedRepo.timestamp) + assertEquals(TESTY_CANONICAL_URL, updatedRepo.address) + assertEquals("non-public test repo", updatedRepo.repository.name.values.first()) + assertEquals(18, updatedRepo.version) + assertEquals("/icons/fdroid-icon.png", updatedRepo.repository.icon?.values?.first()?.name) + val description = "This is a repository of apps to be used with F-Droid. " + + "Applications in this repository are either official binaries built " + + "by the original application developers, or are binaries built " + + "from source by the admin of f-droid.org using the tools on " + + "https://gitlab.com/u/fdroid. " + assertEquals(description, updatedRepo.repository.description.values.first()) + assertEquals( + setOf(TESTY_CANONICAL_URL, "http://frkcchxlcvnb4m5a.onion/fdroid/repo"), + updatedRepo.mirrors.map { it.url }.toSet(), + ) + + // Make sure the per-apk anti features which are new in index v1 get added correctly. + val wazeVersion = versionDao.getVersions(listOf("com.waze")).find { + it.manifest.versionCode == 1019841L + } + assertNotNull(wazeVersion) + assertEquals(setOf(ANTI_FEATURE_KNOWN_VULNERABILITY), wazeVersion.antiFeatures?.keys) + + val protoVersion = versionDao.getAppVersions("io.proto.player").getOrFail().find { + it.version.versionCode == 1110L + } + assertNotNull(protoVersion) + assertEquals("/io.proto.player-1.apk", protoVersion.version.file.name) + val perms = protoVersion.usesPermission.map { it.name } + assertTrue(perms.contains(Manifest.permission.READ_EXTERNAL_STORAGE)) + assertTrue(perms.contains(Manifest.permission.WRITE_EXTERNAL_STORAGE)) + assertFalse(perms.contains(Manifest.permission.READ_CALENDAR)) + val icon = appDao.getApp("com.autonavi.minimap").getOrFail()?.icon?.values?.first()?.name + assertEquals("/com.autonavi.minimap/en-US/icon.png", icon) + + // update again and get unchanged + downloadIndex(updatedRepo, TESTY_JAR) + val result2 = indexUpdater.update(updatedRepo).noError() + assertIs(result2) + } + + @Test + fun testIndexV1WithWrongCert() { + val repoId = repoDao.insertEmptyRepo(TESTY_CANONICAL_URL) + val repo = repoDao.getRepository(repoId) ?: fail() + downloadIndex(repo, TESTY_JAR) + val result = indexUpdater.updateNewRepo(repo, "not the right fingerprint") + assertIs(result) + assertIs(result.e) + + // check that the DB transaction was rolled back and the DB wasn't changed + assertEquals(repo, repoDao.getRepository(repoId) ?: fail()) + assertEquals(0, appDao.countApps()) + assertEquals(0, versionDao.countAppVersions()) + } + + @Test + fun testIndexV1WithOldTimestamp() { + val repoId = repoDao.insertEmptyRepo(TESTY_CANONICAL_URL) + val repo = repoDao.getRepository(repoId) ?: fail() + val futureRepo = + repo.copy(repository = repo.repository.copy(timestamp = System.currentTimeMillis())) + downloadIndex(futureRepo, TESTY_JAR) + val result = indexUpdater.updateNewRepo(futureRepo, TESTY_FINGERPRINT) + assertIs(result) + assertIs(result.e) + assertFalse((result.e as OldIndexException).isSameTimestamp) + } + + @Test + fun testIndexV1WithCorruptAppPackageName() { + val result = testBadTestyJar("testy.at.or.at_corrupt_app_package_name_index-v1.jar") + assertIs(result) + } + + @Test + fun testIndexV1WithCorruptPackageName() { + val result = testBadTestyJar("testy.at.or.at_corrupt_package_name_index-v1.jar") + assertIs(result) + } + + @Test + fun testIndexV1WithBadTestyJarNoManifest() { + val result = testBadTestyJar("testy.at.or.at_no-MANIFEST.MF_index-v1.jar") + assertIs(result) + assertIs(result.e) + } + + @Test + fun testIndexV1WithBadTestyJarNoSigningCert() { + val result = testBadTestyJar("testy.at.or.at_no-.RSA_index-v1.jar") + assertIs(result) + } + + @Test + fun testIndexV1WithBadTestyJarNoSignature() { + val result = testBadTestyJar("testy.at.or.at_no-.SF_index-v1.jar") + assertIs(result) + } + + @Test + fun testIndexV1WithBadTestyJarNoSignatureFiles() { + val result = testBadTestyJar("testy.at.or.at_no-signature_index-v1.jar") + assertIs(result) + assertIs(result.e) + } + + @Suppress("DEPRECATION") + private fun downloadIndex(repo: Repository, jar: String) { + val uri = Uri.parse("${repo.address}/index-v1.jar") + + val jarFile = tmpFolder.newFile() + assets.open(jar).use { inputStream -> + jarFile.outputStream().use { inputStream.copyTo(it) } + } + every { tempFileProvider.createTempFile() } returns jarFile + every { + downloaderFactory.createWithTryFirstMirror(repo, uri, jarFile) + } returns downloader + every { downloader.cacheTag = null } just Runs + every { downloader.download() } just Runs + every { downloader.hasChanged() } returns true + every { downloader.cacheTag } returns null + } + + private fun testBadTestyJar(jar: String): IndexUpdateResult { + val repoId = repoDao.insertEmptyRepo("http://example.org") + val repo = repoDao.getRepository(repoId) ?: fail() + downloadIndex(repo, jar) + return indexUpdater.updateNewRepo(repo, null) + } + + /** + * Easier for debugging, if we throw the index error. + */ + private fun IndexUpdateResult.noError(): IndexUpdateResult { + if (this is IndexUpdateResult.Error) throw e + return this + } + +} + +private const val TESTY_CANONICAL_URL = "http://testy.at.or.at/fdroid/repo" +private const val TESTY_JAR = "testy.at.or.at_index-v1.jar" +private const val TESTY_FINGERPRINT = + "818e469465f96b704e27be2fee4c63ab9f83ddf30e7a34c7371a4728d83b0bc1" +private const val TESTY_CERT = "308204e1308202c9a0030201020204483450fa300d06092a864886f70d01010b" + + "050030213110300e060355040b1307462d44726f6964310d300b060355040313" + + "04736f7661301e170d3136303832333133333131365a170d3434303130393133" + + "333131365a30213110300e060355040b1307462d44726f6964310d300b060355" + + "04031304736f766130820222300d06092a864886f70d01010105000382020f00" + + "3082020a0282020100dfdcd120f3ab224999dddf4ea33ea588d295e4d7130bef" + + "48c143e9d76e5c0e0e9e5d45e64208e35feebc79a83f08939dd6a343b7d1e217" + + "9930a105a1249ccd36d88ff3feffc6e4dc53dae0163a7876dd45ecc1ddb0adf5" + + "099aa56c1a84b52affcd45d0711ffa4de864f35ac0333ebe61ea8673eeda35a8" + + "8f6af678cc4d0f80b089338ac8f2a8279a64195c611d19445cab3fd1a020afed" + + "9bd739bb95142fb2c00a8f847db5ef3325c814f8eb741bacf86ed3907bfe6e45" + + "64d2de5895df0c263824e0b75407589bae2d3a4666c13b92102d8781a8ee9bb4" + + "a5a1a78c4a9c21efdaf5584da42e84418b28f5a81d0456a3dc5b420991801e6b" + + "21e38c99bbe018a5b2d690894a114bc860d35601416aa4dc52216aff8a288d47" + + "75cddf8b72d45fd2f87303a8e9c0d67e442530be28eaf139894337266e0b33d5" + + "7f949256ab32083bcc545bc18a83c9ab8247c12aea037e2b68dee31c734cb1f0" + + "4f241d3b94caa3a2b258ffaf8e6eae9fbbe029a934dc0a0859c5f12033481269" + + "3a1c09352340a39f2a678dbc1afa2a978bfee43afefcb7e224a58af2f3d647e5" + + "745db59061236b8af6fcfd93b3602f9e456978534f3a7851e800071bf56da804" + + "01c81d91c45f82568373af0576b1cc5eef9b85654124b6319770be3cdba3fbeb" + + "e3715e8918fb6c8966624f3d0e815effac3d2ee06dd34ab9c693218b2c7c06ba" + + "99d6b74d4f17b8c3cb0203010001a321301f301d0603551d0e04160414d62bee" + + "9f3798509546acc62eb1de14b08b954d4f300d06092a864886f70d01010b0500" + + "0382020100743f7c5692085895f9d1fffad390fb4202c15f123ed094df259185" + + "960fd6dadf66cb19851070f180297bba4e6996a4434616573b375cfee94fee73" + + "a4505a7ec29136b7e6c22e6436290e3686fe4379d4e3140ec6a08e70cfd3ed5b" + + "634a5eb5136efaaabf5f38e0432d3d79568a556970b8cfba2972f5d23a3856d8" + + "a981b9e9bbbbb88f35e708bde9cbc5f681cbd974085b9da28911296fe2579fa6" + + "4bbe9fa0b93475a7a8db051080b0c5fade0d1c018e7858cd4cbe95145b0620e2" + + "f632cbe0f8af9cbf22e2fdaa72245ae31b0877b07181cc69dd2df74454251d8d" + + "e58d25e76354abe7eb690f22e59b08795a8f2c98c578e0599503d90859276340" + + "72c82c9f82abd50fd12b8fd1a9d1954eb5cc0b4cfb5796b5aaec0356643b4a65" + + "a368442d92ef94edd3ac6a2b7fe3571b8cf9f462729228aab023ef9183f73792" + + "f5379633ccac51079177d604c6bc1873ada6f07d8da6d68c897e88a5fa5d63fd" + + "b8df820f46090e0716e7562dd3c140ba279a65b996f60addb0abe29d4bf2f5ab" + + "e89480771d492307b926d91f02f341b2148502903c43d40f3c6c86a811d06071" + + "1f0698b384acdcc0add44eb54e42962d3d041accc715afd49407715adc09350c" + + "b55e8d9281a3b0b6b5fcd91726eede9b7c8b13afdebb2c2b377629595f1096ba" + + "62fb14946dbac5f3c5f0b4e5b712e7acc7dcf6c46cdc5e6d6dfdeee55a0c92c2" + + "d70f080ac6" diff --git a/libs/database/src/dbTest/java/org/fdroid/index/v2/IndexV2UpdaterTest.kt b/libs/database/src/dbTest/java/org/fdroid/index/v2/IndexV2UpdaterTest.kt new file mode 100644 index 000000000..d24a5d6c8 --- /dev/null +++ b/libs/database/src/dbTest/java/org/fdroid/index/v2/IndexV2UpdaterTest.kt @@ -0,0 +1,294 @@ +package org.fdroid.index.v2 + +import android.net.Uri +import androidx.test.ext.junit.runners.AndroidJUnit4 +import io.mockk.Runs +import io.mockk.every +import io.mockk.just +import io.mockk.mockk +import org.fdroid.CompatibilityChecker +import org.fdroid.database.DbTest +import org.fdroid.database.Repository +import org.fdroid.database.TestUtils.assertTimestampRecent +import org.fdroid.download.Downloader +import org.fdroid.download.DownloaderFactory +import org.fdroid.index.IndexFormatVersion.TWO +import org.fdroid.index.IndexUpdateResult +import org.fdroid.index.SigningException +import org.fdroid.index.TempFileProvider +import org.fdroid.test.TestDataEntryV2 +import org.fdroid.test.TestDataMaxV2 +import org.fdroid.test.TestDataMidV2 +import org.fdroid.test.TestDataMinV2 +import org.fdroid.test.VerifierConstants.CERTIFICATE +import org.fdroid.test.VerifierConstants.FINGERPRINT +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import kotlin.test.assertEquals +import kotlin.test.assertNull +import kotlin.test.assertTrue +import kotlin.test.fail + +@RunWith(AndroidJUnit4::class) +internal class IndexV2UpdaterTest : DbTest() { + + @get:Rule + var tmpFolder: TemporaryFolder = TemporaryFolder() + + private val tempFileProvider: TempFileProvider = mockk() + private val downloaderFactory: DownloaderFactory = mockk() + private val downloader: Downloader = mockk() + private val compatibilityChecker: CompatibilityChecker = CompatibilityChecker { true } + private lateinit var indexUpdater: IndexV2Updater + + @Before + override fun createDb() { + super.createDb() + indexUpdater = IndexV2Updater( + database = db, + tempFileProvider = tempFileProvider, + downloaderFactory = downloaderFactory, + compatibilityChecker = compatibilityChecker, + ) + } + + @Test + fun testFullIndexEmptyToMin() { + val repoId = repoDao.insertEmptyRepo("http://example.org") + val repo = prepareUpdate( + repoId = repoId, + entryPath = "diff-empty-min/entry.jar", + jsonPath = "index-min-v2.json", + entryFileV2 = TestDataEntryV2.emptyToMin.index + ) + val result = indexUpdater.updateNewRepo(repo, FINGERPRINT).noError() + assertEquals(IndexUpdateResult.Processed, result) + assertDbEquals(repoId, TestDataMinV2.index) + + // check that certificate and format version got entered + val updatedRepo = repoDao.getRepository(repoId) ?: fail() + assertEquals(TWO, updatedRepo.formatVersion) + assertEquals(CERTIFICATE, updatedRepo.certificate) + assertTimestampRecent(repoDao.getRepository(repoId)?.lastUpdated) + } + + @Test + fun testFullIndexEmptyToMid() { + val repoId = repoDao.insertEmptyRepo("http://example.org") + val repo = prepareUpdate( + repoId = repoId, + entryPath = "diff-empty-mid/entry.jar", + jsonPath = "index-mid-v2.json", + entryFileV2 = TestDataEntryV2.emptyToMid.index + ) + val result = indexUpdater.updateNewRepo(repo, FINGERPRINT).noError() + assertEquals(IndexUpdateResult.Processed, result) + assertDbEquals(repoId, TestDataMidV2.index) + assertTimestampRecent(repoDao.getRepository(repoId)?.lastUpdated) + } + + @Test + fun testFullIndexEmptyToMax() { + val repoId = repoDao.insertEmptyRepo("http://example.org") + val repo = prepareUpdate( + repoId = repoId, + entryPath = "diff-empty-max/entry.jar", + jsonPath = "index-max-v2.json", + entryFileV2 = TestDataEntryV2.emptyToMax.index + ) + val result = indexUpdater.updateNewRepo(repo, FINGERPRINT).noError() + assertEquals(IndexUpdateResult.Processed, result) + assertDbEquals(repoId, TestDataMaxV2.index) + assertTimestampRecent(repoDao.getRepository(repoId)?.lastUpdated) + } + + @Test + fun testDiffMinToMid() { + val repoId = streamIndexV2IntoDb("index-min-v2.json") + val repo = prepareUpdate( + repoId = repoId, + entryPath = "diff-empty-mid/entry.jar", + jsonPath = "diff-empty-mid/42.json", + entryFileV2 = TestDataEntryV2.emptyToMid.diffs["42"] ?: fail() + ) + val result = indexUpdater.update(repo).noError() + assertEquals(IndexUpdateResult.Processed, result) + assertDbEquals(repoId, TestDataMidV2.index) + assertTimestampRecent(repoDao.getRepository(repoId)?.lastUpdated) + } + + @Test + fun testDiffEmptyToMin() { + val repoId = streamIndexV2IntoDb("index-empty-v2.json") + repoDao.updateRepository(repoId, CERTIFICATE) + val repo = prepareUpdate( + repoId = repoId, + entryPath = "diff-empty-min/entry.jar", + jsonPath = "diff-empty-min/23.json", + entryFileV2 = TestDataEntryV2.emptyToMin.diffs["23"] ?: fail() + ) + val result = indexUpdater.update(repo).noError() + assertEquals(IndexUpdateResult.Processed, result) + assertDbEquals(repoId, TestDataMinV2.index) + assertTimestampRecent(repoDao.getRepository(repoId)?.lastUpdated) + } + + @Test + fun testDiffMidToMax() { + val repoId = streamIndexV2IntoDb("index-mid-v2.json") + repoDao.updateRepository(repoId, CERTIFICATE) + val repo = prepareUpdate( + repoId = repoId, + entryPath = "diff-empty-max/entry.jar", + jsonPath = "diff-empty-max/1337.json", + entryFileV2 = TestDataEntryV2.emptyToMax.diffs["1337"] ?: fail() + ) + val result = indexUpdater.update(repo).noError() + assertEquals(IndexUpdateResult.Processed, result) + assertDbEquals(repoId, TestDataMaxV2.index) + assertTimestampRecent(repoDao.getRepository(repoId)?.lastUpdated) + } + + @Test + fun testSameTimestampUnchanged() { + val repoId = streamIndexV2IntoDb("index-min-v2.json") + repoDao.updateRepository(repoId, CERTIFICATE) + val repo = prepareUpdate( + repoId = repoId, + entryPath = "diff-empty-min/entry.jar", + jsonPath = "diff-empty-min/23.json", + entryFileV2 = TestDataEntryV2.emptyToMin.diffs["23"] ?: fail() + ) + val result = indexUpdater.update(repo).noError() + assertEquals(IndexUpdateResult.Unchanged, result) + assertDbEquals(repoId, TestDataMinV2.index) + assertNull(repoDao.getRepository(repoId)?.lastUpdated) + } + + @Test + fun testHigherTimestampUnchanged() { + val repoId = streamIndexV2IntoDb("index-mid-v2.json") + repoDao.updateRepository(repoId, CERTIFICATE) + val repo = prepareUpdate( + repoId = repoId, + entryPath = "diff-empty-min/entry.jar", + jsonPath = "diff-empty-min/23.json", + entryFileV2 = TestDataEntryV2.emptyToMin.diffs["23"] ?: fail() + ) + val result = indexUpdater.update(repo).noError() + assertEquals(IndexUpdateResult.Unchanged, result) + assertDbEquals(repoId, TestDataMidV2.index) + } + + @Test + fun testNoDiffFoundIndexFallback() { + val repoId = streamIndexV2IntoDb("index-empty-v2.json") + repoDao.updateRepository(repoId, CERTIFICATE) + // fake timestamp of internal repo, so we will fail to find a diff in entry.json + val newRepo = repoDao.getRepository(repoId)?.repository?.copy(timestamp = 22) ?: fail() + repoDao.updateRepository(newRepo) + val repo = prepareUpdate( + repoId = repoId, + entryPath = "diff-empty-min/entry.jar", + jsonPath = "index-min-v2.json", + entryFileV2 = TestDataEntryV2.emptyToMin.index + ) + val result = indexUpdater.update(repo).noError() + assertEquals(IndexUpdateResult.Processed, result) + assertDbEquals(repoId, TestDataMinV2.index) + } + + @Test + fun testWrongFingerprint() { + val repoId = repoDao.insertEmptyRepo("http://example.org") + val repo = prepareUpdate( + repoId = repoId, + entryPath = "diff-empty-min/entry.jar", + jsonPath = "index-min-v2.json", + entryFileV2 = TestDataEntryV2.emptyToMin.index + ) + val result = indexUpdater.updateNewRepo(repo, "wrong fingerprint") + assertTrue(result is IndexUpdateResult.Error) + assertTrue(result.e is SigningException) + } + + @Test + fun testNormalUpdateOnRepoWithMissingFingerprint() { + val repoId = repoDao.insertEmptyRepo("http://example.org") + val repo = prepareUpdate( + repoId = repoId, + entryPath = "diff-empty-min/entry.jar", + jsonPath = "index-min-v2.json", + entryFileV2 = TestDataEntryV2.emptyToMin.index + ) + val result = indexUpdater.update(repo) + assertTrue(result is IndexUpdateResult.Error) + assertTrue(result.e is IllegalArgumentException) + } + + /** + * Ensures that a v1 repo can't use a diff when upgrading to v1, + * but must use a full index update. + */ + @Test + fun testV1ToV2ForcesFullUpdateEvenIfDiffExists() { + val repoId = streamIndexV1IntoDb("index-min-v1.json") + val repo = prepareUpdate( + repoId = repoId, + entryPath = "diff-empty-mid/entry.jar", + jsonPath = "index-mid-v2.json", + entryFileV2 = TestDataEntryV2.emptyToMid.index, + ) + val result = indexUpdater.update(repo).noError() + assertEquals(IndexUpdateResult.Processed, result) + assertDbEquals(repoId, TestDataMidV2.index) + + // check that format version got upgraded + val updatedRepo = repoDao.getRepository(repoId) ?: fail() + assertEquals(TWO, updatedRepo.formatVersion) + } + + private fun prepareUpdate( + repoId: Long, + entryPath: String, + jsonPath: String, + entryFileV2: EntryFileV2, + ): Repository { + val entryFile = tmpFolder.newFile() + val indexFile = tmpFolder.newFile() + val repo = repoDao.getRepository(repoId) ?: fail() + val entryUri = Uri.parse("${repo.address}/entry.jar") + val indexUri = Uri.parse("${repo.address}/${entryFileV2.name.trimStart('/')}") + + assets.open(entryPath).use { inputStream -> + entryFile.outputStream().use { inputStream.copyTo(it) } + } + assets.open(jsonPath).use { inputStream -> + indexFile.outputStream().use { inputStream.copyTo(it) } + } + + every { tempFileProvider.createTempFile() } returnsMany listOf(entryFile, indexFile) + every { + downloaderFactory.createWithTryFirstMirror(repo, entryUri, entryFile) + } returns downloader + every { downloader.download(-1) } just Runs + every { + downloaderFactory.createWithTryFirstMirror(repo, indexUri, indexFile) + } returns downloader + every { downloader.download(entryFileV2.size, entryFileV2.sha256) } just Runs + + return repo + } + + /** + * Easier for debugging, if we throw the index error. + */ + private fun IndexUpdateResult.noError(): IndexUpdateResult { + if (this is IndexUpdateResult.Error) throw e + return this + } + +} diff --git a/libs/database/src/main/AndroidManifest.xml b/libs/database/src/main/AndroidManifest.xml new file mode 100644 index 000000000..07beb0bec --- /dev/null +++ b/libs/database/src/main/AndroidManifest.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/libs/database/src/main/java/org/fdroid/database/App.kt b/libs/database/src/main/java/org/fdroid/database/App.kt new file mode 100644 index 000000000..ae6a1cbf1 --- /dev/null +++ b/libs/database/src/main/java/org/fdroid/database/App.kt @@ -0,0 +1,463 @@ +package org.fdroid.database + +import android.content.res.Resources +import androidx.core.os.ConfigurationCompat.getLocales +import androidx.core.os.LocaleListCompat +import androidx.room.ColumnInfo +import androidx.room.DatabaseView +import androidx.room.Embedded +import androidx.room.Entity +import androidx.room.ForeignKey +import androidx.room.Fts4 +import androidx.room.Ignore +import androidx.room.Relation +import org.fdroid.LocaleChooser.getBestLocale +import org.fdroid.database.Converters.fromStringToMapOfLocalizedTextV2 +import org.fdroid.index.v2.FileV2 +import org.fdroid.index.v2.LocalizedFileListV2 +import org.fdroid.index.v2.LocalizedFileV2 +import org.fdroid.index.v2.LocalizedTextV2 +import org.fdroid.index.v2.MetadataV2 +import org.fdroid.index.v2.Screenshots + +public interface MinimalApp { + public val repoId: Long + public val packageName: String + public val name: String? + public val summary: String? + public fun getIcon(localeList: LocaleListCompat): FileV2? +} + +/** + * The detailed metadata for an app. + * Almost all fields are optional. + * This largely represents [MetadataV2] in a database table. + */ +@Entity( + tableName = AppMetadata.TABLE, + primaryKeys = ["repoId", "packageName"], + foreignKeys = [ForeignKey( + entity = CoreRepository::class, + parentColumns = ["repoId"], + childColumns = ["repoId"], + onDelete = ForeignKey.CASCADE, + )], +) +public data class AppMetadata( + public val repoId: Long, + public val packageName: String, + public val added: Long, + public val lastUpdated: Long, + public val name: LocalizedTextV2? = null, + public val summary: LocalizedTextV2? = null, + public val description: LocalizedTextV2? = null, + public val localizedName: String? = null, + public val localizedSummary: String? = null, + public val webSite: String? = null, + public val changelog: String? = null, + public val license: String? = null, + public val sourceCode: String? = null, + public val issueTracker: String? = null, + public val translation: String? = null, + public val preferredSigner: String? = null, + public val video: LocalizedTextV2? = null, + public val authorName: String? = null, + public val authorEmail: String? = null, + public val authorWebSite: String? = null, + public val authorPhone: String? = null, + public val donate: List? = null, + public val liberapayID: String? = null, + public val liberapay: String? = null, + public val openCollective: String? = null, + public val bitcoin: String? = null, + public val litecoin: String? = null, + public val flattrID: String? = null, + public val categories: List? = null, + /** + * Whether the app is compatible with the current device. + * This value will be computed and is always false until that happened. + * So to always get correct data, this MUST happen within the same transaction + * that adds the [AppMetadata]. + */ + public val isCompatible: Boolean, +) { + internal companion object { + const val TABLE = "AppMetadata" + } +} + +internal fun MetadataV2.toAppMetadata( + repoId: Long, + packageName: String, + isCompatible: Boolean = false, + locales: LocaleListCompat = getLocales(Resources.getSystem().configuration), +) = AppMetadata( + repoId = repoId, + packageName = packageName, + added = added, + lastUpdated = lastUpdated, + name = name, + summary = summary, + description = description, + localizedName = name.getBestLocale(locales), + localizedSummary = summary.getBestLocale(locales), + webSite = webSite, + changelog = changelog, + license = license, + sourceCode = sourceCode, + issueTracker = issueTracker, + translation = translation, + preferredSigner = preferredSigner, + video = video, + authorName = authorName, + authorEmail = authorEmail, + authorWebSite = authorWebSite, + authorPhone = authorPhone, + donate = donate, + liberapayID = liberapayID, + liberapay = liberapay, + openCollective = openCollective, + bitcoin = bitcoin, + litecoin = litecoin, + flattrID = flattrID, + categories = categories, + isCompatible = isCompatible, +) + +@Entity(tableName = AppMetadataFts.TABLE) +@Fts4(contentEntity = AppMetadata::class) +internal data class AppMetadataFts( + val repoId: Long, + val packageName: String, + @ColumnInfo(name = "localizedName") + val name: String? = null, + @ColumnInfo(name = "localizedSummary") + val summary: String? = null, +) { + internal companion object { + const val TABLE = "AppMetadataFts" + } +} + +/** + * A class to represent all data of an App. + * It combines the metadata and localized filed such as icons and screenshots. + */ +public data class App internal constructor( + @Embedded public val metadata: AppMetadata, + @Relation( + parentColumn = "packageName", + entityColumn = "packageName", + ) + private val localizedFiles: List? = null, + @Relation( + parentColumn = "packageName", + entityColumn = "packageName", + ) + private val localizedFileLists: List? = null, +) : MinimalApp { + public override val repoId: Long get() = metadata.repoId + override val packageName: String get() = metadata.packageName + internal val icon: LocalizedFileV2? get() = getLocalizedFile("icon") + internal val featureGraphic: LocalizedFileV2? get() = getLocalizedFile("featureGraphic") + internal val promoGraphic: LocalizedFileV2? get() = getLocalizedFile("promoGraphic") + internal val tvBanner: LocalizedFileV2? get() = getLocalizedFile("tvBanner") + internal val screenshots: Screenshots? + get() = if (localizedFileLists.isNullOrEmpty()) null else Screenshots( + phone = getLocalizedFileList("phone"), + sevenInch = getLocalizedFileList("sevenInch"), + tenInch = getLocalizedFileList("tenInch"), + wear = getLocalizedFileList("wear"), + tv = getLocalizedFileList("tv"), + ).takeIf { !it.isNull } + + private fun getLocalizedFile(type: String): LocalizedFileV2? { + return localizedFiles?.filter { localizedFile -> + localizedFile.repoId == metadata.repoId && localizedFile.type == type + }?.toLocalizedFileV2() + } + + private fun getLocalizedFileList(type: String): LocalizedFileListV2? { + val map = HashMap>() + localizedFileLists?.iterator()?.forEach { file -> + if (file.repoId != metadata.repoId || file.type != type) return@forEach + val list = map.getOrPut(file.locale) { ArrayList() } as ArrayList + list.add(FileV2( + name = file.name, + sha256 = file.sha256, + size = file.size, + )) + } + return map.ifEmpty { null } + } + + public override val name: String? get() = metadata.localizedName + public override val summary: String? get() = metadata.localizedSummary + public fun getDescription(localeList: LocaleListCompat): String? = + metadata.description.getBestLocale(localeList) + + public fun getVideo(localeList: LocaleListCompat): String? = + metadata.video.getBestLocale(localeList) + + public override fun getIcon(localeList: LocaleListCompat): FileV2? = + icon.getBestLocale(localeList) + + public fun getFeatureGraphic(localeList: LocaleListCompat): FileV2? = + featureGraphic.getBestLocale(localeList) + + public fun getPromoGraphic(localeList: LocaleListCompat): FileV2? = + promoGraphic.getBestLocale(localeList) + + public fun getTvBanner(localeList: LocaleListCompat): FileV2? = + tvBanner.getBestLocale(localeList) + + public fun getPhoneScreenshots(localeList: LocaleListCompat): List = + screenshots?.phone.getBestLocale(localeList) ?: emptyList() + + public fun getSevenInchScreenshots(localeList: LocaleListCompat): List = + screenshots?.sevenInch.getBestLocale(localeList) ?: emptyList() + + public fun getTenInchScreenshots(localeList: LocaleListCompat): List = + screenshots?.tenInch.getBestLocale(localeList) ?: emptyList() + + public fun getTvScreenshots(localeList: LocaleListCompat): List = + screenshots?.tv.getBestLocale(localeList) ?: emptyList() + + public fun getWearScreenshots(localeList: LocaleListCompat): List = + screenshots?.wear.getBestLocale(localeList) ?: emptyList() +} + +/** + * A lightweight variant of [App] with minimal data, usually used to provide an overview of apps + * without going into all details that get presented on a dedicated screen. + * The reduced data footprint helps with fast loading many items at once. + * + * It includes [antiFeatureKeys] so some clients can apply filters to them. + */ +public data class AppOverviewItem internal constructor( + public override val repoId: Long, + public override val packageName: String, + public val added: Long, + public val lastUpdated: Long, + @ColumnInfo(name = "localizedName") + public override val name: String? = null, + @ColumnInfo(name = "localizedSummary") + public override val summary: String? = null, + internal val antiFeatures: Map? = null, + @Relation( + parentColumn = "packageName", + entityColumn = "packageName", + ) + internal val localizedIcon: List? = null, +) : MinimalApp { + public override fun getIcon(localeList: LocaleListCompat): FileV2? { + return localizedIcon?.filter { icon -> + icon.repoId == repoId + }?.toLocalizedFileV2().getBestLocale(localeList) + } + + public val antiFeatureKeys: List get() = antiFeatures?.map { it.key } ?: emptyList() +} + +/** + * Similar to [AppOverviewItem], this is a lightweight version of [App] + * meant to show a list of apps. + * + * There is additional information about [installedVersionCode] and [installedVersionName] + * as well as [isCompatible]. + * + * It includes [antiFeatureKeys] of the highest version, so some clients can apply filters to them. + */ +public data class AppListItem internal constructor( + public override val repoId: Long, + public override val packageName: String, + @ColumnInfo(name = "localizedName") + public override val name: String? = null, + @ColumnInfo(name = "localizedSummary") + public override val summary: String? = null, + internal val antiFeatures: String?, + @Relation( + parentColumn = "packageName", + entityColumn = "packageName", + ) + internal val localizedIcon: List?, + /** + * If true, this this app has at least one version that is compatible with this device. + */ + public val isCompatible: Boolean, + /** + * The name of the installed version, null if this app is not installed. + */ + @get:Ignore + public val installedVersionName: String? = null, + /** + * The version code of the installed version, null if this app is not installed. + */ + @get:Ignore + public val installedVersionCode: Long? = null, +) : MinimalApp { + @delegate:Ignore + private val antiFeaturesDecoded by lazy { + fromStringToMapOfLocalizedTextV2(antiFeatures) + } + + public override fun getIcon(localeList: LocaleListCompat): FileV2? { + return localizedIcon?.filter { icon -> + icon.repoId == repoId + }?.toLocalizedFileV2().getBestLocale(localeList) + } + + public val antiFeatureKeys: List + get() = antiFeaturesDecoded?.map { it.key } ?: emptyList() + + public fun getAntiFeatureReason(antiFeatureKey: String, localeList: LocaleListCompat): String? { + return antiFeaturesDecoded?.get(antiFeatureKey)?.getBestLocale(localeList) + } +} + +/** + * An app that has an [update] available. + * It is meant to display available updates in the UI. + */ +public data class UpdatableApp internal constructor( + public override val repoId: Long, + public override val packageName: String, + public val installedVersionCode: Long, + public val update: AppVersion, + /** + * If true, this is not necessarily an update (contrary to the class name), + * but an app with the `KnownVuln` anti-feature. + */ + public val hasKnownVulnerability: Boolean, + public override val name: String? = null, + public override val summary: String? = null, + internal val localizedIcon: List? = null, +) : MinimalApp { + public override fun getIcon(localeList: LocaleListCompat): FileV2? { + return localizedIcon?.filter { icon -> + icon.repoId == update.repoId + }?.toLocalizedFileV2().getBestLocale(localeList) + } +} + +internal interface IFile { + val type: String + val locale: String + val name: String + val sha256: String? + val size: Long? +} + +@Entity( + tableName = LocalizedFile.TABLE, + primaryKeys = ["repoId", "packageName", "type", "locale"], + foreignKeys = [ForeignKey( + entity = AppMetadata::class, + parentColumns = ["repoId", "packageName"], + childColumns = ["repoId", "packageName"], + onDelete = ForeignKey.CASCADE, + )], +) +internal data class LocalizedFile( + val repoId: Long, + val packageName: String, + override val type: String, + override val locale: String, + override val name: String, + override val sha256: String? = null, + override val size: Long? = null, +) : IFile { + internal companion object { + const val TABLE = "LocalizedFile" + } +} + +internal fun LocalizedFileV2.toLocalizedFile( + repoId: Long, + packageName: String, + type: String, +): List = map { (locale, file) -> + LocalizedFile( + repoId = repoId, + packageName = packageName, + type = type, + locale = locale, + name = file.name, + sha256 = file.sha256, + size = file.size, + ) +} + +internal fun List.toLocalizedFileV2(): LocalizedFileV2? = associate { file -> + file.locale to FileV2( + name = file.name, + sha256 = file.sha256, + size = file.size, + ) +}.ifEmpty { null } + +// We can't restrict this query further (e.g. only from enabled repos or max weight), +// because we are using this via @Relation on packageName for specific repos. +// When filtering the result for only the repoId we are interested in, we'd get no icons. +@DatabaseView(viewName = LocalizedIcon.TABLE, + value = "SELECT * FROM ${LocalizedFile.TABLE} WHERE type='icon'") +internal data class LocalizedIcon( + val repoId: Long, + val packageName: String, + override val type: String, + override val locale: String, + override val name: String, + override val sha256: String? = null, + override val size: Long? = null, +) : IFile { + internal companion object { + const val TABLE = "LocalizedIcon" + } +} + +@Entity( + tableName = LocalizedFileList.TABLE, + primaryKeys = ["repoId", "packageName", "type", "locale", "name"], + foreignKeys = [ForeignKey( + entity = AppMetadata::class, + parentColumns = ["repoId", "packageName"], + childColumns = ["repoId", "packageName"], + onDelete = ForeignKey.CASCADE, + )], +) +internal data class LocalizedFileList( + val repoId: Long, + val packageName: String, + val type: String, + val locale: String, + val name: String, + val sha256: String? = null, + val size: Long? = null, +) { + internal companion object { + const val TABLE = "LocalizedFileList" + } +} + +internal fun LocalizedFileListV2.toLocalizedFileList( + repoId: Long, + packageName: String, + type: String, +): List = flatMap { (locale, files) -> + files.map { file -> file.toLocalizedFileList(repoId, packageName, type, locale) } +} + +internal fun FileV2.toLocalizedFileList( + repoId: Long, + packageName: String, + type: String, + locale: String, +) = LocalizedFileList( + repoId = repoId, + packageName = packageName, + type = type, + locale = locale, + name = name, + sha256 = sha256, + size = size, +) diff --git a/libs/database/src/main/java/org/fdroid/database/AppDao.kt b/libs/database/src/main/java/org/fdroid/database/AppDao.kt new file mode 100644 index 000000000..211980706 --- /dev/null +++ b/libs/database/src/main/java/org/fdroid/database/AppDao.kt @@ -0,0 +1,541 @@ +package org.fdroid.database + +import android.content.pm.PackageInfo +import android.content.pm.PackageManager +import android.content.res.Resources +import androidx.annotation.VisibleForTesting +import androidx.core.os.ConfigurationCompat.getLocales +import androidx.core.os.LocaleListCompat +import androidx.lifecycle.LiveData +import androidx.lifecycle.map +import androidx.room.Dao +import androidx.room.Insert +import androidx.room.OnConflictStrategy.REPLACE +import androidx.room.Query +import androidx.room.RoomWarnings.CURSOR_MISMATCH +import androidx.room.Transaction +import androidx.room.Update +import kotlinx.serialization.SerializationException +import kotlinx.serialization.json.JsonNull +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.decodeFromJsonElement +import org.fdroid.LocaleChooser.getBestLocale +import org.fdroid.database.AppListSortOrder.LAST_UPDATED +import org.fdroid.database.AppListSortOrder.NAME +import org.fdroid.database.DbDiffUtils.diffAndUpdateListTable +import org.fdroid.database.DbDiffUtils.diffAndUpdateTable +import org.fdroid.index.IndexParser.json +import org.fdroid.index.v2.FileV2 +import org.fdroid.index.v2.LocalizedFileListV2 +import org.fdroid.index.v2.LocalizedFileV2 +import org.fdroid.index.v2.MetadataV2 +import org.fdroid.index.v2.ReflectionDiffer.applyDiff + +public interface AppDao { + /** + * Inserts an app into the DB. + * This is usually from a full index v2 via [MetadataV2]. + * + * Note: The app is considered to be not compatible until [Version]s are added + * and [updateCompatibility] was called. + * + * @param locales supported by the current system configuration. + */ + public fun insert( + repoId: Long, + packageName: String, + app: MetadataV2, + locales: LocaleListCompat = getLocales(Resources.getSystem().configuration), + ) + + /** + * Updates the [AppMetadata.isCompatible] flag + * based on whether at least one [AppVersion] is compatible. + * This needs to run within the transaction that adds [AppMetadata] to the DB (e.g. [insert]). + * Otherwise the compatibility is wrong. + */ + public fun updateCompatibility(repoId: Long) + + /** + * Gets the app from the DB. If more than one app with this [packageName] exists, + * the one from the repository with the highest weight is returned. + */ + public fun getApp(packageName: String): LiveData + + /** + * Gets an app from a specific [Repository] or null, + * if none is found with the given [packageName], + */ + public fun getApp(repoId: Long, packageName: String): App? + + /** + * Returns a limited number of apps with limited data. + * Apps without name, icon or summary are at the end (or excluded if limit is too small). + * Includes anti-features from the version with the highest version code. + */ + public fun getAppOverviewItems(limit: Int = 200): LiveData> + + /** + * Returns a limited number of apps with limited data within the given [category]. + */ + public fun getAppOverviewItems( + category: String, + limit: Int = 50, + ): LiveData> + + /** + * Returns a list of all [AppListItem] sorted by the given [sortOrder], + * or a subset of [AppListItem]s filtered by the given [searchQuery] if it is non-null. + * In the later case, the [sortOrder] gets ignored. + */ + public fun getAppListItems( + packageManager: PackageManager, + searchQuery: String?, + sortOrder: AppListSortOrder, + ): LiveData> + + /** + * Like [getAppListItems], but further filter items by the given [category]. + */ + public fun getAppListItems( + packageManager: PackageManager, + category: String, + searchQuery: String?, + sortOrder: AppListSortOrder, + ): LiveData> + + public fun getInstalledAppListItems(packageManager: PackageManager): LiveData> + + public fun getNumberOfAppsInCategory(category: String): Int + + public fun getNumberOfAppsInRepository(repoId: Long): Int +} + +public enum class AppListSortOrder { + LAST_UPDATED, NAME +} + +/** + * A list of unknown fields in [MetadataV2] that we don't allow for [AppMetadata]. + * + * We are applying reflection diffs against internal database classes + * and need to prevent the untrusted external JSON input to modify internal fields in those classes. + * This list must always hold the names of all those internal FIELDS for [AppMetadata]. + */ +private val DENY_LIST = listOf("packageName", "repoId") + +/** + * A list of unknown fields in [LocalizedFileV2] or [LocalizedFileListV2] + * that we don't allow for [LocalizedFile] or [LocalizedFileList]. + * + * Similar to [DENY_LIST]. + */ +private val DENY_FILE_LIST = listOf("packageName", "repoId", "type") + +@Dao +internal interface AppDaoInt : AppDao { + + @Transaction + override fun insert( + repoId: Long, + packageName: String, + app: MetadataV2, + locales: LocaleListCompat, + ) { + insert(app.toAppMetadata(repoId, packageName, false, locales)) + app.icon.insert(repoId, packageName, "icon") + app.featureGraphic.insert(repoId, packageName, "featureGraphic") + app.promoGraphic.insert(repoId, packageName, "promoGraphic") + app.tvBanner.insert(repoId, packageName, "tvBanner") + app.screenshots?.let { + it.phone.insert(repoId, packageName, "phone") + it.sevenInch.insert(repoId, packageName, "sevenInch") + it.tenInch.insert(repoId, packageName, "tenInch") + it.wear.insert(repoId, packageName, "wear") + it.tv.insert(repoId, packageName, "tv") + } + } + + private fun LocalizedFileV2?.insert(repoId: Long, packageName: String, type: String) { + this?.toLocalizedFile(repoId, packageName, type)?.let { files -> + insert(files) + } + } + + @JvmName("insertLocalizedFileListV2") + private fun LocalizedFileListV2?.insert(repoId: Long, packageName: String, type: String) { + this?.toLocalizedFileList(repoId, packageName, type)?.let { files -> + insertLocalizedFileLists(files) + } + } + + @Insert(onConflict = REPLACE) + fun insert(appMetadata: AppMetadata) + + @Insert(onConflict = REPLACE) + fun insert(localizedFiles: List) + + @Insert(onConflict = REPLACE) + fun insertLocalizedFileLists(localizedFiles: List) + + @Transaction + fun updateApp( + repoId: Long, + packageName: String, + jsonObject: JsonObject?, + locales: LocaleListCompat, + ) { + if (jsonObject == null) { + // this app is gone, we need to delete it + deleteAppMetadata(repoId, packageName) + return + } + val metadata = getAppMetadata(repoId, packageName) + if (metadata == null) { // new app + val metadataV2: MetadataV2 = json.decodeFromJsonElement(jsonObject) + insert(repoId, packageName, metadataV2) + } else { // diff against existing app + // ensure that diff does not include internal keys + DENY_LIST.forEach { forbiddenKey -> + if (jsonObject.containsKey(forbiddenKey)) throw SerializationException(forbiddenKey) + } + // diff metadata + val diffedApp = applyDiff(metadata, jsonObject) + val updatedApp = + if (jsonObject.containsKey("name") || jsonObject.containsKey("summary")) { + diffedApp.copy( + localizedName = diffedApp.name.getBestLocale(locales), + localizedSummary = diffedApp.summary.getBestLocale(locales), + ) + } else diffedApp + updateAppMetadata(updatedApp) + // diff localizedFiles + val localizedFiles = getLocalizedFiles(repoId, packageName) + localizedFiles.diffAndUpdate(repoId, packageName, "icon", jsonObject) + localizedFiles.diffAndUpdate(repoId, packageName, "featureGraphic", jsonObject) + localizedFiles.diffAndUpdate(repoId, packageName, "promoGraphic", jsonObject) + localizedFiles.diffAndUpdate(repoId, packageName, "tvBanner", jsonObject) + // diff localizedFileLists + val screenshots = jsonObject["screenshots"] + if (screenshots is JsonNull) { + deleteLocalizedFileLists(repoId, packageName) + } else if (screenshots is JsonObject) { + diffAndUpdateLocalizedFileList(repoId, packageName, "phone", screenshots) + diffAndUpdateLocalizedFileList(repoId, packageName, "sevenInch", screenshots) + diffAndUpdateLocalizedFileList(repoId, packageName, "tenInch", screenshots) + diffAndUpdateLocalizedFileList(repoId, packageName, "wear", screenshots) + diffAndUpdateLocalizedFileList(repoId, packageName, "tv", screenshots) + } + } + } + + private fun List.diffAndUpdate( + repoId: Long, + packageName: String, + type: String, + jsonObject: JsonObject, + ) = diffAndUpdateTable( + jsonObject = jsonObject, + jsonObjectKey = type, + itemList = filter { it.type == type }, + itemFinder = { locale, item -> item.locale == locale }, + newItem = { locale -> LocalizedFile(repoId, packageName, type, locale, "") }, + deleteAll = { deleteLocalizedFiles(repoId, packageName, type) }, + deleteOne = { locale -> deleteLocalizedFile(repoId, packageName, type, locale) }, + insertReplace = { list -> insert(list) }, + isNewItemValid = { it.name.isNotEmpty() }, + keyDenyList = DENY_FILE_LIST, + ) + + private fun diffAndUpdateLocalizedFileList( + repoId: Long, + packageName: String, + type: String, + jsonObject: JsonObject, + ) { + diffAndUpdateListTable( + jsonObject = jsonObject, + jsonObjectKey = type, + listParser = { locale, jsonArray -> + json.decodeFromJsonElement>(jsonArray).map { + it.toLocalizedFileList(repoId, packageName, type, locale) + } + }, + deleteAll = { deleteLocalizedFileLists(repoId, packageName, type) }, + deleteList = { locale -> deleteLocalizedFileList(repoId, packageName, type, locale) }, + insertNewList = { _, fileLists -> insertLocalizedFileLists(fileLists) }, + ) + } + + /** + * This is needed to support v1 streaming and shouldn't be used for something else. + */ + @Deprecated("Only for v1 index") + @Query("""UPDATE ${AppMetadata.TABLE} SET preferredSigner = :preferredSigner + WHERE repoId = :repoId AND packageName = :packageName""") + fun updatePreferredSigner(repoId: Long, packageName: String, preferredSigner: String?) + + @Query("""UPDATE ${AppMetadata.TABLE} + SET isCompatible = ( + SELECT TOTAL(isCompatible) > 0 FROM ${Version.TABLE} + WHERE repoId = :repoId AND ${AppMetadata.TABLE}.packageName = ${Version.TABLE}.packageName + ) + WHERE repoId = :repoId""") + override fun updateCompatibility(repoId: Long) + + @Query("""UPDATE ${AppMetadata.TABLE} SET localizedName = :name, localizedSummary = :summary + WHERE repoId = :repoId AND packageName = :packageName""") + fun updateAppMetadata(repoId: Long, packageName: String, name: String?, summary: String?) + + @Update + fun updateAppMetadata(appMetadata: AppMetadata): Int + + @Transaction + @Query("""SELECT ${AppMetadata.TABLE}.* FROM ${AppMetadata.TABLE} + JOIN RepositoryPreferences AS pref USING (repoId) + WHERE packageName = :packageName + ORDER BY pref.weight DESC LIMIT 1""") + override fun getApp(packageName: String): LiveData + + @Transaction + @Query("""SELECT * FROM ${AppMetadata.TABLE} + WHERE repoId = :repoId AND packageName = :packageName""") + override fun getApp(repoId: Long, packageName: String): App? + + /** + * Used for diffing. + */ + @Query("""SELECT * FROM ${AppMetadata.TABLE} + WHERE repoId = :repoId AND packageName = :packageName""") + fun getAppMetadata(repoId: Long, packageName: String): AppMetadata? + + /** + * Used for updating best locales. + */ + @Query("SELECT * FROM ${AppMetadata.TABLE}") + fun getAppMetadata(): List + + /** + * used for diffing + */ + @Query("""SELECT * FROM ${LocalizedFile.TABLE} + WHERE repoId = :repoId AND packageName = :packageName""") + fun getLocalizedFiles(repoId: Long, packageName: String): List + + @Transaction + @Query("""SELECT repoId, packageName, app.added, app.lastUpdated, localizedName, + localizedSummary, version.antiFeatures + FROM ${AppMetadata.TABLE} AS app + JOIN ${RepositoryPreferences.TABLE} AS pref USING (repoId) + LEFT JOIN ${HighestVersion.TABLE} AS version USING (repoId, packageName) + LEFT JOIN ${LocalizedIcon.TABLE} AS icon USING (repoId, packageName) + WHERE pref.enabled = 1 + GROUP BY packageName HAVING MAX(pref.weight) + ORDER BY localizedName IS NULL ASC, icon.packageName IS NULL ASC, + localizedSummary IS NULL ASC, app.lastUpdated DESC + LIMIT :limit""") + override fun getAppOverviewItems(limit: Int): LiveData> + + @Transaction + @Query("""SELECT repoId, packageName, app.added, app.lastUpdated, localizedName, + localizedSummary, version.antiFeatures + FROM ${AppMetadata.TABLE} AS app + JOIN ${RepositoryPreferences.TABLE} AS pref USING (repoId) + LEFT JOIN ${HighestVersion.TABLE} AS version USING (repoId, packageName) + LEFT JOIN ${LocalizedIcon.TABLE} AS icon USING (repoId, packageName) + WHERE pref.enabled = 1 AND categories LIKE '%,' || :category || ',%' + GROUP BY packageName HAVING MAX(pref.weight) + ORDER BY localizedName IS NULL ASC, icon.packageName IS NULL ASC, + localizedSummary IS NULL ASC, app.lastUpdated DESC + LIMIT :limit""") + override fun getAppOverviewItems(category: String, limit: Int): LiveData> + + /** + * Used by [DbUpdateChecker] to get specific apps with available updates. + */ + @Transaction + @SuppressWarnings(CURSOR_MISMATCH) // no anti-features needed here + @Query("""SELECT repoId, packageName, added, app.lastUpdated, localizedName, + localizedSummary + FROM ${AppMetadata.TABLE} AS app WHERE repoId = :repoId AND packageName = :packageName""") + fun getAppOverviewItem(repoId: Long, packageName: String): AppOverviewItem? + + // + // AppListItems + // + + override fun getAppListItems( + packageManager: PackageManager, + searchQuery: String?, + sortOrder: AppListSortOrder, + ): LiveData> { + return if (searchQuery.isNullOrEmpty()) when (sortOrder) { + LAST_UPDATED -> getAppListItemsByLastUpdated().map(packageManager) + NAME -> getAppListItemsByName().map(packageManager) + } else getAppListItems(searchQuery) + } + + override fun getAppListItems( + packageManager: PackageManager, + category: String, + searchQuery: String?, + sortOrder: AppListSortOrder, + ): LiveData> { + return if (searchQuery.isNullOrEmpty()) when (sortOrder) { + LAST_UPDATED -> getAppListItemsByLastUpdated(category).map(packageManager) + NAME -> getAppListItemsByName(category).map(packageManager) + } else getAppListItems(category, searchQuery) + } + + private fun LiveData>.map( + packageManager: PackageManager, + installedPackages: Map = packageManager.getInstalledPackages(0) + .associateBy { packageInfo -> packageInfo.packageName }, + ) = map { items -> + items.map { item -> + val packageInfo = installedPackages[item.packageName] + if (packageInfo == null) item else item.copy( + installedVersionName = packageInfo.versionName, + installedVersionCode = packageInfo.getVersionCode(), + ) + } + } + + @Transaction + @Query(""" + SELECT repoId, packageName, app.localizedName, app.localizedSummary, version.antiFeatures, + app.isCompatible + FROM ${AppMetadata.TABLE} AS app + JOIN ${AppMetadataFts.TABLE} USING (repoId, packageName) + LEFT JOIN ${HighestVersion.TABLE} AS version USING (repoId, packageName) + JOIN ${RepositoryPreferences.TABLE} AS pref USING (repoId) + WHERE pref.enabled = 1 AND ${AppMetadataFts.TABLE} MATCH '"*' || :searchQuery || '*"' + GROUP BY packageName HAVING MAX(pref.weight)""") + fun getAppListItems(searchQuery: String): LiveData> + + @Transaction + @Query(""" + SELECT repoId, packageName, app.localizedName, app.localizedSummary, version.antiFeatures, + app.isCompatible + FROM ${AppMetadata.TABLE} AS app + JOIN ${AppMetadataFts.TABLE} USING (repoId, packageName) + LEFT JOIN ${HighestVersion.TABLE} AS version USING (repoId, packageName) + JOIN ${RepositoryPreferences.TABLE} AS pref USING (repoId) + WHERE pref.enabled = 1 AND categories LIKE '%,' || :category || ',%' AND + ${AppMetadataFts.TABLE} MATCH '"*' || :searchQuery || '*"' + GROUP BY packageName HAVING MAX(pref.weight)""") + fun getAppListItems(category: String, searchQuery: String): LiveData> + + @Transaction + @Query(""" + SELECT repoId, packageName, localizedName, localizedSummary, version.antiFeatures, + app.isCompatible + FROM ${AppMetadata.TABLE} AS app + LEFT JOIN ${HighestVersion.TABLE} AS version USING (repoId, packageName) + JOIN ${RepositoryPreferences.TABLE} AS pref USING (repoId) + WHERE pref.enabled = 1 + GROUP BY packageName HAVING MAX(pref.weight) + ORDER BY localizedName COLLATE NOCASE ASC""") + fun getAppListItemsByName(): LiveData> + + @Transaction + @Query(""" + SELECT repoId, packageName, localizedName, localizedSummary, version.antiFeatures, + app.isCompatible + FROM ${AppMetadata.TABLE} AS app + JOIN ${RepositoryPreferences.TABLE} AS pref USING (repoId) + LEFT JOIN ${HighestVersion.TABLE} AS version USING (repoId, packageName) + WHERE pref.enabled = 1 + GROUP BY packageName HAVING MAX(pref.weight) + ORDER BY app.lastUpdated DESC""") + fun getAppListItemsByLastUpdated(): LiveData> + + @Transaction + @Query(""" + SELECT repoId, packageName, localizedName, localizedSummary, version.antiFeatures, + app.isCompatible + FROM ${AppMetadata.TABLE} AS app + JOIN ${RepositoryPreferences.TABLE} AS pref USING (repoId) + LEFT JOIN ${HighestVersion.TABLE} AS version USING (repoId, packageName) + WHERE pref.enabled = 1 AND categories LIKE '%,' || :category || ',%' + GROUP BY packageName HAVING MAX(pref.weight) + ORDER BY app.lastUpdated DESC""") + fun getAppListItemsByLastUpdated(category: String): LiveData> + + @Transaction + @Query(""" + SELECT repoId, packageName, localizedName, localizedSummary, version.antiFeatures, + app.isCompatible + FROM ${AppMetadata.TABLE} AS app + JOIN ${RepositoryPreferences.TABLE} AS pref USING (repoId) + LEFT JOIN ${HighestVersion.TABLE} AS version USING (repoId, packageName) + WHERE pref.enabled = 1 AND categories LIKE '%,' || :category || ',%' + GROUP BY packageName HAVING MAX(pref.weight) + ORDER BY localizedName COLLATE NOCASE ASC""") + fun getAppListItemsByName(category: String): LiveData> + + @Transaction + @SuppressWarnings(CURSOR_MISMATCH) // no anti-features needed here + @Query("""SELECT repoId, packageName, localizedName, localizedSummary, app.isCompatible + FROM ${AppMetadata.TABLE} AS app + JOIN ${RepositoryPreferences.TABLE} AS pref USING (repoId) + WHERE pref.enabled = 1 AND packageName IN (:packageNames) + GROUP BY packageName HAVING MAX(pref.weight) + ORDER BY localizedName COLLATE NOCASE ASC""") + fun getAppListItems(packageNames: List): LiveData> + + override fun getInstalledAppListItems( + packageManager: PackageManager, + ): LiveData> { + val installedPackages = packageManager.getInstalledPackages(0) + .associateBy { packageInfo -> packageInfo.packageName } + val packageNames = installedPackages.keys.toList() + return getAppListItems(packageNames).map(packageManager, installedPackages) + } + + @Query("""SELECT COUNT(DISTINCT packageName) FROM ${AppMetadata.TABLE} + JOIN ${RepositoryPreferences.TABLE} AS pref USING (repoId) + WHERE pref.enabled = 1 AND categories LIKE '%,' || :category || ',%'""") + override fun getNumberOfAppsInCategory(category: String): Int + + @Query("SELECT COUNT(*) FROM ${AppMetadata.TABLE} WHERE repoId = :repoId") + override fun getNumberOfAppsInRepository(repoId: Long): Int + + @Query("DELETE FROM ${AppMetadata.TABLE} WHERE repoId = :repoId AND packageName = :packageName") + fun deleteAppMetadata(repoId: Long, packageName: String) + + @Query("""DELETE FROM ${LocalizedFile.TABLE} + WHERE repoId = :repoId AND packageName = :packageName AND type = :type""") + fun deleteLocalizedFiles(repoId: Long, packageName: String, type: String) + + @Query("""DELETE FROM ${LocalizedFile.TABLE} + WHERE repoId = :repoId AND packageName = :packageName AND type = :type + AND locale = :locale""") + fun deleteLocalizedFile(repoId: Long, packageName: String, type: String, locale: String) + + @Query("""DELETE FROM ${LocalizedFileList.TABLE} + WHERE repoId = :repoId AND packageName = :packageName""") + fun deleteLocalizedFileLists(repoId: Long, packageName: String) + + @Query("""DELETE FROM ${LocalizedFileList.TABLE} + WHERE repoId = :repoId AND packageName = :packageName AND type = :type""") + fun deleteLocalizedFileLists(repoId: Long, packageName: String, type: String) + + @Query("""DELETE FROM ${LocalizedFileList.TABLE} + WHERE repoId = :repoId AND packageName = :packageName AND type = :type + AND locale = :locale""") + fun deleteLocalizedFileList(repoId: Long, packageName: String, type: String, locale: String) + + @VisibleForTesting + @Query("SELECT COUNT(*) FROM ${AppMetadata.TABLE}") + fun countApps(): Int + + @VisibleForTesting + @Query("SELECT COUNT(*) FROM ${LocalizedFile.TABLE}") + fun countLocalizedFiles(): Int + + @VisibleForTesting + @Query("SELECT COUNT(*) FROM ${LocalizedFileList.TABLE}") + fun countLocalizedFileLists(): Int + +} diff --git a/libs/database/src/main/java/org/fdroid/database/AppPrefs.kt b/libs/database/src/main/java/org/fdroid/database/AppPrefs.kt new file mode 100644 index 000000000..3b08bdc09 --- /dev/null +++ b/libs/database/src/main/java/org/fdroid/database/AppPrefs.kt @@ -0,0 +1,55 @@ +package org.fdroid.database + +import androidx.room.Entity +import androidx.room.PrimaryKey +import org.fdroid.PackagePreference + +/** + * User-defined preferences related to [App]s that get stored in the database, + * so they can be used for queries. + */ +@Entity(tableName = AppPrefs.TABLE) +public data class AppPrefs( + @PrimaryKey + val packageName: String, + override val ignoreVersionCodeUpdate: Long = 0, + // This is named like this, because it hit a Room bug when joining with Version table + // which had exactly the same field. + internal val appPrefReleaseChannels: List? = null, +) : PackagePreference { + internal companion object { + const val TABLE = "AppPrefs" + } + + public val ignoreAllUpdates: Boolean get() = ignoreVersionCodeUpdate == Long.MAX_VALUE + public override val releaseChannels: List get() = appPrefReleaseChannels ?: emptyList() + public fun shouldIgnoreUpdate(versionCode: Long): Boolean = + ignoreVersionCodeUpdate >= versionCode + + /** + * Returns a new instance of [AppPrefs] toggling [ignoreAllUpdates]. + */ + public fun toggleIgnoreAllUpdates(): AppPrefs = copy( + ignoreVersionCodeUpdate = if (ignoreAllUpdates) 0 else Long.MAX_VALUE, + ) + + /** + * Returns a new instance of [AppPrefs] ignoring the given [versionCode] or stop ignoring it + * if it was already ignored. + */ + public fun toggleIgnoreVersionCodeUpdate(versionCode: Long): AppPrefs = copy( + ignoreVersionCodeUpdate = if (shouldIgnoreUpdate(versionCode)) 0 else versionCode, + ) + + /** + * Returns a new instance of [AppPrefs] enabling the given [releaseChannel] or disabling it + * if it was already enabled. + */ + public fun toggleReleaseChannel(releaseChannel: String): AppPrefs = copy( + appPrefReleaseChannels = if (appPrefReleaseChannels?.contains(releaseChannel) == true) { + appPrefReleaseChannels.toMutableList().apply { remove(releaseChannel) } + } else { + (appPrefReleaseChannels?.toMutableList() ?: ArrayList()).apply { add(releaseChannel) } + }, + ) +} diff --git a/libs/database/src/main/java/org/fdroid/database/AppPrefsDao.kt b/libs/database/src/main/java/org/fdroid/database/AppPrefsDao.kt new file mode 100644 index 000000000..137a6dd7d --- /dev/null +++ b/libs/database/src/main/java/org/fdroid/database/AppPrefsDao.kt @@ -0,0 +1,33 @@ +package org.fdroid.database + +import androidx.lifecycle.LiveData +import androidx.lifecycle.distinctUntilChanged +import androidx.lifecycle.map +import androidx.room.Dao +import androidx.room.Insert +import androidx.room.OnConflictStrategy.REPLACE +import androidx.room.Query + +public interface AppPrefsDao { + public fun getAppPrefs(packageName: String): LiveData + public fun update(appPrefs: AppPrefs) +} + +@Dao +internal interface AppPrefsDaoInt : AppPrefsDao { + + override fun getAppPrefs(packageName: String): LiveData { + return getLiveAppPrefs(packageName).distinctUntilChanged().map { data -> + data ?: AppPrefs(packageName) + } + } + + @Query("SELECT * FROM ${AppPrefs.TABLE} WHERE packageName = :packageName") + fun getLiveAppPrefs(packageName: String): LiveData + + @Query("SELECT * FROM ${AppPrefs.TABLE} WHERE packageName = :packageName") + fun getAppPrefsOrNull(packageName: String): AppPrefs? + + @Insert(onConflict = REPLACE) + override fun update(appPrefs: AppPrefs) +} diff --git a/libs/database/src/main/java/org/fdroid/database/Converters.kt b/libs/database/src/main/java/org/fdroid/database/Converters.kt new file mode 100644 index 000000000..774e2e2e9 --- /dev/null +++ b/libs/database/src/main/java/org/fdroid/database/Converters.kt @@ -0,0 +1,62 @@ +package org.fdroid.database + +import androidx.room.TypeConverter +import kotlinx.serialization.builtins.MapSerializer +import kotlinx.serialization.builtins.serializer +import org.fdroid.index.IndexParser.json +import org.fdroid.index.v2.FileV2 +import org.fdroid.index.v2.LocalizedFileV2 +import org.fdroid.index.v2.LocalizedTextV2 + +internal object Converters { + + private val localizedTextV2Serializer = MapSerializer(String.serializer(), String.serializer()) + private val localizedFileV2Serializer = MapSerializer(String.serializer(), FileV2.serializer()) + private val mapOfLocalizedTextV2Serializer = + MapSerializer(String.serializer(), localizedTextV2Serializer) + + @TypeConverter + fun fromStringToLocalizedTextV2(value: String?): LocalizedTextV2? { + return value?.let { json.decodeFromString(localizedTextV2Serializer, it) } + } + + @TypeConverter + fun localizedTextV2toString(text: LocalizedTextV2?): String? { + return text?.let { json.encodeToString(localizedTextV2Serializer, it) } + } + + @TypeConverter + fun fromStringToLocalizedFileV2(value: String?): LocalizedFileV2? { + return value?.let { json.decodeFromString(localizedFileV2Serializer, it) } + } + + @TypeConverter + fun localizedFileV2toString(file: LocalizedFileV2?): String? { + return file?.let { json.encodeToString(localizedFileV2Serializer, it) } + } + + @TypeConverter + fun fromStringToMapOfLocalizedTextV2(value: String?): Map? { + return value?.let { json.decodeFromString(mapOfLocalizedTextV2Serializer, it) } + } + + @TypeConverter + fun mapOfLocalizedTextV2toString(text: Map?): String? { + return text?.let { json.encodeToString(mapOfLocalizedTextV2Serializer, it) } + } + + @TypeConverter + fun fromStringToListString(value: String?): List { + return value?.split(',')?.filter { it.isNotEmpty() } ?: emptyList() + } + + @TypeConverter + fun listStringToString(text: List?): String? { + if (text.isNullOrEmpty()) return null + return text.joinToString( + prefix = ",", + separator = ",", + postfix = ",", + ) { it.replace(',', '_') } + } +} diff --git a/libs/database/src/main/java/org/fdroid/database/DbDiffUtils.kt b/libs/database/src/main/java/org/fdroid/database/DbDiffUtils.kt new file mode 100644 index 000000000..cfc1374f5 --- /dev/null +++ b/libs/database/src/main/java/org/fdroid/database/DbDiffUtils.kt @@ -0,0 +1,125 @@ +package org.fdroid.database + +import kotlinx.serialization.SerializationException +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonNull +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonObject +import org.fdroid.index.v2.ReflectionDiffer + +internal object DbDiffUtils { + + /** + * Applies the diff from the given [jsonObject] identified by the given [jsonObjectKey] + * to [itemList] and updates the DB as needed. + * + * @param newItem A function to produce a new [T] which typically contains the primary key(s). + */ + @Throws(SerializationException::class) + fun diffAndUpdateTable( + jsonObject: JsonObject, + jsonObjectKey: String, + itemList: List, + itemFinder: (String, T) -> Boolean, + newItem: (String) -> T, + deleteAll: () -> Unit, + deleteOne: (String) -> Unit, + insertReplace: (List) -> Unit, + isNewItemValid: (T) -> Boolean = { true }, + keyDenyList: List? = null, + ) { + if (!jsonObject.containsKey(jsonObjectKey)) return + if (jsonObject[jsonObjectKey] == JsonNull) { + deleteAll() + } else { + val obj = jsonObject[jsonObjectKey]?.jsonObject ?: error("no $jsonObjectKey object") + val list = itemList.toMutableList() + obj.entries.forEach { (key, value) -> + if (value is JsonNull) { + list.removeAll { itemFinder(key, it) } + deleteOne(key) + } else { + value.jsonObject.checkDenyList(keyDenyList) + val index = list.indexOfFirst { itemFinder(key, it) } + val item = if (index == -1) null else list[index] + if (item == null) { + val itemToInsert = + ReflectionDiffer.applyDiff(newItem(key), value.jsonObject) + if (!isNewItemValid(itemToInsert)) throw SerializationException("$newItem") + list.add(itemToInsert) + } else { + list[index] = ReflectionDiffer.applyDiff(item, value.jsonObject) + } + } + } + insertReplace(list) + } + } + + /** + * Applies a list diff from a map of lists. + * The map is identified by the given [jsonObjectKey] in the given [jsonObject]. + * The diff is applied for each key + * by replacing the existing list using [deleteList] and [insertNewList]. + * + * @param listParser returns a list of [T] from the given [JsonArray]. + */ + @Throws(SerializationException::class) + fun diffAndUpdateListTable( + jsonObject: JsonObject, + jsonObjectKey: String, + listParser: (String, JsonArray) -> List, + deleteAll: () -> Unit, + deleteList: (String) -> Unit, + insertNewList: (String, List) -> Unit, + ) { + if (!jsonObject.containsKey(jsonObjectKey)) return + if (jsonObject[jsonObjectKey] == JsonNull) { + deleteAll() + } else { + val obj = jsonObject[jsonObjectKey]?.jsonObject ?: error("no $jsonObjectKey object") + obj.entries.forEach { (key, list) -> + if (list is JsonNull) { + deleteList(key) + } else { + val newList = listParser(key, list.jsonArray) + deleteList(key) + insertNewList(key, newList) + } + } + } + } + + /** + * Applies the list diff from the given [jsonObject] identified by the given [jsonObjectKey] + * by replacing an existing list using [deleteList] and [insertNewList]. + * + * @param listParser returns a list of [T] from the given [JsonArray]. + */ + @Throws(SerializationException::class) + fun diffAndUpdateListTable( + jsonObject: JsonObject, + jsonObjectKey: String, + listParser: (JsonArray) -> List, + deleteList: () -> Unit, + insertNewList: (List) -> Unit, + ) { + if (!jsonObject.containsKey(jsonObjectKey)) return + if (jsonObject[jsonObjectKey] == JsonNull) { + deleteList() + } else { + val jsonArray = jsonObject[jsonObjectKey]?.jsonArray ?: error("no $jsonObjectKey array") + val list = listParser(jsonArray) + deleteList() + insertNewList(list) + } + } + + private fun JsonObject.checkDenyList(list: List?) { + list?.forEach { forbiddenKey -> + if (containsKey(forbiddenKey)) throw SerializationException(forbiddenKey) + } + } + +} diff --git a/libs/database/src/main/java/org/fdroid/database/DbUpdateChecker.kt b/libs/database/src/main/java/org/fdroid/database/DbUpdateChecker.kt new file mode 100644 index 000000000..05bf16c2e --- /dev/null +++ b/libs/database/src/main/java/org/fdroid/database/DbUpdateChecker.kt @@ -0,0 +1,137 @@ +package org.fdroid.database + +import android.annotation.SuppressLint +import android.content.pm.PackageInfo +import android.content.pm.PackageManager +import android.content.pm.PackageManager.GET_SIGNATURES +import android.os.Build +import org.fdroid.CompatibilityCheckerImpl +import org.fdroid.PackagePreference +import org.fdroid.UpdateChecker + +public class DbUpdateChecker( + db: FDroidDatabase, + private val packageManager: PackageManager, +) { + + private val appDao = db.getAppDao() as AppDaoInt + private val versionDao = db.getVersionDao() as VersionDaoInt + private val appPrefsDao = db.getAppPrefsDao() as AppPrefsDaoInt + private val compatibilityChecker = CompatibilityCheckerImpl(packageManager) + private val updateChecker = UpdateChecker(compatibilityChecker) + + /** + * Returns a list of apps that can be updated. + * @param releaseChannels optional list of release channels to consider on top of stable. + * If this is null or empty, only versions without channel (stable) will be considered. + */ + public fun getUpdatableApps(releaseChannels: List? = null): List { + val updatableApps = ArrayList() + + @Suppress("DEPRECATION") // we'll use this as long as it works, new one was broken + val installedPackages = packageManager.getInstalledPackages(GET_SIGNATURES) + val packageNames = installedPackages.map { it.packageName } + val versionsByPackage = HashMap>(packageNames.size) + versionDao.getVersions(packageNames).forEach { version -> + val list = versionsByPackage.getOrPut(version.packageName) { ArrayList() } + list.add(version) + } + installedPackages.iterator().forEach { packageInfo -> + val packageName = packageInfo.packageName + val versions = versionsByPackage[packageName] ?: return@forEach // continue + val version = getVersion(versions, packageName, packageInfo, null, releaseChannels) + if (version != null) { + val versionCode = packageInfo.getVersionCode() + val app = getUpdatableApp(version, versionCode) + if (app != null) updatableApps.add(app) + } + } + return updatableApps + } + + /** + * Returns an [AppVersion] for the given [packageName] that is an update or new install + * or null if there is none. + * @param releaseChannels optional list of release channels to consider on top of stable. + * If this is null or empty, only versions without channel (stable) will be considered. + */ + @SuppressLint("PackageManagerGetSignatures") + public fun getSuggestedVersion( + packageName: String, + preferredSigner: String? = null, + releaseChannels: List? = null, + ): AppVersion? { + val versions = versionDao.getVersions(listOf(packageName)) + if (versions.isEmpty()) return null + val packageInfo = try { + @Suppress("DEPRECATION") + packageManager.getPackageInfo(packageName, GET_SIGNATURES) + } catch (e: PackageManager.NameNotFoundException) { + null + } + val version = getVersion(versions, packageName, packageInfo, preferredSigner, + releaseChannels) ?: return null + val versionedStrings = versionDao.getVersionedStrings( + repoId = version.repoId, + packageName = version.packageName, + versionId = version.versionId, + ) + return version.toAppVersion(versionedStrings) + } + + private fun getVersion( + versions: List, + packageName: String, + packageInfo: PackageInfo?, + preferredSigner: String?, + releaseChannels: List?, + ): Version? { + val preferencesGetter: (() -> PackagePreference?) = { + appPrefsDao.getAppPrefsOrNull(packageName) + } + return if (packageInfo == null) { + updateChecker.getSuggestedVersion( + versions = versions, + preferredSigner = preferredSigner, + releaseChannels = releaseChannels, + preferencesGetter = preferencesGetter, + ) + } else { + updateChecker.getUpdate( + versions = versions, + packageInfo = packageInfo, + releaseChannels = releaseChannels, + preferencesGetter = preferencesGetter, + ) + } + } + + private fun getUpdatableApp(version: Version, installedVersionCode: Long): UpdatableApp? { + val versionedStrings = versionDao.getVersionedStrings( + repoId = version.repoId, + packageName = version.packageName, + versionId = version.versionId, + ) + val appOverviewItem = + appDao.getAppOverviewItem(version.repoId, version.packageName) ?: return null + return UpdatableApp( + repoId = version.repoId, + packageName = version.packageName, + installedVersionCode = installedVersionCode, + update = version.toAppVersion(versionedStrings), + hasKnownVulnerability = version.hasKnownVulnerability, + name = appOverviewItem.name, + summary = appOverviewItem.summary, + localizedIcon = appOverviewItem.localizedIcon, + ) + } +} + +internal fun PackageInfo.getVersionCode(): Long { + return if (Build.VERSION.SDK_INT >= 28) { + longVersionCode + } else { + @Suppress("DEPRECATION") // we use the new one above, if available + versionCode.toLong() + } +} diff --git a/libs/database/src/main/java/org/fdroid/database/DbV1StreamReceiver.kt b/libs/database/src/main/java/org/fdroid/database/DbV1StreamReceiver.kt new file mode 100644 index 000000000..5fc39bd59 --- /dev/null +++ b/libs/database/src/main/java/org/fdroid/database/DbV1StreamReceiver.kt @@ -0,0 +1,61 @@ +package org.fdroid.database + +import android.content.res.Resources +import androidx.core.os.ConfigurationCompat.getLocales +import androidx.core.os.LocaleListCompat +import org.fdroid.CompatibilityChecker +import org.fdroid.index.IndexFormatVersion.ONE +import org.fdroid.index.v1.IndexV1StreamReceiver +import org.fdroid.index.v2.AntiFeatureV2 +import org.fdroid.index.v2.CategoryV2 +import org.fdroid.index.v2.MetadataV2 +import org.fdroid.index.v2.PackageVersionV2 +import org.fdroid.index.v2.ReleaseChannelV2 +import org.fdroid.index.v2.RepoV2 + +/** + * Note that this class expects that its [receive] method with [RepoV2] gets called first. + * A different order of calls is not supported. + */ +@Deprecated("Use DbV2StreamReceiver instead") +internal class DbV1StreamReceiver( + private val db: FDroidDatabaseInt, + private val repoId: Long, + private val compatibilityChecker: CompatibilityChecker, +) : IndexV1StreamReceiver { + + private val locales: LocaleListCompat = getLocales(Resources.getSystem().configuration) + + override fun receive(repo: RepoV2, version: Long, certificate: String?) { + db.getRepositoryDao().clear(repoId) + db.getRepositoryDao().update(repoId, repo, version, ONE, certificate) + } + + override fun receive(packageName: String, m: MetadataV2) { + db.getAppDao().insert(repoId, packageName, m, locales) + } + + override fun receive(packageName: String, v: Map) { + db.getVersionDao().insert(repoId, packageName, v) { + compatibilityChecker.isCompatible(it.manifest) + } + } + + override fun updateRepo( + antiFeatures: Map, + categories: Map, + releaseChannels: Map, + ) { + val repoDao = db.getRepositoryDao() + repoDao.insertAntiFeatures(antiFeatures.toRepoAntiFeatures(repoId)) + repoDao.insertCategories(categories.toRepoCategories(repoId)) + repoDao.insertReleaseChannels(releaseChannels.toRepoReleaseChannel(repoId)) + + db.afterUpdatingRepo(repoId) + } + + override fun updateAppMetadata(packageName: String, preferredSigner: String?) { + db.getAppDao().updatePreferredSigner(repoId, packageName, preferredSigner) + } + +} diff --git a/libs/database/src/main/java/org/fdroid/database/DbV2DiffStreamReceiver.kt b/libs/database/src/main/java/org/fdroid/database/DbV2DiffStreamReceiver.kt new file mode 100644 index 000000000..ed24dede2 --- /dev/null +++ b/libs/database/src/main/java/org/fdroid/database/DbV2DiffStreamReceiver.kt @@ -0,0 +1,40 @@ +package org.fdroid.database + +import android.content.res.Resources +import androidx.core.os.ConfigurationCompat.getLocales +import androidx.core.os.LocaleListCompat +import kotlinx.serialization.json.JsonObject +import org.fdroid.CompatibilityChecker +import org.fdroid.index.v2.IndexV2DiffStreamReceiver + +internal class DbV2DiffStreamReceiver( + private val db: FDroidDatabaseInt, + private val repoId: Long, + private val compatibilityChecker: CompatibilityChecker, +) : IndexV2DiffStreamReceiver { + + private val locales: LocaleListCompat = getLocales(Resources.getSystem().configuration) + + override fun receiveRepoDiff(version: Long, repoJsonObject: JsonObject) { + db.getRepositoryDao().updateRepository(repoId, version, repoJsonObject) + } + + override fun receivePackageMetadataDiff(packageName: String, packageJsonObject: JsonObject?) { + db.getAppDao().updateApp(repoId, packageName, packageJsonObject, locales) + } + + override fun receiveVersionsDiff( + packageName: String, + versionsDiffMap: Map?, + ) { + db.getVersionDao().update(repoId, packageName, versionsDiffMap) { + compatibilityChecker.isCompatible(it) + } + } + + @Synchronized + override fun onStreamEnded() { + db.afterUpdatingRepo(repoId) + } + +} diff --git a/libs/database/src/main/java/org/fdroid/database/DbV2StreamReceiver.kt b/libs/database/src/main/java/org/fdroid/database/DbV2StreamReceiver.kt new file mode 100644 index 000000000..e4bce8e99 --- /dev/null +++ b/libs/database/src/main/java/org/fdroid/database/DbV2StreamReceiver.kt @@ -0,0 +1,72 @@ +package org.fdroid.database + +import android.content.res.Resources +import androidx.core.os.ConfigurationCompat.getLocales +import androidx.core.os.LocaleListCompat +import kotlinx.serialization.SerializationException +import org.fdroid.CompatibilityChecker +import org.fdroid.index.IndexFormatVersion.TWO +import org.fdroid.index.v2.FileV2 +import org.fdroid.index.v2.IndexV2StreamReceiver +import org.fdroid.index.v2.PackageV2 +import org.fdroid.index.v2.RepoV2 + +/** + * Receives a stream of IndexV2 data and stores it in the DB. + * + * Note: This should only be used once. + * If you want to process a second stream, create a new instance. + */ +internal class DbV2StreamReceiver( + private val db: FDroidDatabaseInt, + private val repoId: Long, + private val compatibilityChecker: CompatibilityChecker, +) : IndexV2StreamReceiver { + + private val locales: LocaleListCompat = getLocales(Resources.getSystem().configuration) + private var clearedRepoData = false + private val nonNullFileV2: (FileV2?) -> Unit = { fileV2 -> + if (fileV2 != null) { + if (fileV2.sha256 == null) throw SerializationException("${fileV2.name} has no sha256") + if (fileV2.size == null) throw SerializationException("${fileV2.name} has no size") + if (!fileV2.name.startsWith('/')) { + throw SerializationException("${fileV2.name} does not start with /") + } + } + } + + @Synchronized + override fun receive(repo: RepoV2, version: Long, certificate: String) { + repo.walkFiles(nonNullFileV2) + clearRepoDataIfNeeded() + db.getRepositoryDao().update(repoId, repo, version, TWO, certificate) + } + + @Synchronized + override fun receive(packageName: String, p: PackageV2) { + p.walkFiles(nonNullFileV2) + clearRepoDataIfNeeded() + db.getAppDao().insert(repoId, packageName, p.metadata, locales) + db.getVersionDao().insert(repoId, packageName, p.versions) { + compatibilityChecker.isCompatible(it.manifest) + } + } + + @Synchronized + override fun onStreamEnded() { + db.afterUpdatingRepo(repoId) + } + + /** + * As it is a valid index to receive packages before the repo, + * we can not clear all repo data when receiving the repo, + * but need to do it once at the beginning. + */ + private fun clearRepoDataIfNeeded() { + if (!clearedRepoData) { + db.getRepositoryDao().clear(repoId) + clearedRepoData = true + } + } + +} diff --git a/libs/database/src/main/java/org/fdroid/database/FDroidDatabase.kt b/libs/database/src/main/java/org/fdroid/database/FDroidDatabase.kt new file mode 100644 index 000000000..7d32c8d2c --- /dev/null +++ b/libs/database/src/main/java/org/fdroid/database/FDroidDatabase.kt @@ -0,0 +1,96 @@ +package org.fdroid.database + +import android.content.res.Resources +import androidx.core.os.ConfigurationCompat.getLocales +import androidx.core.os.LocaleListCompat +import androidx.room.Database +import androidx.room.RoomDatabase +import androidx.room.TypeConverters +import org.fdroid.LocaleChooser.getBestLocale +import java.util.Locale + +@Database( + // When bumping this version, please make sure to add one (or more) migration(s) below! + // Consider also providing tests for that migration. + // Don't forget to commit the new schema to the git repo as well. + version = 1, + entities = [ + // repo + CoreRepository::class, + Mirror::class, + AntiFeature::class, + Category::class, + ReleaseChannel::class, + RepositoryPreferences::class, + // packages + AppMetadata::class, + AppMetadataFts::class, + LocalizedFile::class, + LocalizedFileList::class, + // versions + Version::class, + VersionedString::class, + // app user preferences + AppPrefs::class, + ], + views = [ + LocalizedIcon::class, + HighestVersion::class, + ], + exportSchema = true, + autoMigrations = [ + // add future migrations here (if they are easy enough to be done automatically) + ], +) +@TypeConverters(Converters::class) +internal abstract class FDroidDatabaseInt internal constructor() : RoomDatabase(), FDroidDatabase { + abstract override fun getRepositoryDao(): RepositoryDaoInt + abstract override fun getAppDao(): AppDaoInt + abstract override fun getVersionDao(): VersionDaoInt + abstract override fun getAppPrefsDao(): AppPrefsDaoInt + override fun afterLocalesChanged(locales: LocaleListCompat) { + val appDao = getAppDao() + runInTransaction { + appDao.getAppMetadata().forEach { appMetadata -> + appDao.updateAppMetadata( + repoId = appMetadata.repoId, + packageName = appMetadata.packageName, + name = appMetadata.name.getBestLocale(locales), + summary = appMetadata.summary.getBestLocale(locales), + ) + } + } + } + + /** + * Call this after updating the data belonging to the given [repoId], + * so the [AppMetadata.isCompatible] can be recalculated in case new versions were added. + */ + fun afterUpdatingRepo(repoId: Long) { + getAppDao().updateCompatibility(repoId) + } +} + +/** + * The F-Droid database offering methods to retrieve the various data access objects. + */ +public interface FDroidDatabase { + public fun getRepositoryDao(): RepositoryDao + public fun getAppDao(): AppDao + public fun getVersionDao(): VersionDao + public fun getAppPrefsDao(): AppPrefsDao + + /** + * Call this after the system [Locale]s have changed. + * If this isn't called, the cached localized app metadata (e.g. name, summary) will be wrong. + */ + public fun afterLocalesChanged( + locales: LocaleListCompat = getLocales(Resources.getSystem().configuration), + ) + + /** + * Call this to run all of the given [body] inside a database transaction. + * Please run as little code as possible to keep the time the database is blocked minimal. + */ + public fun runInTransaction(body: Runnable) +} diff --git a/libs/database/src/main/java/org/fdroid/database/FDroidDatabaseHolder.kt b/libs/database/src/main/java/org/fdroid/database/FDroidDatabaseHolder.kt new file mode 100644 index 000000000..620bd1a82 --- /dev/null +++ b/libs/database/src/main/java/org/fdroid/database/FDroidDatabaseHolder.kt @@ -0,0 +1,93 @@ +package org.fdroid.database + +import android.content.Context +import android.util.Log +import androidx.annotation.GuardedBy +import androidx.room.Room +import androidx.room.RoomDatabase +import androidx.sqlite.db.SupportSQLiteDatabase +import kotlinx.coroutines.DelicateCoroutinesApi +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.GlobalScope +import kotlinx.coroutines.launch + +/** + * A way to pre-populate the database with a fixture. + * This can be supplied to [FDroidDatabaseHolder.getDb] + * and will then be called when a new database is created. + */ +public fun interface FDroidFixture { + /** + * Called when a new database gets created. + * Multiple DB operations should use [FDroidDatabase.runInTransaction]. + */ + public fun prePopulateDb(db: FDroidDatabase) +} + +/** + * A database holder using a singleton pattern to ensure + * that only one database is open at the same time. + */ +public object FDroidDatabaseHolder { + // Singleton prevents multiple instances of database opening at the same time. + @Volatile + @GuardedBy("lock") + private var INSTANCE: FDroidDatabaseInt? = null + private val lock = Object() + + internal val TAG = FDroidDatabase::class.simpleName + internal val dispatcher get() = Dispatchers.IO + + /** + * Give you an existing instance of [FDroidDatabase] or creates/opens a new one if none exists. + * Note: The given [name] is only used when calling this for the first time. + * Subsequent calls with a different name will return the instance created by the first call. + */ + @JvmStatic + @JvmOverloads + public fun getDb( + context: Context, + name: String = "fdroid_db", + fixture: FDroidFixture? = null, + ): FDroidDatabase { + // if the INSTANCE is not null, then return it, + // if it is, then create the database + return INSTANCE ?: synchronized(lock) { + val builder = Room.databaseBuilder( + context.applicationContext, + FDroidDatabaseInt::class.java, + name, + ).apply { + // We allow destructive migration (if no real migration was provided), + // so we have the option to nuke the DB in production (if that will ever be needed). + fallbackToDestructiveMigration() + // Add our [FixtureCallback] if a fixture was provided + if (fixture != null) addCallback(FixtureCallback(fixture)) + } + val instance = builder.build() + INSTANCE = instance + // return instance + instance + } + } + + private class FixtureCallback(private val fixture: FDroidFixture) : RoomDatabase.Callback() { + override fun onCreate(db: SupportSQLiteDatabase) { + super.onCreate(db) + @OptIn(DelicateCoroutinesApi::class) + GlobalScope.launch(dispatcher) { + val database: FDroidDatabase + synchronized(lock) { + database = INSTANCE ?: error("DB not yet initialized") + } + fixture.prePopulateDb(database) + Log.d(TAG, "Loaded fixtures") + } + } + + override fun onDestructiveMigration(db: SupportSQLiteDatabase) { + onCreate(db) + } + } + +} diff --git a/libs/database/src/main/java/org/fdroid/database/Repository.kt b/libs/database/src/main/java/org/fdroid/database/Repository.kt new file mode 100644 index 000000000..c585d7bf0 --- /dev/null +++ b/libs/database/src/main/java/org/fdroid/database/Repository.kt @@ -0,0 +1,383 @@ +package org.fdroid.database + +import androidx.core.os.LocaleListCompat +import androidx.room.Embedded +import androidx.room.Entity +import androidx.room.ForeignKey +import androidx.room.Ignore +import androidx.room.PrimaryKey +import androidx.room.Relation +import org.fdroid.LocaleChooser.getBestLocale +import org.fdroid.index.IndexFormatVersion +import org.fdroid.index.IndexUtils.getFingerprint +import org.fdroid.index.v2.AntiFeatureV2 +import org.fdroid.index.v2.CategoryV2 +import org.fdroid.index.v2.FileV2 +import org.fdroid.index.v2.LocalizedFileV2 +import org.fdroid.index.v2.LocalizedTextV2 +import org.fdroid.index.v2.MirrorV2 +import org.fdroid.index.v2.ReleaseChannelV2 +import org.fdroid.index.v2.RepoV2 + +@Entity(tableName = CoreRepository.TABLE) +internal data class CoreRepository( + @PrimaryKey(autoGenerate = true) val repoId: Long = 0, + val name: LocalizedTextV2 = emptyMap(), + val icon: LocalizedFileV2?, + val address: String, + val webBaseUrl: String? = null, + val timestamp: Long, + val version: Long?, + val formatVersion: IndexFormatVersion?, + val maxAge: Int?, + val description: LocalizedTextV2 = emptyMap(), + val certificate: String?, +) { + internal companion object { + const val TABLE = "CoreRepository" + } +} + +internal fun RepoV2.toCoreRepository( + repoId: Long = 0, + version: Long, + formatVersion: IndexFormatVersion? = null, + certificate: String? = null, +) = CoreRepository( + repoId = repoId, + name = name, + icon = icon, + address = address, + webBaseUrl = webBaseUrl, + timestamp = timestamp, + version = version, + formatVersion = formatVersion, + maxAge = null, + description = description, + certificate = certificate, +) + +public data class Repository internal constructor( + @Embedded internal val repository: CoreRepository, + @Relation( + parentColumn = "repoId", + entityColumn = "repoId", + ) + internal val mirrors: List, + @Relation( + parentColumn = "repoId", + entityColumn = "repoId", + ) + internal val antiFeatures: List, + @Relation( + parentColumn = "repoId", + entityColumn = "repoId", + ) + internal val categories: List, + @Relation( + parentColumn = "repoId", + entityColumn = "repoId", + ) + internal val releaseChannels: List, + @Relation( + parentColumn = "repoId", + entityColumn = "repoId", + ) + internal val preferences: RepositoryPreferences, +) { + /** + * Used to create a minimal version of a [Repository]. + */ + public constructor( + repoId: Long, + address: String, + timestamp: Long, + formatVersion: IndexFormatVersion, + certificate: String?, + version: Long, + weight: Int, + lastUpdated: Long, + ) : this( + repository = CoreRepository( + repoId = repoId, + icon = null, + address = address, + timestamp = timestamp, + formatVersion = formatVersion, + maxAge = 42, + certificate = certificate, + version = version, + ), + mirrors = emptyList(), + antiFeatures = emptyList(), + categories = emptyList(), + releaseChannels = emptyList(), + preferences = RepositoryPreferences( + repoId = repoId, + weight = weight, + lastUpdated = lastUpdated, + ) + ) + + public val repoId: Long get() = repository.repoId + public val address: String get() = repository.address + public val webBaseUrl: String? get() = repository.webBaseUrl + public val timestamp: Long get() = repository.timestamp + public val version: Long get() = repository.version ?: 0 + public val formatVersion: IndexFormatVersion? get() = repository.formatVersion + public val certificate: String? get() = repository.certificate + + public fun getName(localeList: LocaleListCompat): String? = + repository.name.getBestLocale(localeList) + + public fun getDescription(localeList: LocaleListCompat): String? = + repository.description.getBestLocale(localeList) + + public fun getIcon(localeList: LocaleListCompat): FileV2? = + repository.icon.getBestLocale(localeList) + + public fun getAntiFeatures(): Map { + return antiFeatures.associateBy { antiFeature -> antiFeature.id } + } + + public fun getCategories(): Map { + return categories.associateBy { category -> category.id } + } + + public fun getReleaseChannels(): Map { + return releaseChannels.associateBy { releaseChannel -> releaseChannel.id } + } + + public val weight: Int get() = preferences.weight + public val enabled: Boolean get() = preferences.enabled + public val lastUpdated: Long? get() = preferences.lastUpdated + public val userMirrors: List get() = preferences.userMirrors ?: emptyList() + public val disabledMirrors: List get() = preferences.disabledMirrors ?: emptyList() + public val username: String? get() = preferences.username + public val password: String? get() = preferences.password + + @Suppress("DEPRECATION") + @Deprecated("Only used for v1 index", ReplaceWith("")) + public val lastETag: String? + get() = preferences.lastETag + + /** + * The fingerprint for the [certificate]. + * This gets calculated on first call and is an expensive operation. + * Subsequent calls re-use the + */ + @delegate:Ignore + public val fingerprint: String? by lazy { + certificate?.let { getFingerprint(it) } + } + + /** + * Returns official and user-added mirrors without the [disabledMirrors]. + */ + public fun getMirrors(): List { + return getAllMirrors(true).filter { + !disabledMirrors.contains(it.baseUrl) + } + } + + /** + * Returns all mirrors, including [disabledMirrors]. + */ + @JvmOverloads + public fun getAllMirrors(includeUserMirrors: Boolean = true): List { + val all = mirrors.map { + it.toDownloadMirror() + } + if (includeUserMirrors) userMirrors.map { + org.fdroid.download.Mirror(it) + } else emptyList() + // whether or not the repo address is part of the mirrors is not yet standardized, + // so we may need to add it to the list ourselves + val hasCanonicalMirror = all.find { it.baseUrl == address } != null + return if (hasCanonicalMirror) all else all.toMutableList().apply { + add(0, org.fdroid.download.Mirror(address)) + } + } +} + +/** + * A database table to store repository mirror information. + */ +@Entity( + tableName = Mirror.TABLE, + primaryKeys = ["repoId", "url"], + foreignKeys = [ForeignKey( + entity = CoreRepository::class, + parentColumns = ["repoId"], + childColumns = ["repoId"], + onDelete = ForeignKey.CASCADE, + )], +) +internal data class Mirror( + val repoId: Long, + val url: String, + val location: String? = null, +) { + internal companion object { + const val TABLE = "Mirror" + } + + fun toDownloadMirror(): org.fdroid.download.Mirror = org.fdroid.download.Mirror( + baseUrl = url, + location = location, + ) +} + +internal fun MirrorV2.toMirror(repoId: Long) = Mirror( + repoId = repoId, + url = url, + location = location, +) + +/** + * An attribute belonging to a [Repository]. + */ +public abstract class RepoAttribute { + public abstract val icon: FileV2? + internal abstract val name: LocalizedTextV2 + internal abstract val description: LocalizedTextV2 + + public fun getName(localeList: LocaleListCompat): String? = + name.getBestLocale(localeList) + + public fun getDescription(localeList: LocaleListCompat): String? = + description.getBestLocale(localeList) +} + +/** + * An anti-feature belonging to a [Repository]. + */ +@Entity( + tableName = AntiFeature.TABLE, + primaryKeys = ["repoId", "id"], + foreignKeys = [ForeignKey( + entity = CoreRepository::class, + parentColumns = ["repoId"], + childColumns = ["repoId"], + onDelete = ForeignKey.CASCADE, + )], +) +public data class AntiFeature internal constructor( + internal val repoId: Long, + internal val id: String, + @Embedded(prefix = "icon_") public override val icon: FileV2? = null, + override val name: LocalizedTextV2, + override val description: LocalizedTextV2, +) : RepoAttribute() { + internal companion object { + const val TABLE = "AntiFeature" + } +} + +internal fun Map.toRepoAntiFeatures(repoId: Long) = map { + AntiFeature( + repoId = repoId, + id = it.key, + icon = it.value.icon, + name = it.value.name, + description = it.value.description, + ) +} + +/** + * A category of apps belonging to a [Repository]. + */ +@Entity( + tableName = Category.TABLE, + primaryKeys = ["repoId", "id"], + foreignKeys = [ForeignKey( + entity = CoreRepository::class, + parentColumns = ["repoId"], + childColumns = ["repoId"], + onDelete = ForeignKey.CASCADE, + )], +) +public data class Category internal constructor( + internal val repoId: Long, + public val id: String, + @Embedded(prefix = "icon_") public override val icon: FileV2? = null, + override val name: LocalizedTextV2, + override val description: LocalizedTextV2, +) : RepoAttribute() { + internal companion object { + const val TABLE = "Category" + } +} + +internal fun Map.toRepoCategories(repoId: Long) = map { + Category( + repoId = repoId, + id = it.key, + icon = it.value.icon, + name = it.value.name, + description = it.value.description, + ) +} + +/** + * A release-channel for apps belonging to a [Repository]. + */ +@Entity( + tableName = ReleaseChannel.TABLE, + primaryKeys = ["repoId", "id"], + foreignKeys = [ForeignKey( + entity = CoreRepository::class, + parentColumns = ["repoId"], + childColumns = ["repoId"], + onDelete = ForeignKey.CASCADE, + )], +) +public data class ReleaseChannel( + internal val repoId: Long, + internal val id: String, + @Embedded(prefix = "icon_") public override val icon: FileV2? = null, + override val name: LocalizedTextV2, + override val description: LocalizedTextV2, +) : RepoAttribute() { + internal companion object { + const val TABLE = "ReleaseChannel" + } +} + +internal fun Map.toRepoReleaseChannel(repoId: Long) = map { + ReleaseChannel( + repoId = repoId, + id = it.key, + name = it.value.name, + description = it.value.description, + ) +} + +@Entity(tableName = RepositoryPreferences.TABLE) +internal data class RepositoryPreferences( + @PrimaryKey internal val repoId: Long, + val weight: Int, + val enabled: Boolean = true, + val lastUpdated: Long? = System.currentTimeMillis(), + @Deprecated("Only used for indexV1") val lastETag: String? = null, + val userMirrors: List? = null, + val disabledMirrors: List? = null, + val username: String? = null, + val password: String? = null, +) { + internal companion object { + const val TABLE = "RepositoryPreferences" + } +} + +/** + * A reduced version of [Repository] used to pre-populate the [FDroidDatabase]. + */ +public data class InitialRepository( + val name: String, + val address: String, + val description: String, + val certificate: String, + val version: Long, + val enabled: Boolean, + val weight: Int, +) diff --git a/libs/database/src/main/java/org/fdroid/database/RepositoryDao.kt b/libs/database/src/main/java/org/fdroid/database/RepositoryDao.kt new file mode 100644 index 000000000..e096256cf --- /dev/null +++ b/libs/database/src/main/java/org/fdroid/database/RepositoryDao.kt @@ -0,0 +1,416 @@ +package org.fdroid.database + +import androidx.annotation.VisibleForTesting +import androidx.lifecycle.LiveData +import androidx.room.Dao +import androidx.room.Insert +import androidx.room.OnConflictStrategy.REPLACE +import androidx.room.Query +import androidx.room.RewriteQueriesToDropUnusedColumns +import androidx.room.Transaction +import androidx.room.Update +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.decodeFromJsonElement +import org.fdroid.database.DbDiffUtils.diffAndUpdateListTable +import org.fdroid.database.DbDiffUtils.diffAndUpdateTable +import org.fdroid.index.IndexFormatVersion +import org.fdroid.index.IndexParser.json +import org.fdroid.index.v2.MirrorV2 +import org.fdroid.index.v2.ReflectionDiffer.applyDiff +import org.fdroid.index.v2.RepoV2 + +public interface RepositoryDao { + /** + * Inserts a new [InitialRepository] from a fixture. + * + * @return the [Repository.repoId] of the inserted repo. + */ + public fun insert(initialRepo: InitialRepository): Long + + /** + * Inserts an empty [Repository] for an initial update. + * + * @return the [Repository.repoId] of the inserted repo. + */ + public fun insertEmptyRepo( + address: String, + username: String? = null, + password: String? = null, + ): Long + + /** + * Returns the repository with the given [repoId] or null, if none was found with that ID. + */ + public fun getRepository(repoId: Long): Repository? + + /** + * Returns a list of all [Repository]s in the database. + */ + public fun getRepositories(): List + + /** + * Same as [getRepositories], but does return a [LiveData]. + */ + public fun getLiveRepositories(): LiveData> + + /** + * Returns a live data of all categories declared by all [Repository]s. + */ + public fun getLiveCategories(): LiveData> + + /** + * Enables or disables the repository with the given [repoId]. + * Data from disabled repositories is ignored in many queries. + */ + public fun setRepositoryEnabled(repoId: Long, enabled: Boolean) + + /** + * Updates the user-defined mirrors of the repository with the given [repoId]. + * The existing mirrors get overwritten with the given [mirrors]. + */ + public fun updateUserMirrors(repoId: Long, mirrors: List) + + /** + * Updates the user name and password (for basic authentication) + * of the repository with the given [repoId]. + * The existing user name and password get overwritten with the given [username] and [password]. + */ + public fun updateUsernameAndPassword(repoId: Long, username: String?, password: String?) + + /** + * Updates the disabled mirrors of the repository with the given [repoId]. + * The existing disabled mirrors get overwritten with the given [disabledMirrors]. + */ + public fun updateDisabledMirrors(repoId: Long, disabledMirrors: List) + + /** + * Removes a [Repository] with the given [repoId] with all associated data from the database. + */ + public fun deleteRepository(repoId: Long) + + /** + * Removes all repos and their preferences. + */ + public fun clearAll() +} + +@Dao +internal interface RepositoryDaoInt : RepositoryDao { + + @Insert(onConflict = REPLACE) + fun insertOrReplace(repository: CoreRepository): Long + + @Update + fun update(repository: CoreRepository) + + @Insert(onConflict = REPLACE) + fun insertMirrors(mirrors: List) + + @Insert(onConflict = REPLACE) + fun insertAntiFeatures(repoFeature: List) + + @Insert(onConflict = REPLACE) + fun insertCategories(repoFeature: List) + + @Insert(onConflict = REPLACE) + fun insertReleaseChannels(repoFeature: List) + + @Insert(onConflict = REPLACE) + fun insert(repositoryPreferences: RepositoryPreferences) + + @Transaction + override fun insert(initialRepo: InitialRepository): Long { + val repo = CoreRepository( + name = mapOf("en-US" to initialRepo.name), + address = initialRepo.address, + icon = null, + timestamp = -1, + version = initialRepo.version, + formatVersion = null, + maxAge = null, + description = mapOf("en-US" to initialRepo.description), + certificate = initialRepo.certificate, + ) + val repoId = insertOrReplace(repo) + val repositoryPreferences = RepositoryPreferences( + repoId = repoId, + weight = initialRepo.weight, + lastUpdated = null, + enabled = initialRepo.enabled, + ) + insert(repositoryPreferences) + return repoId + } + + @Transaction + override fun insertEmptyRepo( + address: String, + username: String?, + password: String?, + ): Long { + val repo = CoreRepository( + name = mapOf("en-US" to address), + icon = null, + address = address, + timestamp = -1, + version = null, + formatVersion = null, + maxAge = null, + certificate = null, + ) + val repoId = insertOrReplace(repo) + val currentMaxWeight = getMaxRepositoryWeight() + val repositoryPreferences = RepositoryPreferences( + repoId = repoId, + weight = currentMaxWeight + 1, + lastUpdated = null, + username = username, + password = password, + ) + insert(repositoryPreferences) + return repoId + } + + @Transaction + @VisibleForTesting + fun insertOrReplace(repository: RepoV2, version: Long = 0): Long { + val repoId = insertOrReplace(repository.toCoreRepository(version = version)) + val currentMaxWeight = getMaxRepositoryWeight() + val repositoryPreferences = RepositoryPreferences(repoId, currentMaxWeight + 1) + insert(repositoryPreferences) + insertRepoTables(repoId, repository) + return repoId + } + + @Query("SELECT MAX(weight) FROM ${RepositoryPreferences.TABLE}") + fun getMaxRepositoryWeight(): Int + + @Transaction + @Query("SELECT * FROM ${CoreRepository.TABLE} WHERE repoId = :repoId") + override fun getRepository(repoId: Long): Repository? + + @Transaction + @Query("SELECT * FROM ${CoreRepository.TABLE}") + override fun getRepositories(): List + + @Transaction + @Query("SELECT * FROM ${CoreRepository.TABLE}") + override fun getLiveRepositories(): LiveData> + + @Query("SELECT * FROM ${RepositoryPreferences.TABLE} WHERE repoId = :repoId") + fun getRepositoryPreferences(repoId: Long): RepositoryPreferences? + + @RewriteQueriesToDropUnusedColumns + @Query("""SELECT * FROM ${Category.TABLE} + JOIN ${RepositoryPreferences.TABLE} AS pref USING (repoId) + WHERE pref.enabled = 1 GROUP BY id HAVING MAX(pref.weight)""") + override fun getLiveCategories(): LiveData> + + /** + * Updates an existing repo with new data from a full index update. + * Call [clear] first to ensure old data was removed. + */ + @Transaction + fun update( + repoId: Long, + repository: RepoV2, + version: Long, + formatVersion: IndexFormatVersion, + certificate: String?, + ) { + update(repository.toCoreRepository(repoId, version, formatVersion, certificate)) + insertRepoTables(repoId, repository) + } + + private fun insertRepoTables(repoId: Long, repository: RepoV2) { + insertMirrors(repository.mirrors.map { it.toMirror(repoId) }) + insertAntiFeatures(repository.antiFeatures.toRepoAntiFeatures(repoId)) + insertCategories(repository.categories.toRepoCategories(repoId)) + insertReleaseChannels(repository.releaseChannels.toRepoReleaseChannel(repoId)) + } + + @Update + fun updateRepository(repo: CoreRepository): Int + + /** + * Updates the certificate for the [Repository] with the given [repoId]. + * This should be used for V1 index updating where we only get the full cert + * after reading the entire index file. + * V2 index should use [update] instead as there the certificate is known + * before reading full index. + */ + @Query("UPDATE ${CoreRepository.TABLE} SET certificate = :certificate WHERE repoId = :repoId") + fun updateRepository(repoId: Long, certificate: String) + + @Update + fun updateRepositoryPreferences(preferences: RepositoryPreferences) + + /** + * Used to update an existing repository with a given [jsonObject] JSON diff. + */ + @Transaction + fun updateRepository(repoId: Long, version: Long, jsonObject: JsonObject) { + // get existing repo + val repo = getRepository(repoId) ?: error("Repo $repoId does not exist") + // update repo with JSON diff + updateRepository(applyDiff(repo.repository, jsonObject).copy(version = version)) + // replace mirror list (if it is in the diff) + diffAndUpdateListTable( + jsonObject = jsonObject, + jsonObjectKey = "mirrors", + listParser = { mirrorArray -> + json.decodeFromJsonElement>(mirrorArray).map { + it.toMirror(repoId) + } + }, + deleteList = { deleteMirrors(repoId) }, + insertNewList = { mirrors -> insertMirrors(mirrors) }, + ) + // diff and update the antiFeatures + diffAndUpdateTable( + jsonObject = jsonObject, + jsonObjectKey = "antiFeatures", + itemList = repo.antiFeatures, + itemFinder = { key, item -> item.id == key }, + newItem = { key -> AntiFeature(repoId, key, null, emptyMap(), emptyMap()) }, + deleteAll = { deleteAntiFeatures(repoId) }, + deleteOne = { key -> deleteAntiFeature(repoId, key) }, + insertReplace = { list -> insertAntiFeatures(list) }, + ) + // diff and update the categories + diffAndUpdateTable( + jsonObject = jsonObject, + jsonObjectKey = "categories", + itemList = repo.categories, + itemFinder = { key, item -> item.id == key }, + newItem = { key -> Category(repoId, key, null, emptyMap(), emptyMap()) }, + deleteAll = { deleteCategories(repoId) }, + deleteOne = { key -> deleteCategory(repoId, key) }, + insertReplace = { list -> insertCategories(list) }, + ) + // diff and update the releaseChannels + diffAndUpdateTable( + jsonObject = jsonObject, + jsonObjectKey = "releaseChannels", + itemList = repo.releaseChannels, + itemFinder = { key, item -> item.id == key }, + newItem = { key -> ReleaseChannel(repoId, key, null, emptyMap(), emptyMap()) }, + deleteAll = { deleteReleaseChannels(repoId) }, + deleteOne = { key -> deleteReleaseChannel(repoId, key) }, + insertReplace = { list -> insertReleaseChannels(list) }, + ) + } + + @Query("UPDATE ${RepositoryPreferences.TABLE} SET enabled = :enabled WHERE repoId = :repoId") + override fun setRepositoryEnabled(repoId: Long, enabled: Boolean) + + @Query("""UPDATE ${RepositoryPreferences.TABLE} SET userMirrors = :mirrors + WHERE repoId = :repoId""") + override fun updateUserMirrors(repoId: Long, mirrors: List) + + @Query("""UPDATE ${RepositoryPreferences.TABLE} SET username = :username, password = :password + WHERE repoId = :repoId""") + override fun updateUsernameAndPassword(repoId: Long, username: String?, password: String?) + + @Query("""UPDATE ${RepositoryPreferences.TABLE} SET disabledMirrors = :disabledMirrors + WHERE repoId = :repoId""") + override fun updateDisabledMirrors(repoId: Long, disabledMirrors: List) + + @Transaction + override fun deleteRepository(repoId: Long) { + deleteCoreRepository(repoId) + // we don't use cascading delete for preferences, + // so we can replace index data on full updates + deleteRepositoryPreferences(repoId) + } + + @Query("DELETE FROM ${CoreRepository.TABLE} WHERE repoId = :repoId") + fun deleteCoreRepository(repoId: Long) + + @Query("DELETE FROM ${RepositoryPreferences.TABLE} WHERE repoId = :repoId") + fun deleteRepositoryPreferences(repoId: Long) + + @Query("DELETE FROM ${CoreRepository.TABLE}") + fun deleteAllCoreRepositories() + + @Query("DELETE FROM ${RepositoryPreferences.TABLE}") + fun deleteAllRepositoryPreferences() + + /** + * Used for diffing. + */ + @Query("DELETE FROM ${Mirror.TABLE} WHERE repoId = :repoId") + fun deleteMirrors(repoId: Long) + + /** + * Used for diffing. + */ + @Query("DELETE FROM ${AntiFeature.TABLE} WHERE repoId = :repoId") + fun deleteAntiFeatures(repoId: Long) + + /** + * Used for diffing. + */ + @Query("DELETE FROM ${AntiFeature.TABLE} WHERE repoId = :repoId AND id = :id") + fun deleteAntiFeature(repoId: Long, id: String) + + /** + * Used for diffing. + */ + @Query("DELETE FROM ${Category.TABLE} WHERE repoId = :repoId") + fun deleteCategories(repoId: Long) + + /** + * Used for diffing. + */ + @Query("DELETE FROM ${Category.TABLE} WHERE repoId = :repoId AND id = :id") + fun deleteCategory(repoId: Long, id: String) + + /** + * Used for diffing. + */ + @Query("DELETE FROM ${ReleaseChannel.TABLE} WHERE repoId = :repoId") + fun deleteReleaseChannels(repoId: Long) + + /** + * Used for diffing. + */ + @Query("DELETE FROM ${ReleaseChannel.TABLE} WHERE repoId = :repoId AND id = :id") + fun deleteReleaseChannel(repoId: Long, id: String) + + /** + * Use when replacing an existing repo with a full index. + * This removes all existing index data associated with this repo from the database, + * but does not touch repository preferences. + * @throws IllegalStateException if no repo with the given [repoId] exists. + */ + @Transaction + fun clear(repoId: Long) { + val repo = getRepository(repoId) ?: error("repo with id $repoId does not exist") + // this clears all foreign key associated data since the repo gets replaced + insertOrReplace(repo.repository) + } + + @Transaction + override fun clearAll() { + deleteAllCoreRepositories() + deleteAllRepositoryPreferences() + } + + @VisibleForTesting + @Query("SELECT COUNT(*) FROM ${Mirror.TABLE}") + fun countMirrors(): Int + + @VisibleForTesting + @Query("SELECT COUNT(*) FROM ${AntiFeature.TABLE}") + fun countAntiFeatures(): Int + + @VisibleForTesting + @Query("SELECT COUNT(*) FROM ${Category.TABLE}") + fun countCategories(): Int + + @VisibleForTesting + @Query("SELECT COUNT(*) FROM ${ReleaseChannel.TABLE}") + fun countReleaseChannels(): Int + +} diff --git a/libs/database/src/main/java/org/fdroid/database/Version.kt b/libs/database/src/main/java/org/fdroid/database/Version.kt new file mode 100644 index 000000000..3b1227572 --- /dev/null +++ b/libs/database/src/main/java/org/fdroid/database/Version.kt @@ -0,0 +1,236 @@ +package org.fdroid.database + +import androidx.core.os.LocaleListCompat +import androidx.room.DatabaseView +import androidx.room.Embedded +import androidx.room.Entity +import androidx.room.ForeignKey +import androidx.room.Relation +import org.fdroid.LocaleChooser.getBestLocale +import org.fdroid.database.VersionedStringType.PERMISSION +import org.fdroid.database.VersionedStringType.PERMISSION_SDK_23 +import org.fdroid.index.v2.ANTI_FEATURE_KNOWN_VULNERABILITY +import org.fdroid.index.v2.FileV1 +import org.fdroid.index.v2.FileV2 +import org.fdroid.index.v2.LocalizedTextV2 +import org.fdroid.index.v2.ManifestV2 +import org.fdroid.index.v2.PackageManifest +import org.fdroid.index.v2.PackageVersion +import org.fdroid.index.v2.PackageVersionV2 +import org.fdroid.index.v2.PermissionV2 +import org.fdroid.index.v2.SignerV2 +import org.fdroid.index.v2.UsesSdkV2 + +/** + * A database table entity representing the version of an [App] + * identified by its [versionCode] and [signer]. + * This holds the data of [PackageVersionV2]. + */ +@Entity( + tableName = Version.TABLE, + primaryKeys = ["repoId", "packageName", "versionId"], + foreignKeys = [ForeignKey( + entity = AppMetadata::class, + parentColumns = ["repoId", "packageName"], + childColumns = ["repoId", "packageName"], + onDelete = ForeignKey.CASCADE, + )], +) +internal data class Version( + val repoId: Long, + val packageName: String, + val versionId: String, + val added: Long, + @Embedded(prefix = "file_") val file: FileV1, + @Embedded(prefix = "src_") val src: FileV2? = null, + @Embedded(prefix = "manifest_") val manifest: AppManifest, + override val releaseChannels: List? = emptyList(), + val antiFeatures: Map? = null, + val whatsNew: LocalizedTextV2? = null, + val isCompatible: Boolean, +) : PackageVersion { + internal companion object { + const val TABLE = "Version" + } + + override val versionCode: Long get() = manifest.versionCode + override val signer: SignerV2? get() = manifest.signer + override val packageManifest: PackageManifest get() = manifest + override val hasKnownVulnerability: Boolean + get() = antiFeatures?.contains(ANTI_FEATURE_KNOWN_VULNERABILITY) == true + + internal fun toAppVersion(versionedStrings: List): AppVersion = AppVersion( + version = this, + versionedStrings = versionedStrings, + ) +} + +internal fun PackageVersionV2.toVersion( + repoId: Long, + packageName: String, + versionId: String, + isCompatible: Boolean, +) = Version( + repoId = repoId, + packageName = packageName, + versionId = versionId, + added = added, + file = file, + src = src, + manifest = manifest.toManifest(), + releaseChannels = releaseChannels, + antiFeatures = antiFeatures, + whatsNew = whatsNew, + isCompatible = isCompatible, +) + +/** + * A version of an [App] identified by [AppManifest.versionCode] and [AppManifest.signer]. + */ +public data class AppVersion internal constructor( + @Embedded internal val version: Version, + @Relation( + parentColumn = "versionId", + entityColumn = "versionId", + ) + internal val versionedStrings: List?, +) { + public val repoId: Long get() = version.repoId + public val packageName: String get() = version.packageName + public val added: Long get() = version.added + public val isCompatible: Boolean get() = version.isCompatible + public val manifest: AppManifest get() = version.manifest + public val file: FileV1 get() = version.file + public val src: FileV2? get() = version.src + public val usesPermission: List + get() = versionedStrings?.getPermissions(version) ?: emptyList() + public val usesPermissionSdk23: List + get() = versionedStrings?.getPermissionsSdk23(version) ?: emptyList() + public val featureNames: List get() = version.manifest.features ?: emptyList() + public val nativeCode: List get() = version.manifest.nativecode ?: emptyList() + public val releaseChannels: List get() = version.releaseChannels ?: emptyList() + public val antiFeatureKeys: List + get() = version.antiFeatures?.map { it.key } ?: emptyList() + + public fun getWhatsNew(localeList: LocaleListCompat): String? = + version.whatsNew.getBestLocale(localeList) + + public fun getAntiFeatureReason(antiFeatureKey: String, localeList: LocaleListCompat): String? { + return version.antiFeatures?.get(antiFeatureKey)?.getBestLocale(localeList) + } +} + +/** + * The manifest information of an [AppVersion]. + */ +public data class AppManifest( + public val versionName: String, + public val versionCode: Long, + @Embedded(prefix = "usesSdk_") public val usesSdk: UsesSdkV2? = null, + public override val maxSdkVersion: Int? = null, + @Embedded(prefix = "signer_") public val signer: SignerV2? = null, + public override val nativecode: List? = emptyList(), + public val features: List? = emptyList(), +) : PackageManifest { + public override val minSdkVersion: Int? get() = usesSdk?.minSdkVersion + public override val featureNames: List? get() = features +} + +internal fun ManifestV2.toManifest() = AppManifest( + versionName = versionName, + versionCode = versionCode, + usesSdk = usesSdk, + maxSdkVersion = maxSdkVersion, + signer = signer, + nativecode = nativecode, + features = features.map { it.name }, +) + +@DatabaseView(viewName = HighestVersion.TABLE, + value = """SELECT repoId, packageName, antiFeatures FROM ${Version.TABLE} + GROUP BY repoId, packageName HAVING MAX(manifest_versionCode)""") +internal class HighestVersion( + val repoId: Long, + val packageName: String, + val antiFeatures: Map? = null, +) { + internal companion object { + const val TABLE = "HighestVersion" + } +} + +internal enum class VersionedStringType { + PERMISSION, + PERMISSION_SDK_23, +} + +@Entity( + tableName = VersionedString.TABLE, + primaryKeys = ["repoId", "packageName", "versionId", "type", "name"], + foreignKeys = [ForeignKey( + entity = Version::class, + parentColumns = ["repoId", "packageName", "versionId"], + childColumns = ["repoId", "packageName", "versionId"], + onDelete = ForeignKey.CASCADE, + )], +) +internal data class VersionedString( + val repoId: Long, + val packageName: String, + val versionId: String, + val type: VersionedStringType, + val name: String, + val version: Int? = null, +) { + internal companion object { + const val TABLE = "VersionedString" + } +} + +internal fun List.toVersionedString( + version: Version, + type: VersionedStringType, +) = map { permission -> + VersionedString( + repoId = version.repoId, + packageName = version.packageName, + versionId = version.versionId, + type = type, + name = permission.name, + version = permission.maxSdkVersion, + ) +} + +internal fun ManifestV2.getVersionedStrings(version: Version): List { + return usesPermission.toVersionedString(version, PERMISSION) + + usesPermissionSdk23.toVersionedString(version, PERMISSION_SDK_23) +} + +internal fun List.getPermissions(version: Version) = mapNotNull { v -> + v.map(version, PERMISSION) { + PermissionV2( + name = v.name, + maxSdkVersion = v.version, + ) + } +} + +internal fun List.getPermissionsSdk23(version: Version) = mapNotNull { v -> + v.map(version, PERMISSION_SDK_23) { + PermissionV2( + name = v.name, + maxSdkVersion = v.version, + ) + } +} + +private fun VersionedString.map( + v: Version, + wantedType: VersionedStringType, + factory: () -> T, +): T? { + return if (repoId != v.repoId || packageName != v.packageName || versionId != v.versionId || + type != wantedType + ) null + else factory() +} diff --git a/libs/database/src/main/java/org/fdroid/database/VersionDao.kt b/libs/database/src/main/java/org/fdroid/database/VersionDao.kt new file mode 100644 index 000000000..dd5369aeb --- /dev/null +++ b/libs/database/src/main/java/org/fdroid/database/VersionDao.kt @@ -0,0 +1,228 @@ +package org.fdroid.database + +import androidx.lifecycle.LiveData +import androidx.room.Dao +import androidx.room.Insert +import androidx.room.OnConflictStrategy.REPLACE +import androidx.room.Query +import androidx.room.RewriteQueriesToDropUnusedColumns +import androidx.room.Transaction +import androidx.room.Update +import kotlinx.serialization.SerializationException +import kotlinx.serialization.json.JsonNull +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.decodeFromJsonElement +import org.fdroid.database.VersionedStringType.PERMISSION +import org.fdroid.database.VersionedStringType.PERMISSION_SDK_23 +import org.fdroid.index.IndexParser.json +import org.fdroid.index.v2.PackageManifest +import org.fdroid.index.v2.PackageVersionV2 +import org.fdroid.index.v2.PermissionV2 +import org.fdroid.index.v2.ReflectionDiffer + +public interface VersionDao { + /** + * Inserts new versions for a given [packageName] from a full index. + */ + public fun insert( + repoId: Long, + packageName: String, + packageVersions: Map, + checkIfCompatible: (PackageVersionV2) -> Boolean, + ) + + /** + * Returns a list of versions for the given [packageName] sorting by highest version code first. + */ + public fun getAppVersions(packageName: String): LiveData> +} + +/** + * A list of unknown fields in [PackageVersionV2] that we don't allow for [Version]. + * + * We are applying reflection diffs against internal database classes + * and need to prevent the untrusted external JSON input to modify internal fields in those classes. + * This list must always hold the names of all those internal FIELDS for [Version]. + */ +private val DENY_LIST = listOf("packageName", "repoId", "versionId") + +@Dao +internal interface VersionDaoInt : VersionDao { + + @Transaction + override fun insert( + repoId: Long, + packageName: String, + packageVersions: Map, + checkIfCompatible: (PackageVersionV2) -> Boolean, + ) { + packageVersions.entries.iterator().forEach { (versionId, packageVersion) -> + val isCompatible = checkIfCompatible(packageVersion) + insert(repoId, packageName, versionId, packageVersion, isCompatible) + } + } + + @Transaction + fun insert( + repoId: Long, + packageName: String, + versionId: String, + packageVersion: PackageVersionV2, + isCompatible: Boolean, + ) { + val version = packageVersion.toVersion(repoId, packageName, versionId, isCompatible) + insert(version) + insert(packageVersion.manifest.getVersionedStrings(version)) + } + + @Insert(onConflict = REPLACE) + fun insert(version: Version) + + @Insert(onConflict = REPLACE) + fun insert(versionedString: List) + + @Update + fun update(version: Version) + + fun update( + repoId: Long, + packageName: String, + versionsDiffMap: Map?, + checkIfCompatible: (PackageManifest) -> Boolean, + ) { + if (versionsDiffMap == null) { // no more versions, delete all + deleteAppVersion(repoId, packageName) + } else versionsDiffMap.forEach { (versionId, jsonObject) -> + if (jsonObject == null) { // delete individual version + deleteAppVersion(repoId, packageName, versionId) + } else { + val version = getVersion(repoId, packageName, versionId) + if (version == null) { // new version, parse normally + val packageVersionV2: PackageVersionV2 = + json.decodeFromJsonElement(jsonObject) + val isCompatible = checkIfCompatible(packageVersionV2.packageManifest) + insert(repoId, packageName, versionId, packageVersionV2, isCompatible) + } else { // diff against existing version + diffVersion(version, jsonObject, checkIfCompatible) + } + } + } // end forEach + } + + private fun diffVersion( + version: Version, + jsonObject: JsonObject, + checkIfCompatible: (PackageManifest) -> Boolean, + ) { + // ensure that diff does not include internal keys + DENY_LIST.forEach { forbiddenKey -> + if (jsonObject.containsKey(forbiddenKey)) { + throw SerializationException(forbiddenKey) + } + } + // diff version + val diffedVersion = ReflectionDiffer.applyDiff(version, jsonObject) + val isCompatible = checkIfCompatible(diffedVersion.packageManifest) + update(diffedVersion.copy(isCompatible = isCompatible)) + // diff versioned strings + val manifest = jsonObject["manifest"] + if (manifest is JsonNull) { // no more manifest, delete all versionedStrings + deleteVersionedStrings(version.repoId, version.packageName, version.versionId) + } else if (manifest is JsonObject) { + diffVersionedStrings(version, manifest, "usesPermission", PERMISSION) + diffVersionedStrings(version, manifest, "usesPermissionSdk23", + PERMISSION_SDK_23) + } + } + + private fun diffVersionedStrings( + version: Version, + jsonObject: JsonObject, + key: String, + type: VersionedStringType, + ) = DbDiffUtils.diffAndUpdateListTable( + jsonObject = jsonObject, + jsonObjectKey = key, + listParser = { permissionArray -> + val list: List = json.decodeFromJsonElement(permissionArray) + list.toVersionedString(version, type) + }, + deleteList = { + deleteVersionedStrings(version.repoId, version.packageName, version.versionId, type) + }, + insertNewList = { versionedStrings -> insert(versionedStrings) }, + ) + + @Transaction + @RewriteQueriesToDropUnusedColumns + @Query("""SELECT * FROM ${Version.TABLE} + JOIN ${RepositoryPreferences.TABLE} AS pref USING (repoId) + WHERE pref.enabled = 1 AND packageName = :packageName + ORDER BY manifest_versionCode DESC, pref.weight DESC""") + override fun getAppVersions(packageName: String): LiveData> + + /** + * Only use for testing, not sorted, does take disabled repos into account. + */ + @Transaction + @Query("""SELECT * FROM ${Version.TABLE} + WHERE repoId = :repoId AND packageName = :packageName""") + fun getAppVersions(repoId: Long, packageName: String): List + + @Query("""SELECT * FROM ${Version.TABLE} + WHERE repoId = :repoId AND packageName = :packageName AND versionId = :versionId""") + fun getVersion(repoId: Long, packageName: String, versionId: String): Version? + + /** + * Used for finding versions that are an update, + * so takes [AppPrefs.ignoreVersionCodeUpdate] into account. + */ + @RewriteQueriesToDropUnusedColumns + @Query("""SELECT * FROM ${Version.TABLE} + JOIN ${RepositoryPreferences.TABLE} AS pref USING (repoId) + LEFT JOIN ${AppPrefs.TABLE} AS appPrefs USING (packageName) + WHERE pref.enabled = 1 AND + manifest_versionCode > COALESCE(appPrefs.ignoreVersionCodeUpdate, 0) AND + packageName IN (:packageNames) + ORDER BY manifest_versionCode DESC, pref.weight DESC""") + fun getVersions(packageNames: List): List + + @Query("""SELECT * FROM ${VersionedString.TABLE} + WHERE repoId = :repoId AND packageName = :packageName""") + fun getVersionedStrings(repoId: Long, packageName: String): List + + @Query("""SELECT * FROM ${VersionedString.TABLE} + WHERE repoId = :repoId AND packageName = :packageName AND versionId = :versionId""") + fun getVersionedStrings( + repoId: Long, + packageName: String, + versionId: String, + ): List + + @Query("""DELETE FROM ${Version.TABLE} WHERE repoId = :repoId AND packageName = :packageName""") + fun deleteAppVersion(repoId: Long, packageName: String) + + @Query("""DELETE FROM ${Version.TABLE} + WHERE repoId = :repoId AND packageName = :packageName AND versionId = :versionId""") + fun deleteAppVersion(repoId: Long, packageName: String, versionId: String) + + @Query("""DELETE FROM ${VersionedString.TABLE} + WHERE repoId = :repoId AND packageName = :packageName AND versionId = :versionId""") + fun deleteVersionedStrings(repoId: Long, packageName: String, versionId: String) + + @Query("""DELETE FROM ${VersionedString.TABLE} WHERE repoId = :repoId + AND packageName = :packageName AND versionId = :versionId AND type = :type""") + fun deleteVersionedStrings( + repoId: Long, + packageName: String, + versionId: String, + type: VersionedStringType, + ) + + @Query("SELECT COUNT(*) FROM ${Version.TABLE}") + fun countAppVersions(): Int + + @Query("SELECT COUNT(*) FROM ${VersionedString.TABLE}") + fun countVersionedStrings(): Int + +} diff --git a/libs/database/src/main/java/org/fdroid/download/DownloaderFactory.kt b/libs/database/src/main/java/org/fdroid/download/DownloaderFactory.kt new file mode 100644 index 000000000..57450df20 --- /dev/null +++ b/libs/database/src/main/java/org/fdroid/download/DownloaderFactory.kt @@ -0,0 +1,43 @@ +package org.fdroid.download + +import android.net.Uri +import android.util.Log +import org.fdroid.database.Repository +import java.io.File +import java.io.IOException + +/** + * This is in the database library, because only that knows about the [Repository] class. + */ +public abstract class DownloaderFactory { + + /** + * Same as [create], but trying canonical address first. + * + * See https://gitlab.com/fdroid/fdroidclient/-/issues/1708 for why this is still needed. + */ + @Throws(IOException::class) + public fun createWithTryFirstMirror(repo: Repository, uri: Uri, destFile: File): Downloader { + val tryFirst = repo.getMirrors().find { mirror -> + mirror.baseUrl == repo.address + } + if (tryFirst == null) { + Log.w("DownloaderFactory", "Try-first mirror not found, disabled by user?") + } + val mirrors: List = repo.getMirrors() + return create(repo, mirrors, uri, destFile, tryFirst) + } + + @Throws(IOException::class) + public abstract fun create(repo: Repository, uri: Uri, destFile: File): Downloader + + @Throws(IOException::class) + protected abstract fun create( + repo: Repository, + mirrors: List, + uri: Uri, + destFile: File, + tryFirst: Mirror?, + ): Downloader + +} diff --git a/libs/database/src/main/java/org/fdroid/index/IndexUpdater.kt b/libs/database/src/main/java/org/fdroid/index/IndexUpdater.kt new file mode 100644 index 000000000..f2eaf7286 --- /dev/null +++ b/libs/database/src/main/java/org/fdroid/index/IndexUpdater.kt @@ -0,0 +1,103 @@ +package org.fdroid.index + +import android.net.Uri +import org.fdroid.database.Repository +import org.fdroid.download.Downloader +import org.fdroid.download.NotFoundException +import java.io.File +import java.io.IOException + +/** + * The currently known (and supported) format versions of the F-Droid index. + */ +public enum class IndexFormatVersion { ONE, TWO } + +public sealed class IndexUpdateResult { + public object Unchanged : IndexUpdateResult() + public object Processed : IndexUpdateResult() + public object NotFound : IndexUpdateResult() + public class Error(public val e: Exception) : IndexUpdateResult() +} + +public interface IndexUpdateListener { + public fun onDownloadProgress(repo: Repository, bytesRead: Long, totalBytes: Long) + public fun onUpdateProgress(repo: Repository, appsProcessed: Int, totalApps: Int) +} + +public fun interface RepoUriBuilder { + /** + * Returns an [Uri] for downloading a file from the [Repository]. + * Allowing different implementations for this is useful for exotic repository locations + * that do not allow for simple concatenation. + */ + public fun getUri(repo: Repository, vararg pathElements: String): Uri +} + +internal val defaultRepoUriBuilder = RepoUriBuilder { repo, pathElements -> + val builder = Uri.parse(repo.address).buildUpon() + pathElements.forEach { builder.appendEncodedPath(it) } + builder.build() +} + +public fun interface TempFileProvider { + @Throws(IOException::class) + public fun createTempFile(): File +} + +/** + * A class to update information of a [Repository] in the database with a new downloaded index. + */ +public abstract class IndexUpdater { + + /** + * The [IndexFormatVersion] used by this updater. + * One updater usually handles exactly one format version. + * If you need a higher level of abstraction, check [RepoUpdater]. + */ + public abstract val formatVersion: IndexFormatVersion + + /** + * Updates a new [repo] for the first time. + */ + public fun updateNewRepo( + repo: Repository, + expectedSigningFingerprint: String?, + ): IndexUpdateResult = catchExceptions { + update(repo, null, expectedSigningFingerprint) + } + + /** + * Updates an existing [repo] with a known [Repository.certificate]. + */ + public fun update( + repo: Repository, + ): IndexUpdateResult = catchExceptions { + require(repo.certificate != null) { "Repo ${repo.address} had no certificate" } + update(repo, repo.certificate, null) + } + + private fun catchExceptions(block: () -> IndexUpdateResult): IndexUpdateResult { + return try { + block() + } catch (e: NotFoundException) { + IndexUpdateResult.NotFound + } catch (e: Exception) { + IndexUpdateResult.Error(e) + } + } + + protected abstract fun update( + repo: Repository, + certificate: String?, + fingerprint: String?, + ): IndexUpdateResult +} + +internal fun Downloader.setIndexUpdateListener( + listener: IndexUpdateListener?, + repo: Repository, +) { + if (listener != null) setListener { bytesRead, totalBytes -> + listener.onDownloadProgress(repo, bytesRead, totalBytes) + } +} diff --git a/libs/database/src/main/java/org/fdroid/index/RepoUpdater.kt b/libs/database/src/main/java/org/fdroid/index/RepoUpdater.kt new file mode 100644 index 000000000..fce2a125a --- /dev/null +++ b/libs/database/src/main/java/org/fdroid/index/RepoUpdater.kt @@ -0,0 +1,98 @@ +package org.fdroid.index + +import mu.KotlinLogging +import org.fdroid.CompatibilityChecker +import org.fdroid.database.FDroidDatabase +import org.fdroid.database.Repository +import org.fdroid.download.DownloaderFactory +import org.fdroid.index.v1.IndexV1Updater +import org.fdroid.index.v2.IndexV2Updater +import java.io.File +import java.io.FileNotFoundException + +/** + * Updates a [Repository] with a downloaded index, detects changes and chooses the right + * [IndexUpdater] automatically. + */ +public class RepoUpdater( + tempDir: File, + db: FDroidDatabase, + downloaderFactory: DownloaderFactory, + repoUriBuilder: RepoUriBuilder = defaultRepoUriBuilder, + compatibilityChecker: CompatibilityChecker, + listener: IndexUpdateListener, +) { + private val log = KotlinLogging.logger {} + private val tempFileProvider = TempFileProvider { + File.createTempFile("dl-", "", tempDir) + } + + /** + * A list of [IndexUpdater]s to try, sorted by newest first. + */ + private val indexUpdater = listOf( + IndexV2Updater( + database = db, + tempFileProvider = tempFileProvider, + downloaderFactory = downloaderFactory, + repoUriBuilder = repoUriBuilder, + compatibilityChecker = compatibilityChecker, + listener = listener, + ), + IndexV1Updater( + database = db, + tempFileProvider = tempFileProvider, + downloaderFactory = downloaderFactory, + repoUriBuilder = repoUriBuilder, + compatibilityChecker = compatibilityChecker, + listener = listener, + ), + ) + + /** + * Updates the given [repo]. + * If [Repository.certificate] is null, + * the repo is considered to be new this being the first update. + */ + public fun update( + repo: Repository, + fingerprint: String? = null, + ): IndexUpdateResult { + return if (repo.certificate == null) { + // This is a new repo without a certificate + updateNewRepo(repo, fingerprint) + } else { + update(repo) + } + } + + private fun updateNewRepo( + repo: Repository, + expectedSigningFingerprint: String?, + ): IndexUpdateResult = update(repo) { updater -> + updater.updateNewRepo(repo, expectedSigningFingerprint) + } + + private fun update(repo: Repository): IndexUpdateResult = update(repo) { updater -> + updater.update(repo) + } + + private fun update( + repo: Repository, + doUpdate: (IndexUpdater) -> IndexUpdateResult, + ): IndexUpdateResult { + indexUpdater.forEach { updater -> + // don't downgrade to older updaters if repo used new format already + val repoFormatVersion = repo.formatVersion + if (repoFormatVersion != null && repoFormatVersion > updater.formatVersion) { + val updaterVersion = updater.formatVersion.name + log.warn { "Not using updater $updaterVersion for repo ${repo.address}" } + return@forEach + } + val result = doUpdate(updater) + if (result != IndexUpdateResult.NotFound) return result + } + return IndexUpdateResult.Error(FileNotFoundException("No files found for ${repo.address}")) + } + +} diff --git a/libs/database/src/main/java/org/fdroid/index/v1/IndexV1Updater.kt b/libs/database/src/main/java/org/fdroid/index/v1/IndexV1Updater.kt new file mode 100644 index 000000000..36471e6ff --- /dev/null +++ b/libs/database/src/main/java/org/fdroid/index/v1/IndexV1Updater.kt @@ -0,0 +1,88 @@ +@file:Suppress("DEPRECATION") + +package org.fdroid.index.v1 + +import mu.KotlinLogging +import org.fdroid.CompatibilityChecker +import org.fdroid.database.DbV1StreamReceiver +import org.fdroid.database.FDroidDatabase +import org.fdroid.database.FDroidDatabaseInt +import org.fdroid.database.Repository +import org.fdroid.download.DownloaderFactory +import org.fdroid.index.IndexFormatVersion +import org.fdroid.index.IndexFormatVersion.ONE +import org.fdroid.index.IndexUpdateListener +import org.fdroid.index.IndexUpdateResult +import org.fdroid.index.IndexUpdater +import org.fdroid.index.RepoUriBuilder +import org.fdroid.index.TempFileProvider +import org.fdroid.index.defaultRepoUriBuilder +import org.fdroid.index.setIndexUpdateListener + +internal const val SIGNED_FILE_NAME = "index-v1.jar" + +public class IndexV1Updater( + database: FDroidDatabase, + private val tempFileProvider: TempFileProvider, + private val downloaderFactory: DownloaderFactory, + private val repoUriBuilder: RepoUriBuilder = defaultRepoUriBuilder, + private val compatibilityChecker: CompatibilityChecker, + private val listener: IndexUpdateListener? = null, +) : IndexUpdater() { + + private val log = KotlinLogging.logger {} + public override val formatVersion: IndexFormatVersion = ONE + private val db: FDroidDatabaseInt = database as FDroidDatabaseInt + + override fun update( + repo: Repository, + certificate: String?, + fingerprint: String?, + ): IndexUpdateResult { + // Normally, we shouldn't allow repository downgrades and assert the condition below. + // However, F-Droid is concerned that late v2 bugs will require users to downgrade to v1, + // as it happened already with the migration from v0 to v1. + if (repo.formatVersion != null && repo.formatVersion != ONE) { + log.error { "Format downgrade for ${repo.address}" } + } + val uri = repoUriBuilder.getUri(repo, SIGNED_FILE_NAME) + val file = tempFileProvider.createTempFile() + val downloader = downloaderFactory.createWithTryFirstMirror(repo, uri, file).apply { + cacheTag = repo.lastETag + setIndexUpdateListener(listener, repo) + } + try { + downloader.download() + if (!downloader.hasChanged()) return IndexUpdateResult.Unchanged + val eTag = downloader.cacheTag + + val verifier = IndexV1Verifier(file, certificate, fingerprint) + db.runInTransaction { + val (cert, _) = verifier.getStreamAndVerify { inputStream -> + listener?.onUpdateProgress(repo, 0, 0) + val streamReceiver = DbV1StreamReceiver(db, repo.repoId, compatibilityChecker) + val streamProcessor = + IndexV1StreamProcessor(streamReceiver, certificate, repo.timestamp) + streamProcessor.process(inputStream) + } + // update certificate, if we didn't have any before + val repoDao = db.getRepositoryDao() + if (certificate == null) { + repoDao.updateRepository(repo.repoId, cert) + } + // update RepositoryPreferences with timestamp and ETag (for v1) + val updatedPrefs = repo.preferences.copy( + lastUpdated = System.currentTimeMillis(), + lastETag = eTag, + ) + repoDao.updateRepositoryPreferences(updatedPrefs) + } + } catch (e: OldIndexException) { + if (e.isSameTimestamp) return IndexUpdateResult.Unchanged + else throw e + } finally { + file.delete() + } + return IndexUpdateResult.Processed + } +} diff --git a/libs/database/src/main/java/org/fdroid/index/v2/IndexV2Updater.kt b/libs/database/src/main/java/org/fdroid/index/v2/IndexV2Updater.kt new file mode 100644 index 000000000..c4482038e --- /dev/null +++ b/libs/database/src/main/java/org/fdroid/index/v2/IndexV2Updater.kt @@ -0,0 +1,114 @@ +package org.fdroid.index.v2 + +import org.fdroid.CompatibilityChecker +import org.fdroid.database.DbV2DiffStreamReceiver +import org.fdroid.database.DbV2StreamReceiver +import org.fdroid.database.FDroidDatabase +import org.fdroid.database.FDroidDatabaseInt +import org.fdroid.database.Repository +import org.fdroid.download.DownloaderFactory +import org.fdroid.index.IndexFormatVersion +import org.fdroid.index.IndexFormatVersion.ONE +import org.fdroid.index.IndexFormatVersion.TWO +import org.fdroid.index.IndexParser +import org.fdroid.index.IndexUpdateListener +import org.fdroid.index.IndexUpdateResult +import org.fdroid.index.IndexUpdater +import org.fdroid.index.RepoUriBuilder +import org.fdroid.index.TempFileProvider +import org.fdroid.index.defaultRepoUriBuilder +import org.fdroid.index.parseEntryV2 +import org.fdroid.index.setIndexUpdateListener + +internal const val SIGNED_FILE_NAME = "entry.jar" + +public class IndexV2Updater( + database: FDroidDatabase, + private val tempFileProvider: TempFileProvider, + private val downloaderFactory: DownloaderFactory, + private val repoUriBuilder: RepoUriBuilder = defaultRepoUriBuilder, + private val compatibilityChecker: CompatibilityChecker, + private val listener: IndexUpdateListener? = null, +) : IndexUpdater() { + + public override val formatVersion: IndexFormatVersion = TWO + private val db: FDroidDatabaseInt = database as FDroidDatabaseInt + + override fun update( + repo: Repository, + certificate: String?, + fingerprint: String?, + ): IndexUpdateResult { + val (cert, entry) = getCertAndEntryV2(repo, certificate, fingerprint) + // don't process repos that we already did process in the past + if (entry.timestamp <= repo.timestamp) return IndexUpdateResult.Unchanged + // get diff, if available + val diff = entry.getDiff(repo.timestamp) + return if (diff == null || repo.formatVersion == ONE) { + // no diff found (or this is upgrade from v1 repo), so do full index update + val streamReceiver = DbV2StreamReceiver(db, repo.repoId, compatibilityChecker) + val streamProcessor = IndexV2FullStreamProcessor(streamReceiver, cert) + processStream(repo, entry.index, entry.version, streamProcessor) + } else { + // use available diff + val streamReceiver = DbV2DiffStreamReceiver(db, repo.repoId, compatibilityChecker) + val streamProcessor = IndexV2DiffStreamProcessor(streamReceiver) + processStream(repo, diff, entry.version, streamProcessor) + } + } + + private fun getCertAndEntryV2( + repo: Repository, + certificate: String?, + fingerprint: String?, + ): Pair { + val uri = repoUriBuilder.getUri(repo, SIGNED_FILE_NAME) + val file = tempFileProvider.createTempFile() + val downloader = downloaderFactory.createWithTryFirstMirror(repo, uri, file).apply { + setIndexUpdateListener(listener, repo) + } + try { + downloader.download(-1L) + val verifier = EntryVerifier(file, certificate, fingerprint) + return verifier.getStreamAndVerify { inputStream -> + IndexParser.parseEntryV2(inputStream) + } + } finally { + file.delete() + } + } + + private fun processStream( + repo: Repository, + entryFile: EntryFileV2, + repoVersion: Long, + streamProcessor: IndexV2StreamProcessor, + ): IndexUpdateResult { + val uri = repoUriBuilder.getUri(repo, entryFile.name.trimStart('/')) + val file = tempFileProvider.createTempFile() + val downloader = downloaderFactory.createWithTryFirstMirror(repo, uri, file).apply { + setIndexUpdateListener(listener, repo) + } + try { + downloader.download(entryFile.size, entryFile.sha256) + file.inputStream().use { inputStream -> + val repoDao = db.getRepositoryDao() + db.runInTransaction { + streamProcessor.process(repoVersion, inputStream) { i -> + listener?.onUpdateProgress(repo, i, entryFile.numPackages) + } + // update RepositoryPreferences with timestamp + val repoPrefs = repoDao.getRepositoryPreferences(repo.repoId) + ?: error("No repo prefs for ${repo.repoId}") + val updatedPrefs = repoPrefs.copy( + lastUpdated = System.currentTimeMillis(), + ) + repoDao.updateRepositoryPreferences(updatedPrefs) + } + } + } finally { + file.delete() + } + return IndexUpdateResult.Processed + } +} diff --git a/libs/database/src/sharedTest b/libs/database/src/sharedTest new file mode 120000 index 000000000..2b1905dab --- /dev/null +++ b/libs/database/src/sharedTest @@ -0,0 +1 @@ +../../index/src/sharedTest \ No newline at end of file diff --git a/libs/database/src/test/java/org/fdroid/database/AppPrefsTest.kt b/libs/database/src/test/java/org/fdroid/database/AppPrefsTest.kt new file mode 100644 index 000000000..1e2944774 --- /dev/null +++ b/libs/database/src/test/java/org/fdroid/database/AppPrefsTest.kt @@ -0,0 +1,72 @@ +package org.fdroid.database + +import org.fdroid.test.TestUtils.getRandomString +import org.junit.Test +import kotlin.random.Random +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +internal class AppPrefsTest { + + @Test + fun testDefaults() { + val prefs = AppPrefs(getRandomString()) + assertFalse(prefs.ignoreAllUpdates) + for (i in 1..1337L) assertFalse(prefs.shouldIgnoreUpdate(i)) + assertEquals(emptyList(), prefs.releaseChannels) + } + + @Test + fun testIgnoreVersionCodeUpdate() { + val ignoredCode = Random.nextLong(1, Long.MAX_VALUE - 1) + val prefs = AppPrefs(getRandomString(), ignoredCode) + assertFalse(prefs.ignoreAllUpdates) + assertTrue(prefs.shouldIgnoreUpdate(ignoredCode - 1)) + assertTrue(prefs.shouldIgnoreUpdate(ignoredCode)) + assertFalse(prefs.shouldIgnoreUpdate(ignoredCode + 1)) + + // after toggling, it is not ignored anymore + assertFalse(prefs.toggleIgnoreVersionCodeUpdate(ignoredCode) + .shouldIgnoreUpdate(ignoredCode)) + } + + @Test + fun testIgnoreAllUpdates() { + val prefs = AppPrefs(getRandomString()).toggleIgnoreAllUpdates() + assertTrue(prefs.ignoreAllUpdates) + assertTrue(prefs.shouldIgnoreUpdate(Random.nextLong())) + assertTrue(prefs.shouldIgnoreUpdate(Random.nextLong())) + assertTrue(prefs.shouldIgnoreUpdate(Random.nextLong())) + + // after toggling, all are not ignored anymore + val toggled = prefs.toggleIgnoreAllUpdates() + assertFalse(toggled.ignoreAllUpdates) + assertFalse(toggled.shouldIgnoreUpdate(Random.nextLong(1, Long.MAX_VALUE - 1))) + assertFalse(toggled.shouldIgnoreUpdate(Random.nextLong(1, Long.MAX_VALUE - 1))) + assertFalse(toggled.shouldIgnoreUpdate(Random.nextLong(1, Long.MAX_VALUE - 1))) + } + + @Test + fun testReleaseChannels() { + // no release channels initially + val prefs = AppPrefs(getRandomString()) + assertEquals(emptyList(), prefs.releaseChannels) + + // A gets toggled and is then in channels + val a = prefs.toggleReleaseChannel("A") + assertEquals(listOf("A"), a.releaseChannels) + + // toggling it off returns empty list again + assertEquals(emptyList(), a.toggleReleaseChannel("A").releaseChannels) + + // toggling A and B returns both + val ab = prefs.toggleReleaseChannel("A").toggleReleaseChannel("B") + assertEquals(setOf("A", "B"), ab.releaseChannels.toSet()) + + // toggling both off returns empty list again + assertEquals(emptyList(), + ab.toggleReleaseChannel("A").toggleReleaseChannel("B").releaseChannels) + } + +} diff --git a/libs/database/src/test/java/org/fdroid/database/ConvertersTest.kt b/libs/database/src/test/java/org/fdroid/database/ConvertersTest.kt new file mode 100644 index 000000000..37e46bfb1 --- /dev/null +++ b/libs/database/src/test/java/org/fdroid/database/ConvertersTest.kt @@ -0,0 +1,41 @@ +package org.fdroid.database + +import org.fdroid.test.TestRepoUtils.getRandomLocalizedFileV2 +import org.fdroid.test.TestUtils.getRandomList +import org.fdroid.test.TestUtils.getRandomString +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull + +internal class ConvertersTest { + + @Test + fun testListConversion() { + val list = getRandomList { getRandomString() } + + val str = Converters.listStringToString(list) + val convertedList = Converters.fromStringToListString(str) + assertEquals(list, convertedList) + } + + @Test + fun testEmptyListConversion() { + val list = emptyList() + + val str = Converters.listStringToString(list) + assertNull(str) + assertNull(Converters.listStringToString(null)) + val convertedList = Converters.fromStringToListString(str) + assertEquals(list, convertedList) + } + + @Test + fun testFileV2Conversion() { + val file = getRandomLocalizedFileV2() + + val str = Converters.localizedFileV2toString(file) + val convertedFile = Converters.fromStringToLocalizedFileV2(str) + assertEquals(file, convertedFile) + } + +} diff --git a/libs/database/src/test/java/org/fdroid/database/DbV2StreamReceiverTest.kt b/libs/database/src/test/java/org/fdroid/database/DbV2StreamReceiverTest.kt new file mode 100644 index 000000000..004720c30 --- /dev/null +++ b/libs/database/src/test/java/org/fdroid/database/DbV2StreamReceiverTest.kt @@ -0,0 +1,51 @@ +package org.fdroid.database + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import io.mockk.every +import io.mockk.mockk +import kotlinx.serialization.SerializationException +import org.fdroid.CompatibilityChecker +import org.fdroid.index.v2.FileV2 +import org.fdroid.index.v2.RepoV2 +import org.junit.Test +import org.junit.runner.RunWith +import kotlin.test.assertFailsWith + +@RunWith(AndroidJUnit4::class) +internal class DbV2StreamReceiverTest { + + private val db: FDroidDatabaseInt = mockk() + private val compatChecker: CompatibilityChecker = mockk() + private val dbV2StreamReceiver = DbV2StreamReceiver(db, 42L, compatChecker) + + @Test + fun testFileV2Verified() { + // proper icon file passes + val repoV2 = RepoV2( + icon = mapOf("en" to FileV2(name = "/foo", sha256 = "bar", size = 23L)), + address = "http://example.org", + timestamp = 42L, + ) + every { db.getRepositoryDao() } returns mockk(relaxed = true) + dbV2StreamReceiver.receive(repoV2, 42L, "cert") + + // icon file without leading / does not pass + val repoV2NoSlash = + repoV2.copy(icon = mapOf("en" to FileV2(name = "foo", sha256 = "bar", size = 23L))) + assertFailsWith { + dbV2StreamReceiver.receive(repoV2NoSlash, 42L, "cert") + } + + // icon file without sha256 hash fails + val repoNoSha256 = repoV2.copy(icon = mapOf("en" to FileV2(name = "/foo", size = 23L))) + assertFailsWith { + dbV2StreamReceiver.receive(repoNoSha256, 42L, "cert") + } + + // icon file without size fails + val repoNoSize = repoV2.copy(icon = mapOf("en" to FileV2(name = "/foo", sha256 = "bar"))) + assertFailsWith { + dbV2StreamReceiver.receive(repoNoSize, 42L, "cert") + } + } +} diff --git a/index/.gitignore b/libs/download/.gitignore similarity index 100% rename from index/.gitignore rename to libs/download/.gitignore diff --git a/download/build.gradle b/libs/download/build.gradle similarity index 99% rename from download/build.gradle rename to libs/download/build.gradle index 9fbceb6e0..d8ddeb5cd 100644 --- a/download/build.gradle +++ b/libs/download/build.gradle @@ -99,7 +99,6 @@ android { } defaultConfig { minSdkVersion 21 - targetSdkVersion 25 testInstrumentationRunner 'androidx.test.runner.AndroidJUnitRunner' testInstrumentationRunnerArguments disableAnalytics: 'true' } diff --git a/download/gradle.properties b/libs/download/gradle.properties similarity index 100% rename from download/gradle.properties rename to libs/download/gradle.properties diff --git a/download/lint.xml b/libs/download/lint.xml similarity index 52% rename from download/lint.xml rename to libs/download/lint.xml index ce5cc7387..f8ca69ebc 100644 --- a/download/lint.xml +++ b/libs/download/lint.xml @@ -1,7 +1,5 @@ - - \ No newline at end of file diff --git a/download/src/androidAndroidTest/kotlin/org/fdroid/download/HttpManagerInstrumentationTest.kt b/libs/download/src/androidAndroidTest/kotlin/org/fdroid/download/HttpManagerInstrumentationTest.kt similarity index 100% rename from download/src/androidAndroidTest/kotlin/org/fdroid/download/HttpManagerInstrumentationTest.kt rename to libs/download/src/androidAndroidTest/kotlin/org/fdroid/download/HttpManagerInstrumentationTest.kt diff --git a/download/src/androidMain/AndroidManifest.xml b/libs/download/src/androidMain/AndroidManifest.xml similarity index 100% rename from download/src/androidMain/AndroidManifest.xml rename to libs/download/src/androidMain/AndroidManifest.xml diff --git a/download/src/androidMain/kotlin/org/fdroid/download/Downloader.kt b/libs/download/src/androidMain/kotlin/org/fdroid/download/Downloader.kt similarity index 75% rename from download/src/androidMain/kotlin/org/fdroid/download/Downloader.kt rename to libs/download/src/androidMain/kotlin/org/fdroid/download/Downloader.kt index 555335307..0a8d165fd 100644 --- a/download/src/androidMain/kotlin/org/fdroid/download/Downloader.kt +++ b/libs/download/src/androidMain/kotlin/org/fdroid/download/Downloader.kt @@ -2,11 +2,13 @@ package org.fdroid.download import mu.KotlinLogging import org.fdroid.fdroid.ProgressListener +import org.fdroid.fdroid.isMatching import java.io.File import java.io.FileOutputStream import java.io.IOException import java.io.InputStream import java.io.OutputStream +import java.security.MessageDigest public abstract class Downloader constructor( @JvmField @@ -17,6 +19,13 @@ public abstract class Downloader constructor( private val log = KotlinLogging.logger {} } + protected var fileSize: Long? = null + + /** + * If not null, this is the expected sha256 hash of the [outputFile] after download. + */ + protected var sha256: String? = null + /** * If you ask for the cacheTag before calling download(), you will get the * same one you passed in (if any). If you call it after download(), you @@ -25,6 +34,7 @@ public abstract class Downloader constructor( * If this cacheTag matches that returned by the server, then no download will * take place, and a status code of 304 will be returned by download(). */ + @Deprecated("Used only for v1 repos") public var cacheTag: String? = null @Volatile @@ -36,13 +46,24 @@ public abstract class Downloader constructor( /** * Call this to start the download. * Never call this more than once. Create a new [Downloader], if you need to download again! + * + * @totalSize must be set to what the index tells us the size will be + * @sha256 must be set to the sha256 hash from the index and only be null for `entry.jar`. */ @Throws(IOException::class, InterruptedException::class) + public abstract fun download(totalSize: Long, sha256: String? = null) + + /** + * Call this to start the download. + * Never call this more than once. Create a new [Downloader], if you need to download again! + */ + @Deprecated("Use only for v1 repos") + @Throws(IOException::class, InterruptedException::class) public abstract fun download() - @Throws(IOException::class) + @Throws(IOException::class, NotFoundException::class) protected abstract fun getInputStream(resumable: Boolean): InputStream - protected open suspend fun getBytes(resumable: Boolean, receiver: (ByteArray) -> Unit) { + protected open suspend fun getBytes(resumable: Boolean, receiver: BytesReceiver) { throw NotImplementedError() } @@ -57,6 +78,7 @@ public abstract class Downloader constructor( * After calling [download], this returns true if a new file was downloaded and * false if the file on the server has not changed and thus was not downloaded. */ + @Deprecated("Only for v1 repos") public abstract fun hasChanged(): Boolean public abstract fun close() @@ -88,17 +110,28 @@ public abstract class Downloader constructor( @Throws(InterruptedException::class, IOException::class, NoResumeException::class) protected suspend fun downloadFromBytesReceiver(isResume: Boolean) { try { + val messageDigest: MessageDigest? = if (sha256 == null) null else { + MessageDigest.getInstance("SHA-256") + } FileOutputStream(outputFile, isResume).use { outputStream -> var bytesCopied = outputFile.length() var lastTimeReported = 0L val bytesTotal = totalDownloadSize() - getBytes(isResume) { bytes -> + getBytes(isResume) { bytes, numTotalBytes -> // Getting the input stream is slow(ish) for HTTP downloads, so we'll check if // we were interrupted before proceeding to the download. throwExceptionIfInterrupted() outputStream.write(bytes) + messageDigest?.update(bytes) bytesCopied += bytes.size - lastTimeReported = reportProgress(lastTimeReported, bytesCopied, bytesTotal) + val total = if (bytesTotal == -1L) numTotalBytes ?: -1L else bytesTotal + lastTimeReported = reportProgress(lastTimeReported, bytesCopied, total) + } + // check if expected sha256 hash matches + sha256?.let { expectedHash -> + if (!messageDigest.isMatching(expectedHash)) { + throw IOException("Hash not matching") + } } // force progress reporting at the end reportProgress(0L, bytesCopied, bytesTotal) @@ -119,6 +152,9 @@ public abstract class Downloader constructor( */ @Throws(IOException::class, InterruptedException::class) private fun copyInputToOutputStream(input: InputStream, output: OutputStream) { + val messageDigest: MessageDigest? = if (sha256 == null) null else { + MessageDigest.getInstance("SHA-256") + } try { var bytesCopied = outputFile.length() var lastTimeReported = 0L @@ -128,10 +164,17 @@ public abstract class Downloader constructor( while (numBytes >= 0) { throwExceptionIfInterrupted() output.write(buffer, 0, numBytes) + messageDigest?.update(buffer, 0, numBytes) bytesCopied += numBytes lastTimeReported = reportProgress(lastTimeReported, bytesCopied, bytesTotal) numBytes = input.read(buffer) } + // check if expected sha256 hash matches + sha256?.let { expectedHash -> + if (!messageDigest.isMatching(expectedHash)) { + throw IOException("Hash not matching") + } + } // force progress reporting at the end reportProgress(0L, bytesCopied, bytesTotal) } finally { @@ -176,3 +219,7 @@ public abstract class Downloader constructor( } } + +public fun interface BytesReceiver { + public suspend fun receive(bytes: ByteArray, numTotalBytes: Long?) +} diff --git a/download/src/androidMain/kotlin/org/fdroid/download/HttpDownloader.kt b/libs/download/src/androidMain/kotlin/org/fdroid/download/HttpDownloader.kt similarity index 90% rename from download/src/androidMain/kotlin/org/fdroid/download/HttpDownloader.kt rename to libs/download/src/androidMain/kotlin/org/fdroid/download/HttpDownloader.kt index 68621f2cc..bfcaef7e0 100644 --- a/download/src/androidMain/kotlin/org/fdroid/download/HttpDownloader.kt +++ b/libs/download/src/androidMain/kotlin/org/fdroid/download/HttpDownloader.kt @@ -21,10 +21,8 @@ */ package org.fdroid.download -import android.annotation.TargetApi -import android.os.Build.VERSION.SDK_INT import io.ktor.client.plugins.ResponseException -import kotlinx.coroutines.DelicateCoroutinesApi +import io.ktor.http.HttpStatusCode.Companion.NotFound import kotlinx.coroutines.runBlocking import mu.KotlinLogging import java.io.File @@ -45,23 +43,30 @@ public class HttpDownloader constructor( val log = KotlinLogging.logger {} } + @Deprecated("Only for v1 repos") private var hasChanged = false - private var fileSize = -1L override fun getInputStream(resumable: Boolean): InputStream { throw NotImplementedError("Use getInputStreamSuspend instead.") } @Throws(IOException::class, NoResumeException::class) - override suspend fun getBytes(resumable: Boolean, receiver: (ByteArray) -> Unit) { + protected override suspend fun getBytes(resumable: Boolean, receiver: BytesReceiver) { val skipBytes = if (resumable) outputFile.length() else null return try { httpManager.get(request, skipBytes, receiver) } catch (e: ResponseException) { - throw IOException(e) + if (e.response.status == NotFound) throw NotFoundException(e) + else throw IOException(e) } } + public override fun download(totalSize: Long, sha256: String?) { + this.fileSize = totalSize + this.sha256 = sha256 + downloadToFile() + } + /** * Get a remote file, checking the HTTP response code, if it has changed since * the last time a download was tried. @@ -98,9 +103,9 @@ public class HttpDownloader constructor( * * @see [Cookieless cookies](http://lucb1e.com/rp/cookielesscookies) */ - @OptIn(DelicateCoroutinesApi::class) + @Suppress("DEPRECATION") @Throws(IOException::class, InterruptedException::class) - override fun download() { + public override fun download() { val headInfo = runBlocking { httpManager.head(request, cacheTag) ?: throw IOException() } @@ -137,9 +142,13 @@ public class HttpDownloader constructor( } hasChanged = true + downloadToFile() + } + + private fun downloadToFile() { var resumable = false val fileLength = outputFile.length() - if (fileLength > fileSize) { + if (fileLength > fileSize ?: -1) { if (!outputFile.delete()) log.warn { "Warning: " + outputFile.absolutePath + " not deleted" } @@ -163,15 +172,10 @@ public class HttpDownloader constructor( } } - @TargetApi(24) - public override fun totalDownloadSize(): Long { - return if (SDK_INT < 24) { - fileSize.toInt().toLong() // TODO why? - } else { - fileSize - } - } + protected override fun totalDownloadSize(): Long = fileSize ?: -1L + @Suppress("DEPRECATION") + @Deprecated("Only for v1 repos") override fun hasChanged(): Boolean { return hasChanged } diff --git a/download/src/androidMain/kotlin/org/fdroid/download/HttpManager.kt b/libs/download/src/androidMain/kotlin/org/fdroid/download/HttpManager.kt similarity index 100% rename from download/src/androidMain/kotlin/org/fdroid/download/HttpManager.kt rename to libs/download/src/androidMain/kotlin/org/fdroid/download/HttpManager.kt diff --git a/download/src/androidMain/kotlin/org/fdroid/download/HttpPoster.kt b/libs/download/src/androidMain/kotlin/org/fdroid/download/HttpPoster.kt similarity index 100% rename from download/src/androidMain/kotlin/org/fdroid/download/HttpPoster.kt rename to libs/download/src/androidMain/kotlin/org/fdroid/download/HttpPoster.kt diff --git a/download/src/androidMain/kotlin/org/fdroid/download/glide/DownloadRequestLoader.kt b/libs/download/src/androidMain/kotlin/org/fdroid/download/glide/DownloadRequestLoader.kt similarity index 100% rename from download/src/androidMain/kotlin/org/fdroid/download/glide/DownloadRequestLoader.kt rename to libs/download/src/androidMain/kotlin/org/fdroid/download/glide/DownloadRequestLoader.kt diff --git a/download/src/androidMain/kotlin/org/fdroid/download/glide/HttpFetcher.kt b/libs/download/src/androidMain/kotlin/org/fdroid/download/glide/HttpFetcher.kt similarity index 100% rename from download/src/androidMain/kotlin/org/fdroid/download/glide/HttpFetcher.kt rename to libs/download/src/androidMain/kotlin/org/fdroid/download/glide/HttpFetcher.kt diff --git a/download/src/androidMain/kotlin/org/fdroid/download/glide/HttpGlideUrlLoader.kt b/libs/download/src/androidMain/kotlin/org/fdroid/download/glide/HttpGlideUrlLoader.kt similarity index 100% rename from download/src/androidMain/kotlin/org/fdroid/download/glide/HttpGlideUrlLoader.kt rename to libs/download/src/androidMain/kotlin/org/fdroid/download/glide/HttpGlideUrlLoader.kt diff --git a/libs/download/src/androidMain/kotlin/org/fdroid/fdroid/HashUtils.kt b/libs/download/src/androidMain/kotlin/org/fdroid/fdroid/HashUtils.kt new file mode 100644 index 000000000..64e421b35 --- /dev/null +++ b/libs/download/src/androidMain/kotlin/org/fdroid/fdroid/HashUtils.kt @@ -0,0 +1,13 @@ +package org.fdroid.fdroid + +import java.security.MessageDigest + +internal fun MessageDigest?.isMatching(sha256: String): Boolean { + if (this == null) return false + val hexDigest = digest().toHex() + return hexDigest.equals(sha256, ignoreCase = true) +} + +internal fun ByteArray.toHex(): String = joinToString(separator = "") { eachByte -> + "%02x".format(eachByte) +} diff --git a/download/src/androidTest/kotlin/org/fdroid/download/HttpDownloaderTest.kt b/libs/download/src/androidTest/kotlin/org/fdroid/download/HttpDownloaderTest.kt similarity index 81% rename from download/src/androidTest/kotlin/org/fdroid/download/HttpDownloaderTest.kt rename to libs/download/src/androidTest/kotlin/org/fdroid/download/HttpDownloaderTest.kt index bca53c4c8..ab4b65ceb 100644 --- a/download/src/androidTest/kotlin/org/fdroid/download/HttpDownloaderTest.kt +++ b/libs/download/src/androidTest/kotlin/org/fdroid/download/HttpDownloaderTest.kt @@ -19,6 +19,7 @@ import org.fdroid.runSuspend import org.junit.Assume.assumeTrue import org.junit.Rule import org.junit.rules.TemporaryFolder +import java.io.IOException import java.net.BindException import java.net.ServerSocket import kotlin.random.Random @@ -31,7 +32,7 @@ import kotlin.test.fail private const val TOR_SOCKS_PORT = 9050 -@Suppress("BlockingMethodInNonBlockingContext") +@Suppress("BlockingMethodInNonBlockingContext", "DEPRECATION") internal class HttpDownloaderTest { @get:Rule @@ -55,6 +56,39 @@ internal class HttpDownloaderTest { assertContentEquals(bytes, file.readBytes()) } + @Test + fun testDownloadWithCorrectHash() = runSuspend { + val file = folder.newFile() + val bytes = "We know the hash for this string".encodeToByteArray() + var progressReported = false + + val mockEngine = MockEngine { respond(bytes) } + val httpManager = HttpManager(userAgent, null, httpClientEngineFactory = get(mockEngine)) + val httpDownloader = HttpDownloader(httpManager, downloadRequest, file) + httpDownloader.setListener { _, totalBytes -> + assertEquals(bytes.size.toLong(), totalBytes) + progressReported = true + } + httpDownloader.download(bytes.size.toLong(), + "e3802e5f8ae3dc7bbf5f1f4f7fb825d9bce9d1ddce50ac564fcbcfdeb31f1b90") + + assertContentEquals(bytes, file.readBytes()) + assertTrue(progressReported) + } + + @Test(expected = IOException::class) + fun testDownloadWithWrongHash() = runSuspend { + val file = folder.newFile() + val bytes = "We know the hash for this string".encodeToByteArray() + + val mockEngine = MockEngine { respond(bytes) } + val httpManager = HttpManager(userAgent, null, httpClientEngineFactory = get(mockEngine)) + val httpDownloader = HttpDownloader(httpManager, downloadRequest, file) + httpDownloader.download(bytes.size.toLong(), "This is not the right hash") + + assertContentEquals(bytes, file.readBytes()) + } + @Test fun testResumeSuccess() = runSuspend { val file = folder.newFile() diff --git a/download/src/androidTest/kotlin/org/fdroid/download/HttpPosterTest.kt b/libs/download/src/androidTest/kotlin/org/fdroid/download/HttpPosterTest.kt similarity index 100% rename from download/src/androidTest/kotlin/org/fdroid/download/HttpPosterTest.kt rename to libs/download/src/androidTest/kotlin/org/fdroid/download/HttpPosterTest.kt diff --git a/download/src/commonMain/kotlin/org/fdroid/download/DownloadRequest.kt b/libs/download/src/commonMain/kotlin/org/fdroid/download/DownloadRequest.kt similarity index 100% rename from download/src/commonMain/kotlin/org/fdroid/download/DownloadRequest.kt rename to libs/download/src/commonMain/kotlin/org/fdroid/download/DownloadRequest.kt diff --git a/download/src/commonMain/kotlin/org/fdroid/download/HeadInfo.kt b/libs/download/src/commonMain/kotlin/org/fdroid/download/HeadInfo.kt similarity index 100% rename from download/src/commonMain/kotlin/org/fdroid/download/HeadInfo.kt rename to libs/download/src/commonMain/kotlin/org/fdroid/download/HeadInfo.kt diff --git a/download/src/commonMain/kotlin/org/fdroid/download/HttpManager.kt b/libs/download/src/commonMain/kotlin/org/fdroid/download/HttpManager.kt similarity index 93% rename from download/src/commonMain/kotlin/org/fdroid/download/HttpManager.kt rename to libs/download/src/commonMain/kotlin/org/fdroid/download/HttpManager.kt index 5d488c3e3..60ed5e9df 100644 --- a/download/src/commonMain/kotlin/org/fdroid/download/HttpManager.kt +++ b/libs/download/src/commonMain/kotlin/org/fdroid/download/HttpManager.kt @@ -119,9 +119,10 @@ public open class HttpManager @JvmOverloads constructor( public suspend fun get( request: DownloadRequest, skipFirstBytes: Long? = null, - receiver: suspend (ByteArray) -> Unit, + receiver: BytesReceiver, ): Unit = mirrorChooser.mirrorRequest(request) { mirror, url -> getHttpStatement(request, mirror, url, skipFirstBytes).execute { response -> + val contentLength = response.contentLength() if (skipFirstBytes != null && response.status != PartialContent) { throw NoResumeException() } @@ -130,7 +131,7 @@ public open class HttpManager @JvmOverloads constructor( while (!channel.isClosedForRead) { val packet = channel.readRemaining(limit) while (!packet.isEmpty) { - receiver(packet.readBytes()) + receiver.receive(packet.readBytes(), contentLength) } } } @@ -179,7 +180,7 @@ public open class HttpManager @JvmOverloads constructor( skipFirstBytes: Long? = null, ): ByteArray { val channel = ByteChannel() - get(request, skipFirstBytes) { bytes -> + get(request, skipFirstBytes) { bytes, _ -> channel.writeFully(bytes) } channel.close() @@ -224,4 +225,14 @@ public open class HttpManager @JvmOverloads constructor( } } +/** + * Thrown if we tried to resume a download, but the current mirror server does not offer resuming. + */ public class NoResumeException : Exception() + +/** + * Thrown when a file was not found. + * Catching this is useful when checking if a new index version exists + * and then falling back to an older version. + */ +public class NotFoundException(e: Throwable? = null) : Exception(e) diff --git a/download/src/commonMain/kotlin/org/fdroid/download/Mirror.kt b/libs/download/src/commonMain/kotlin/org/fdroid/download/Mirror.kt similarity index 95% rename from download/src/commonMain/kotlin/org/fdroid/download/Mirror.kt rename to libs/download/src/commonMain/kotlin/org/fdroid/download/Mirror.kt index 79d03d3ee..4235cf13a 100644 --- a/download/src/commonMain/kotlin/org/fdroid/download/Mirror.kt +++ b/libs/download/src/commonMain/kotlin/org/fdroid/download/Mirror.kt @@ -7,7 +7,7 @@ import io.ktor.http.appendPathSegments import mu.KotlinLogging public data class Mirror @JvmOverloads constructor( - private val baseUrl: String, + val baseUrl: String, val location: String? = null, ) { public val url: Url by lazy { @@ -34,6 +34,8 @@ public data class Mirror @JvmOverloads constructor( public fun isLocal(): Boolean = url.isLocal() + public fun isHttp(): Boolean = url.protocol.name.startsWith("http") + public companion object { @JvmStatic public fun fromStrings(list: List): List = list.map { Mirror(it) } diff --git a/download/src/commonMain/kotlin/org/fdroid/download/MirrorChooser.kt b/libs/download/src/commonMain/kotlin/org/fdroid/download/MirrorChooser.kt similarity index 92% rename from download/src/commonMain/kotlin/org/fdroid/download/MirrorChooser.kt rename to libs/download/src/commonMain/kotlin/org/fdroid/download/MirrorChooser.kt index fad2a438b..18b67b25a 100644 --- a/download/src/commonMain/kotlin/org/fdroid/download/MirrorChooser.kt +++ b/libs/download/src/commonMain/kotlin/org/fdroid/download/MirrorChooser.kt @@ -2,6 +2,7 @@ package org.fdroid.download import io.ktor.client.plugins.ResponseException import io.ktor.http.HttpStatusCode.Companion.Forbidden +import io.ktor.http.HttpStatusCode.Companion.NotFound import io.ktor.http.Url import io.ktor.utils.io.errors.IOException import mu.KotlinLogging @@ -43,6 +44,8 @@ internal abstract class MirrorChooserImpl : MirrorChooser { } catch (e: ResponseException) { // don't try other mirrors if we got Forbidden response, but supplied credentials if (downloadRequest.hasCredentials && e.response.status == Forbidden) throw e + // don't try other mirrors if we got NotFount response and downloaded a repo + if (downloadRequest.tryFirstMirror != null && e.response.status == NotFound) throw e // also throw if this is the last mirror to try, otherwise try next throwOnLastMirror(e, index == downloadRequest.mirrors.size - 1) } catch (e: IOException) { diff --git a/download/src/commonMain/kotlin/org/fdroid/download/Proxy.kt b/libs/download/src/commonMain/kotlin/org/fdroid/download/Proxy.kt similarity index 100% rename from download/src/commonMain/kotlin/org/fdroid/download/Proxy.kt rename to libs/download/src/commonMain/kotlin/org/fdroid/download/Proxy.kt diff --git a/download/src/commonMain/kotlin/org/fdroid/fdroid/ProgressListener.kt b/libs/download/src/commonMain/kotlin/org/fdroid/fdroid/ProgressListener.kt similarity index 93% rename from download/src/commonMain/kotlin/org/fdroid/fdroid/ProgressListener.kt rename to libs/download/src/commonMain/kotlin/org/fdroid/fdroid/ProgressListener.kt index 8d988c6c3..b4a47b3b8 100644 --- a/download/src/commonMain/kotlin/org/fdroid/fdroid/ProgressListener.kt +++ b/libs/download/src/commonMain/kotlin/org/fdroid/fdroid/ProgressListener.kt @@ -16,6 +16,6 @@ package org.fdroid.fdroid * * `int`s, i.e. [String.hashCode] * */ -public interface ProgressListener { +public fun interface ProgressListener { public fun onProgress(bytesRead: Long, totalBytes: Long) } diff --git a/download/src/commonTest/kotlin/org/fdroid/TestUtils.kt b/libs/download/src/commonTest/kotlin/org/fdroid/TestUtils.kt similarity index 100% rename from download/src/commonTest/kotlin/org/fdroid/TestUtils.kt rename to libs/download/src/commonTest/kotlin/org/fdroid/TestUtils.kt diff --git a/download/src/commonTest/kotlin/org/fdroid/download/HttpManagerIntegrationTest.kt b/libs/download/src/commonTest/kotlin/org/fdroid/download/HttpManagerIntegrationTest.kt similarity index 100% rename from download/src/commonTest/kotlin/org/fdroid/download/HttpManagerIntegrationTest.kt rename to libs/download/src/commonTest/kotlin/org/fdroid/download/HttpManagerIntegrationTest.kt diff --git a/download/src/commonTest/kotlin/org/fdroid/download/HttpManagerTest.kt b/libs/download/src/commonTest/kotlin/org/fdroid/download/HttpManagerTest.kt similarity index 91% rename from download/src/commonTest/kotlin/org/fdroid/download/HttpManagerTest.kt rename to libs/download/src/commonTest/kotlin/org/fdroid/download/HttpManagerTest.kt index 38d8adf29..3d64b5435 100644 --- a/download/src/commonTest/kotlin/org/fdroid/download/HttpManagerTest.kt +++ b/libs/download/src/commonTest/kotlin/org/fdroid/download/HttpManagerTest.kt @@ -18,6 +18,7 @@ import io.ktor.http.HttpHeaders.Range import io.ktor.http.HttpHeaders.UserAgent import io.ktor.http.HttpStatusCode.Companion.Forbidden import io.ktor.http.HttpStatusCode.Companion.InternalServerError +import io.ktor.http.HttpStatusCode.Companion.NotFound import io.ktor.http.HttpStatusCode.Companion.OK import io.ktor.http.HttpStatusCode.Companion.PartialContent import io.ktor.http.HttpStatusCode.Companion.TemporaryRedirect @@ -37,7 +38,7 @@ import kotlin.test.assertNull import kotlin.test.assertTrue import kotlin.test.fail -class HttpManagerTest { +internal class HttpManagerTest { private val userAgent = getRandomString() private val mirrors = listOf(Mirror("http://example.org"), Mirror("http://example.net/")) @@ -195,7 +196,29 @@ class HttpManagerTest { // assert there is only one request per API call using one of the mirrors assertEquals(2, mockEngine.requestHistory.size) mockEngine.requestHistory.forEach { request -> - println(mockEngine.requestHistory) + val url = request.url.toString() + assertTrue(url == "http://example.org/foo" || url == "http://example.net/foo") + } + } + + @Test + fun testNoMoreMirrorsWhenRepoDownloadNotFound() = runSuspend { + val downloadRequest = downloadRequest.copy(tryFirstMirror = mirrors[0]) + val mockEngine = MockEngine { respond("", NotFound) } + val httpManager = HttpManager(userAgent, null, httpClientEngineFactory = get(mockEngine)) + + assertTrue(downloadRequest.tryFirstMirror != null) + + assertNull(httpManager.head(downloadRequest)) + val e = assertFailsWith { + httpManager.getBytes(downloadRequest) + } + + // assert that the exception reflects the NotFound error + assertEquals(NotFound, e.response.status) + // assert there is only one request per API call using one of the mirrors + assertEquals(2, mockEngine.requestHistory.size) + mockEngine.requestHistory.forEach { request -> val url = request.url.toString() assertTrue(url == "http://example.org/foo" || url == "http://example.net/foo") } diff --git a/download/src/commonTest/kotlin/org/fdroid/download/MirrorChooserTest.kt b/libs/download/src/commonTest/kotlin/org/fdroid/download/MirrorChooserTest.kt similarity index 100% rename from download/src/commonTest/kotlin/org/fdroid/download/MirrorChooserTest.kt rename to libs/download/src/commonTest/kotlin/org/fdroid/download/MirrorChooserTest.kt diff --git a/download/src/commonTest/kotlin/org/fdroid/download/MirrorTest.kt b/libs/download/src/commonTest/kotlin/org/fdroid/download/MirrorTest.kt similarity index 100% rename from download/src/commonTest/kotlin/org/fdroid/download/MirrorTest.kt rename to libs/download/src/commonTest/kotlin/org/fdroid/download/MirrorTest.kt diff --git a/download/src/jvmMain/kotlin/org/fdroid/download/HttpManager.kt b/libs/download/src/jvmMain/kotlin/org/fdroid/download/HttpManager.kt similarity index 100% rename from download/src/jvmMain/kotlin/org/fdroid/download/HttpManager.kt rename to libs/download/src/jvmMain/kotlin/org/fdroid/download/HttpManager.kt diff --git a/download/src/nativeMain/kotlin/org/fdroid/download/HttpManager.kt b/libs/download/src/nativeMain/kotlin/org/fdroid/download/HttpManager.kt similarity index 100% rename from download/src/nativeMain/kotlin/org/fdroid/download/HttpManager.kt rename to libs/download/src/nativeMain/kotlin/org/fdroid/download/HttpManager.kt diff --git a/libs/index/.gitignore b/libs/index/.gitignore new file mode 100644 index 000000000..42afabfd2 --- /dev/null +++ b/libs/index/.gitignore @@ -0,0 +1 @@ +/build \ No newline at end of file diff --git a/index/build.gradle b/libs/index/build.gradle similarity index 96% rename from index/build.gradle rename to libs/index/build.gradle index 37d40e84a..34353d10a 100644 --- a/index/build.gradle +++ b/libs/index/build.gradle @@ -39,19 +39,17 @@ kotlin { implementation "org.jetbrains.kotlinx:kotlinx-serialization-json:1.3.2" implementation 'io.github.microutils:kotlin-logging:2.1.21' - implementation project(":download") + implementation project(":libs:download") implementation "io.ktor:ktor-io:2.0.0" } } - sharedTest { + commonTest { dependencies { + implementation project(":libs:sharedTest") implementation kotlin('test') implementation "com.goncalossilva:resources:0.2.1" } } - commonTest { - dependsOn(sharedTest) - } // JVM is disabled for now, because Android app is including it instead of Android library jvmMain { dependencies { diff --git a/index/consumer-rules.pro b/libs/index/consumer-rules.pro similarity index 100% rename from index/consumer-rules.pro rename to libs/index/consumer-rules.pro diff --git a/index/gradle.properties b/libs/index/gradle.properties similarity index 100% rename from index/gradle.properties rename to libs/index/gradle.properties diff --git a/index/src/androidAndroidTest/kotlin/org/fdroid/BestLocaleTest.kt b/libs/index/src/androidAndroidTest/kotlin/org/fdroid/BestLocaleTest.kt similarity index 100% rename from index/src/androidAndroidTest/kotlin/org/fdroid/BestLocaleTest.kt rename to libs/index/src/androidAndroidTest/kotlin/org/fdroid/BestLocaleTest.kt diff --git a/index/src/androidAndroidTest/kotlin/org/fdroid/index/v1/IndexV1CreatorTest.kt b/libs/index/src/androidAndroidTest/kotlin/org/fdroid/index/v1/IndexV1CreatorTest.kt similarity index 100% rename from index/src/androidAndroidTest/kotlin/org/fdroid/index/v1/IndexV1CreatorTest.kt rename to libs/index/src/androidAndroidTest/kotlin/org/fdroid/index/v1/IndexV1CreatorTest.kt diff --git a/index/src/androidMain/AndroidManifest.xml b/libs/index/src/androidMain/AndroidManifest.xml similarity index 100% rename from index/src/androidMain/AndroidManifest.xml rename to libs/index/src/androidMain/AndroidManifest.xml diff --git a/index/src/androidMain/kotlin/org/fdroid/CompatibilityChecker.kt b/libs/index/src/androidMain/kotlin/org/fdroid/CompatibilityChecker.kt similarity index 100% rename from index/src/androidMain/kotlin/org/fdroid/CompatibilityChecker.kt rename to libs/index/src/androidMain/kotlin/org/fdroid/CompatibilityChecker.kt diff --git a/index/src/androidMain/kotlin/org/fdroid/LocaleChooser.kt b/libs/index/src/androidMain/kotlin/org/fdroid/LocaleChooser.kt similarity index 100% rename from index/src/androidMain/kotlin/org/fdroid/LocaleChooser.kt rename to libs/index/src/androidMain/kotlin/org/fdroid/LocaleChooser.kt diff --git a/index/src/androidMain/kotlin/org/fdroid/UpdateChecker.kt b/libs/index/src/androidMain/kotlin/org/fdroid/UpdateChecker.kt similarity index 100% rename from index/src/androidMain/kotlin/org/fdroid/UpdateChecker.kt rename to libs/index/src/androidMain/kotlin/org/fdroid/UpdateChecker.kt diff --git a/index/src/androidMain/kotlin/org/fdroid/index/IndexCreator.kt b/libs/index/src/androidMain/kotlin/org/fdroid/index/IndexCreator.kt similarity index 100% rename from index/src/androidMain/kotlin/org/fdroid/index/IndexCreator.kt rename to libs/index/src/androidMain/kotlin/org/fdroid/index/IndexCreator.kt diff --git a/index/src/androidMain/kotlin/org/fdroid/index/IndexParser.kt b/libs/index/src/androidMain/kotlin/org/fdroid/index/IndexParser.kt similarity index 100% rename from index/src/androidMain/kotlin/org/fdroid/index/IndexParser.kt rename to libs/index/src/androidMain/kotlin/org/fdroid/index/IndexParser.kt diff --git a/index/src/androidMain/kotlin/org/fdroid/index/IndexUtils.kt b/libs/index/src/androidMain/kotlin/org/fdroid/index/IndexUtils.kt similarity index 100% rename from index/src/androidMain/kotlin/org/fdroid/index/IndexUtils.kt rename to libs/index/src/androidMain/kotlin/org/fdroid/index/IndexUtils.kt diff --git a/index/src/androidMain/kotlin/org/fdroid/index/JarIndexVerifier.kt b/libs/index/src/androidMain/kotlin/org/fdroid/index/JarIndexVerifier.kt similarity index 100% rename from index/src/androidMain/kotlin/org/fdroid/index/JarIndexVerifier.kt rename to libs/index/src/androidMain/kotlin/org/fdroid/index/JarIndexVerifier.kt diff --git a/index/src/androidMain/kotlin/org/fdroid/index/v1/IndexV1Creator.kt b/libs/index/src/androidMain/kotlin/org/fdroid/index/v1/IndexV1Creator.kt similarity index 100% rename from index/src/androidMain/kotlin/org/fdroid/index/v1/IndexV1Creator.kt rename to libs/index/src/androidMain/kotlin/org/fdroid/index/v1/IndexV1Creator.kt diff --git a/index/src/androidMain/kotlin/org/fdroid/index/v1/IndexV1StreamProcessor.kt b/libs/index/src/androidMain/kotlin/org/fdroid/index/v1/IndexV1StreamProcessor.kt similarity index 100% rename from index/src/androidMain/kotlin/org/fdroid/index/v1/IndexV1StreamProcessor.kt rename to libs/index/src/androidMain/kotlin/org/fdroid/index/v1/IndexV1StreamProcessor.kt diff --git a/index/src/androidMain/kotlin/org/fdroid/index/v1/IndexV1Verifier.kt b/libs/index/src/androidMain/kotlin/org/fdroid/index/v1/IndexV1Verifier.kt similarity index 100% rename from index/src/androidMain/kotlin/org/fdroid/index/v1/IndexV1Verifier.kt rename to libs/index/src/androidMain/kotlin/org/fdroid/index/v1/IndexV1Verifier.kt diff --git a/index/src/androidMain/kotlin/org/fdroid/index/v2/EntryVerifier.kt b/libs/index/src/androidMain/kotlin/org/fdroid/index/v2/EntryVerifier.kt similarity index 100% rename from index/src/androidMain/kotlin/org/fdroid/index/v2/EntryVerifier.kt rename to libs/index/src/androidMain/kotlin/org/fdroid/index/v2/EntryVerifier.kt diff --git a/index/src/androidMain/kotlin/org/fdroid/index/v2/IndexV2DiffStreamProcessor.kt b/libs/index/src/androidMain/kotlin/org/fdroid/index/v2/IndexV2DiffStreamProcessor.kt similarity index 100% rename from index/src/androidMain/kotlin/org/fdroid/index/v2/IndexV2DiffStreamProcessor.kt rename to libs/index/src/androidMain/kotlin/org/fdroid/index/v2/IndexV2DiffStreamProcessor.kt diff --git a/index/src/androidMain/kotlin/org/fdroid/index/v2/IndexV2FullStreamProcessor.kt b/libs/index/src/androidMain/kotlin/org/fdroid/index/v2/IndexV2FullStreamProcessor.kt similarity index 100% rename from index/src/androidMain/kotlin/org/fdroid/index/v2/IndexV2FullStreamProcessor.kt rename to libs/index/src/androidMain/kotlin/org/fdroid/index/v2/IndexV2FullStreamProcessor.kt diff --git a/index/src/androidMain/kotlin/org/fdroid/index/v2/IndexV2StreamProcessor.kt b/libs/index/src/androidMain/kotlin/org/fdroid/index/v2/IndexV2StreamProcessor.kt similarity index 100% rename from index/src/androidMain/kotlin/org/fdroid/index/v2/IndexV2StreamProcessor.kt rename to libs/index/src/androidMain/kotlin/org/fdroid/index/v2/IndexV2StreamProcessor.kt diff --git a/index/src/androidMain/kotlin/org/fdroid/index/v2/ReflectionDiffer.kt b/libs/index/src/androidMain/kotlin/org/fdroid/index/v2/ReflectionDiffer.kt similarity index 100% rename from index/src/androidMain/kotlin/org/fdroid/index/v2/ReflectionDiffer.kt rename to libs/index/src/androidMain/kotlin/org/fdroid/index/v2/ReflectionDiffer.kt diff --git a/index/src/androidTest/kotlin/org/fdroid/CompatibilityCheckerTest.kt b/libs/index/src/androidTest/kotlin/org/fdroid/CompatibilityCheckerTest.kt similarity index 100% rename from index/src/androidTest/kotlin/org/fdroid/CompatibilityCheckerTest.kt rename to libs/index/src/androidTest/kotlin/org/fdroid/CompatibilityCheckerTest.kt diff --git a/index/src/androidTest/kotlin/org/fdroid/UpdateCheckerTest.kt b/libs/index/src/androidTest/kotlin/org/fdroid/UpdateCheckerTest.kt similarity index 100% rename from index/src/androidTest/kotlin/org/fdroid/UpdateCheckerTest.kt rename to libs/index/src/androidTest/kotlin/org/fdroid/UpdateCheckerTest.kt diff --git a/index/src/androidTest/kotlin/org/fdroid/index/v1/IndexV1StreamProcessorTest.kt b/libs/index/src/androidTest/kotlin/org/fdroid/index/v1/IndexV1StreamProcessorTest.kt similarity index 90% rename from index/src/androidTest/kotlin/org/fdroid/index/v1/IndexV1StreamProcessorTest.kt rename to libs/index/src/androidTest/kotlin/org/fdroid/index/v1/IndexV1StreamProcessorTest.kt index 143da029c..1240762b1 100644 --- a/index/src/androidTest/kotlin/org/fdroid/index/v1/IndexV1StreamProcessorTest.kt +++ b/libs/index/src/androidTest/kotlin/org/fdroid/index/v1/IndexV1StreamProcessorTest.kt @@ -1,6 +1,7 @@ package org.fdroid.index.v1 import kotlinx.serialization.SerializationException +import org.fdroid.index.assetPath import org.fdroid.index.v2.AntiFeatureV2 import org.fdroid.index.v2.CategoryV2 import org.fdroid.index.v2.IndexV2 @@ -28,38 +29,34 @@ internal class IndexV1StreamProcessorTest { @Test fun testEmpty() { - testStreamProcessing("src/sharedTest/resources/index-empty-v1.json", - TestDataEmptyV2.index.v1compat()) + testStreamProcessing("$assetPath/index-empty-v1.json", TestDataEmptyV2.index.v1compat()) } @Test(expected = OldIndexException::class) fun testEmptyEqualTimestamp() { - testStreamProcessing("src/sharedTest/resources/index-empty-v1.json", + testStreamProcessing("$assetPath/index-empty-v1.json", TestDataEmptyV2.index.v1compat(), TestDataEmptyV2.index.repo.timestamp) } @Test(expected = OldIndexException::class) fun testEmptyHigherTimestamp() { - testStreamProcessing("src/sharedTest/resources/index-empty-v1.json", + testStreamProcessing("$assetPath/index-empty-v1.json", TestDataEmptyV2.index.v1compat(), TestDataEmptyV2.index.repo.timestamp + 1) } @Test fun testMin() { - testStreamProcessing("src/sharedTest/resources/index-min-v1.json", - TestDataMinV2.index.v1compat()) + testStreamProcessing("$assetPath/index-min-v1.json", TestDataMinV2.index.v1compat()) } @Test fun testMid() { - testStreamProcessing("src/sharedTest/resources/index-mid-v1.json", - TestDataMidV2.indexCompat) + testStreamProcessing("$assetPath/index-mid-v1.json", TestDataMidV2.indexCompat) } @Test fun testMax() { - testStreamProcessing("src/sharedTest/resources/index-max-v1.json", - TestDataMaxV2.indexCompat) + testStreamProcessing("$assetPath/index-max-v1.json", TestDataMaxV2.indexCompat) } @Test diff --git a/index/src/androidTest/kotlin/org/fdroid/index/v1/IndexV1VerifierTest.kt b/libs/index/src/androidTest/kotlin/org/fdroid/index/v1/IndexV1VerifierTest.kt similarity index 100% rename from index/src/androidTest/kotlin/org/fdroid/index/v1/IndexV1VerifierTest.kt rename to libs/index/src/androidTest/kotlin/org/fdroid/index/v1/IndexV1VerifierTest.kt diff --git a/index/src/androidTest/kotlin/org/fdroid/index/v2/EntryVerifierTest.kt b/libs/index/src/androidTest/kotlin/org/fdroid/index/v2/EntryVerifierTest.kt similarity index 100% rename from index/src/androidTest/kotlin/org/fdroid/index/v2/EntryVerifierTest.kt rename to libs/index/src/androidTest/kotlin/org/fdroid/index/v2/EntryVerifierTest.kt diff --git a/index/src/androidTest/kotlin/org/fdroid/index/v2/IndexV2FullStreamProcessorTest.kt b/libs/index/src/androidTest/kotlin/org/fdroid/index/v2/IndexV2FullStreamProcessorTest.kt similarity index 90% rename from index/src/androidTest/kotlin/org/fdroid/index/v2/IndexV2FullStreamProcessorTest.kt rename to libs/index/src/androidTest/kotlin/org/fdroid/index/v2/IndexV2FullStreamProcessorTest.kt index fcc7c3df2..5ddb65f86 100644 --- a/index/src/androidTest/kotlin/org/fdroid/index/v2/IndexV2FullStreamProcessorTest.kt +++ b/libs/index/src/androidTest/kotlin/org/fdroid/index/v2/IndexV2FullStreamProcessorTest.kt @@ -2,6 +2,7 @@ package org.fdroid.index.v2 import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.SerializationException +import org.fdroid.index.assetPath import org.fdroid.test.TestDataEmptyV2 import org.fdroid.test.TestDataMaxV2 import org.fdroid.test.TestDataMidV2 @@ -27,29 +28,27 @@ internal class IndexV2FullStreamProcessorTest { @Test fun testEmpty() { - testStreamProcessing("src/sharedTest/resources/index-empty-v2.json", - TestDataEmptyV2.index, 0) + testStreamProcessing("$assetPath/index-empty-v2.json", TestDataEmptyV2.index, 0) } @Test fun testMin() { - testStreamProcessing("src/sharedTest/resources/index-min-v2.json", TestDataMinV2.index, 1) + testStreamProcessing("$assetPath/index-min-v2.json", TestDataMinV2.index, 1) } @Test fun testMinReordered() { - testStreamProcessing("src/sharedTest/resources/index-min-reordered-v2.json", - TestDataMinV2.index, 1) + testStreamProcessing("$assetPath/index-min-reordered-v2.json", TestDataMinV2.index, 1) } @Test fun testMid() { - testStreamProcessing("src/sharedTest/resources/index-mid-v2.json", TestDataMidV2.index, 2) + testStreamProcessing("$assetPath/index-mid-v2.json", TestDataMidV2.index, 2) } @Test fun testMax() { - testStreamProcessing("src/sharedTest/resources/index-max-v2.json", TestDataMaxV2.index, 3) + testStreamProcessing("$assetPath/index-max-v2.json", TestDataMaxV2.index, 3) } @Test diff --git a/index/src/androidTest/kotlin/org/fdroid/index/v2/ReflectionDifferTest.kt b/libs/index/src/androidTest/kotlin/org/fdroid/index/v2/ReflectionDifferTest.kt similarity index 83% rename from index/src/androidTest/kotlin/org/fdroid/index/v2/ReflectionDifferTest.kt rename to libs/index/src/androidTest/kotlin/org/fdroid/index/v2/ReflectionDifferTest.kt index fe55f2219..735302ebb 100644 --- a/index/src/androidTest/kotlin/org/fdroid/index/v2/ReflectionDifferTest.kt +++ b/libs/index/src/androidTest/kotlin/org/fdroid/index/v2/ReflectionDifferTest.kt @@ -8,6 +8,7 @@ import kotlinx.serialization.json.JsonPrimitive import kotlinx.serialization.json.jsonObject import org.fdroid.index.IndexParser import org.fdroid.index.IndexParser.json +import org.fdroid.index.assetPath import org.fdroid.index.parseV2 import org.fdroid.test.DiffUtils.clean import org.fdroid.test.DiffUtils.cleanMetadata @@ -25,44 +26,44 @@ internal class ReflectionDifferTest { @Test fun testEmptyToMin() = testDiff( - diffPath = "src/sharedTest/resources/diff-empty-min/23.json", - startPath = "src/sharedTest/resources/index-empty-v2.json", - endPath = "src/sharedTest/resources/index-min-v2.json", + diffPath = "$assetPath/diff-empty-min/23.json", + startPath = "$assetPath/index-empty-v2.json", + endPath = "$assetPath/index-min-v2.json", ) @Test fun testEmptyToMid() = testDiff( - diffPath = "src/sharedTest/resources/diff-empty-mid/23.json", - startPath = "src/sharedTest/resources/index-empty-v2.json", - endPath = "src/sharedTest/resources/index-mid-v2.json", + diffPath = "$assetPath/diff-empty-mid/23.json", + startPath = "$assetPath/index-empty-v2.json", + endPath = "$assetPath/index-mid-v2.json", ) @Test fun testEmptyToMax() = testDiff( - diffPath = "src/sharedTest/resources/diff-empty-max/23.json", - startPath = "src/sharedTest/resources/index-empty-v2.json", - endPath = "src/sharedTest/resources/index-max-v2.json", + diffPath = "$assetPath/diff-empty-max/23.json", + startPath = "$assetPath/index-empty-v2.json", + endPath = "$assetPath/index-max-v2.json", ) @Test fun testMinToMid() = testDiff( - diffPath = "src/sharedTest/resources/diff-empty-mid/42.json", - startPath = "src/sharedTest/resources/index-min-v2.json", - endPath = "src/sharedTest/resources/index-mid-v2.json", + diffPath = "$assetPath/diff-empty-mid/42.json", + startPath = "$assetPath/index-min-v2.json", + endPath = "$assetPath/index-mid-v2.json", ) @Test fun testMinToMax() = testDiff( - diffPath = "src/sharedTest/resources/diff-empty-max/42.json", - startPath = "src/sharedTest/resources/index-min-v2.json", - endPath = "src/sharedTest/resources/index-max-v2.json", + diffPath = "$assetPath/diff-empty-max/42.json", + startPath = "$assetPath/index-min-v2.json", + endPath = "$assetPath/index-max-v2.json", ) @Test fun testMidToMax() = testDiff( - diffPath = "src/sharedTest/resources/diff-empty-max/1337.json", - startPath = "src/sharedTest/resources/index-mid-v2.json", - endPath = "src/sharedTest/resources/index-max-v2.json", + diffPath = "$assetPath/diff-empty-max/1337.json", + startPath = "$assetPath/index-mid-v2.json", + endPath = "$assetPath/index-max-v2.json", ) @Test diff --git a/index/src/commonMain/kotlin/org/fdroid/index/IndexConverter.kt b/libs/index/src/commonMain/kotlin/org/fdroid/index/IndexConverter.kt similarity index 100% rename from index/src/commonMain/kotlin/org/fdroid/index/IndexConverter.kt rename to libs/index/src/commonMain/kotlin/org/fdroid/index/IndexConverter.kt diff --git a/index/src/commonMain/kotlin/org/fdroid/index/IndexParser.kt b/libs/index/src/commonMain/kotlin/org/fdroid/index/IndexParser.kt similarity index 100% rename from index/src/commonMain/kotlin/org/fdroid/index/IndexParser.kt rename to libs/index/src/commonMain/kotlin/org/fdroid/index/IndexParser.kt diff --git a/index/src/commonMain/kotlin/org/fdroid/index/v1/AppV1.kt b/libs/index/src/commonMain/kotlin/org/fdroid/index/v1/AppV1.kt similarity index 100% rename from index/src/commonMain/kotlin/org/fdroid/index/v1/AppV1.kt rename to libs/index/src/commonMain/kotlin/org/fdroid/index/v1/AppV1.kt diff --git a/index/src/commonMain/kotlin/org/fdroid/index/v1/IndexV1.kt b/libs/index/src/commonMain/kotlin/org/fdroid/index/v1/IndexV1.kt similarity index 100% rename from index/src/commonMain/kotlin/org/fdroid/index/v1/IndexV1.kt rename to libs/index/src/commonMain/kotlin/org/fdroid/index/v1/IndexV1.kt diff --git a/index/src/commonMain/kotlin/org/fdroid/index/v1/IndexV1StreamReceiver.kt b/libs/index/src/commonMain/kotlin/org/fdroid/index/v1/IndexV1StreamReceiver.kt similarity index 100% rename from index/src/commonMain/kotlin/org/fdroid/index/v1/IndexV1StreamReceiver.kt rename to libs/index/src/commonMain/kotlin/org/fdroid/index/v1/IndexV1StreamReceiver.kt diff --git a/index/src/commonMain/kotlin/org/fdroid/index/v1/PackageV1.kt b/libs/index/src/commonMain/kotlin/org/fdroid/index/v1/PackageV1.kt similarity index 100% rename from index/src/commonMain/kotlin/org/fdroid/index/v1/PackageV1.kt rename to libs/index/src/commonMain/kotlin/org/fdroid/index/v1/PackageV1.kt diff --git a/index/src/commonMain/kotlin/org/fdroid/index/v2/IndexV2.kt b/libs/index/src/commonMain/kotlin/org/fdroid/index/v2/IndexV2.kt similarity index 100% rename from index/src/commonMain/kotlin/org/fdroid/index/v2/IndexV2.kt rename to libs/index/src/commonMain/kotlin/org/fdroid/index/v2/IndexV2.kt diff --git a/index/src/commonMain/kotlin/org/fdroid/index/v2/IndexV2DiffStreamReceiver.kt b/libs/index/src/commonMain/kotlin/org/fdroid/index/v2/IndexV2DiffStreamReceiver.kt similarity index 100% rename from index/src/commonMain/kotlin/org/fdroid/index/v2/IndexV2DiffStreamReceiver.kt rename to libs/index/src/commonMain/kotlin/org/fdroid/index/v2/IndexV2DiffStreamReceiver.kt diff --git a/index/src/commonMain/kotlin/org/fdroid/index/v2/IndexV2StreamReceiver.kt b/libs/index/src/commonMain/kotlin/org/fdroid/index/v2/IndexV2StreamReceiver.kt similarity index 100% rename from index/src/commonMain/kotlin/org/fdroid/index/v2/IndexV2StreamReceiver.kt rename to libs/index/src/commonMain/kotlin/org/fdroid/index/v2/IndexV2StreamReceiver.kt diff --git a/index/src/commonMain/kotlin/org/fdroid/index/v2/PackageV2.kt b/libs/index/src/commonMain/kotlin/org/fdroid/index/v2/PackageV2.kt similarity index 100% rename from index/src/commonMain/kotlin/org/fdroid/index/v2/PackageV2.kt rename to libs/index/src/commonMain/kotlin/org/fdroid/index/v2/PackageV2.kt diff --git a/index/src/commonTest/kotlin/org/fdroid/index/IndexConverterTest.kt b/libs/index/src/commonTest/kotlin/org/fdroid/index/IndexConverterTest.kt similarity index 67% rename from index/src/commonTest/kotlin/org/fdroid/index/IndexConverterTest.kt rename to libs/index/src/commonTest/kotlin/org/fdroid/index/IndexConverterTest.kt index b3cf9d28e..6a5133d6a 100644 --- a/index/src/commonTest/kotlin/org/fdroid/index/IndexConverterTest.kt +++ b/libs/index/src/commonTest/kotlin/org/fdroid/index/IndexConverterTest.kt @@ -12,30 +12,28 @@ import org.fdroid.test.v1compat import kotlin.test.Test import kotlin.test.assertEquals +internal const val assetPath = "../sharedTest/src/main/assets" + internal class IndexConverterTest { @Test fun testEmpty() { - testConversation("src/sharedTest/resources/index-empty-v1.json", - TestDataEmptyV2.index.v1compat()) + testConversation("$assetPath/index-empty-v1.json", TestDataEmptyV2.index.v1compat()) } @Test fun testMin() { - testConversation("src/sharedTest/resources/index-min-v1.json", - TestDataMinV2.index.v1compat()) + testConversation("$assetPath/index-min-v1.json", TestDataMinV2.index.v1compat()) } @Test fun testMid() { - testConversation("src/sharedTest/resources/index-mid-v1.json", - TestDataMidV2.indexCompat) + testConversation("$assetPath/index-mid-v1.json", TestDataMidV2.indexCompat) } @Test fun testMax() { - testConversation("src/sharedTest/resources/index-max-v1.json", - TestDataMaxV2.indexCompat) + testConversation("$assetPath/index-max-v1.json", TestDataMaxV2.indexCompat) } private fun testConversation(file: String, expectedIndex: IndexV2) { diff --git a/index/src/commonTest/kotlin/org/fdroid/index/v1/IndexV1Test.kt b/libs/index/src/commonTest/kotlin/org/fdroid/index/v1/IndexV1Test.kt similarity index 90% rename from index/src/commonTest/kotlin/org/fdroid/index/v1/IndexV1Test.kt rename to libs/index/src/commonTest/kotlin/org/fdroid/index/v1/IndexV1Test.kt index 2d7710c5c..b8aa10e7b 100644 --- a/index/src/commonTest/kotlin/org/fdroid/index/v1/IndexV1Test.kt +++ b/libs/index/src/commonTest/kotlin/org/fdroid/index/v1/IndexV1Test.kt @@ -3,6 +3,7 @@ package org.fdroid.index.v1 import com.goncalossilva.resources.Resource import kotlinx.serialization.SerializationException import org.fdroid.index.IndexParser.parseV1 +import org.fdroid.index.assetPath import org.fdroid.test.TestDataEmptyV1 import org.fdroid.test.TestDataMaxV1 import org.fdroid.test.TestDataMidV1 @@ -16,7 +17,7 @@ internal class IndexV1Test { @Test fun testIndexEmptyV1() { - val indexRes = Resource("src/sharedTest/resources/index-empty-v1.json") + val indexRes = Resource("$assetPath/index-empty-v1.json") val indexStr = indexRes.readText() val index = parseV1(indexStr) assertEquals(TestDataEmptyV1.index, index) @@ -24,7 +25,7 @@ internal class IndexV1Test { @Test fun testIndexMinV1() { - val indexRes = Resource("src/sharedTest/resources/index-min-v1.json") + val indexRes = Resource("$assetPath/index-min-v1.json") val indexStr = indexRes.readText() val index = parseV1(indexStr) assertEquals(TestDataMinV1.index, index) @@ -32,7 +33,7 @@ internal class IndexV1Test { @Test fun testIndexMidV1() { - val indexRes = Resource("src/sharedTest/resources/index-mid-v1.json") + val indexRes = Resource("$assetPath/index-mid-v1.json") val indexStr = indexRes.readText() val index = parseV1(indexStr) assertEquals(TestDataMidV1.index, index) @@ -40,7 +41,7 @@ internal class IndexV1Test { @Test fun testIndexMaxV1() { - val indexRes = Resource("src/sharedTest/resources/index-max-v1.json") + val indexRes = Resource("$assetPath/index-max-v1.json") val indexStr = indexRes.readText() val index = parseV1(indexStr) assertEquals(TestDataMaxV1.index, index) diff --git a/index/src/commonTest/kotlin/org/fdroid/index/v2/EntryTest.kt b/libs/index/src/commonTest/kotlin/org/fdroid/index/v2/EntryTest.kt similarity index 86% rename from index/src/commonTest/kotlin/org/fdroid/index/v2/EntryTest.kt rename to libs/index/src/commonTest/kotlin/org/fdroid/index/v2/EntryTest.kt index db078b6b2..af1b24c1a 100644 --- a/index/src/commonTest/kotlin/org/fdroid/index/v2/EntryTest.kt +++ b/libs/index/src/commonTest/kotlin/org/fdroid/index/v2/EntryTest.kt @@ -3,6 +3,7 @@ package org.fdroid.index.v2 import com.goncalossilva.resources.Resource import kotlinx.serialization.SerializationException import org.fdroid.index.IndexParser +import org.fdroid.index.assetPath import org.fdroid.test.TestDataEntryV2 import org.junit.Test import kotlin.test.assertContains @@ -13,26 +14,22 @@ internal class EntryTest { @Test fun testEmpty() { - testEntryEquality("src/sharedTest/resources/entry-empty-v2.json", - TestDataEntryV2.empty) + testEntryEquality("$assetPath/entry-empty-v2.json", TestDataEntryV2.empty) } @Test fun testEmptyToMin() { - testEntryEquality("src/sharedTest/resources/diff-empty-min/entry.json", - TestDataEntryV2.emptyToMin) + testEntryEquality("$assetPath/diff-empty-min/entry.json", TestDataEntryV2.emptyToMin) } @Test fun testEmptyToMid() { - testEntryEquality("src/sharedTest/resources/diff-empty-mid/entry.json", - TestDataEntryV2.emptyToMid) + testEntryEquality("$assetPath/diff-empty-mid/entry.json", TestDataEntryV2.emptyToMid) } @Test fun testEmptyToMax() { - testEntryEquality("src/sharedTest/resources/diff-empty-max/entry.json", - TestDataEntryV2.emptyToMax) + testEntryEquality("$assetPath/diff-empty-max/entry.json", TestDataEntryV2.emptyToMax) } @Test diff --git a/index/src/commonTest/kotlin/org/fdroid/index/v2/IndexV2DiffStreamProcessorTest.kt b/libs/index/src/commonTest/kotlin/org/fdroid/index/v2/IndexV2DiffStreamProcessorTest.kt similarity index 86% rename from index/src/commonTest/kotlin/org/fdroid/index/v2/IndexV2DiffStreamProcessorTest.kt rename to libs/index/src/commonTest/kotlin/org/fdroid/index/v2/IndexV2DiffStreamProcessorTest.kt index 28cd60753..1f146e417 100644 --- a/index/src/commonTest/kotlin/org/fdroid/index/v2/IndexV2DiffStreamProcessorTest.kt +++ b/libs/index/src/commonTest/kotlin/org/fdroid/index/v2/IndexV2DiffStreamProcessorTest.kt @@ -5,6 +5,7 @@ import kotlinx.serialization.json.JsonNull import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.jsonObject import org.fdroid.index.IndexParser +import org.fdroid.index.assetPath import org.junit.Test import java.io.ByteArrayInputStream import java.io.File @@ -16,22 +17,22 @@ import kotlin.test.fail internal class IndexV2DiffStreamProcessorTest { @Test - fun testEmptyToMin() = testDiff("src/sharedTest/resources/diff-empty-min/23.json", 1) + fun testEmptyToMin() = testDiff("$assetPath/diff-empty-min/23.json", 1) @Test - fun testEmptyToMid() = testDiff("src/sharedTest/resources/diff-empty-mid/23.json", 2) + fun testEmptyToMid() = testDiff("$assetPath/diff-empty-mid/23.json", 2) @Test - fun testEmptyToMax() = testDiff("src/sharedTest/resources/diff-empty-max/23.json", 3) + fun testEmptyToMax() = testDiff("$assetPath/diff-empty-max/23.json", 3) @Test - fun testMinToMid() = testDiff("src/sharedTest/resources/diff-empty-mid/42.json", 2) + fun testMinToMid() = testDiff("$assetPath/diff-empty-mid/42.json", 2) @Test - fun testMinToMax() = testDiff("src/sharedTest/resources/diff-empty-max/42.json", 3) + fun testMinToMax() = testDiff("$assetPath/diff-empty-max/42.json", 3) @Test - fun testMidToMax() = testDiff("src/sharedTest/resources/diff-empty-max/1337.json", 2) + fun testMidToMax() = testDiff("$assetPath/diff-empty-max/1337.json", 2) @Test fun testRemovePackage() { diff --git a/index/src/commonTest/kotlin/org/fdroid/index/v2/IndexV2Test.kt b/libs/index/src/commonTest/kotlin/org/fdroid/index/v2/IndexV2Test.kt similarity index 82% rename from index/src/commonTest/kotlin/org/fdroid/index/v2/IndexV2Test.kt rename to libs/index/src/commonTest/kotlin/org/fdroid/index/v2/IndexV2Test.kt index ae161840e..06bfd7601 100644 --- a/index/src/commonTest/kotlin/org/fdroid/index/v2/IndexV2Test.kt +++ b/libs/index/src/commonTest/kotlin/org/fdroid/index/v2/IndexV2Test.kt @@ -3,6 +3,7 @@ package org.fdroid.index.v2 import com.goncalossilva.resources.Resource import kotlinx.serialization.SerializationException import org.fdroid.index.IndexParser.parseV2 +import org.fdroid.index.assetPath import org.fdroid.test.TestDataEmptyV2 import org.fdroid.test.TestDataMaxV2 import org.fdroid.test.TestDataMidV2 @@ -16,28 +17,27 @@ internal class IndexV2Test { @Test fun testEmpty() { - testIndexEquality("src/sharedTest/resources/index-empty-v2.json", TestDataEmptyV2.index) + testIndexEquality("$assetPath/index-empty-v2.json", TestDataEmptyV2.index) } @Test fun testMin() { - testIndexEquality("src/sharedTest/resources/index-min-v2.json", TestDataMinV2.index) + testIndexEquality("$assetPath/index-min-v2.json", TestDataMinV2.index) } @Test fun testMinReordered() { - testIndexEquality("src/sharedTest/resources/index-min-reordered-v2.json", - TestDataMinV2.index) + testIndexEquality("$assetPath/index-min-reordered-v2.json", TestDataMinV2.index) } @Test fun testMid() { - testIndexEquality("src/sharedTest/resources/index-mid-v2.json", TestDataMidV2.index) + testIndexEquality("$assetPath/index-mid-v2.json", TestDataMidV2.index) } @Test fun testMax() { - testIndexEquality("src/sharedTest/resources/index-max-v2.json", TestDataMaxV2.index) + testIndexEquality("$assetPath/index-max-v2.json", TestDataMaxV2.index) } @Test diff --git a/index/src/commonTest/resources/verification/invalid-MD5-MD5withRSA-v1.jar b/libs/index/src/commonTest/resources/verification/invalid-MD5-MD5withRSA-v1.jar similarity index 100% rename from index/src/commonTest/resources/verification/invalid-MD5-MD5withRSA-v1.jar rename to libs/index/src/commonTest/resources/verification/invalid-MD5-MD5withRSA-v1.jar diff --git a/index/src/commonTest/resources/verification/invalid-MD5-MD5withRSA-v2.jar b/libs/index/src/commonTest/resources/verification/invalid-MD5-MD5withRSA-v2.jar similarity index 100% rename from index/src/commonTest/resources/verification/invalid-MD5-MD5withRSA-v2.jar rename to libs/index/src/commonTest/resources/verification/invalid-MD5-MD5withRSA-v2.jar diff --git a/index/src/commonTest/resources/verification/invalid-MD5-SHA1withRSA-v1.jar b/libs/index/src/commonTest/resources/verification/invalid-MD5-SHA1withRSA-v1.jar similarity index 100% rename from index/src/commonTest/resources/verification/invalid-MD5-SHA1withRSA-v1.jar rename to libs/index/src/commonTest/resources/verification/invalid-MD5-SHA1withRSA-v1.jar diff --git a/index/src/commonTest/resources/verification/invalid-MD5-SHA1withRSA-v2.jar b/libs/index/src/commonTest/resources/verification/invalid-MD5-SHA1withRSA-v2.jar similarity index 100% rename from index/src/commonTest/resources/verification/invalid-MD5-SHA1withRSA-v2.jar rename to libs/index/src/commonTest/resources/verification/invalid-MD5-SHA1withRSA-v2.jar diff --git a/index/src/commonTest/resources/verification/invalid-SHA1-SHA1withRSA-v2.jar b/libs/index/src/commonTest/resources/verification/invalid-SHA1-SHA1withRSA-v2.jar similarity index 100% rename from index/src/commonTest/resources/verification/invalid-SHA1-SHA1withRSA-v2.jar rename to libs/index/src/commonTest/resources/verification/invalid-SHA1-SHA1withRSA-v2.jar diff --git a/index/src/commonTest/resources/verification/invalid-v1.jar b/libs/index/src/commonTest/resources/verification/invalid-v1.jar similarity index 100% rename from index/src/commonTest/resources/verification/invalid-v1.jar rename to libs/index/src/commonTest/resources/verification/invalid-v1.jar diff --git a/index/src/commonTest/resources/verification/invalid-v2.jar b/libs/index/src/commonTest/resources/verification/invalid-v2.jar similarity index 100% rename from index/src/commonTest/resources/verification/invalid-v2.jar rename to libs/index/src/commonTest/resources/verification/invalid-v2.jar diff --git a/index/src/commonTest/resources/verification/invalid-wrong-entry-v1.jar b/libs/index/src/commonTest/resources/verification/invalid-wrong-entry-v1.jar similarity index 100% rename from index/src/commonTest/resources/verification/invalid-wrong-entry-v1.jar rename to libs/index/src/commonTest/resources/verification/invalid-wrong-entry-v1.jar diff --git a/index/src/commonTest/resources/verification/unsigned.jar b/libs/index/src/commonTest/resources/verification/unsigned.jar similarity index 100% rename from index/src/commonTest/resources/verification/unsigned.jar rename to libs/index/src/commonTest/resources/verification/unsigned.jar diff --git a/index/src/commonTest/resources/verification/valid-apksigner-v2.jar b/libs/index/src/commonTest/resources/verification/valid-apksigner-v2.jar similarity index 100% rename from index/src/commonTest/resources/verification/valid-apksigner-v2.jar rename to libs/index/src/commonTest/resources/verification/valid-apksigner-v2.jar diff --git a/index/src/commonTest/resources/verification/valid-v1.jar b/libs/index/src/commonTest/resources/verification/valid-v1.jar similarity index 100% rename from index/src/commonTest/resources/verification/valid-v1.jar rename to libs/index/src/commonTest/resources/verification/valid-v1.jar diff --git a/index/src/commonTest/resources/verification/valid-v2.jar b/libs/index/src/commonTest/resources/verification/valid-v2.jar similarity index 100% rename from index/src/commonTest/resources/verification/valid-v2.jar rename to libs/index/src/commonTest/resources/verification/valid-v2.jar diff --git a/libs/sharedTest/.gitignore b/libs/sharedTest/.gitignore new file mode 100644 index 000000000..42afabfd2 --- /dev/null +++ b/libs/sharedTest/.gitignore @@ -0,0 +1 @@ +/build \ No newline at end of file diff --git a/libs/sharedTest/build.gradle b/libs/sharedTest/build.gradle new file mode 100644 index 000000000..25be3d328 --- /dev/null +++ b/libs/sharedTest/build.gradle @@ -0,0 +1,27 @@ +plugins { + id 'kotlin-android' + id 'com.android.library' + id "org.jlleitschuh.gradle.ktlint" version "10.2.1" +} + +java { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 +} + +// not really an Android library, but index is not publishing for JVM at the moment +android { + compileSdkVersion 31 + defaultConfig { + minSdkVersion 21 + } +} + +dependencies { + implementation project(":libs:index") + + implementation 'org.jetbrains.kotlin:kotlin-test' + implementation "org.jetbrains.kotlinx:kotlinx-serialization-json:1.3.2" +} + +apply from: "${rootProject.rootDir}/gradle/ktlint.gradle" diff --git a/libs/sharedTest/src/main/AndroidManifest.xml b/libs/sharedTest/src/main/AndroidManifest.xml new file mode 100644 index 000000000..0955a1977 --- /dev/null +++ b/libs/sharedTest/src/main/AndroidManifest.xml @@ -0,0 +1,2 @@ + + diff --git a/index/src/sharedTest/resources/diff-empty-max/1337.json b/libs/sharedTest/src/main/assets/diff-empty-max/1337.json similarity index 100% rename from index/src/sharedTest/resources/diff-empty-max/1337.json rename to libs/sharedTest/src/main/assets/diff-empty-max/1337.json diff --git a/index/src/sharedTest/resources/diff-empty-max/23.json b/libs/sharedTest/src/main/assets/diff-empty-max/23.json similarity index 100% rename from index/src/sharedTest/resources/diff-empty-max/23.json rename to libs/sharedTest/src/main/assets/diff-empty-max/23.json diff --git a/index/src/sharedTest/resources/diff-empty-max/42.json b/libs/sharedTest/src/main/assets/diff-empty-max/42.json similarity index 100% rename from index/src/sharedTest/resources/diff-empty-max/42.json rename to libs/sharedTest/src/main/assets/diff-empty-max/42.json diff --git a/index/src/sharedTest/resources/diff-empty-max/entry.jar b/libs/sharedTest/src/main/assets/diff-empty-max/entry.jar similarity index 100% rename from index/src/sharedTest/resources/diff-empty-max/entry.jar rename to libs/sharedTest/src/main/assets/diff-empty-max/entry.jar diff --git a/index/src/sharedTest/resources/diff-empty-max/entry.json b/libs/sharedTest/src/main/assets/diff-empty-max/entry.json similarity index 100% rename from index/src/sharedTest/resources/diff-empty-max/entry.json rename to libs/sharedTest/src/main/assets/diff-empty-max/entry.json diff --git a/index/src/sharedTest/resources/diff-empty-mid/23.json b/libs/sharedTest/src/main/assets/diff-empty-mid/23.json similarity index 100% rename from index/src/sharedTest/resources/diff-empty-mid/23.json rename to libs/sharedTest/src/main/assets/diff-empty-mid/23.json diff --git a/index/src/sharedTest/resources/diff-empty-mid/42.json b/libs/sharedTest/src/main/assets/diff-empty-mid/42.json similarity index 100% rename from index/src/sharedTest/resources/diff-empty-mid/42.json rename to libs/sharedTest/src/main/assets/diff-empty-mid/42.json diff --git a/index/src/sharedTest/resources/diff-empty-mid/entry.jar b/libs/sharedTest/src/main/assets/diff-empty-mid/entry.jar similarity index 100% rename from index/src/sharedTest/resources/diff-empty-mid/entry.jar rename to libs/sharedTest/src/main/assets/diff-empty-mid/entry.jar diff --git a/index/src/sharedTest/resources/diff-empty-mid/entry.json b/libs/sharedTest/src/main/assets/diff-empty-mid/entry.json similarity index 100% rename from index/src/sharedTest/resources/diff-empty-mid/entry.json rename to libs/sharedTest/src/main/assets/diff-empty-mid/entry.json diff --git a/index/src/sharedTest/resources/diff-empty-min/23.json b/libs/sharedTest/src/main/assets/diff-empty-min/23.json similarity index 100% rename from index/src/sharedTest/resources/diff-empty-min/23.json rename to libs/sharedTest/src/main/assets/diff-empty-min/23.json diff --git a/index/src/sharedTest/resources/diff-empty-min/entry.jar b/libs/sharedTest/src/main/assets/diff-empty-min/entry.jar similarity index 100% rename from index/src/sharedTest/resources/diff-empty-min/entry.jar rename to libs/sharedTest/src/main/assets/diff-empty-min/entry.jar diff --git a/index/src/sharedTest/resources/diff-empty-min/entry.json b/libs/sharedTest/src/main/assets/diff-empty-min/entry.json similarity index 100% rename from index/src/sharedTest/resources/diff-empty-min/entry.json rename to libs/sharedTest/src/main/assets/diff-empty-min/entry.json diff --git a/index/src/sharedTest/resources/entry-empty-v2.json b/libs/sharedTest/src/main/assets/entry-empty-v2.json similarity index 100% rename from index/src/sharedTest/resources/entry-empty-v2.json rename to libs/sharedTest/src/main/assets/entry-empty-v2.json diff --git a/index/src/sharedTest/resources/index-empty-v1.json b/libs/sharedTest/src/main/assets/index-empty-v1.json similarity index 100% rename from index/src/sharedTest/resources/index-empty-v1.json rename to libs/sharedTest/src/main/assets/index-empty-v1.json diff --git a/index/src/sharedTest/resources/index-empty-v2.json b/libs/sharedTest/src/main/assets/index-empty-v2.json similarity index 100% rename from index/src/sharedTest/resources/index-empty-v2.json rename to libs/sharedTest/src/main/assets/index-empty-v2.json diff --git a/index/src/sharedTest/resources/index-max-v1.json b/libs/sharedTest/src/main/assets/index-max-v1.json similarity index 100% rename from index/src/sharedTest/resources/index-max-v1.json rename to libs/sharedTest/src/main/assets/index-max-v1.json diff --git a/index/src/sharedTest/resources/index-max-v2.json b/libs/sharedTest/src/main/assets/index-max-v2.json similarity index 100% rename from index/src/sharedTest/resources/index-max-v2.json rename to libs/sharedTest/src/main/assets/index-max-v2.json diff --git a/index/src/sharedTest/resources/index-mid-v1.json b/libs/sharedTest/src/main/assets/index-mid-v1.json similarity index 100% rename from index/src/sharedTest/resources/index-mid-v1.json rename to libs/sharedTest/src/main/assets/index-mid-v1.json diff --git a/index/src/sharedTest/resources/index-mid-v2.json b/libs/sharedTest/src/main/assets/index-mid-v2.json similarity index 100% rename from index/src/sharedTest/resources/index-mid-v2.json rename to libs/sharedTest/src/main/assets/index-mid-v2.json diff --git a/index/src/sharedTest/resources/index-min-reordered-v2.json b/libs/sharedTest/src/main/assets/index-min-reordered-v2.json similarity index 100% rename from index/src/sharedTest/resources/index-min-reordered-v2.json rename to libs/sharedTest/src/main/assets/index-min-reordered-v2.json diff --git a/index/src/sharedTest/resources/index-min-v1.json b/libs/sharedTest/src/main/assets/index-min-v1.json similarity index 100% rename from index/src/sharedTest/resources/index-min-v1.json rename to libs/sharedTest/src/main/assets/index-min-v1.json diff --git a/index/src/sharedTest/resources/index-min-v2.json b/libs/sharedTest/src/main/assets/index-min-v2.json similarity index 100% rename from index/src/sharedTest/resources/index-min-v2.json rename to libs/sharedTest/src/main/assets/index-min-v2.json diff --git a/libs/sharedTest/src/main/assets/testy.at.or.at_corrupt_app_package_name_index-v1.jar b/libs/sharedTest/src/main/assets/testy.at.or.at_corrupt_app_package_name_index-v1.jar new file mode 100644 index 000000000..44611cede Binary files /dev/null and b/libs/sharedTest/src/main/assets/testy.at.or.at_corrupt_app_package_name_index-v1.jar differ diff --git a/libs/sharedTest/src/main/assets/testy.at.or.at_corrupt_package_name_index-v1.jar b/libs/sharedTest/src/main/assets/testy.at.or.at_corrupt_package_name_index-v1.jar new file mode 100644 index 000000000..a087e8397 Binary files /dev/null and b/libs/sharedTest/src/main/assets/testy.at.or.at_corrupt_package_name_index-v1.jar differ diff --git a/libs/sharedTest/src/main/assets/testy.at.or.at_index-v1.jar b/libs/sharedTest/src/main/assets/testy.at.or.at_index-v1.jar new file mode 100644 index 000000000..3554722f5 Binary files /dev/null and b/libs/sharedTest/src/main/assets/testy.at.or.at_index-v1.jar differ diff --git a/libs/sharedTest/src/main/assets/testy.at.or.at_no-.RSA_index-v1.jar b/libs/sharedTest/src/main/assets/testy.at.or.at_no-.RSA_index-v1.jar new file mode 100644 index 000000000..9b3267a73 Binary files /dev/null and b/libs/sharedTest/src/main/assets/testy.at.or.at_no-.RSA_index-v1.jar differ diff --git a/libs/sharedTest/src/main/assets/testy.at.or.at_no-.SF_index-v1.jar b/libs/sharedTest/src/main/assets/testy.at.or.at_no-.SF_index-v1.jar new file mode 100644 index 000000000..113a17e55 Binary files /dev/null and b/libs/sharedTest/src/main/assets/testy.at.or.at_no-.SF_index-v1.jar differ diff --git a/libs/sharedTest/src/main/assets/testy.at.or.at_no-MANIFEST.MF_index-v1.jar b/libs/sharedTest/src/main/assets/testy.at.or.at_no-MANIFEST.MF_index-v1.jar new file mode 100644 index 000000000..feee0f85d Binary files /dev/null and b/libs/sharedTest/src/main/assets/testy.at.or.at_no-MANIFEST.MF_index-v1.jar differ diff --git a/libs/sharedTest/src/main/assets/testy.at.or.at_no-signature_index-v1.jar b/libs/sharedTest/src/main/assets/testy.at.or.at_no-signature_index-v1.jar new file mode 100644 index 000000000..17faf77a0 Binary files /dev/null and b/libs/sharedTest/src/main/assets/testy.at.or.at_no-signature_index-v1.jar differ diff --git a/index/src/sharedTest/kotlin/org/fdroid/test/DiffUtils.kt b/libs/sharedTest/src/main/kotlin/org/fdroid/test/DiffUtils.kt similarity index 83% rename from index/src/sharedTest/kotlin/org/fdroid/test/DiffUtils.kt rename to libs/sharedTest/src/main/kotlin/org/fdroid/test/DiffUtils.kt index c84c2afb8..8fa84a34f 100644 --- a/index/src/sharedTest/kotlin/org/fdroid/test/DiffUtils.kt +++ b/libs/sharedTest/src/main/kotlin/org/fdroid/test/DiffUtils.kt @@ -7,12 +7,12 @@ import org.fdroid.index.v2.PackageVersionV2 import org.fdroid.index.v2.RepoV2 import kotlin.random.Random -public object DiffUtils { +object DiffUtils { /** * Create a map diff by adding or removing keys. Note that this does not change keys. */ - public fun Map.randomDiff(factory: () -> T): Map = buildMap { + fun Map.randomDiff(factory: () -> T): Map = buildMap { if (this@randomDiff.isNotEmpty()) { // remove random keys while (Random.nextBoolean()) put(this@randomDiff.keys.random(), null) @@ -25,13 +25,13 @@ public object DiffUtils { /** * Removes keys from a JSON object representing a [RepoV2] which need special handling. */ - internal fun JsonObject.cleanRepo(): JsonObject { + fun JsonObject.cleanRepo(): JsonObject { val keysToFilter = listOf("mirrors", "antiFeatures", "categories", "releaseChannels") val newMap = filterKeys { it !in keysToFilter } return JsonObject(newMap) } - internal fun RepoV2.clean() = copy( + fun RepoV2.clean() = copy( mirrors = emptyList(), antiFeatures = emptyMap(), categories = emptyMap(), @@ -41,14 +41,14 @@ public object DiffUtils { /** * Removes keys from a JSON object representing a [MetadataV2] which need special handling. */ - internal fun JsonObject.cleanMetadata(): JsonObject { + fun JsonObject.cleanMetadata(): JsonObject { val keysToFilter = listOf("icon", "featureGraphic", "promoGraphic", "tvBanner", "screenshots") val newMap = filterKeys { it !in keysToFilter } return JsonObject(newMap) } - internal fun MetadataV2.clean() = copy( + fun MetadataV2.clean() = copy( icon = null, featureGraphic = null, promoGraphic = null, @@ -59,7 +59,7 @@ public object DiffUtils { /** * Removes keys from a JSON object representing a [PackageVersionV2] which need special handling. */ - internal fun JsonObject.cleanVersion(): JsonObject { + fun JsonObject.cleanVersion(): JsonObject { if (!containsKey("manifest")) return this val keysToFilter = listOf("features", "usesPermission", "usesPermissionSdk23") val newMap = toMutableMap() @@ -68,7 +68,7 @@ public object DiffUtils { return JsonObject(newMap) } - internal fun PackageVersionV2.clean() = copy( + fun PackageVersionV2.clean() = copy( manifest = manifest.copy( features = emptyList(), usesPermission = emptyList(), @@ -76,7 +76,7 @@ public object DiffUtils { ), ) - public fun Map.applyDiff(diff: Map): Map = + fun Map.applyDiff(diff: Map): Map = toMutableMap().apply { diff.entries.forEach { (key, value) -> if (value == null) remove(key) diff --git a/index/src/sharedTest/kotlin/org/fdroid/test/TestAppUtils.kt b/libs/sharedTest/src/main/kotlin/org/fdroid/test/TestAppUtils.kt similarity index 91% rename from index/src/sharedTest/kotlin/org/fdroid/test/TestAppUtils.kt rename to libs/sharedTest/src/main/kotlin/org/fdroid/test/TestAppUtils.kt index 29cf469db..4d7ee9830 100644 --- a/index/src/sharedTest/kotlin/org/fdroid/test/TestAppUtils.kt +++ b/libs/sharedTest/src/main/kotlin/org/fdroid/test/TestAppUtils.kt @@ -13,9 +13,9 @@ import org.fdroid.test.TestUtils.orNull import kotlin.random.Random import kotlin.test.assertEquals -public object TestAppUtils { +object TestAppUtils { - public fun getRandomMetadataV2(): MetadataV2 = MetadataV2( + fun getRandomMetadataV2(): MetadataV2 = MetadataV2( added = Random.nextLong(), lastUpdated = Random.nextLong(), name = getRandomLocalizedTextV2().orNull(), @@ -49,7 +49,7 @@ public object TestAppUtils { screenshots = getRandomScreenshots().orNull(), ) - public fun getRandomScreenshots(): Screenshots? = Screenshots( + fun getRandomScreenshots(): Screenshots? = Screenshots( phone = getRandomLocalizedFileListV2().orNull(), sevenInch = getRandomLocalizedFileListV2().orNull(), tenInch = getRandomLocalizedFileListV2().orNull(), @@ -57,7 +57,7 @@ public object TestAppUtils { tv = getRandomLocalizedFileListV2().orNull(), ).takeIf { !it.isNull } - public fun getRandomLocalizedFileListV2(): Map> = + fun getRandomLocalizedFileListV2(): Map> = TestUtils.getRandomMap(Random.nextInt(1, 3)) { getRandomString() to getRandomList(Random.nextInt(1, 7)) { getRandomFileV2() @@ -68,7 +68,7 @@ public object TestAppUtils { * [Screenshots] include lists which can be ordered differently, * so we need to ignore order when comparing them. */ - public fun assertScreenshotsEqual(s1: Screenshots?, s2: Screenshots?) { + fun assertScreenshotsEqual(s1: Screenshots?, s2: Screenshots?) { if (s1 != null && s2 != null) { assertLocalizedFileListV2Equal(s1.phone, s2.phone) assertLocalizedFileListV2Equal(s1.sevenInch, s2.sevenInch) diff --git a/index/src/sharedTest/kotlin/org/fdroid/test/TestDataEntryV2.kt b/libs/sharedTest/src/main/kotlin/org/fdroid/test/TestDataEntryV2.kt similarity index 93% rename from index/src/sharedTest/kotlin/org/fdroid/test/TestDataEntryV2.kt rename to libs/sharedTest/src/main/kotlin/org/fdroid/test/TestDataEntryV2.kt index 3bda91a92..d209b65ee 100644 --- a/index/src/sharedTest/kotlin/org/fdroid/test/TestDataEntryV2.kt +++ b/libs/sharedTest/src/main/kotlin/org/fdroid/test/TestDataEntryV2.kt @@ -3,9 +3,9 @@ package org.fdroid.test import org.fdroid.index.v2.EntryFileV2 import org.fdroid.index.v2.EntryV2 -internal object TestDataEntryV2 { +object TestDataEntryV2 { - internal val empty = EntryV2( + val empty = EntryV2( timestamp = 23, version = 20001, index = EntryFileV2( @@ -16,7 +16,7 @@ internal object TestDataEntryV2 { ), ) - internal val emptyToMin = EntryV2( + val emptyToMin = EntryV2( timestamp = 42, version = 20001, maxAge = 7, @@ -36,7 +36,7 @@ internal object TestDataEntryV2 { ), ) - internal val emptyToMid = EntryV2( + val emptyToMid = EntryV2( timestamp = 1337, version = 20001, index = EntryFileV2( @@ -61,7 +61,7 @@ internal object TestDataEntryV2 { ), ) - internal val emptyToMax = EntryV2( + val emptyToMax = EntryV2( timestamp = Long.MAX_VALUE, version = Long.MAX_VALUE, maxAge = Int.MAX_VALUE, diff --git a/index/src/sharedTest/kotlin/org/fdroid/test/TestDataV1.kt b/libs/sharedTest/src/main/kotlin/org/fdroid/test/TestDataV1.kt similarity index 99% rename from index/src/sharedTest/kotlin/org/fdroid/test/TestDataV1.kt rename to libs/sharedTest/src/main/kotlin/org/fdroid/test/TestDataV1.kt index 2c2109cc2..9d4e88cd1 100644 --- a/index/src/sharedTest/kotlin/org/fdroid/test/TestDataV1.kt +++ b/libs/sharedTest/src/main/kotlin/org/fdroid/test/TestDataV1.kt @@ -8,7 +8,7 @@ import org.fdroid.index.v1.PermissionV1 import org.fdroid.index.v1.RepoV1 import org.fdroid.index.v1.Requests -internal object TestDataEmptyV1 { +object TestDataEmptyV1 { val repo = RepoV1( timestamp = 23, version = 23, @@ -22,7 +22,7 @@ internal object TestDataEmptyV1 { ) } -internal object TestDataMinV1 { +object TestDataMinV1 { val repo = RepoV1( timestamp = 42, @@ -61,7 +61,7 @@ internal object TestDataMinV1 { ) } -internal object TestDataMidV1 { +object TestDataMidV1 { val repo = RepoV1( timestamp = 1337, @@ -408,7 +408,7 @@ internal object TestDataMidV1 { } -internal object TestDataMaxV1 { +object TestDataMaxV1 { val repo = RepoV1( timestamp = Long.MAX_VALUE, diff --git a/index/src/sharedTest/kotlin/org/fdroid/test/TestDataV2.kt b/libs/sharedTest/src/main/kotlin/org/fdroid/test/TestDataV2.kt similarity index 99% rename from index/src/sharedTest/kotlin/org/fdroid/test/TestDataV2.kt rename to libs/sharedTest/src/main/kotlin/org/fdroid/test/TestDataV2.kt index a7d6f3495..0fc40ec79 100644 --- a/index/src/sharedTest/kotlin/org/fdroid/test/TestDataV2.kt +++ b/libs/sharedTest/src/main/kotlin/org/fdroid/test/TestDataV2.kt @@ -20,13 +20,13 @@ import org.fdroid.index.v2.Screenshots import org.fdroid.index.v2.SignerV2 import org.fdroid.index.v2.UsesSdkV2 -internal const val LOCALE = "en-US" +const val LOCALE = "en-US" -internal fun IndexV2.v1compat() = copy( +fun IndexV2.v1compat() = copy( repo = repo.v1compat(), ) -internal fun RepoV2.v1compat() = copy( +fun RepoV2.v1compat() = copy( name = name.filterKeys { it == LOCALE }, description = description.filterKeys { it == LOCALE }, icon = icon.filterKeys { it == LOCALE }.mapValues { it.value.v1compat() }, @@ -37,7 +37,7 @@ internal fun RepoV2.v1compat() = copy( antiFeatures = antiFeatures.mapValues { AntiFeatureV2(name = mapOf(LOCALE to it.key)) }, ) -internal fun PackageV2.v1compat(overrideLocale: Boolean = false) = copy( +fun PackageV2.v1compat(overrideLocale: Boolean = false) = copy( metadata = metadata.copy( name = if (overrideLocale) metadata.name?.filterKeys { it == LOCALE } else metadata.name, summary = if (overrideLocale) metadata.summary?.filterKeys { it == LOCALE } @@ -68,7 +68,7 @@ internal fun PackageV2.v1compat(overrideLocale: Boolean = false) = copy( ) ) -internal fun PackageVersionV2.v1compat() = copy( +fun PackageVersionV2.v1compat() = copy( src = src?.v1compat(), manifest = manifest.copy( signer = if (manifest.signer?.sha256?.size ?: 0 <= 1) manifest.signer else { @@ -79,12 +79,12 @@ internal fun PackageVersionV2.v1compat() = copy( antiFeatures = antiFeatures.mapValues { emptyMap() } ) -internal fun FileV2.v1compat() = copy( +fun FileV2.v1compat() = copy( sha256 = null, size = null, ) -internal object TestDataEmptyV2 { +object TestDataEmptyV2 { val repo = RepoV2( timestamp = 23, @@ -110,7 +110,7 @@ internal object TestDataEmptyV2 { ) } -internal object TestDataMinV2 { +object TestDataMinV2 { val repo = RepoV2( timestamp = 42, @@ -158,7 +158,7 @@ internal object TestDataMinV2 { ) } -internal object TestDataMidV2 { +object TestDataMidV2 { val repo = RepoV2( timestamp = 1337, @@ -699,7 +699,7 @@ internal object TestDataMidV2 { ) } -internal object TestDataMaxV2 { +object TestDataMaxV2 { val repo = RepoV2( timestamp = Long.MAX_VALUE, diff --git a/index/src/sharedTest/kotlin/org/fdroid/test/TestRepoUtils.kt b/libs/sharedTest/src/main/kotlin/org/fdroid/test/TestRepoUtils.kt similarity index 84% rename from index/src/sharedTest/kotlin/org/fdroid/test/TestRepoUtils.kt rename to libs/sharedTest/src/main/kotlin/org/fdroid/test/TestRepoUtils.kt index d4d275870..e4726d8ea 100644 --- a/index/src/sharedTest/kotlin/org/fdroid/test/TestRepoUtils.kt +++ b/libs/sharedTest/src/main/kotlin/org/fdroid/test/TestRepoUtils.kt @@ -12,32 +12,32 @@ import org.fdroid.test.TestUtils.getRandomString import org.fdroid.test.TestUtils.orNull import kotlin.random.Random -public object TestRepoUtils { +object TestRepoUtils { - public fun getRandomMirror(): MirrorV2 = MirrorV2( + fun getRandomMirror(): MirrorV2 = MirrorV2( url = getRandomString(), location = getRandomString().orNull() ) - public fun getRandomLocalizedTextV2(size: Int = Random.nextInt(0, 23)): LocalizedTextV2 = + fun getRandomLocalizedTextV2(size: Int = Random.nextInt(0, 23)): LocalizedTextV2 = buildMap { repeat(size) { put(getRandomString(4), getRandomString()) } } - public fun getRandomFileV2(sha256Nullable: Boolean = true): FileV2 = FileV2( + fun getRandomFileV2(sha256Nullable: Boolean = true): FileV2 = FileV2( name = getRandomString(), sha256 = getRandomString(64).also { if (sha256Nullable) orNull() }, size = Random.nextLong(-1, Long.MAX_VALUE) ) - public fun getRandomLocalizedFileV2(): Map = + fun getRandomLocalizedFileV2(): Map = TestUtils.getRandomMap(Random.nextInt(1, 8)) { getRandomString(4) to getRandomFileV2() } - public fun getRandomRepo(): RepoV2 = RepoV2( + fun getRandomRepo(): RepoV2 = RepoV2( name = getRandomLocalizedTextV2(), icon = getRandomLocalizedFileV2(), address = getRandomString(), diff --git a/index/src/sharedTest/kotlin/org/fdroid/test/TestUtils.kt b/libs/sharedTest/src/main/kotlin/org/fdroid/test/TestUtils.kt similarity index 82% rename from index/src/sharedTest/kotlin/org/fdroid/test/TestUtils.kt rename to libs/sharedTest/src/main/kotlin/org/fdroid/test/TestUtils.kt index eca7685d0..af79d6d8c 100644 --- a/index/src/sharedTest/kotlin/org/fdroid/test/TestUtils.kt +++ b/libs/sharedTest/src/main/kotlin/org/fdroid/test/TestUtils.kt @@ -6,16 +6,16 @@ import org.fdroid.index.v2.MetadataV2 import org.fdroid.index.v2.Screenshots import kotlin.random.Random -public object TestUtils { +object TestUtils { private val charPool: List = ('a'..'z') + ('A'..'Z') + ('0'..'9') - public fun getRandomString(length: Int = Random.nextInt(1, 128)): String = (1..length) + fun getRandomString(length: Int = Random.nextInt(1, 128)): String = (1..length) .map { Random.nextInt(0, charPool.size) } .map(charPool::get) .joinToString("") - public fun getRandomList( + fun getRandomList( size: Int = Random.nextInt(0, 23), factory: () -> T, ): List = if (size == 0) emptyList() else buildList { @@ -24,7 +24,7 @@ public object TestUtils { } } - public fun getRandomMap( + fun getRandomMap( size: Int = Random.nextInt(0, 23), factory: () -> Pair, ): Map = if (size == 0) emptyMap() else buildMap { @@ -34,11 +34,11 @@ public object TestUtils { } } - public fun T.orNull(): T? { + fun T.orNull(): T? { return if (Random.nextBoolean()) null else this } - public fun IndexV2.sorted(): IndexV2 = copy( + fun IndexV2.sorted(): IndexV2 = copy( packages = packages.toSortedMap().mapValues { entry -> entry.value.copy( metadata = entry.value.metadata.sort(), @@ -57,7 +57,7 @@ public object TestUtils { } ) - public fun MetadataV2.sort(): MetadataV2 = copy( + fun MetadataV2.sort(): MetadataV2 = copy( name = name?.toSortedMap(), summary = summary?.toSortedMap(), description = description?.toSortedMap(), @@ -65,7 +65,7 @@ public object TestUtils { screenshots = screenshots?.sort(), ) - public fun Screenshots.sort(): Screenshots = copy( + fun Screenshots.sort(): Screenshots = copy( phone = phone?.sort(), sevenInch = sevenInch?.sort(), tenInch = tenInch?.sort(), @@ -73,7 +73,7 @@ public object TestUtils { tv = tv?.sort(), ) - public fun LocalizedFileListV2.sort(): LocalizedFileListV2 { + fun LocalizedFileListV2.sort(): LocalizedFileListV2 { return toSortedMap().mapValues { entry -> entry.value.sortedBy { it.name } } } diff --git a/index/src/sharedTest/kotlin/org/fdroid/test/TestVersionUtils.kt b/libs/sharedTest/src/main/kotlin/org/fdroid/test/TestVersionUtils.kt similarity index 98% rename from index/src/sharedTest/kotlin/org/fdroid/test/TestVersionUtils.kt rename to libs/sharedTest/src/main/kotlin/org/fdroid/test/TestVersionUtils.kt index 063bb7ff4..2416c72f2 100644 --- a/index/src/sharedTest/kotlin/org/fdroid/test/TestVersionUtils.kt +++ b/libs/sharedTest/src/main/kotlin/org/fdroid/test/TestVersionUtils.kt @@ -15,7 +15,7 @@ import org.fdroid.test.TestUtils.getRandomString import org.fdroid.test.TestUtils.orNull import kotlin.random.Random -internal object TestVersionUtils { +object TestVersionUtils { fun getRandomPackageVersionV2( versionCode: Long = Random.nextLong(1, Long.MAX_VALUE), diff --git a/index/src/sharedTest/kotlin/org/fdroid/test/VerifierConstants.kt b/libs/sharedTest/src/main/kotlin/org/fdroid/test/VerifierConstants.kt similarity index 90% rename from index/src/sharedTest/kotlin/org/fdroid/test/VerifierConstants.kt rename to libs/sharedTest/src/main/kotlin/org/fdroid/test/VerifierConstants.kt index 388c26cfe..273e9dcb0 100644 --- a/index/src/sharedTest/kotlin/org/fdroid/test/VerifierConstants.kt +++ b/libs/sharedTest/src/main/kotlin/org/fdroid/test/VerifierConstants.kt @@ -1,9 +1,9 @@ package org.fdroid.test -public object VerifierConstants { +object VerifierConstants { - public const val VERIFICATION_DIR: String = "src/commonTest/resources/verification/" - public const val CERTIFICATE: String = + const val VERIFICATION_DIR: String = "src/commonTest/resources/verification/" + const val CERTIFICATE: String = "308202cf308201b7a0030201020204410b599a300d06092a864886f70d01010b" + "05003018311630140603550403130d546f727374656e2047726f7465301e170d" + "3134303631363139303332305a170d3431313130313139303332305a30183116" + @@ -27,7 +27,7 @@ public object VerifierConstants { "b93eff762c4b3b4fb05f8b26256570607a1400cddad2ebd4762bcf4efe703248" + "fa5b9ab455e3a5c98cb46f10adb6979aed8f96a688fd1d2a3beab380308e2ebe" + "0a4a880615567aafc0bfe344c5d7ef677e060f" - public const val FINGERPRINT: String = + const val FINGERPRINT: String = "28e14fb3b280bce8ff1e0f8e82726ff46923662cecff2a0689108ce19e8b347c" } diff --git a/settings.gradle b/settings.gradle index f7d6d2151..85eab6ce8 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,3 +1,5 @@ include ':app' -include ':download' -include ':index' \ No newline at end of file +include ':libs:sharedTest' +include ':libs:download' +include ':libs:index' +include ':libs:database'