mirror/fdroidclient
1
0
Fork 0
mirror of https://github.com/f-droid/fdroidclient.git synced 2026-08-02 10:36:57 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'lib-cleanup' into 'master'

Browse Source 
Library maintenance chores

Closes #3117, #3099, #3098, #3097, and #3089

See merge request fdroid/fdroidclient!1598
This commit is contained in:
Michael Pöhn
2025-11-04 15:09:11 +00:00
parent 1cb34b1e63 07b56f0516
commit 5e881ee54a
73 changed files with 5029 additions and 672 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.gitlab-ci.yml
View File

@@ -152,7 +152,7 @@ libs lint ktlintCheck:
script:
# always report on lint errors to the build log
- sed -i -e 's,textReport .*,textReport true,' app/build.gradle
- ./gradlew :libs:database:lint :libs:download:lint :libs:index:lint ktlintCheck
- ./gradlew :libs:database:lint :libs:download:lint :libs:index:lint ktlintCheck checkLegacyAbi
# Reference: https://gitlab.com/components/code-quality-oss/codequality-os-scanners-integration/-/blob/4121970daed111dda84cab4547e1f2951684653c/templates/pmd.yml#L52-92
app lint pmd:
@@ -347,9 +347,10 @@ pages:
only:
- master
script:
- ./gradlew :libs:download:dokkaHtml :libs:index:dokkaHtml :libs:database:dokkaHtml
- ./gradlew :libs:core:dokkaGenerateHtml :libs:download:dokkaGenerateHtml :libs:index:dokkaGenerateHtml :libs:database:dokkaGenerateHtml
- mkdir -p public/libs
- touch public/index.html public/libs/index.html
- cp -r libs/core/build/dokka/html public/libs/core
- cp -r libs/download/build/dokka/html public/libs/download
- cp -r libs/index/build/dokka/html public/libs/index
- cp -r libs/database/build/dokka/html public/libs/database

7
build.gradle
View File

@@ -6,6 +6,7 @@ buildscript {
}
plugins {
alias libs.plugins.android.application apply false
alias libs.plugins.android.library apply false
alias libs.plugins.android.ksp apply false
alias libs.plugins.jetbrains.kotlin.android apply false
alias libs.plugins.jetbrains.kotlin.multiplatform apply false
@@ -20,12 +21,6 @@ allprojects {
mavenCentral()
maven { url 'https://maven.google.com/' }
}
// use new Sonatype server for mavenCentral publishing
plugins.withId("com.vanniktech.maven.publish") {
mavenPublish {
sonatypeHost = "S01"
}
}
}
subprojects {
apply plugin: "org.jlleitschuh.gradle.ktlint"

5
gradle.properties
View File

@@ -8,6 +8,9 @@ kotlin.mpp.androidSourceSetLayoutVersion=2
# Gradle Maven Publish Info below (https://github.com/vanniktech/gradle-maven-publish-plugin)
# These are common for all libraries in gradle submodules.
# Specifics for each library are defined in the submodule's gradle.properties file.
mavenCentralPublishing=true
mavenCentralAutomaticPublishing=true
signAllPublications=true
GROUP=org.fdroid
@@ -22,5 +25,3 @@ POM_SCM_DEV_CONNECTION=scm:git:ssh://git@gitlab.com:fdroid/fdroidclient.git
POM_DEVELOPER_ID=grote
POM_DEVELOPER_NAME=Torsten Grote
POM_DEVELOPER_URL=https://github.com/grote/
SONATYPE_AUTOMATIC_RELEASE=true

17
gradle/libs.versions.toml
View File

@@ -1,18 +1,18 @@
[versions]
compileSdk = "36"
kotlin = "2.2.0"
kotlin = "2.2.21"
androidGradlePlugin = "8.11.1" # 8.12.0 pulls in aapt2 which has issue on buildserver
androidKspPlugin = "2.2.0-2.0.2" # first version needs to match kotlin version
dokka = "2.0.0"
mavenPublish = "0.18.0"
androidKspPlugin = "2.2.21-2.0.4" # first version needs to match kotlin version
dokka = "2.1.0"
mavenPublish = "0.34.0"
jlleitschuhKtlint = "13.1.0"
kotlinxSerializationJson = "1.9.0" # 1.4.1 because https://github.com/Kotlin/kotlinx.serialization/issues/2231
kotlinxSerializationJson = "1.9.0"
kotlinxCoroutinesTest = "1.10.2"
ktor = "3.3.0"
ktor = "3.3.1"
okhttp = "4.12.0"
room = "2.8.1"
room = "2.8.3"
glide = "5.0.5"
glideCompose = "1.0.0-beta08"
@@ -130,11 +130,9 @@ microutils-kotlin-logging = { module = "io.github.microutils:kotlin-logging", ve
commons-io = { module = "commons-io:commons-io", version.ref = "commonsIo" }
commons-net = { module = "commons-net:commons-net", version.ref = "commonsNet" }
ktor-io = { module = "io.ktor:ktor-io", version.ref = "ktor" }
ktor-client-cio = { module = "io.ktor:ktor-client-cio", version.ref = "ktor" }
ktor-client-core = { module = "io.ktor:ktor-client-core", version.ref = "ktor" }
ktor-client-mock = { module = "io.ktor:ktor-client-mock", version.ref = "ktor" }
ktor-client-curl = { module = "io.ktor:ktor-client-curl", version.ref = "ktor" }
ktor-client-okhttp = { module = "io.ktor:ktor-client-okhttp", version.ref = "ktor" }
okhttp = { module = "com.squareup.okhttp3:okhttp", version.ref = "okhttp" }
goncalossilva-resources = { module = "com.goncalossilva:resources", version.ref = "goncalossilvaResources" }
@@ -167,6 +165,7 @@ json = { module = "org.json:json", version.ref = "json" }
[plugins]
android-application = { id = "com.android.application", version.ref = "androidGradlePlugin" }
android-library = { id = "com.android.library", version.ref = "androidGradlePlugin" }
android-ksp = { id = "com.google.devtools.ksp", version.ref = "androidKspPlugin" }
jetbrains-kotlin-android = { id = "org.jetbrains.kotlin.android", version.ref = "kotlin" }
jetbrains-kotlin-multiplatform = { id = "org.jetbrains.kotlin.multiplatform", version.ref = "kotlin" }

624
gradle/verification-metadata.xml
View File

File diff suppressed because it is too large Load Diff

1
libs/core/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
/build

8
libs/core/api/android/core.api Normal file
View File

@@ -0,0 +1,8 @@
public abstract interface class org/fdroid/IndexFile {
public abstract fun getIpfsCidV1 ()Ljava/lang/String;
public abstract fun getName ()Ljava/lang/String;
public abstract fun getSha256 ()Ljava/lang/String;
public abstract fun getSize ()Ljava/lang/Long;
public abstract fun serialize ()Ljava/lang/String;
}

79
libs/core/build.gradle.kts Normal file
View File

@@ -0,0 +1,79 @@
plugins {
alias(libs.plugins.jetbrains.kotlin.multiplatform)
alias(libs.plugins.android.library)
alias(libs.plugins.jetbrains.dokka)
alias(libs.plugins.vanniktech.maven.publish)
}
kotlin {
androidTarget {
compilerOptions {
jvmTarget = org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_17
}
publishLibraryVariants("release")
}
explicitApi()
@OptIn(org.jetbrains.kotlin.gradle.dsl.abi.ExperimentalAbiValidation::class)
abiValidation {
enabled = true
}
compilerOptions {
optIn.add("kotlin.RequiresOptIn")
}
sourceSets {
commonMain {
dependencies {
}
}
commonTest {
dependencies {
implementation(kotlin("test"))
}
}
}
}
android {
namespace = "org.fdroid.core"
@Suppress("ktlint:standard:chain-method-continuation")
compileSdk = libs.versions.compileSdk.get().toInt()
defaultConfig {
minSdk = 21
testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
testInstrumentationRunnerArguments["disableAnalytics"] = "true"
}
buildTypes {
getByName("release") {
isMinifyEnabled = false
proguardFiles(
getDefaultProguardFile("proguard-android-optimize.txt"),
"proguard-rules.pro",
)
}
}
compileOptions {
sourceCompatibility = JavaVersion.VERSION_17
targetCompatibility = JavaVersion.VERSION_17
}
}
signing {
useGpgCmd()
}
mavenPublishing {
@Suppress("ktlint:standard:chain-method-continuation")
configure(
com.vanniktech.maven.publish.KotlinMultiplatform(
javadocJar = com.vanniktech.maven.publish.JavadocJar.Dokka("dokkaHtml"),
sourcesJar = true,
androidVariantsToPublish = listOf("release"),
),
)
}
dokka {
pluginsConfiguration.html {
customAssets.from("${file("${rootProject.rootDir}/logo-icon.svg")}")
footerMessage.set("© 2010-2025 F-Droid Limited and Contributors")
}
}

7
libs/core/gradle.properties Normal file
View File

@@ -0,0 +1,7 @@
POM_ARTIFACT_ID=core
VERSION_NAME=0.0.1
POM_NAME=F-Droid core library
POM_DESCRIPTION=A Kotlin multi-platform library to provide core classes.
POM_INCEPTION_YEAR=2025
POM_URL=https://gitlab.com/fdroid/fdroidclient/-/tree/master/libs/core

0
libs/download/src/commonMain/kotlin/org/fdroid/IndexFile.kt → libs/core/src/commonMain/kotlin/org/fdroid/IndexFile.kt
View File

0
libs/database/api/database.api Normal file
View File

136
libs/database/build.gradle
View File

@@ -1,136 +0,0 @@
plugins {
id 'kotlin-android'
id 'com.android.library'
id 'com.google.devtools.ksp'
id 'org.jetbrains.dokka'
id 'com.vanniktech.maven.publish'
}
android {
namespace "org.fdroid.database"
compileSdk libs.versions.compileSdk.get().toInteger()
defaultConfig {
minSdkVersion 23
targetSdk 34 // relevant for instrumentation tests (targetSdk 21 fails on Android 14)
consumerProguardFiles "consumer-rules.pro"
testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
testInstrumentationRunnerArguments disableAnalytics: 'true'
}
buildTypes {
release {
minifyEnabled false
proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
}
}
sourceSets {
androidTest {
java.srcDirs += "src/dbTest/java"
// Adds exported schema location as test app assets.
assets.srcDirs += files("$projectDir/schemas".toString())
}
test {
java.srcDirs += "src/dbTest/java"
// Adds exported schema location as test app assets.
assets.srcDirs += files("$projectDir/schemas".toString())
}
}
compileOptions {
sourceCompatibility JavaVersion.VERSION_17
targetCompatibility JavaVersion.VERSION_17
}
testOptions {
unitTests {
includeAndroidResources = true
}
}
kotlinOptions {
jvmTarget = '17'
freeCompilerArgs += "-Xexplicit-api=strict"
freeCompilerArgs += "-opt-in=kotlin.RequiresOptIn"
}
aaptOptions {
// needed only for instrumentation tests: assets.openFd()
noCompress "json"
}
packagingOptions {
exclude 'META-INF/AL2.0'
exclude 'META-INF/LGPL2.1'
exclude 'META-INF/LICENSE.md'
exclude 'META-INF/LICENSE-notice.md'
}
}
dependencies {
implementation project(":libs:download")
implementation project(":libs:index")
implementation libs.androidx.core.ktx
implementation libs.androidx.lifecycle.livedata.ktx
implementation libs.androidx.room.runtime
implementation libs.androidx.room.ktx
ksp libs.androidx.room.compiler
implementation libs.microutils.kotlin.logging
implementation libs.kotlinx.serialization.json
testImplementation project(":libs:sharedTest")
testImplementation libs.junit
testImplementation libs.mockk
testImplementation libs.kotlin.test
testImplementation libs.androidx.test.core.ktx
testImplementation libs.androidx.test.ext.junit
testImplementation libs.androidx.core.testing
testImplementation libs.androidx.room.testing
testImplementation libs.robolectric
testImplementation libs.commons.io
testImplementation libs.logback.classic
testImplementation libs.kotlinx.coroutines.test
testImplementation libs.turbine
testImplementation libs.okhttp
androidTestImplementation project(":libs:sharedTest")
androidTestImplementation libs.mockk.android
androidTestImplementation libs.kotlin.test
androidTestImplementation libs.androidx.test.ext.junit
androidTestImplementation libs.androidx.core.testing
androidTestImplementation libs.androidx.espresso.core
androidTestImplementation libs.androidx.room.testing
androidTestImplementation libs.commons.io
}
ksp {
arg(new RoomSchemaArgProvider(new File(projectDir, "schemas")))
}
signing {
useGpgCmd()
}
import org.jetbrains.dokka.gradle.DokkaTask
tasks.withType(DokkaTask).configureEach {
pluginsMapConfiguration.set(
["org.jetbrains.dokka.base.DokkaBase": """{
"customAssets": ["${file("${rootProject.rootDir}/logo-icon.svg")}"],
"footerMessage": "© 2010-2022 F-Droid Limited and Contributors"
}"""]
)
}
class RoomSchemaArgProvider implements CommandLineArgumentProvider {
@InputDirectory
@PathSensitive(PathSensitivity.RELATIVE)
File schemaDir
RoomSchemaArgProvider(File schemaDir) {
this.schemaDir = schemaDir
}
@Override
Iterable<String> asArguments() {
return ["room.schemaLocation=${schemaDir.path}".toString()]
}
}

139
libs/database/build.gradle.kts Normal file
View File

@@ -0,0 +1,139 @@
plugins {
alias(libs.plugins.jetbrains.kotlin.android)
alias(libs.plugins.android.library)
alias(libs.plugins.android.ksp)
alias(libs.plugins.jetbrains.dokka)
alias(libs.plugins.vanniktech.maven.publish)
}
android {
namespace = "org.fdroid.database"
@Suppress("ktlint:standard:chain-method-continuation")
compileSdk = libs.versions.compileSdk.get().toInt()
defaultConfig {
minSdk = 23
consumerProguardFiles("consumer-rules.pro")
testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
testInstrumentationRunnerArguments["disableAnalytics"] = "true"
}
buildTypes {
getByName("release") {
isMinifyEnabled = false
proguardFiles(
getDefaultProguardFile("proguard-android-optimize.txt"),
"proguard-rules.pro",
)
}
}
sourceSets {
getByName("androidTest") {
java.srcDirs("src/dbTest/java")
// Adds exported schema location as test app assets.
assets.srcDirs(files("$projectDir/schemas"))
}
getByName("test") {
java.srcDirs("src/dbTest/java")
// Adds exported schema location as test app assets.
assets.srcDirs(files("$projectDir/schemas"))
}
}
compileOptions {
sourceCompatibility = JavaVersion.VERSION_17
targetCompatibility = JavaVersion.VERSION_17
}
testOptions {
targetSdk = 34 // relevant for instrumentation tests (targetSdk 21 fails on Android 14)
unitTests {
isIncludeAndroidResources = true
}
}
androidResources {
// needed only for instrumentation tests: assets.openFd()
noCompress += "json"
}
packaging {
resources {
excludes.add("META-INF/AL2.0")
excludes.add("META-INF/LGPL2.1")
excludes.add("META-INF/LICENSE.md")
excludes.add("META-INF/LICENSE-notice.md")
}
}
}
kotlin {
explicitApi()
@OptIn(org.jetbrains.kotlin.gradle.dsl.abi.ExperimentalAbiValidation::class)
abiValidation {
enabled = true
}
compilerOptions {
jvmTarget = org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_17
optIn.add("kotlin.RequiresOptIn")
}
}
dependencies {
implementation(project(":libs:core"))
implementation(project(":libs:index"))
implementation(project(":libs:download")) // needed for updater code
implementation(libs.androidx.core.ktx)
implementation(libs.androidx.lifecycle.livedata.ktx)
implementation(libs.androidx.room.runtime)
implementation(libs.androidx.room.ktx)
ksp(libs.androidx.room.compiler)
implementation(libs.microutils.kotlin.logging)
implementation(libs.kotlinx.serialization.json)
testImplementation(project(":libs:sharedTest"))
testImplementation(libs.junit)
testImplementation(libs.mockk)
testImplementation(libs.kotlin.test)
testImplementation(libs.androidx.test.core.ktx)
testImplementation(libs.androidx.test.ext.junit)
testImplementation(libs.androidx.core.testing)
testImplementation(libs.androidx.room.testing)
testImplementation(libs.robolectric)
testImplementation(libs.commons.io)
testImplementation(libs.logback.classic)
testImplementation(libs.kotlinx.coroutines.test)
testImplementation(libs.turbine)
testImplementation(libs.okhttp)
androidTestImplementation(project(":libs:sharedTest"))
androidTestImplementation(libs.mockk.android)
androidTestImplementation(libs.kotlin.test)
androidTestImplementation(libs.androidx.test.ext.junit)
androidTestImplementation(libs.androidx.core.testing)
androidTestImplementation(libs.androidx.espresso.core)
androidTestImplementation(libs.androidx.room.testing)
androidTestImplementation(libs.commons.io)
}
ksp {
arg(RoomSchemaArgProvider(File(projectDir, "schemas")))
}
signing {
useGpgCmd()
}
dokka {
pluginsConfiguration.html {
customAssets.from("${file("${rootProject.rootDir}/logo-icon.svg")}")
footerMessage.set("© 2010-2025 F-Droid Limited and Contributors")
}
}
class RoomSchemaArgProvider(
@get:InputDirectory
@get:PathSensitive(PathSensitivity.RELATIVE)
val schemaDir: File,
) : CommandLineArgumentProvider {
override fun asArguments(): Iterable<String> = listOf("room.schemaLocation=${schemaDir.path}")
}

2
libs/database/gradle.properties
View File

@@ -1,5 +1,5 @@
POM_ARTIFACT_ID=database
VERSION_NAME=0.1.0
VERSION_NAME=0.2.0
POM_NAME=F-Droid database library
POM_DESCRIPTION=An Android library to store F-Droid related info in Room such as repositories, apps and their versions.

1033
libs/database/schemas/org.fdroid.database.FDroidDatabaseInt/9.json Normal file
View File

File diff suppressed because it is too large Load Diff

137
libs/database/src/dbTest/java/org/fdroid/database/AppOverviewItemsTest.kt
View File

@@ -1,16 +1,20 @@
package org.fdroid.database
import androidx.test.ext.junit.runners.AndroidJUnit4
import kotlinx.coroutines.runBlocking
import org.fdroid.LocaleChooser.getBestLocale
import org.fdroid.database.TestUtils.getOrAwaitValue
import org.fdroid.database.TestUtils.getOrFail
import org.fdroid.index.v2.MetadataV2
import org.fdroid.test.TestAppUtils.getRandomMetadataV2
import org.fdroid.test.TestRepoUtils.getRandomRepo
import org.fdroid.test.TestUtils.getRandomString
import org.fdroid.test.TestVersionUtils.getRandomPackageVersionV2
import org.junit.Test
import org.junit.runner.RunWith
import kotlin.test.assertEquals
import kotlin.test.assertFalse
import kotlin.test.assertNotEquals
import kotlin.test.assertNotNull
import kotlin.test.assertNull
import kotlin.test.assertTrue
@@ -19,13 +23,13 @@ import kotlin.test.assertTrue
internal class AppOverviewItemsTest : AppTest() {
@Test
fun testAntiFeatures() {
// insert one apps with without version
fun testAntiFeatures() = runBlocking {
// insert one app with without version
val repoId = repoDao.insertOrReplace(getRandomRepo())
appDao.insert(repoId, packageName, app1, locales)
// without version, anti-features are empty
appDao.getAppOverviewItems().getOrFail().let { apps ->
getItems().forEach { apps ->
assertEquals(1, apps.size)
assertNull(apps[0].antiFeatures)
}
@@ -33,7 +37,7 @@ internal class AppOverviewItemsTest : AppTest() {
// with one version, the app has those anti-features
val version = getRandomPackageVersionV2(versionCode = 42)
versionDao.insert(repoId, packageName, "1", version, true)
appDao.getAppOverviewItems().getOrFail().let { apps ->
getItems().forEach { apps ->
assertEquals(1, apps.size)
assertEquals(version.antiFeatures, apps[0].antiFeatures)
}
@@ -41,7 +45,7 @@ internal class AppOverviewItemsTest : AppTest() {
// with two versions, the app has the anti-features of the highest version
val version2 = getRandomPackageVersionV2(versionCode = 23)
versionDao.insert(repoId, packageName, "2", version2, true)
appDao.getAppOverviewItems().getOrFail().let { apps ->
getItems().forEach { apps ->
assertEquals(1, apps.size)
assertEquals(version.antiFeatures, apps[0].antiFeatures)
}
@@ -49,20 +53,20 @@ internal class AppOverviewItemsTest : AppTest() {
// with three versions, the app has the anti-features of the highest version
val version3 = getRandomPackageVersionV2(versionCode = 1337)
versionDao.insert(repoId, packageName, "3", version3, true)
appDao.getAppOverviewItems().getOrFail().let { apps ->
getItems().forEach { apps ->
assertEquals(1, apps.size)
assertEquals(version3.antiFeatures, apps[0].antiFeatures)
}
}
@Test
fun testIcons() {
fun testIcons() = runBlocking {
// insert one app
val repoId = repoDao.insertOrReplace(getRandomRepo())
appDao.insert(repoId, packageName, app1, locales)
// icon is returned correctly
appDao.getAppOverviewItems().getOrFail().let { apps ->
getItems().forEach { apps ->
assertEquals(1, apps.size)
assertEquals(app1.icon.getBestLocale(locales), apps[0].getIcon(locales))
}
@@ -72,14 +76,14 @@ internal class AppOverviewItemsTest : AppTest() {
appDao.insert(repoId2, packageName, app2, locales)
// app is still returned as before
appDao.getAppOverviewItems().getOrFail().let { apps ->
getItems().forEach { apps ->
assertEquals(1, apps.size)
assertEquals(app1.icon.getBestLocale(locales), apps[0].getIcon(locales))
}
// after preferring second repo, icon is returned from app in second repo
appPrefsDao.update(AppPrefs(packageName, preferredRepoId = repoId2))
appDao.getAppOverviewItems().getOrFail().let { apps ->
getItems().forEach { apps ->
assertEquals(1, apps.size)
assertEquals(app2.icon.getBestLocale(locales), apps[0].getIcon(locales))
}
@@ -99,17 +103,18 @@ internal class AppOverviewItemsTest : AppTest() {
}
@Test
fun testIncompatibleFlag() {
fun testIncompatibleFlag() = runBlocking {
// insert two apps
val repoId = repoDao.insertOrReplace(getRandomRepo())
appDao.insert(repoId, packageName1, app1, locales)
appDao.insert(repoId, packageName2, app2, locales)
// both apps are not compatible
appDao.getAppOverviewItems().getOrFail().also {
assertEquals(2, it.size)
}.forEach {
assertFalse(it.isCompatible)
getItems().forEach { apps ->
assertEquals(2, apps.size)
apps.forEach {
assertFalse(it.isCompatible)
}
}
// both apps, in the same category, are not compatible
appDao.getAppOverviewItems("A").getOrFail().also {
@@ -128,10 +133,10 @@ internal class AppOverviewItemsTest : AppTest() {
appDao.updateCompatibility(repoId)
// now only one is not compatible
appDao.getAppOverviewItems().getOrFail().also {
assertEquals(2, it.size)
assertFalse(it[0].isCompatible)
assertTrue(it[1].isCompatible)
getItems().forEach { apps ->
assertEquals(2, apps.size)
assertFalse(apps[0].isCompatible)
assertTrue(apps[1].isCompatible)
}
appDao.getAppOverviewItems("A").getOrFail().also {
assertEquals(2, it.size)
@@ -143,14 +148,14 @@ internal class AppOverviewItemsTest : AppTest() {
}
@Test
fun testGetByRepoWeight() {
fun testGetByRepoWeight() = runBlocking {
// insert one app with one version
val repoId = repoDao.insertOrReplace(getRandomRepo())
appDao.insert(repoId, packageName, app1, locales)
versionDao.insert(repoId, packageName, "1", getRandomPackageVersionV2(2), true)
// app is returned correctly
appDao.getAppOverviewItems().getOrFail().let { apps ->
getItems().forEach { apps ->
assertEquals(1, apps.size)
assertEquals(app1, apps[0])
}
@@ -160,21 +165,21 @@ internal class AppOverviewItemsTest : AppTest() {
appDao.insert(repoId2, packageName, app2, locales)
// app is still returned as before, new repo doesn't override old one
appDao.getAppOverviewItems().getOrFail().let { apps ->
getItems().forEach { apps ->
assertEquals(1, apps.size)
assertEquals(app1, apps[0])
}
// now second app from second repo is returned after preferring it explicitly
appPrefsDao.update(AppPrefs(packageName, preferredRepoId = repoId2))
appDao.getAppOverviewItems().getOrFail().let { apps ->
getItems().forEach { apps ->
assertEquals(1, apps.size)
assertEquals(app2, apps[0])
}
}
@Test
fun testGetByRepoPref() {
fun testGetByRepoPref() = runBlocking {
// insert same app into three repos (repoId1 has highest weight)
val repoId1 = repoDao.insertOrReplace(getRandomRepo())
val repoId3 = repoDao.insertOrReplace(getRandomRepo())
@@ -184,7 +189,7 @@ internal class AppOverviewItemsTest : AppTest() {
appDao.insert(repoId3, packageName, app3, locales)
// app is returned correctly from repo1
appDao.getAppOverviewItems().getOrFail().let { apps ->
getItems().forEach { apps ->
assertEquals(1, apps.size)
assertEquals(app1, apps[0])
}
@@ -195,7 +200,7 @@ internal class AppOverviewItemsTest : AppTest() {
// prefer repo3 for this app
appPrefsDao.update(AppPrefs(packageName, preferredRepoId = repoId3))
appDao.getAppOverviewItems().getOrFail().let { apps ->
getItems().forEach { apps ->
assertEquals(1, apps.size)
assertEquals(app3, apps[0])
}
@@ -206,7 +211,7 @@ internal class AppOverviewItemsTest : AppTest() {
// prefer repo2 for this app
appPrefsDao.update(AppPrefs(packageName, preferredRepoId = repoId2))
appDao.getAppOverviewItems().getOrFail().let { apps ->
getItems().forEach { apps ->
assertEquals(1, apps.size)
assertEquals(app2, apps[0])
}
@@ -220,7 +225,7 @@ internal class AppOverviewItemsTest : AppTest() {
// prefer repo1 for this app
appPrefsDao.update(AppPrefs(packageName, preferredRepoId = repoId1))
appDao.getAppOverviewItems().getOrFail().let { apps ->
getItems().forEach { apps ->
assertEquals(1, apps.size)
assertEquals(app1, apps[0])
}
@@ -341,7 +346,7 @@ internal class AppOverviewItemsTest : AppTest() {
}
@Test
fun testOnlyFromEnabledRepos() {
fun testOnlyFromEnabledRepos() = runBlocking {
val repoId = repoDao.insertOrReplace(getRandomRepo())
appDao.insert(repoId, packageName1, app1, locales)
appDao.insert(repoId, packageName2, app2, locales)
@@ -349,18 +354,24 @@ internal class AppOverviewItemsTest : AppTest() {
appDao.insert(repoId2, packageName3, app3, locales)
// 3 apps from 2 repos
assertEquals(3, appDao.getAppOverviewItems().getOrAwaitValue()?.size)
getItems().forEach { apps ->
assertEquals(3, apps.size)
}
assertEquals(3, appDao.getAppOverviewItems("A").getOrAwaitValue()?.size)
// only 1 app after disabling first repo
repoDao.setRepositoryEnabled(repoId, false)
assertEquals(1, appDao.getAppOverviewItems().getOrAwaitValue()?.size)
getItems().forEach { apps ->
assertEquals(1, apps.size)
}
assertEquals(1, appDao.getAppOverviewItems("A").getOrAwaitValue()?.size)
assertEquals(1, appDao.getAppOverviewItems("B").getOrAwaitValue()?.size)
// no more apps after disabling all repos
repoDao.setRepositoryEnabled(repoId2, false)
assertEquals(0, appDao.getAppOverviewItems().getOrAwaitValue()?.size)
getItems().forEach { apps ->
assertEquals(0, apps.size)
}
assertEquals(0, appDao.getAppOverviewItems("A").getOrAwaitValue()?.size)
assertEquals(0, appDao.getAppOverviewItems("B").getOrAwaitValue()?.size)
}
@@ -405,13 +416,73 @@ internal class AppOverviewItemsTest : AppTest() {
assertEquals(app2, appDao.getAppOverviewItem(repoId2, packageName))
}
@Test
fun testByAuthor() = runBlocking {
val author = getRandomString()
val packageName = getRandomString()
val repoId = repoDao.insertOrReplace(getRandomRepo())
appDao.insert(repoId, packageName, getRandomMetadataV2(author), locales)
// author has only one app
assertFalse(appDao.hasAuthorMoreThanOneApp(author).getOrFail())
assertEquals(0, appDao.getAppsByAuthor("foo bar").size)
appDao.getAppsByAuthor(author).let { apps ->
assertEquals(1, apps.size)
assertEquals(packageName, apps[0].packageName)
}
// now add 49 more apps
(1 until 50).forEach { _ ->
appDao.insert(repoId, getRandomString(), getRandomMetadataV2(author), locales)
}
assertTrue(appDao.hasAuthorMoreThanOneApp(author).getOrFail())
assertEquals(50, appDao.getAppsByAuthor(author).size)
}
@Test
fun testByCategory() = runBlocking {
// insert three apps
val repoId = repoDao.insertOrReplace(getRandomRepo())
appDao.insert(repoId, packageName1, app1, locales)
appDao.insert(repoId, packageName2, app2, locales)
appDao.insert(repoId, packageName3, app3, locales)
// only two apps are in category B
appDao.getAppsByCategory("B").let { apps ->
assertEquals(2, apps.size)
assertNotEquals(packageName2, apps[0].packageName)
assertNotEquals(packageName2, apps[1].packageName)
}
// no app is in category C
assertEquals(0, appDao.getAppsByCategory("C").size)
// we'll add app1 as a variant of app2, but its repo has lower weight, so no effect
val repoId2 = repoDao.insertOrReplace(getRandomRepo())
appDao.insert(repoId2, packageName2, app1, locales)
appDao.getAppsByCategory("B").let { apps ->
assertEquals(2, apps.size)
assertNotEquals(packageName2, apps[0].packageName)
assertNotEquals(packageName2, apps[1].packageName)
}
}
private suspend fun getItems(): List<List<AppOverviewItem>> {
return listOf(
appDao.getAppOverviewItems().getOrFail(),
// manually sort the second list, so both results are comparable
appDao.getAllApps().sortedByDescending { it.lastUpdated },
)
}
private fun assertEquals(expected: MetadataV2, actual: AppOverviewItem?) {
assertNotNull(actual)
assertEquals(expected.added, actual.added)
assertEquals(expected.lastUpdated, actual.lastUpdated)
assertEquals(expected.name.getBestLocale(locales), actual.name)
assertEquals(expected.summary.getBestLocale(locales), actual.summary)
assertEquals(expected.summary.getBestLocale(locales), actual.summary)
assertEquals(expected.name.getBestLocale(locales), actual.getName(locales))
assertEquals(expected.summary.getBestLocale(locales), actual.getSummary(locales))
assertEquals(expected.icon.getBestLocale(locales), actual.getIcon(locales))
}

2
libs/database/src/dbTest/java/org/fdroid/database/CountryCodeMigrationTest.kt
View File

@@ -80,7 +80,7 @@ internal class CountryCodeMigrationTest {
FDroidDatabaseInt::class.java,
TEST_DB
)
.addMigrations(MIGRATION_2_3, MIGRATION_5_6)
.addMigrations(MIGRATION_2_3, MIGRATION_5_6, MIGRATION_8_9)
.allowMainThreadQueries()
.build().use { db ->
// check repo got timestamp and etag reset

7
libs/database/src/dbTest/java/org/fdroid/database/DbTest.kt
View File

@@ -114,7 +114,12 @@ internal abstract class DbTest {
assertEquals(sortedIndex.packages.size, appDao.countApps(), "number of packages")
sortedIndex.packages.forEach { (packageName, packageV2) ->
assertEquals(
packageV2.metadata,
// zero-whitespace hack needs to get applied to expected data
packageV2.metadata.copy(
name = packageV2.metadata.name.zero(),
summary = packageV2.metadata.summary.zero(),
description = packageV2.metadata.description.zero(),
),
appDao.getApp(repoId, packageName)?.toMetadataV2()?.sort()
)
val versions = versionDao.getAppVersions(repoId, packageName).getOrFail().map {

2
libs/database/src/dbTest/java/org/fdroid/database/DbUpdateCheckerTest.kt
View File

@@ -40,7 +40,7 @@ internal class DbUpdateCheckerTest : AppTest() {
override fun createDb() {
super.createDb()
every { packageManager.systemAvailableFeatures } returns emptyArray()
updateChecker = DbUpdateChecker(db, packageManager) { true }
updateChecker = DbUpdateChecker(db, packageManager, { true })
}
@Test

240
libs/database/src/dbTest/java/org/fdroid/database/FtsAddColumnsMigrationTest.kt Normal file
View File

@@ -0,0 +1,240 @@
package org.fdroid.database
import android.content.ContentValues
import android.content.Context
import android.database.sqlite.SQLiteDatabase.CONFLICT_FAIL
import androidx.arch.core.executor.testing.InstantTaskExecutorRule
import androidx.room.Room
import androidx.room.testing.MigrationTestHelper
import androidx.sqlite.db.SupportSQLiteDatabase
import androidx.sqlite.db.framework.FrameworkSQLiteOpenHelperFactory
import androidx.test.core.app.ApplicationProvider
import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.platform.app.InstrumentationRegistry
import kotlinx.coroutines.runBlocking
import org.fdroid.database.TestUtils.getOrFail
import org.fdroid.test.TestUtils.getRandomString
import org.junit.Rule
import org.junit.Test
import org.junit.runner.RunWith
import kotlin.random.Random
import kotlin.test.assertEquals
private const val TEST_DB = "fts-test"
@RunWith(AndroidJUnit4::class)
internal class FtsAddColumnsMigrationTest {
@get:Rule
val helper: MigrationTestHelper = MigrationTestHelper(
instrumentation = InstrumentationRegistry.getInstrumentation(),
databaseClass = FDroidDatabaseInt::class.java,
specs = emptyList(),
openFactory = FrameworkSQLiteOpenHelperFactory(),
)
@get:Rule
val instantExec = InstantTaskExecutorRule()
private val context: Context = ApplicationProvider.getApplicationContext()
private val repo = ContentValues().apply {
put("repoId", 1)
put("name", Converters.localizedTextV2toString(mapOf("de" to "a", "en-US" to "b")))
put("address", getRandomString())
put("certificate", "abcdef")
put("description", Converters.localizedTextV2toString(mapOf("de" to "aa", "en-US" to "bb")))
put("version", Random.nextLong())
put("timestamp", Random.nextLong())
}
private val repoPrefs = ContentValues().apply {
put("repoId", 1)
put("enabled", true)
put("weight", 1)
}
private val oeffiMetadata = ContentValues().apply {
put("packageName", "de.schildbach.oeffi")
put("repoId", 1)
put(
"name", Converters.localizedTextV2toString(
mapOf(
"de" to "Öffi", "en-US" to "Offi"
)
)
)
put(
"description", Converters.localizedTextV2toString(
mapOf(
"de" to "Öffentlicher Nahverkehr", "en-US" to "Public Transport"
)
)
)
put("license", "GPL-3.0")
put(
"summary", Converters.localizedTextV2toString(
mapOf(
"de" to "Der König des Fahrplandschungels!",
"en-US" to " King of public transit planning!"
)
)
)
put("localizedName", "Öffi")
put("localizedSummary", "Der König des Fahrplandschungels!")
put("added", Random.nextLong())
put("lastUpdated", Random.nextLong())
put("isCompatible", true)
}
private val oeffiVersion = ContentValues().apply {
put("repoId", 1)
put("packageName", "de.schildbach.oeffi")
put("versionId", 1)
put("added", 1000)
put("isCompatible", true)
put("file_name", "transportr.apk")
put("file_sha256", "73475CB40A568E8DA8A045CED110137E159F890AC4DA883B6B17DC651B3A8049")
put("manifest_versionName", "1.0.0")
put("manifest_versionCode", 1)
}
private val transportrMetadata = ContentValues().apply {
put("packageName", "de.grobox.liberario")
put("repoId", 1)
put(
"name", Converters.localizedTextV2toString(
mapOf(
"de" to "Transportr", "en-US" to "Transportr"
)
)
)
put(
"description", Converters.localizedTextV2toString(
mapOf(
"de" to "Öffentlicher Nahverkehr", "en-US" to "Public Transport"
)
)
)
put("license", "GPL-3.0")
put(
"summary", Converters.localizedTextV2toString(
mapOf(
"de" to "Freier Assistent für den öffentlichen Nahverkehr ohne Werbung",
"en-US" to "Free Public Transport Assistant without Ads or Tracking"
)
)
)
put("localizedName", "Transportr")
put(
"localizedSummary",
"Freier Assistent für den öffentlichen Nahverkehr ohne Werbung und Tracking"
)
put("added", Random.nextLong())
put("lastUpdated", Random.nextLong())
put("isCompatible", true)
}
private val transportrVersion = ContentValues().apply {
put("repoId", 1)
put("packageName", "de.grobox.liberario")
put("versionId", 1)
put("added", 1000)
put("isCompatible", true)
put("file_name", "transportr.apk")
put("file_sha256", "73475CB40A568E8DA8A045CED110137E159F890AC4DA883B6B17DC651B3A8049")
put("manifest_versionName", "1.0.0")
put("manifest_versionCode", 1)
}
@Test
fun testMigration() = runBlocking {
helper.createDatabase(TEST_DB, 8).use { db ->
// Database has schema version 8. Insert some data using SQL queries.
// We can't use DAO classes because they expect the latest schema.
db.insert(CoreRepository.TABLE, CONFLICT_FAIL, repo)
db.insert(RepositoryPreferences.TABLE, CONFLICT_FAIL, repoPrefs)
db.insert(AppMetadata.TABLE, CONFLICT_FAIL, oeffiMetadata)
db.insert(AppMetadata.TABLE, CONFLICT_FAIL, transportrMetadata)
db.insert(Version.TABLE, CONFLICT_FAIL, oeffiVersion)
db.insert(Version.TABLE, CONFLICT_FAIL, transportrVersion)
// default search with no diacritics
assertSearch(db, "*Transport*", 1)
assertSearch(db, "Transportr", 1)
assertSearch(db, "*f*", 2)
// no or wrong diacritics
assertSearch(db, "Offi", 0)
assertSearch(db, "offi", 0)
assertSearch(db, "õffi", 0)
assertSearch(db, "*Offi*", 0)
assertSearch(db, "*offi*", 0)
assertSearch(db, "Konig", 0)
// correct diacritics
assertSearch(db, "*Öffi*", 1)
assertSearch(db, "*öffi*", 1)
assertSearch(db, "Öffi", 1)
assertSearch(db, "öffi", 1)
// both apps have "öff" in their name or summary
assertSearch(db, "*öff*", 2)
assertSearch(db, "König", 1)
}
helper.runMigrationsAndValidate(TEST_DB, 9, true, MIGRATION_8_9).close()
// now get the Room DB, so we can use our DAOs for verifying the migration
Room.databaseBuilder(context, FDroidDatabaseInt::class.java, TEST_DB)
.allowMainThreadQueries()
.build().use { db ->
// assert that apps are still there
val metadata = db.getAppDao().getAppMetadata()
assertEquals(2, metadata.size)
// default search with no diacritics
assertGetAppListItems(db, "*Transport*", 2)
assertGetAppListItems(db, "Transportr", 1)
assertGetAppListItems(db, "*f*", 2)
// no or wrong diacritics also produces results now
assertGetAppListItems(db, "Offi", 1)
assertGetAppListItems(db, "offi", 1)
assertGetAppListItems(db, "õffi", 1)
assertGetAppListItems(db, "*Offi*", 1)
assertGetAppListItems(db, "*offi*", 1)
assertGetAppListItems(db, "Konig", 1)
// correct diacritics still produces results
assertGetAppListItems(db, "*Öffi*", 1)
assertGetAppListItems(db, "*öffi*", 1)
assertGetAppListItems(db, "Öffi", 1)
assertGetAppListItems(db, "öffi", 1)
// both apps have "öff" in their name or summary
assertGetAppListItems(db, "*öff*", 2)
assertGetAppListItems(db, "König", 1)
// a new app also gets updated in the index
assertGetAppListItems(db, "Rosa", 0)
assertGetAppListItems(db, "Elefant", 0)
val newApp = AppMetadata(
repoId = 1,
packageName = "org.example",
added = 23,
lastUpdated = 42,
name = mapOf("de_DE" to "Rosa Elefant"),
isCompatible = true
)
db.getAppDao().insert(newApp)
assertGetAppListItems(db, "Rosa", 1)
assertGetAppListItems(db, "Elefant", 1)
}
}
private fun assertSearch(db: SupportSQLiteDatabase, query: String, expected: Int) {
db.query("SELECT * FROM AppMetadataFts WHERE AppMetadataFts MATCH '$query'")
.use { cursor -> assertEquals(expected, cursor.count) }
}
private fun assertGetAppListItems(db: FDroidDatabaseInt, query: String, expected: Int) {
db.getAppDao().getAppListItems(query).getOrFail().let { result ->
assertEquals(expected, result.size, "${result.map { it.name }}")
}
}
}

23
libs/database/src/dbTest/java/org/fdroid/database/FtsCaseInsensitiveMigrationTest.kt
View File

@@ -176,29 +176,16 @@ internal class FtsCaseInsensitiveMigrationTest {
// now get the Room DB, so we can use our DAOs for verifying the migration
Room.databaseBuilder(context, FDroidDatabaseInt::class.java, TEST_DB)
.allowMainThreadQueries()
.addMigrations(MIGRATION_8_9) // was added later
.build().use { db ->
// assert that apps are still there
val metadata = db.getAppDao().getAppMetadata()
assertEquals(2, metadata.size)
// default search with no diacritics
assertGetAppListItems(db, "*Transport*", 1)
assertGetAppListItems(db, "Transportr", 1)
assertGetAppListItems(db, "*f*", 2)
// no or wrong diacritics
assertGetAppListItems(db, "Offi", 0)
assertGetAppListItems(db, "offi", 0)
assertGetAppListItems(db, "õffi", 0)
assertGetAppListItems(db, "*Offi*", 0)
assertGetAppListItems(db, "*offi*", 0)
assertGetAppListItems(db, "Konig", 0)
// correct diacritics
assertGetAppListItems(db, "*Öffi*", 1)
assertGetAppListItems(db, "*öffi*", 1)
assertGetAppListItems(db, "Öffi", 1)
assertGetAppListItems(db, "öffi", 1)
// both apps have "öff" in their name or summary
assertGetAppListItems(db, "*öff*", 2)
assertGetAppListItems(db, "König", 1)
assertGetAppListItems(db, "*Transport*", 2)
// other tests here were removed, because MIGRATION_8_9 changed this once again
// and has its own test
}
}

31
libs/database/src/dbTest/java/org/fdroid/database/IndexV2DiffTest.kt
View File

@@ -390,6 +390,37 @@ internal class IndexV2DiffTest : DbTest() {
)
}
@Test
fun testMinAddChinese() {
val diffJson = """{
"packages": {
"org.fdroid.min1": {
"metadata": {
"name": { "zh-CN": "自由软件仓库" },
"summary": { "ja": "这个仓库中的" },
"description": { "ko-KR": "切始终是从" }
}
}
}
}""".trimIndent()
testJsonDiff(
startPath = "index-min-v2.json",
diff = diffJson,
endIndex = TestDataMinV2.index.copy(
packages = TestDataMinV2.index.packages.mapValues {
it.value.copy(
metadata = it.value.metadata.copy(
// zero whitespaces (to separate tokens) will be added in testJsonDiff()
name = mapOf("zh-CN" to "自由软件仓库"),
summary = mapOf("ja" to "这个仓库中的"),
description = mapOf("ko-KR" to "切始终是从"),
)
)
}
),
)
}
private fun testJsonDiff(startPath: String, diff: String, endIndex: IndexV2) {
testDiff(startPath, ByteArrayInputStream(diff.toByteArray()), endIndex)
}

6
libs/database/src/dbTest/java/org/fdroid/database/MultiRepoMigrationTest.kt
View File

@@ -232,7 +232,7 @@ internal class MultiRepoMigrationTest {
// now get the Room DB, so we can use our DAOs for verifying the migration
databaseBuilder(getApplicationContext(), FDroidDatabaseInt::class.java, TEST_DB)
.addMigrations(MIGRATION_2_3, MIGRATION_5_6)
.addMigrations(MIGRATION_2_3, MIGRATION_5_6, MIGRATION_8_9)
.allowMainThreadQueries()
.build()
.use { db ->
@@ -275,7 +275,7 @@ internal class MultiRepoMigrationTest {
// now get the Room DB, so we can use our DAOs for verifying the migration
databaseBuilder(getApplicationContext(), FDroidDatabaseInt::class.java, TEST_DB)
.addMigrations(MIGRATION_2_3, MIGRATION_5_6)
.addMigrations(MIGRATION_2_3, MIGRATION_5_6, MIGRATION_8_9)
.allowMainThreadQueries()
.build().use { db ->
// repo without cert did not get migrated, because we auto-migrate to latest version
@@ -311,7 +311,7 @@ internal class MultiRepoMigrationTest {
// now get the Room DB, so we can use our DAOs for verifying the migration
databaseBuilder(getApplicationContext(), FDroidDatabaseInt::class.java, TEST_DB)
.addMigrations(MIGRATION_2_3, MIGRATION_5_6)
.addMigrations(MIGRATION_2_3, MIGRATION_5_6, MIGRATION_8_9)
.allowMainThreadQueries()
.build().use { db ->
check(db)

2
libs/database/src/dbTest/java/org/fdroid/database/PreferredRepoMigrationTest.kt
View File

@@ -80,7 +80,7 @@ internal class PreferredRepoMigrationTest {
FDroidDatabaseInt::class.java,
TEST_DB
)
.addMigrations(MIGRATION_2_3, MIGRATION_5_6)
.addMigrations(MIGRATION_2_3, MIGRATION_5_6, MIGRATION_8_9)
.allowMainThreadQueries()
.build().use { db ->
// repo without cert did not get migrated, the other one did

2
libs/database/src/dbTest/java/org/fdroid/database/RepoCertNonNullMigrationTest.kt
View File

@@ -78,7 +78,7 @@ internal class RepoCertNonNullMigrationTest {
// now get the Room DB, so we can use our DAOs for verifying the migration
Room.databaseBuilder(getApplicationContext(), FDroidDatabaseInt::class.java, TEST_DB)
.addMigrations(MIGRATION_2_3, MIGRATION_5_6)
.addMigrations(MIGRATION_2_3, MIGRATION_5_6, MIGRATION_8_9)
.allowMainThreadQueries()
.build().use { db ->
// repo without cert did not get migrated, the other one did

2
libs/database/src/dbTest/java/org/fdroid/index/v1/IndexV1UpdaterTest.kt
View File

@@ -194,7 +194,7 @@ internal class IndexV1UpdaterTest : DbTest() {
assets.open(jar).use { inputStream ->
jarFile.outputStream().use { inputStream.copyTo(it) }
}
every { tempFileProvider.createTempFile() } returns jarFile
every { tempFileProvider.createTempFile(null) } returns jarFile
every {
downloaderFactory.createWithTryFirstMirror(repo, uri, indexFile, jarFile)
} returns downloader

6
libs/database/src/dbTest/java/org/fdroid/index/v2/IndexV2UpdaterTest.kt
View File

@@ -283,7 +283,9 @@ internal class IndexV2UpdaterTest : DbTest() {
assets.open("diff-empty-min/23.json").use { inputStream ->
indexFile.outputStream().use { inputStream.copyTo(it) }
}
every { tempFileProvider.createTempFile() } returnsMany listOf(entryFile, indexFile)
every {
tempFileProvider.createTempFile(any())
} returnsMany listOf(entryFile, indexFile)
val result2 = indexUpdater.update(repo2)
assertIs<IndexUpdateResult.Error>(result2)
@@ -312,7 +314,7 @@ internal class IndexV2UpdaterTest : DbTest() {
indexFile.outputStream().use { inputStream.copyTo(it) }
}
every { tempFileProvider.createTempFile() } returnsMany listOf(entryFile, indexFile)
every { tempFileProvider.createTempFile(any()) } returnsMany listOf(entryFile, indexFile)
every {
downloaderFactory.createWithTryFirstMirror(repo, entryUri, entryFileV2, any())
} returns downloader

68
libs/database/src/main/java/org/fdroid/database/App.kt
View File

@@ -9,6 +9,7 @@ import androidx.room.Embedded
import androidx.room.Entity
import androidx.room.ForeignKey
import androidx.room.Fts4
import androidx.room.FtsOptions
import androidx.room.Ignore
import androidx.room.Relation
import org.fdroid.LocaleChooser.getBestLocale
@@ -96,9 +97,9 @@ internal fun MetadataV2.toAppMetadata(
packageName = packageName,
added = added,
lastUpdated = lastUpdated,
name = name,
summary = summary,
description = description,
name = name.zero(),
summary = summary.zero(),
description = description.zero(),
localizedName = name.getBestLocale(locales),
localizedSummary = summary.getBestLocale(locales),
webSite = webSite,
@@ -124,19 +125,53 @@ internal fun MetadataV2.toAppMetadata(
isCompatible = isCompatible,
)
/**
* Introduce zero whitespace for CJK (Chinese, Japanese, Korean) languages.
* This is needed, because the sqlite tokenizers available to us either handle those languages
* or do diacritics removals.
* Since we can't remove diacritics here ourselves,
* we help the tokenizer for CJK languages instead.
*/
internal fun LocalizedTextV2?.zero(): LocalizedTextV2? {
if (this == null) return null
return toMutableMap().mapValues { (locale, text) ->
if (locale.startsWith("zh") || locale.startsWith("ja") || locale.startsWith("ko")) {
StringBuilder().apply {
text.forEachIndexed { i, char ->
if (Character.isIdeographic(char.code) && i + 1 < text.length) {
append(char)
append("\u200B")
} else {
append(char)
}
}
}.toString()
} else {
text
}
}
}
@Entity(tableName = AppMetadataFts.TABLE)
@Fts4(
contentEntity = AppMetadata::class,
// make FTS for non-ASCII characters case insensitive, but do not remove diacritics
tokenizer = "unicode61 \"remove_diacritics=0\""
// make FTS for non-ASCII characters case insensitive, CJK languages are handled separately,
// because there's no tokenizer available that handles everything
tokenizer = FtsOptions.TOKENIZER_UNICODE61,
// can't use remove_diacritics=2 because it is SDK_INT >=30
// see: https://www.twisterrob.net/blog/2023/10/sqlite-unicode61-remove-diacritics-2.html
// separators=. is mainly for package name search
// tokenchars=- is so that searching for F-Droid works as expected
tokenizerArgs = ["remove_diacritics=1", "separators=.", "tokenchars=-"],
notIndexed = ["repoId"],
)
internal data class AppMetadataFts(
val repoId: Long,
val packageName: String,
@ColumnInfo(name = "localizedName")
val name: String? = null,
@ColumnInfo(name = "localizedSummary")
val summary: String? = null,
val description: String? = null,
val authorName: String? = null,
val packageName: String,
) {
internal companion object {
const val TABLE = "AppMetadataFts"
@@ -250,9 +285,16 @@ public data class AppOverviewItem internal constructor(
public val added: Long,
public val lastUpdated: Long,
@ColumnInfo(name = "localizedName")
@Deprecated("Use getName() method instead.")
public override val name: String? = null,
@ColumnInfo(name = "localizedSummary")
@Deprecated("Use getSummary() method instead.")
public override val summary: String? = null,
@ColumnInfo(name = "name")
internal val internalName: LocalizedTextV2? = null,
@ColumnInfo(name = "summary")
internal val internalSummary: LocalizedTextV2? = null,
public val categories: List<String>? = null,
internal val antiFeatures: Map<String, LocalizedTextV2>? = null,
@Relation(
parentColumn = "packageName",
@@ -264,6 +306,14 @@ public data class AppOverviewItem internal constructor(
*/
public val isCompatible: Boolean,
) : MinimalApp {
public fun getName(localeList: LocaleListCompat): String? {
return internalName.getBestLocale(localeList)
}
public fun getSummary(localeList: LocaleListCompat): String? {
return internalSummary.getBestLocale(localeList)
}
public override fun getIcon(localeList: LocaleListCompat): FileV2? {
return localizedIcon?.filter { icon ->
icon.repoId == repoId
@@ -291,6 +341,7 @@ public data class AppListItem internal constructor(
@ColumnInfo(name = "localizedSummary")
public override val summary: String? = null,
public val lastUpdated: Long,
public val categories: List<String>? = null,
internal val antiFeatures: String?,
@Relation(
parentColumn = "packageName",
@@ -344,6 +395,7 @@ public data class UpdatableApp internal constructor(
public override val repoId: Long,
public override val packageName: String,
public val installedVersionCode: Long,
public val installedVersionName: String,
public val update: AppVersion,
public val isFromPreferredRepo: Boolean,
/**

217
libs/database/src/main/java/org/fdroid/database/AppDao.kt
View File

@@ -19,6 +19,8 @@ import androidx.room.RoomRawQuery
import androidx.room.RoomWarnings.Companion.QUERY_MISMATCH
import androidx.room.Transaction
import androidx.room.Update
import androidx.sqlite.SQLiteStatement
import kotlinx.coroutines.flow.Flow
import kotlinx.serialization.SerializationException
import kotlinx.serialization.json.JsonNull
import kotlinx.serialization.json.JsonObject
@@ -34,6 +36,7 @@ import org.fdroid.index.v2.LocalizedFileListV2
import org.fdroid.index.v2.LocalizedFileV2
import org.fdroid.index.v2.MetadataV2
import org.fdroid.index.v2.ReflectionDiffer.applyDiff
import java.util.concurrent.TimeUnit
public interface AppDao {
/**
@@ -83,16 +86,62 @@ public interface AppDao {
* Apps without name, icon or summary are at the end (or excluded if limit is too small).
* Includes anti-features from the version with the highest version code.
*/
@Deprecated("Use getNewAppsFlow and getRecentlyUpdatedAppsFlow instead")
public fun getAppOverviewItems(limit: Int = 200): LiveData<List<AppOverviewItem>>
/**
* Returns a limited number of apps with limited data within the given [category].
*/
@Deprecated("Use getAppsByCategory instead")
public fun getAppOverviewItems(
category: String,
limit: Int = 50,
): LiveData<List<AppOverviewItem>>
/**
* Returns all apps from the database.
*/
public suspend fun getAllApps(): List<AppOverviewItem>
/**
* Returns all apps whose author is set exactly to [authorName].
*/
public suspend fun getAppsByAuthor(authorName: String): List<AppOverviewItem>
/**
* Returns all apps that are in the category with [categoryId].
*/
public suspend fun getAppsByCategory(categoryId: String): List<AppOverviewItem>
/**
* Returns apps that are new. This means that they were added and last updated at the same time.
* @param maxAgeInDays the number of days that is still considered "new".
* Apps older than this won't be returned.
*/
public suspend fun getNewApps(maxAgeInDays: Long = 14): List<AppOverviewItem>
/**
* Get apps that were recently updated.
* This excludes apps returned by [getNewApps].
* @param limit only return that many apps and not more.
*/
public suspend fun getRecentlyUpdatedApps(limit: Int = 200): List<AppOverviewItem>
/**
* Get all apps from the repository identified by [repoId].
*/
public suspend fun getAppsByRepository(repoId: Long): List<AppOverviewItem>
/**
* Same as [getNewApps], but returns an observable [Flow].
*/
public fun getNewAppsFlow(maxAgeInDays: Long = 14): Flow<List<AppOverviewItem>>
/**
* Same as [getRecentlyUpdatedApps], but returns an observable [Flow].
*/
public fun getRecentlyUpdatedAppsFlow(limit: Int = 200): Flow<List<AppOverviewItem>>
/**
* Returns a list of all [AppListItem] sorted by the given [sortOrder],
* or a subset of [AppListItem]s filtered by the given [searchQuery] if it is non-null.
@@ -141,6 +190,8 @@ public interface AppDao {
public fun getInstalledAppListItems(packageManager: PackageManager): LiveData<List<AppListItem>>
public suspend fun getAppSearchItems(searchQuery: String): List<AppSearchItem>
public fun getNumberOfAppsInCategory(category: String): Int
public fun getNumberOfAppsInRepository(repoId: Long): Int
@@ -237,13 +288,19 @@ internal interface AppDaoInt : AppDao {
}
// diff metadata
val diffedApp = applyDiff(metadata, jsonObject)
val updatedApp =
if (jsonObject.containsKey("name") || jsonObject.containsKey("summary")) {
diffedApp.copy(
localizedName = diffedApp.name.getBestLocale(locales),
localizedSummary = diffedApp.summary.getBestLocale(locales),
)
} else diffedApp
val containsName = jsonObject.containsKey("name")
val containsSummary = jsonObject.containsKey("summary")
val containsDescription = jsonObject.containsKey("description")
val updatedApp = if (containsName || containsSummary || containsDescription) {
diffedApp.copy(
name = if (containsName) diffedApp.name.zero() else diffedApp.name,
summary = if (containsSummary) diffedApp.summary.zero() else diffedApp.summary,
description = if (containsDescription) diffedApp.description.zero()
else diffedApp.description,
localizedName = diffedApp.name.getBestLocale(locales),
localizedSummary = diffedApp.summary.getBestLocale(locales),
)
} else diffedApp
updateAppMetadata(updatedApp)
// diff localizedFiles
val localizedFiles = getLocalizedFiles(repoId, packageName)
@@ -319,6 +376,7 @@ internal interface AppDaoInt : AppDao {
WHERE repoId = :repoId""")
override fun updateCompatibility(repoId: Long)
@Deprecated("Will be removed in future version")
@Query("""UPDATE ${AppMetadata.TABLE} SET localizedName = :name, localizedSummary = :summary
WHERE repoId = :repoId AND packageName = :packageName""")
fun updateAppMetadata(repoId: Long, packageName: String, name: String?, summary: String?)
@@ -366,7 +424,7 @@ internal interface AppDaoInt : AppDao {
@Transaction
@Query("""SELECT repoId, packageName, app.added, app.lastUpdated, localizedName,
localizedSummary, version.antiFeatures, app.isCompatible
localizedSummary, app.name, summary, categories, version.antiFeatures, app.isCompatible
FROM ${AppMetadata.TABLE} AS app
JOIN ${RepositoryPreferences.TABLE} AS pref USING (repoId)
JOIN PreferredRepo USING (packageName)
@@ -381,7 +439,7 @@ internal interface AppDaoInt : AppDao {
@Transaction
@Query("""SELECT repoId, packageName, app.added, app.lastUpdated, localizedName,
localizedSummary, version.antiFeatures, app.isCompatible
localizedSummary, app.name, summary, categories, version.antiFeatures, app.isCompatible
FROM ${AppMetadata.TABLE} AS app
JOIN ${RepositoryPreferences.TABLE} AS pref USING (repoId)
JOIN PreferredRepo USING (packageName)
@@ -401,10 +459,115 @@ internal interface AppDaoInt : AppDao {
@Transaction
@SuppressWarnings(QUERY_MISMATCH) // no anti-features needed here
@Query("""SELECT repoId, packageName, added, app.lastUpdated, localizedName,
localizedSummary, app.isCompatible
localizedSummary, name, summary, categories, app.isCompatible
FROM ${AppMetadata.TABLE} AS app WHERE repoId = :repoId AND packageName = :packageName""")
fun getAppOverviewItem(repoId: Long, packageName: String): AppOverviewItem?
@Transaction
override suspend fun getAllApps(): List<AppOverviewItem> {
val query = getAppsQuery("") {}
return getApps(query)
}
@Transaction
override suspend fun getAppsByAuthor(authorName: String): List<AppOverviewItem> {
val query = getAppsQuery("authorName = ?") { statement ->
statement.bindText(1, authorName)
}
return getApps(query)
}
@Transaction
override suspend fun getAppsByCategory(categoryId: String): List<AppOverviewItem> {
val query = getAppsQuery("categories LIKE '%,' || ? || ',%'") { statement ->
statement.bindText(1, categoryId)
}
return getApps(query)
}
@Transaction
override suspend fun getNewApps(maxAgeInDays: Long): List<AppOverviewItem> {
val query =
getAppsQuery("app.added = app.lastUpdated AND app.lastUpdated > ?") { statement ->
statement.bindLong(
1,
System.currentTimeMillis() - TimeUnit.DAYS.toMillis(maxAgeInDays)
)
}
return getApps(query)
}
@Transaction
override suspend fun getRecentlyUpdatedApps(limit: Int): List<AppOverviewItem> {
val query = getAppsQuery(
"app.added != app.lastUpdated ORDER BY app.lastUpdated DESC LIMIT ?"
) { statement ->
statement.bindInt(1, limit)
}
return getApps(query)
}
@Transaction
@Query("""SELECT repoId, packageName, app.added, app.lastUpdated, localizedName,
localizedSummary, name, summary, categories, version.antiFeatures, app.isCompatible
FROM ${AppMetadata.TABLE} AS app
LEFT JOIN ${HighestVersion.TABLE} AS version USING (repoId, packageName)
WHERE repoId = :repoId""")
override suspend fun getAppsByRepository(repoId: Long): List<AppOverviewItem>
override fun getNewAppsFlow(maxAgeInDays: Long): Flow<List<AppOverviewItem>> {
val query =
getAppsQuery(
"app.added = app.lastUpdated AND app.lastUpdated > ? ORDER BY app.added DESC"
) { statement ->
statement.bindLong(
1,
System.currentTimeMillis() - TimeUnit.DAYS.toMillis(maxAgeInDays)
)
}
return getAppsFlow(query)
}
override fun getRecentlyUpdatedAppsFlow(limit: Int): Flow<List<AppOverviewItem>> {
val query = getAppsQuery(
"app.added != app.lastUpdated ORDER BY app.lastUpdated DESC LIMIT ?"
) { statement ->
statement.bindInt(1, limit)
}
return getAppsFlow(query)
}
private fun getAppsQuery(
whereQuery: String,
onBindStatement: (SQLiteStatement) -> Unit,
): RoomRawQuery {
val queryBuilder = StringBuilder(
"""
SELECT repoId, packageName, app.added, app.lastUpdated, localizedName,
localizedSummary, name, summary, categories, version.antiFeatures, app.isCompatible
FROM ${AppMetadata.TABLE} AS app
JOIN PreferredRepo USING (packageName)
LEFT JOIN ${HighestVersion.TABLE} AS version USING (repoId, packageName)
WHERE repoId = preferredRepoId"""
)
if (whereQuery.isNotEmpty()) queryBuilder.append(" AND ").append(whereQuery)
return RoomRawQuery(
sql = queryBuilder.toString().trimIndent(),
onBindStatement = onBindStatement,
)
}
@RawQuery
suspend fun getApps(query: RoomRawQuery): List<AppOverviewItem>
@Transaction
@RawQuery(
observedEntities = [
AppMetadata::class, Version::class, Repository::class, RepositoryPreferences::class,
]
)
fun getAppsFlow(query: RoomRawQuery): Flow<List<AppOverviewItem>>
//
// AppListItems
//
@@ -430,7 +593,7 @@ internal interface AppDaoInt : AppDao {
val queryBuilder =
StringBuilder("""
SELECT repoId, packageName, localizedName, localizedSummary, app.lastUpdated,
version.antiFeatures, app.isCompatible, app.preferredSigner
categories, version.antiFeatures, app.isCompatible, app.preferredSigner
FROM ${AppMetadata.TABLE} AS app
JOIN ${RepositoryPreferences.TABLE} AS pref USING (repoId)
JOIN PreferredRepo USING (packageName)
@@ -461,7 +624,7 @@ internal interface AppDaoInt : AppDao {
val queryBuilder =
StringBuilder("""
SELECT repoId, packageName, localizedName, localizedSummary, app.lastUpdated,
version.antiFeatures, app.isCompatible, app.preferredSigner
categories, version.antiFeatures, app.isCompatible, app.preferredSigner
FROM ${AppMetadata.TABLE} AS app
LEFT JOIN ${HighestVersion.TABLE} AS version USING (repoId, packageName)
WHERE repoId = :repoId""")
@@ -486,7 +649,7 @@ internal interface AppDaoInt : AppDao {
val queryBuilder =
StringBuilder("""
SELECT repoId, packageName, localizedName, localizedSummary, app.lastUpdated,
version.antiFeatures, app.isCompatible, app.preferredSigner
categories, version.antiFeatures, app.isCompatible, app.preferredSigner
FROM ${AppMetadata.TABLE} AS app
JOIN ${RepositoryPreferences.TABLE} AS pref USING (repoId)
JOIN PreferredRepo USING (packageName)
@@ -537,7 +700,7 @@ internal interface AppDaoInt : AppDao {
@Transaction
@Query("""
SELECT repoId, packageName, app.localizedName, app.localizedSummary, app.lastUpdated,
version.antiFeatures, app.isCompatible, app.preferredSigner
categories, version.antiFeatures, app.isCompatible, app.preferredSigner
FROM ${AppMetadata.TABLE} AS app
JOIN PreferredRepo USING (packageName)
JOIN ${AppMetadataFts.TABLE} USING (repoId, packageName)
@@ -554,7 +717,7 @@ internal interface AppDaoInt : AppDao {
@Transaction
@Query("""
SELECT repoId, packageName, app.localizedName, app.localizedSummary, app.lastUpdated,
version.antiFeatures, app.isCompatible, app.preferredSigner
categories, version.antiFeatures, app.isCompatible, app.preferredSigner
FROM ${AppMetadata.TABLE} AS app
JOIN ${AppMetadataFts.TABLE} USING (repoId, packageName)
JOIN PreferredRepo USING (packageName)
@@ -574,7 +737,7 @@ internal interface AppDaoInt : AppDao {
@Transaction
@Query("""
SELECT repoId, packageName, app.localizedName, app.localizedSummary, app.lastUpdated,
version.antiFeatures, app.isCompatible, app.preferredSigner
categories, version.antiFeatures, app.isCompatible, app.preferredSigner
FROM ${AppMetadata.TABLE} AS app
LEFT JOIN ${HighestVersion.TABLE} AS version USING (repoId, packageName)
WHERE repoId = :repoId AND app.rowid IN (
@@ -586,7 +749,7 @@ internal interface AppDaoInt : AppDao {
@Transaction
@Query("""
SELECT repoId, packageName, localizedName, localizedSummary, app.lastUpdated,
version.antiFeatures, app.isCompatible, app.preferredSigner
categories, version.antiFeatures, app.isCompatible, app.preferredSigner
FROM ${AppMetadata.TABLE} AS app
LEFT JOIN ${HighestVersion.TABLE} AS version USING (repoId, packageName)
JOIN PreferredRepo USING (packageName)
@@ -599,7 +762,7 @@ internal interface AppDaoInt : AppDao {
@Transaction
@Query("""
SELECT repoId, packageName, localizedName, localizedSummary, app.lastUpdated,
version.antiFeatures, app.isCompatible, app.preferredSigner
categories, version.antiFeatures, app.isCompatible, app.preferredSigner
FROM ${AppMetadata.TABLE} AS app
JOIN ${RepositoryPreferences.TABLE} AS pref USING (repoId)
JOIN PreferredRepo USING (packageName)
@@ -618,7 +781,7 @@ internal interface AppDaoInt : AppDao {
@Transaction
@SuppressWarnings(QUERY_MISMATCH) // no anti-features needed here
@Query("""SELECT repoId, packageName, localizedName, localizedSummary, app.lastUpdated,
app.isCompatible, app.preferredSigner
categories, app.isCompatible, app.preferredSigner
FROM ${AppMetadata.TABLE} AS app
JOIN ${RepositoryPreferences.TABLE} AS pref USING (repoId)
JOIN PreferredRepo USING (packageName)
@@ -630,7 +793,7 @@ internal interface AppDaoInt : AppDao {
@Transaction
@Query(
"""SELECT repoId, packageName, app.localizedName, app.localizedSummary, app.lastUpdated,
version.antiFeatures, app.isCompatible, app.preferredSigner
categories, version.antiFeatures, app.isCompatible, app.preferredSigner
FROM ${AppMetadata.TABLE} AS app
LEFT JOIN ${HighestVersion.TABLE} AS version USING (repoId, packageName)
WHERE authorName = :authorName AND app.rowid IN (
@@ -700,6 +863,20 @@ internal interface AppDaoInt : AppDao {
}
}
@Transaction
@Query(
"""
SELECT repoId, packageName, app.lastUpdated, app.name, app.summary,
app.description, app.authorName, app.categories,
matchinfo(${AppMetadataFts.TABLE}, 'pcx')
FROM ${AppMetadata.TABLE} AS app
JOIN PreferredRepo USING (packageName)
JOIN ${AppMetadataFts.TABLE} USING (repoId, packageName)
WHERE ${AppMetadataFts.TABLE} MATCH :searchQuery AND
repoId = preferredRepoId"""
)
override suspend fun getAppSearchItems(searchQuery: String): List<AppSearchItem>
//
// Misc Queries
//

98
libs/database/src/main/java/org/fdroid/database/AppSearchItem.kt Normal file
View File

@@ -0,0 +1,98 @@
package org.fdroid.database
import androidx.core.os.LocaleListCompat
import androidx.room.ColumnInfo
import androidx.room.Ignore
import androidx.room.Relation
import org.fdroid.LocaleChooser.getBestLocale
import org.fdroid.index.v2.FileV2
import org.fdroid.index.v2.LocalizedTextV2
import java.nio.ByteBuffer
import java.nio.ByteOrder
import kotlin.math.min
@OptIn(ExperimentalUnsignedTypes::class)
@ConsistentCopyVisibility
public data class AppSearchItem internal constructor(
public val repoId: Long,
public val packageName: String,
public val lastUpdated: Long,
public val name: LocalizedTextV2? = null,
public val summary: LocalizedTextV2? = null,
public val description: LocalizedTextV2? = null,
public val authorName: String? = null,
public val categories: List<String>? = null,
@Relation(
parentColumn = "packageName",
entityColumn = "packageName",
)
internal val localizedIcon: List<LocalizedIcon>? = null,
@Suppress("ArrayInDataClass")
@ColumnInfo("matchinfo(${AppMetadataFts.TABLE}, 'pcx')")
internal val matchInfo: ByteArray,
) : Comparable<AppSearchItem> {
public fun getIcon(localeList: LocaleListCompat): FileV2? {
return localizedIcon?.filter { icon ->
icon.repoId == repoId
}?.toLocalizedFileV2().getBestLocale(localeList)
}
@Ignore
public val score: Double
init {
val info = matchInfo.toIntArray()
val numPhrases = info[0]
val numColumns = info[1]
val scoreMap = mutableMapOf<Int, Int>()
for (phrase in 0 until numPhrases) {
val offset = 2 + phrase * numColumns * 3
// start with 1 below, because we don't care about repoId column
for (column in 1 until numColumns) {
val numHitsInRow = info[offset + 3 * column]
// increase score if this column had a hit
if (numHitsInRow > 0) {
// each hit in a column only contributes to the score once
scoreMap.getOrPut(column) {
weights[column] ?: error("No weight for column $column")
}
}
}
}
val weeksOld = (System.currentTimeMillis() - lastUpdated) / (1000 * 60 * 60 * 24 * 7)
val punishment = min(100, weeksOld / 3)
score = scoreMap.values.sum().toDouble() - punishment
}
private fun ByteArray.toIntArray(skipSize: Int = 4): IntArray {
val intArray = IntArray(size / skipSize)
// go through each 4 bytes to turn them into integers
(indices step skipSize).forEachIndexed { intIndex, byteIndex ->
// we are cutting the first two bytes off, because we don't want to deal with UInt
// and expected integers are small enough
intArray[intIndex] = ByteBuffer.wrap(this, byteIndex, 4)
.order(ByteOrder.LITTLE_ENDIAN)
.int
}
return intArray
}
override fun compareTo(other: AppSearchItem): Int {
val scoreComp = score.compareTo(other.score)
return if (scoreComp == 0) {
lastUpdated.compareTo(other.lastUpdated)
} else {
scoreComp
}
}
}
@Suppress("ktlint:standard:no-multi-spaces")
private val weights = mapOf(
// 0 is repoId which we ignore
1 to 100, // "name"
2 to 50, // "summary"
3 to 25, // "description"
4 to 10, // "authorName"
5 to 5, // "packageName"
)

5
libs/database/src/main/java/org/fdroid/database/DbUpdateChecker.kt
View File

@@ -14,12 +14,12 @@ public class DbUpdateChecker @JvmOverloads constructor(
db: FDroidDatabase,
private val packageManager: PackageManager,
compatibilityChecker: CompatibilityChecker = CompatibilityCheckerImpl(packageManager),
private val updateChecker: UpdateChecker = UpdateChecker(compatibilityChecker),
) {
private val appDao = db.getAppDao() as AppDaoInt
private val versionDao = db.getVersionDao() as VersionDaoInt
private val appPrefsDao = db.getAppPrefsDao() as AppPrefsDaoInt
private val updateChecker = UpdateChecker(compatibilityChecker)
/**
* Returns a list of apps that can be updated.
@@ -69,6 +69,7 @@ public class DbUpdateChecker @JvmOverloads constructor(
val app = getUpdatableApp(
version = version,
installedVersionCode = getLongVersionCode(packageInfo),
installedVersionName = packageInfo.versionName ?: "???", // should never be null
isFromPreferredRepo = preferredRepoId == version.repoId,
)
if (app != null) updatableApps.add(app)
@@ -158,6 +159,7 @@ public class DbUpdateChecker @JvmOverloads constructor(
private fun getUpdatableApp(
version: Version,
installedVersionCode: Long,
installedVersionName: String,
isFromPreferredRepo: Boolean,
): UpdatableApp? {
val versionedStrings = versionDao.getVersionedStrings(
@@ -171,6 +173,7 @@ public class DbUpdateChecker @JvmOverloads constructor(
repoId = version.repoId,
packageName = version.packageName,
installedVersionCode = installedVersionCode,
installedVersionName = installedVersionName,
update = version.toAppVersion(versionedStrings),
isFromPreferredRepo = isFromPreferredRepo,
hasKnownVulnerability = version.hasKnownVulnerability,

6
libs/database/src/main/java/org/fdroid/database/FDroidDatabase.kt
View File

@@ -16,7 +16,7 @@ import java.util.concurrent.Callable
// When bumping this version, please make sure to add one (or more) migration(s) below!
// Consider also providing tests for that migration.
// Don't forget to commit the new schema to the git repo as well.
version = 8,
version = 9,
entities = [
// repo
CoreRepository::class,
@@ -50,6 +50,7 @@ import java.util.concurrent.Callable
// 5 to 6 is a manual migration
AutoMigration(6, 7),
AutoMigration(7, 8, CountryCodeMigration::class),
// 8 to 9 is a manual migration
// add future migrations above!
],
)
@@ -59,6 +60,8 @@ internal abstract class FDroidDatabaseInt : RoomDatabase(), FDroidDatabase, Clos
abstract override fun getAppDao(): AppDaoInt
abstract override fun getVersionDao(): VersionDaoInt
abstract override fun getAppPrefsDao(): AppPrefsDaoInt
@Deprecated("Will be removed in future version")
override fun afterLocalesChanged(locales: LocaleListCompat) {
val appDao = getAppDao()
runInTransaction {
@@ -118,6 +121,7 @@ public interface FDroidDatabase {
* Call this after the system [Locale]s have changed.
* If this isn't called, the cached localized app metadata (e.g. name, summary) will be wrong.
*/
@Deprecated("Will be removed in future version")
public fun afterLocalesChanged(
locales: LocaleListCompat = getLocales(Resources.getSystem().configuration),
)

2
libs/database/src/main/java/org/fdroid/database/FDroidDatabaseHolder.kt
View File

@@ -58,7 +58,7 @@ public object FDroidDatabaseHolder {
FDroidDatabaseInt::class.java,
name,
).apply {
addMigrations(MIGRATION_2_3, MIGRATION_5_6)
addMigrations(MIGRATION_2_3, MIGRATION_5_6, MIGRATION_8_9)
// We allow destructive migration (if no real migration was provided),
// so we have the option to nuke the DB in production (if that will ever be needed).
fallbackToDestructiveMigration(false)

63
libs/database/src/main/java/org/fdroid/database/Migrations.kt
View File

@@ -159,3 +159,66 @@ internal class CountryCodeMigration : AutoMigrationSpec {
}
}
}
/**
* The tokenizer of the FTS4 table for the app metadata was modified.
* This migration is needed to recreate the FTS table to respect the new tokenizer
* and also re-create the triggers to keep the FTS table in sync with the AppMetadata table.
*/
internal val MIGRATION_8_9 = object : Migration(8, 9) {
override fun migrate(db: SupportSQLiteDatabase) {
db.execSQL("DROP TABLE `AppMetadataFts`")
db.execSQL("DROP TRIGGER IF EXISTS `room_fts_content_sync_AppMetadataFts_BEFORE_UPDATE`")
db.execSQL("DROP TRIGGER IF EXISTS `room_fts_content_sync_AppMetadataFts_BEFORE_DELETE`")
db.execSQL("DROP TRIGGER IF EXISTS `room_fts_content_sync_AppMetadataFts_AFTER_UPDATE`")
db.execSQL("DROP TRIGGER IF EXISTS `room_fts_content_sync_AppMetadataFts_AFTER_INSERT`")
// table creation taken from auto-generated code:
// build/generated/ksp/debug/kotlin/org/fdroid/database/FDroidDatabaseInt_Impl.kt
db.execSQL(
"""
CREATE VIRTUAL TABLE IF NOT EXISTS `AppMetadataFts`
USING FTS4(
`repoId` INTEGER NOT NULL,
`name` TEXT, `summary` TEXT,
`description` TEXT,
`authorName` TEXT,
`packageName` TEXT NOT NULL,
tokenize=unicode61 `remove_diacritics=1` `separators=.` `tokenchars=-`,
content=`AppMetadata`,
notindexed=`repoId`
)""".trimIndent()
)
db.execSQL(
"""
CREATE TRIGGER IF NOT EXISTS room_fts_content_sync_AppMetadataFts_BEFORE_UPDATE
BEFORE UPDATE ON `AppMetadata` BEGIN DELETE FROM `AppMetadataFts`
WHERE `docid`=OLD.`rowid`; END
""".trimIndent()
)
db.execSQL(
"""
CREATE TRIGGER IF NOT EXISTS room_fts_content_sync_AppMetadataFts_BEFORE_DELETE
BEFORE DELETE ON `AppMetadata` BEGIN DELETE FROM `AppMetadataFts`
WHERE `docid`=OLD.`rowid`; END
""".trimIndent()
)
db.execSQL(
"""
CREATE TRIGGER IF NOT EXISTS room_fts_content_sync_AppMetadataFts_AFTER_UPDATE
AFTER UPDATE ON `AppMetadata` BEGIN INSERT INTO
`AppMetadataFts`(`docid`, `repoId`, `name`, `summary`, `description`, `authorName`, `packageName`)
VALUES (NEW.`rowid`, NEW.`repoId`, NEW.`name`, NEW.`summary`, NEW.`description`, NEW.`authorName`, NEW.`packageName`)
; END""".trimIndent()
)
db.execSQL(
"""
CREATE TRIGGER IF NOT EXISTS room_fts_content_sync_AppMetadataFts_AFTER_INSERT
AFTER INSERT ON `AppMetadata` BEGIN INSERT INTO
`AppMetadataFts`(`docid`, `repoId`, `name`, `summary`, `description`, `authorName`, `packageName`)
VALUES (NEW.`rowid`, NEW.`repoId`, NEW.`name`, NEW.`summary`, NEW.`description`, NEW.`authorName`, NEW.`packageName`)
; END""".trimIndent()
)
// rebuild the FTS table to populate it with the new tokenizer
db.execSQL("INSERT INTO AppMetadataFts(AppMetadataFts) VALUES('rebuild')")
}
}

13
libs/database/src/main/java/org/fdroid/database/Repository.kt
View File

@@ -1,8 +1,6 @@
package org.fdroid.database
import android.util.Log
import androidx.annotation.WorkerThread
import androidx.core.net.toUri
import androidx.core.os.LocaleListCompat
import androidx.room.Embedded
import androidx.room.Entity
@@ -229,19 +227,14 @@ public data class Repository internal constructor(
}
val shareUri: String
@WorkerThread
@WorkerThread // because fingerprint creation can take time
get() {
var uri = address.toUri()
try {
uri = uri.buildUpon().appendQueryParameter("fingerprint", fingerprint).build()
} catch (e: UnsupportedOperationException) {
Log.e(TAG, "Failed to append fingerprint to URI: $e")
}
return uri.toString()
return "https://fdroid.link/#$address?fingerprint=$fingerprint"
}
}
// Dummy repo to use in Compose Previews and in tests
@Deprecated("Will be removed in future version")
public val DUMMY_TEST_REPO: Repository = Repository(
repoId = 1L,
address = "https://example.com/fdroid/repo",

27
libs/database/src/main/java/org/fdroid/database/Version.kt
View File

@@ -5,6 +5,7 @@ import androidx.room.DatabaseView
import androidx.room.Embedded
import androidx.room.Entity
import androidx.room.ForeignKey
import androidx.room.Ignore
import androidx.room.Relation
import org.fdroid.LocaleChooser.getBestLocale
import org.fdroid.database.VersionedStringType.PERMISSION
@@ -40,7 +41,7 @@ internal data class Version(
val repoId: Long,
val packageName: String,
val versionId: String,
val added: Long,
override val added: Long,
@Embedded(prefix = "file_") val file: FileV1,
@Embedded(prefix = "src_") val src: FileV2? = null,
@Embedded(prefix = "manifest_") val manifest: AppManifest,
@@ -54,6 +55,8 @@ internal data class Version(
}
override val versionCode: Long get() = manifest.versionCode
override val versionName: String get() = manifest.versionName
override val size: Long? get() = file.size
override val signer: SignerV2? get() = manifest.signer
override val packageManifest: PackageManifest get() = manifest
override val hasKnownVulnerability: Boolean
@@ -95,10 +98,11 @@ public data class AppVersion internal constructor(
entityColumn = "versionId",
)
internal val versionedStrings: List<VersionedString>?,
) {
) : PackageVersion {
public val repoId: Long get() = version.repoId
public val packageName: String get() = version.packageName
public val added: Long get() = version.added
public override val added: Long get() = version.added
override val size: Long? get() = version.file.size
public val isCompatible: Boolean get() = version.isCompatible
public val manifest: AppManifest get() = version.manifest
public val file: FileV1 get() = version.file
@@ -109,7 +113,22 @@ public data class AppVersion internal constructor(
get() = versionedStrings?.getPermissionsSdk23(version) ?: emptyList()
public val featureNames: List<String> get() = version.manifest.features ?: emptyList()
public val nativeCode: List<String> get() = version.manifest.nativecode ?: emptyList()
public val releaseChannels: List<String> get() = version.releaseChannels ?: emptyList()
public override val releaseChannels: List<String> get() = version.releaseChannels ?: emptyList()
@get:Ignore
public override val versionCode: Long get() = version.manifest.versionCode
@get:Ignore
override val versionName: String get() = version.manifest.versionName
@get:Ignore
public override val signer: SignerV2? get() = version.manifest.signer
@Ignore
public override val packageManifest: PackageManifest = version.manifest
@Ignore
public override val hasKnownVulnerability: Boolean = version.hasKnownVulnerability
public val antiFeatureKeys: List<String>
get() = version.antiFeatures?.map { it.key } ?: emptyList()

4
libs/database/src/main/java/org/fdroid/index/IndexUpdater.kt
View File

@@ -16,7 +16,7 @@ public sealed class IndexUpdateResult {
public object Unchanged : IndexUpdateResult()
public object Processed : IndexUpdateResult()
public object NotFound : IndexUpdateResult()
public class Error(public val e: Exception) : IndexUpdateResult()
public data class Error(public val e: Exception) : IndexUpdateResult()
}
public interface IndexUpdateListener {
@@ -41,7 +41,7 @@ internal val defaultRepoUriBuilder = RepoUriBuilder { repo, pathElements ->
public fun interface TempFileProvider {
@Throws(IOException::class)
public fun createTempFile(): File
public fun createTempFile(sha256: String?): File
}
/**

36
libs/database/src/main/java/org/fdroid/index/RepoManager.kt
View File

@@ -13,8 +13,11 @@ import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.flow.update
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import org.fdroid.CompatibilityChecker
import org.fdroid.CompatibilityCheckerImpl
import org.fdroid.database.AppPrefs
import org.fdroid.database.AppPrefsDaoInt
import org.fdroid.database.FDroidDatabase
@@ -28,7 +31,6 @@ import org.fdroid.repo.RepoAdder
import org.fdroid.repo.RepoUriGetter
import java.io.File
import java.net.Proxy
import java.util.concurrent.CancellationException
import java.util.concurrent.CountDownLatch
import kotlin.coroutines.CoroutineContext
@@ -39,12 +41,20 @@ public class RepoManager @JvmOverloads constructor(
downloaderFactory: DownloaderFactory,
httpManager: HttpManager,
repoUriBuilder: RepoUriBuilder = defaultRepoUriBuilder,
compatibilityChecker: CompatibilityChecker = CompatibilityCheckerImpl(
packageManager = context.packageManager,
forceTouchApps = false,
),
private val coroutineContext: CoroutineContext = Dispatchers.IO,
) {
private val repositoryDao = db.getRepositoryDao() as RepositoryDaoInt
private val appPrefsDao = db.getAppPrefsDao() as AppPrefsDaoInt
private val tempFileProvider = TempFileProvider {
File.createTempFile("dl-", "", context.cacheDir)
private val tempFileProvider = TempFileProvider { sha256 ->
if (sha256 == null) {
File.createTempFile("dl-", "", context.cacheDir)
} else {
File(context.cacheDir, sha256).apply { createNewFile() }
}
}
private val repoAdder = RepoAdder(
context = context,
@@ -52,6 +62,7 @@ public class RepoManager @JvmOverloads constructor(
tempFileProvider = tempFileProvider,
downloaderFactory = downloaderFactory,
httpManager = httpManager,
compatibilityChecker = compatibilityChecker,
repoUriBuilder = repoUriBuilder,
coroutineContext = coroutineContext,
)
@@ -134,6 +145,14 @@ public class RepoManager @JvmOverloads constructor(
*/
@WorkerThread
public fun deleteRepository(repoId: Long) {
// while this will get updated automatically, getting the update may be slow,
// so to speed up the UI, we emit the state change right away (deletion is unlikely to fail)
_repositoriesState.update {
_repositoriesState.value.filter { repo ->
// keep only repos that are not the deleted one
repo.repoId != repoId
}
}
db.runInTransaction {
// find and remove archive repo if existing
val repository = repositoryDao.getRepository(repoId) ?: return@runInTransaction
@@ -141,11 +160,6 @@ public class RepoManager @JvmOverloads constructor(
if (archiveRepoId != null) repositoryDao.deleteRepository(archiveRepoId)
// delete main repo
repositoryDao.deleteRepository(repoId)
// while this gets updated automatically, getting the update may be slow,
// so to speed up the UI, we emit the state change right away
_repositoriesState.value = _repositoriesState.value.filter { repo ->
repo.repoId == repoId
}
}
}
@@ -240,11 +254,7 @@ public class RepoManager @JvmOverloads constructor(
var archiveRepoId = repositoryDao.getArchiveRepoId(cert)
if (enabled) {
if (archiveRepoId == null) {
try {
archiveRepoId = repoAdder.addArchiveRepo(repository, proxy)
} catch (e: CancellationException) {
if (e.message != "expected") throw e
}
archiveRepoId = repoAdder.addArchiveRepo(repository, proxy)
} else {
repositoryDao.setRepositoryEnabled(archiveRepoId, true)
}

10
libs/database/src/main/java/org/fdroid/index/RepoUpdater.kt
View File

@@ -20,11 +20,15 @@ public class RepoUpdater(
downloaderFactory: DownloaderFactory,
repoUriBuilder: RepoUriBuilder = defaultRepoUriBuilder,
compatibilityChecker: CompatibilityChecker,
listener: IndexUpdateListener,
listener: IndexUpdateListener? = null,
) {
private val log = KotlinLogging.logger {}
private val tempFileProvider = TempFileProvider {
File.createTempFile("dl-", "", tempDir)
private val tempFileProvider = TempFileProvider { sha256 ->
if (sha256 == null) {
File.createTempFile("dl-", "", tempDir)
} else {
File(tempDir, sha256).apply { createNewFile() }
}
}
/**

2
libs/database/src/main/java/org/fdroid/index/v1/IndexV1Updater.kt
View File

@@ -42,7 +42,7 @@ public class IndexV1Updater(
if (repo.formatVersion != null && repo.formatVersion != ONE) {
log.error { "Format downgrade for ${repo.address}" }
}
val file = tempFileProvider.createTempFile()
val file = tempFileProvider.createTempFile(null)
val downloader = downloaderFactory.createWithTryFirstMirror(
repo = repo,
uri = repoUriBuilder.getUri(repo, SIGNED_FILE_NAME),

4
libs/database/src/main/java/org/fdroid/index/v2/IndexV2Updater.kt
View File

@@ -54,7 +54,7 @@ public class IndexV2Updater(
}
private fun getCertAndEntry(repo: Repository, certificate: String): Pair<String, Entry> {
val file = tempFileProvider.createTempFile()
val file = tempFileProvider.createTempFile(null)
val downloader = downloaderFactory.createWithTryFirstMirror(
repo = repo,
uri = repoUriBuilder.getUri(repo, SIGNED_FILE_NAME),
@@ -80,7 +80,7 @@ public class IndexV2Updater(
repoVersion: Long,
streamProcessor: IndexV2StreamProcessor,
): IndexUpdateResult {
val file = tempFileProvider.createTempFile()
val file = tempFileProvider.createTempFile(entryFile.sha256)
val downloader = downloaderFactory.createWithTryFirstMirror(
repo = repo,
uri = repoUriBuilder.getUri(repo, entryFile.name.trimStart('/')),

98
libs/database/src/main/java/org/fdroid/repo/RepoAdder.kt
View File

@@ -10,16 +10,21 @@ import androidx.annotation.AnyThread
import androidx.annotation.VisibleForTesting
import androidx.annotation.WorkerThread
import androidx.core.content.ContextCompat.getSystemService
import androidx.core.net.toUri
import kotlinx.coroutines.DelicateCoroutinesApi
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.Job
import kotlinx.coroutines.cancel
import kotlinx.coroutines.currentCoroutineContext
import kotlinx.coroutines.ensureActive
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.getAndUpdate
import kotlinx.coroutines.isActive
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import kotlinx.serialization.SerializationException
import mu.KotlinLogging
import org.fdroid.CompatibilityChecker
import org.fdroid.database.AppOverviewItem
import org.fdroid.database.FDroidDatabase
import org.fdroid.database.MinimalApp
@@ -31,6 +36,8 @@ import org.fdroid.download.HttpManager
import org.fdroid.download.HttpManager.Companion.isInvalidHttpUrl
import org.fdroid.download.NotFoundException
import org.fdroid.index.IndexFormatVersion
import org.fdroid.index.IndexUpdateResult
import org.fdroid.index.RepoUpdater
import org.fdroid.index.RepoUriBuilder
import org.fdroid.index.SigningException
import org.fdroid.index.TempFileProvider
@@ -39,6 +46,7 @@ import org.fdroid.repo.AddRepoError.ErrorType.INVALID_INDEX
import org.fdroid.repo.AddRepoError.ErrorType.IO_ERROR
import org.fdroid.repo.AddRepoError.ErrorType.IS_ARCHIVE_REPO
import org.fdroid.repo.AddRepoError.ErrorType.UNKNOWN_SOURCES_DISALLOWED
import java.io.File
import java.io.IOException
import java.net.Proxy
import kotlin.coroutines.CoroutineContext
@@ -54,11 +62,12 @@ public class Fetching(
public val receivedRepo: Repository?,
public val apps: List<MinimalApp>,
public val fetchResult: FetchResult?,
public val indexFile: File? = null,
) : AddRepoState() {
/**
* true if fetching is complete.
*/
public val done: Boolean = false,
) : AddRepoState() {
public val done: Boolean = indexFile != null
override fun toString(): String {
return "Fetching(fetchUrl=$fetchUrl, repo=${receivedRepo?.address}, apps=${apps.size}, " +
"fetchResult=$fetchResult, done=$done)"
@@ -69,6 +78,7 @@ public object Adding : AddRepoState()
public class Added(
public val repo: Repository,
public val updateResult: IndexUpdateResult?,
) : AddRepoState()
public data class AddRepoError(
@@ -100,6 +110,7 @@ internal class RepoAdder(
private val tempFileProvider: TempFileProvider,
private val downloaderFactory: DownloaderFactory,
private val httpManager: HttpManager,
private val compatibilityChecker: CompatibilityChecker,
private val repoUriGetter: RepoUriGetter = RepoUriGetter,
private val repoUriBuilder: RepoUriBuilder = defaultRepoUriBuilder,
private val coroutineContext: CoroutineContext = Dispatchers.IO,
@@ -158,46 +169,60 @@ internal class RepoAdder(
)
}
fetchResult = getFetchResult(fetchUrl, repo)
coroutineContext.ensureActive() // ensure active before updating state
addRepoState.value = Fetching(fetchUrl, receivedRepo, apps.toList(), fetchResult)
}
override fun onAppReceived(app: AppOverviewItem) {
apps.add(app)
coroutineContext.ensureActive() // ensure active before updating state
addRepoState.value = Fetching(fetchUrl, receivedRepo, apps.toList(), fetchResult)
}
}
// set a state early, so the ui can show progress animation
coroutineContext.ensureActive() // ensure active before updating state
addRepoState.value = Fetching(fetchUrl, receivedRepo, apps, fetchResult)
// try fetching repo with v2 format first and fallback to v1
try {
val indexFile = try {
fetchRepo(nUri.uri, nUri.fingerprint, proxy, nUri.username, nUri.password, receiver)
} catch (e: SigningException) {
log.error(e) { "Error verifying repo with given fingerprint." }
addRepoState.value = AddRepoError(INVALID_FINGERPRINT, e)
onError(AddRepoError(INVALID_FINGERPRINT, e))
return
} catch (e: IOException) {
log.error(e) { "Error fetching repo." }
addRepoState.value = AddRepoError(IO_ERROR, e)
onError(AddRepoError(IO_ERROR, e))
return
} catch (e: SerializationException) {
log.error(e) { "Error fetching repo." }
addRepoState.value = AddRepoError(INVALID_INDEX, e)
onError(AddRepoError(INVALID_INDEX, e))
return
} catch (e: NotFoundException) { // v1 repos can also have 404
log.error(e) { "Error fetching repo." }
addRepoState.value = AddRepoError(INVALID_INDEX, e)
onError(AddRepoError(INVALID_INDEX, e))
return
}
// set final result
val finalRepo = receivedRepo
coroutineContext.ensureActive() // ensure active before updating state
if (finalRepo == null) {
addRepoState.value = AddRepoError(INVALID_INDEX)
onError(AddRepoError(INVALID_INDEX))
} else {
addRepoState.value = Fetching(fetchUrl, finalRepo, apps, fetchResult, done = true)
addRepoState.value = Fetching(fetchUrl, finalRepo, apps, fetchResult, indexFile)
}
}
private suspend fun onError(state: AddRepoError) {
if (currentCoroutineContext().isActive) {
addRepoState.value = state
}
}
/**
* Fetches the repo from the given [uri] and posts updates to [receiver].
* @return the temporary file the repo was written to.
*/
private suspend fun fetchRepo(
uri: Uri,
fingerprint: String?,
@@ -205,10 +230,9 @@ internal class RepoAdder(
username: String?,
password: String?,
receiver: RepoPreviewReceiver,
) {
try {
val repo =
getTempRepo(uri, IndexFormatVersion.TWO, username, password)
): File {
return try {
val repo = getTempRepo(uri, IndexFormatVersion.TWO, username, password)
val repoFetcher = RepoV2Fetcher(
tempFileProvider, downloaderFactory, httpManager, repoUriBuilder, proxy
)
@@ -216,8 +240,7 @@ internal class RepoAdder(
} catch (e: NotFoundException) {
log.warn(e) { "Did not find v2 repo, trying v1 now." }
// try to fetch v1 repo
val repo =
getTempRepo(uri, IndexFormatVersion.ONE, username, password)
val repo = getTempRepo(uri, IndexFormatVersion.ONE, username, password)
val repoFetcher = RepoV1Fetcher(tempFileProvider, downloaderFactory, repoUriBuilder)
repoFetcher.fetchRepo(uri, repo, receiver, fingerprint)
}
@@ -226,7 +249,7 @@ internal class RepoAdder(
private fun getFetchResult(fetchUrlIn: String, fetchedRepo: Repository): FetchResult {
// Note the delicate difference between fetchedRepo (from the network) and
// existingRepo (from the database) in this function!
val cert = fetchedRepo.certificate ?: error("Certificate was null")
val cert = fetchedRepo.certificate
val existingRepo = repositoryDao.getRepository(cert)
val fetchUrl = fetchUrlIn.trimEnd('/')
@@ -262,22 +285,24 @@ internal class RepoAdder(
@WorkerThread
internal fun addFetchedRepository(): Repository? {
// prevent double calls (e.g. caused by double tapping a UI button)
if (addRepoState.compareAndSet(Adding, Adding)) return null
// cancel fetch preview job, so it stops emitting new states
// first cancel fetch preview job, so it stops emitting new states,
// screwing up the atomicity of getAndUpdate() below.
fetchJob?.cancel()
// get current state before changing it
val state = (addRepoState.value as? Fetching)
?: throw IllegalStateException("Unexpected state: ${addRepoState.value}")
addRepoState.value = Adding
// prevent double calls (e.g. caused by double tapping a UI button)
val state = addRepoState.getAndUpdate {
log.info { "Previous state was $it" }
Adding
} as? Fetching ?: error("Unexpected previous state")
log.info { "Moved to state ${addRepoState.value}, cancelling preview job..." }
val repo = state.receivedRepo
?: throw IllegalStateException("No repo: ${addRepoState.value}")
val fetchResult = state.fetchResult
?: throw IllegalStateException("No fetchResult: ${addRepoState.value}")
var indexUpdateResult: IndexUpdateResult? = null
val modifiedRepo: Repository = when (fetchResult) {
is FetchResult.IsExistingRepository -> error("Repo exists: $fetchResult")
is FetchResult.IsExistingMirror -> error("Mirror exists: $fetchResult")
@@ -289,7 +314,7 @@ internal class RepoAdder(
icon = repo.repository.icon ?: emptyMap(),
address = repo.address,
formatVersion = repo.formatVersion,
certificate = repo.certificate ?: error("Repo had no certificate"),
certificate = repo.certificate,
username = repo.username,
password = repo.password,
)
@@ -305,6 +330,17 @@ internal class RepoAdder(
repositoryDao.updateUserMirrors(repoId, userMirrors)
}
repositoryDao.getRepository(repoId) ?: error("New repository not found in DB")
}.also { repo ->
// Update the repo before returning, so we already have its content.
// This should pick up [indexFile] automatically without re-downloading,
// because we use the sha256 hash as the file name.
indexUpdateResult = RepoUpdater(
tempDir = context.cacheDir,
db = db,
downloaderFactory = downloaderFactory,
compatibilityChecker = compatibilityChecker,
).update(repo)
log.info { "Updated repo: $indexUpdateResult" }
}
}
@@ -321,11 +357,14 @@ internal class RepoAdder(
}
}
}
addRepoState.value = Added(modifiedRepo)
log.info { "Added repository" }
state.indexFile?.delete()
addRepoState.value = Added(modifiedRepo, indexUpdateResult)
return modifiedRepo
}
internal fun abortAddingRepo() {
(addRepoState.value as? Fetching)?.indexFile?.delete()
addRepoState.value = None
fetchJob?.cancel()
}
@@ -348,7 +387,7 @@ internal class RepoAdder(
icon = archiveRepo.repository.icon ?: emptyMap(),
address = archiveRepo.address,
formatVersion = archiveRepo.formatVersion,
certificate = archiveRepo.certificate ?: error("Repo had no certificate"),
certificate = archiveRepo.certificate,
username = archiveRepo.username,
password = archiveRepo.password,
)
@@ -357,14 +396,13 @@ internal class RepoAdder(
repositoryDao.setWeight(repoId, repo.weight - 1)
archiveRepoId = repoId
}
cancel("expected") // no need to continue downloading the entire repo
}
override fun onAppReceived(app: AppOverviewItem) {
// no-op
}
}
val uri = Uri.parse(address)
val uri = address.toUri()
fetchRepo(uri, repo.fingerprint, proxy, repo.username, repo.password, receiver)
return@withContext archiveRepoId
}
@@ -398,7 +436,7 @@ internal class RepoAdder(
}
internal val defaultRepoUriBuilder = RepoUriBuilder { repo, pathElements ->
val builder = Uri.parse(repo.address).buildUpon()
val builder = repo.address.toUri().buildUpon()
pathElements.forEach { builder.appendEncodedPath(it) }
builder.build()
}

8
libs/database/src/main/java/org/fdroid/repo/RepoFetcher.kt
View File

@@ -6,9 +6,15 @@ import org.fdroid.database.AppOverviewItem
import org.fdroid.database.Repository
import org.fdroid.download.NotFoundException
import org.fdroid.index.SigningException
import java.io.File
import java.io.IOException
internal fun interface RepoFetcher {
/**
* Fetches the repo from the given [uri] and posts updates to [receiver].
* @return the temporary file the repo was written to.
* Note that the OS may delete this at any time.
*/
@Throws(
IOException::class,
SigningException::class,
@@ -20,7 +26,7 @@ internal fun interface RepoFetcher {
repo: Repository,
receiver: RepoPreviewReceiver,
fingerprint: String?,
)
): File
}
internal interface RepoPreviewReceiver {

18
libs/database/src/main/java/org/fdroid/repo/RepoV1Fetcher.kt
View File

@@ -15,6 +15,7 @@ import org.fdroid.index.parseV1
import org.fdroid.index.v1.IndexV1Verifier
import org.fdroid.index.v1.SIGNED_FILE_NAME
import org.fdroid.index.v2.FileV2
import java.io.File
internal class RepoV1Fetcher(
private val tempFileProvider: TempFileProvider,
@@ -30,23 +31,19 @@ internal class RepoV1Fetcher(
repo: Repository,
receiver: RepoPreviewReceiver,
fingerprint: String?,
) {
): File {
// download and verify index-v1.jar
val indexFile = tempFileProvider.createTempFile()
val indexFile = tempFileProvider.createTempFile(null)
val entryDownloader = downloaderFactory.create(
repo = repo,
uri = repoUriBuilder.getUri(repo, SIGNED_FILE_NAME),
indexFile = FileV2.fromPath("/$SIGNED_FILE_NAME"),
destFile = indexFile,
)
val (cert, indexV1) = try {
entryDownloader.download()
val verifier = IndexV1Verifier(indexFile, null, fingerprint)
verifier.getStreamAndVerify { inputStream ->
IndexParser.parseV1(inputStream)
}
} finally {
indexFile.delete()
entryDownloader.download()
val verifier = IndexV1Verifier(indexFile, null, fingerprint)
val (cert, indexV1) = verifier.getStreamAndVerify { inputStream ->
IndexParser.parseV1(inputStream)
}
val version = indexV1.repo.version
val indexV2 = IndexConverter().toIndexV2(indexV1)
@@ -63,5 +60,6 @@ internal class RepoV1Fetcher(
val app = RepoV2StreamReceiver.getAppOverViewItem(packageName, packageV2, locales)
receiver.onAppReceived(app)
}
return indexFile
}
}

22
libs/database/src/main/java/org/fdroid/repo/RepoV2Fetcher.kt
View File

@@ -17,6 +17,7 @@ import org.fdroid.index.v2.EntryVerifier
import org.fdroid.index.v2.FileV2
import org.fdroid.index.v2.IndexV2FullStreamProcessor
import org.fdroid.index.v2.SIGNED_FILE_NAME
import java.io.File
import java.net.Proxy
import java.security.DigestInputStream
import java.security.MessageDigest
@@ -36,9 +37,9 @@ internal class RepoV2Fetcher(
repo: Repository,
receiver: RepoPreviewReceiver,
fingerprint: String?,
) {
): File {
// download and verify entry
val entryFile = tempFileProvider.createTempFile()
val entryFile = tempFileProvider.createTempFile(null)
val entryDownloader = downloaderFactory.create(
repo = repo,
uri = repoUriBuilder.getUri(repo, SIGNED_FILE_NAME),
@@ -59,7 +60,8 @@ internal class RepoV2Fetcher(
val streamReceiver = RepoV2StreamReceiver(receiver, cert, repo.username, repo.password)
val streamProcessor = IndexV2FullStreamProcessor(streamReceiver)
val digestInputStream = if (uri.scheme?.startsWith("http") == true) {
val indexFile = tempFileProvider.createTempFile(entry.index.sha256)
val inputStream = if (uri.scheme?.startsWith("http") == true) {
// stream index for http(s) downloads
val indexRequest = DownloadRequest(
indexFile = entry.index,
@@ -68,10 +70,11 @@ internal class RepoV2Fetcher(
username = repo.username,
password = repo.password,
)
httpManager.getDigestInputStream(indexRequest)
val digestInputStream = httpManager.getDigestInputStream(indexRequest)
// wrap stream to exfiltrate index file for later usage
SavingInputStream(digestInputStream, indexFile)
} else {
// no streaming supported, download file first
val indexFile = tempFileProvider.createTempFile()
val indexDownloader = downloaderFactory.create(
repo = repo,
uri = repoUriBuilder.getUri(repo, entry.index.name.trimStart('/')),
@@ -82,12 +85,17 @@ internal class RepoV2Fetcher(
val digest = MessageDigest.getInstance("SHA-256")
DigestInputStream(indexFile.inputStream(), digest)
}
digestInputStream.use { inputStream ->
inputStream.use { inputStream ->
streamProcessor.process(entry.version, inputStream) { }
}
val hexDigest = digestInputStream.getDigestHex()
val hexDigest = when (inputStream) {
is DigestInputStream -> inputStream.getDigestHex()
is SavingInputStream -> inputStream.inputStream.getDigestHex()
else -> error("Unknown InputStream ${inputStream::class.java}")
}
if (!hexDigest.equals(entry.index.sha256, ignoreCase = true)) {
throw SigningException("Invalid ${entry.index.name} hash: $hexDigest")
}
return indexFile
}
}

2
libs/database/src/main/java/org/fdroid/repo/RepoV2StreamReceiver.kt
View File

@@ -61,6 +61,8 @@ internal open class RepoV2StreamReceiver(
lastUpdated = p.metadata.lastUpdated,
name = p.metadata.name.getBestLocale(locales),
summary = p.metadata.summary.getBestLocale(locales),
internalName = p.metadata.name,
internalSummary = p.metadata.summary,
antiFeatures = p.versions.values.lastOrNull()?.antiFeatures,
localizedIcon = p.metadata.icon?.map { (locale, file) ->
LocalizedIcon(

45
libs/database/src/main/java/org/fdroid/repo/SavingInputStream.kt Normal file
View File

@@ -0,0 +1,45 @@
package org.fdroid.repo
import java.io.File
import java.io.InputStream
import java.security.DigestInputStream
internal class SavingInputStream(
val inputStream: DigestInputStream,
outputFile: File,
) : InputStream() {
private val outputStream = outputFile.outputStream()
override fun read(): Int {
val byte = inputStream.read()
if (byte != -1) {
outputStream.write(byte)
}
return byte
}
override fun read(b: ByteArray): Int {
val bytesRead = inputStream.read(b)
if (bytesRead != -1) {
outputStream.write(b, 0, bytesRead)
}
return bytesRead
}
override fun read(b: ByteArray, off: Int, len: Int): Int {
val bytesRead = inputStream.read(b, off, len)
if (bytesRead != -1) {
outputStream.write(b, off, bytesRead)
}
return bytesRead
}
override fun close() {
try {
inputStream.close()
} finally {
outputStream.close()
}
}
}

34
libs/database/src/test/java/org/fdroid/repo/RepoAdderIntegrationTest.kt
View File

@@ -4,16 +4,21 @@ import androidx.core.os.LocaleListCompat
import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.platform.app.InstrumentationRegistry
import app.cash.turbine.test
import io.mockk.MockKException
import io.mockk.every
import io.mockk.mockk
import io.mockk.slot
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.test.runTest
import org.fdroid.database.FDroidDatabase
import org.fdroid.CompatibilityChecker
import org.fdroid.database.FDroidDatabaseInt
import org.fdroid.database.NewRepository
import org.fdroid.database.Repository
import org.fdroid.database.RepositoryDaoInt
import org.fdroid.download.HttpManager
import org.fdroid.download.TestDownloadFactory
import org.fdroid.index.IndexFormatVersion
import org.fdroid.index.IndexUpdateResult
import org.fdroid.index.TempFileProvider
import org.fdroid.repo.AddRepoError.ErrorType.INVALID_FINGERPRINT
import org.junit.Assume.assumeTrue
@@ -22,7 +27,9 @@ import org.junit.Rule
import org.junit.Test
import org.junit.rules.TemporaryFolder
import org.junit.runner.RunWith
import java.util.concurrent.Callable
import kotlin.test.assertEquals
import kotlin.test.assertIs
import kotlin.test.assertNotNull
import kotlin.test.assertNull
import kotlin.test.assertTrue
@@ -34,17 +41,25 @@ internal class RepoAdderIntegrationTest {
var folder: TemporaryFolder = TemporaryFolder()
private val context = InstrumentationRegistry.getInstrumentation().targetContext
private val db = mockk<FDroidDatabase>()
private val db = mockk<FDroidDatabaseInt>()
private val repoDao = mockk<RepositoryDaoInt>()
private val tempFileProvider = TempFileProvider { folder.newFile() }
private val httpManager = HttpManager("test")
private val downloaderFactory = TestDownloadFactory(httpManager)
private val compatibilityChecker = mockk<CompatibilityChecker>()
private val repoAdder: RepoAdder
init {
every { db.getRepositoryDao() } returns repoDao
repoAdder = RepoAdder(context, db, tempFileProvider, downloaderFactory, httpManager)
repoAdder = RepoAdder(
context = context,
db = db,
tempFileProvider = tempFileProvider,
downloaderFactory = downloaderFactory,
httpManager = httpManager,
compatibilityChecker = compatibilityChecker,
)
}
@Before
@@ -80,7 +95,9 @@ internal class RepoAdderIntegrationTest {
assertEquals(1, (awaitItem() as Fetching).apps.size)
assertEquals(2, (awaitItem() as Fetching).apps.size)
assertEquals(3, (awaitItem() as Fetching).apps.size)
assertTrue(awaitItem() is Fetching)
assertEquals(4, (awaitItem() as Fetching).apps.size)
assertEquals(5, (awaitItem() as Fetching).apps.size)
assertTrue((awaitItem() as Fetching).done)
}
val state = repoAdder.addRepoState.value
@@ -90,7 +107,12 @@ internal class RepoAdderIntegrationTest {
println(" ${app.packageName} ${app.summary}")
}
val runSlot = slot<Callable<Any>>()
every { db.runInTransaction(capture(runSlot)) } answers {
runSlot.captured.call()
}
val newRepo: Repository = mockk()
every { newRepo.formatVersion } returns IndexFormatVersion.TWO
every { repoDao.insert(any<NewRepository>()) } returns 42L
every { repoDao.getRepository(42L) } returns newRepo
@@ -99,6 +121,10 @@ internal class RepoAdderIntegrationTest {
val addedState = awaitItem()
assertTrue(addedState is Added, addedState.toString())
assertEquals(newRepo, addedState.repo)
// we are not mocking all the actual repo adding,
// so just assert that this fails due to mocking
assertIs<IndexUpdateResult.Error>(addedState.updateResult)
assertIs<MockKException>(addedState.updateResult.e)
}
}

72
libs/database/src/test/java/org/fdroid/repo/RepoAdderTest.kt
View File

@@ -8,7 +8,9 @@ import android.os.UserManager.DISALLOW_INSTALL_UNKNOWN_SOURCES
import androidx.core.os.LocaleListCompat
import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.platform.app.InstrumentationRegistry
import app.cash.turbine.TurbineTestContext
import app.cash.turbine.test
import io.mockk.MockKException
import io.mockk.Runs
import io.mockk.coEvery
import io.mockk.every
@@ -21,8 +23,9 @@ import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.delay
import kotlinx.coroutines.launch
import kotlinx.coroutines.test.runTest
import org.fdroid.CompatibilityChecker
import org.fdroid.LocaleChooser.getBestLocale
import org.fdroid.database.FDroidDatabase
import org.fdroid.database.FDroidDatabaseInt
import org.fdroid.database.Mirror
import org.fdroid.database.NewRepository
import org.fdroid.database.Repository
@@ -37,6 +40,7 @@ import org.fdroid.download.NotFoundException
import org.fdroid.download.getDigestInputStream
import org.fdroid.index.IndexFormatVersion
import org.fdroid.index.IndexParser.json
import org.fdroid.index.IndexUpdateResult
import org.fdroid.index.SigningException
import org.fdroid.index.TempFileProvider
import org.fdroid.index.v2.IndexV2
@@ -77,10 +81,11 @@ internal class RepoAdderTest {
var folder: TemporaryFolder = TemporaryFolder()
private val context = InstrumentationRegistry.getInstrumentation().targetContext
private val db = mockk<FDroidDatabase>()
private val db = mockk<FDroidDatabaseInt>()
private val repoDao = mockk<RepositoryDaoInt>()
private val tempFileProvider = mockk<TempFileProvider>()
private val httpManager = mockk<HttpManager>()
private val compatibilityChecker = mockk<CompatibilityChecker>()
private val downloaderFactory = mockk<DownloaderFactory>()
private val downloader = mockk<Downloader>()
private val digest = mockk<MessageDigest>()
@@ -96,14 +101,28 @@ internal class RepoAdderTest {
mockkStatic("org.fdroid.download.HttpManagerKt")
repoAdder = RepoAdder(context, db, tempFileProvider, downloaderFactory, httpManager)
repoAdder = RepoAdder(
context = context,
db = db,
tempFileProvider = tempFileProvider,
downloaderFactory = downloaderFactory,
httpManager = httpManager,
compatibilityChecker = compatibilityChecker,
)
}
@Test
fun testDisallowInstallUnknownSources() = runTest {
val context = mockk<Context>()
val userManager = mockk<UserManager>()
val repoAdder = RepoAdder(context, db, tempFileProvider, downloaderFactory, httpManager)
val repoAdder = RepoAdder(
context,
db,
tempFileProvider,
downloaderFactory,
httpManager,
compatibilityChecker,
)
every { context.getSystemService(UserManager::class.java) } returns userManager
every {
@@ -170,6 +189,8 @@ internal class RepoAdderTest {
val fetching: Fetching = awaitItem() as Fetching // still Fetching from last call
assertEquals(expectedResult, fetching.fetchResult)
every { newRepo.formatVersion } returns IndexFormatVersion.TWO
repoAdder.addFetchedRepository()
assertIs<Adding>(awaitItem()) // now moved to Adding
@@ -177,6 +198,10 @@ internal class RepoAdderTest {
val addedState = awaitItem()
assertIs<Added>(addedState)
assertEquals(newRepo, addedState.repo)
// we are not mocking all the actual repo adding,
// so just assert that this fails due to mocking
assertIs<IndexUpdateResult.Error>(addedState.updateResult)
assertIs<MockKException>(addedState.updateResult.e)
}
}
@@ -199,6 +224,8 @@ internal class RepoAdderTest {
val fetching: Fetching = awaitItem() as Fetching // still Fetching from last call
assertIs<IsNewRepository>(fetching.fetchResult)
every { newRepo.formatVersion } returns IndexFormatVersion.TWO
repoAdder.addFetchedRepository()
assertIs<Adding>(awaitItem()) // now moved to Adding
@@ -206,6 +233,10 @@ internal class RepoAdderTest {
val addedState = awaitItem()
assertIs<Added>(addedState)
assertEquals(newRepo, addedState.repo)
// we are not mocking all the actual repo adding,
// so just assert that this fails due to mocking
assertIs<IndexUpdateResult.Error>(addedState.updateResult)
assertIs<MockKException>(addedState.updateResult.e)
}
verify(exactly = 0) {
@@ -409,7 +440,7 @@ internal class RepoAdderTest {
val url = "https://example.org/repo"
val jarFile = folder.newFile()
every { tempFileProvider.createTempFile() } returns jarFile
every { tempFileProvider.createTempFile(any()) } returns jarFile
every {
downloaderFactory.create(
repo = match { it.address == url && it.formatVersion == IndexFormatVersion.TWO },
@@ -445,7 +476,7 @@ internal class RepoAdderTest {
val index = "{ invalid JSON foo bar,".toByteArray()
val indexStream = DigestInputStream(ByteArrayInputStream(index), digest)
every { tempFileProvider.createTempFile() } returns jarFile
every { tempFileProvider.createTempFile(any()) } returns jarFile
every {
downloaderFactory.create(
repo = match {
@@ -496,7 +527,7 @@ internal class RepoAdderTest {
val url = "https://example.org/repo/"
val jarFile = folder.newFile()
every { tempFileProvider.createTempFile() } returns jarFile
every { tempFileProvider.createTempFile(any()) } returns jarFile
every {
downloaderFactory.create(
repo = match {
@@ -563,7 +594,7 @@ internal class RepoAdderTest {
val index = json.encodeToString(IndexV2.serializer(), indexV2).toByteArray()
val indexStream = DigestInputStream(ByteArrayInputStream(index), digest)
every { tempFileProvider.createTempFile() } returns jarFile
every { tempFileProvider.createTempFile(any()) } returns jarFile
every {
downloaderFactory.create(
repo = match {
@@ -623,7 +654,7 @@ internal class RepoAdderTest {
val index = json.encodeToString(IndexV2.serializer(), indexV2).toByteArray()
val indexStream = DigestInputStream(ByteArrayInputStream(index), digest)
every { tempFileProvider.createTempFile() } returns jarFile
every { tempFileProvider.createTempFile(any()) } returns jarFile
every {
downloaderFactory.create(
repo = match {
@@ -686,7 +717,7 @@ internal class RepoAdderTest {
val urlTrimmed = "http://testy.at.or.at/fdroid/repo"
val jarFile = folder.newFile()
every { tempFileProvider.createTempFile() } returns jarFile
every { tempFileProvider.createTempFile(any()) } returns jarFile
every {
downloaderFactory.create(
@@ -743,7 +774,7 @@ internal class RepoAdderTest {
val url = "https://example.org/repo"
val jarFile = folder.newFile()
every { tempFileProvider.createTempFile() } returns jarFile
every { tempFileProvider.createTempFile(any()) } returns jarFile
every {
downloaderFactory.create(
repo = match { it.address == url && it.formatVersion == IndexFormatVersion.TWO },
@@ -820,6 +851,7 @@ internal class RepoAdderTest {
repoAdder.addRepoState.test {
assertIs<Fetching>(awaitItem()) // still Fetching from last call
every { newRepo.formatVersion } returns IndexFormatVersion.TWO
repoAdder.addFetchedRepository()
assertIs<Adding>(awaitItem()) // now moved to Adding
@@ -827,6 +859,10 @@ internal class RepoAdderTest {
val addedState = awaitItem()
assertIs<Added>(addedState)
assertEquals(newRepo, addedState.repo)
// we are not mocking all the actual repo adding,
// so just assert that this fails due to mocking
assertIs<IndexUpdateResult.Error>(addedState.updateResult)
assertIs<MockKException>(addedState.updateResult.e)
}
}
@@ -867,7 +903,7 @@ internal class RepoAdderTest {
val indexInputStream = assets.open(indexFile)
val indexDigestStream = DigestInputStream(indexInputStream, digest)
every { tempFileProvider.createTempFile() } returns jarFile
every { tempFileProvider.createTempFile(any()) } returns jarFile
every {
downloaderFactory.create(
repo = match {
@@ -935,7 +971,7 @@ internal class RepoAdderTest {
url,
expectedFetchResult,
TestDataMinV2.repo,
) { awaitItem ->
) {
val state = awaitItem()
assertIs<Fetching>(state)
assertEquals(TestDataMinV2.packages.size, state.apps.size)
@@ -954,7 +990,7 @@ internal class RepoAdderTest {
url,
expectedFetchResult,
TestDataMidV2.repo,
) { awaitItem ->
) {
val state = awaitItem()
assertIs<Fetching>(state)
assertEquals(1, state.apps.size)
@@ -976,7 +1012,7 @@ internal class RepoAdderTest {
url: String,
expectedFetchResult: FetchResult,
expectedRepo: RepoV2,
onAppsReceived: suspend (suspend () -> AddRepoState) -> Unit,
onAppsReceived: suspend TurbineTestContext<AddRepoState>.() -> Unit,
) {
repoAdder.addRepoState.test {
assertIs<None>(awaitItem())
@@ -984,7 +1020,7 @@ internal class RepoAdderTest {
launch(Dispatchers.IO) {
// FIXME executing this block may emit items too fast, so we might miss one
// causing flaky tests. A short delay may fix it, let's see.
delay(250)
delay(750)
repoAdder.fetchRepository(url = url, proxy = null)
}
@@ -1008,12 +1044,14 @@ internal class RepoAdderTest {
assertFalse(state2.done)
// onAppReceived (state3)
onAppsReceived(::awaitItem)
onAppsReceived(this)
// final result
val state4 = awaitItem()
assertIs<Fetching>(state4)
assertTrue(state4.done)
expectNoEvents()
}
}
}

210
libs/download/api/android/download.api Normal file
View File

@@ -0,0 +1,210 @@
public abstract interface class org/fdroid/download/BytesReceiver {
public abstract fun receive ([BLjava/lang/Long;Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
}
public final class org/fdroid/download/DownloadRequest {
public fun <init> (Ljava/lang/String;Ljava/util/List;)V
public fun <init> (Ljava/lang/String;Ljava/util/List;Ljava/net/Proxy;)V
public fun <init> (Ljava/lang/String;Ljava/util/List;Ljava/net/Proxy;Ljava/lang/String;)V
public fun <init> (Ljava/lang/String;Ljava/util/List;Ljava/net/Proxy;Ljava/lang/String;Ljava/lang/String;)V
public fun <init> (Ljava/lang/String;Ljava/util/List;Ljava/net/Proxy;Ljava/lang/String;Ljava/lang/String;Lorg/fdroid/download/Mirror;)V
public synthetic fun <init> (Ljava/lang/String;Ljava/util/List;Ljava/net/Proxy;Ljava/lang/String;Ljava/lang/String;Lorg/fdroid/download/Mirror;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
public fun <init> (Lorg/fdroid/IndexFile;Ljava/util/List;)V
public fun <init> (Lorg/fdroid/IndexFile;Ljava/util/List;Ljava/net/Proxy;)V
public fun <init> (Lorg/fdroid/IndexFile;Ljava/util/List;Ljava/net/Proxy;Ljava/lang/String;)V
public fun <init> (Lorg/fdroid/IndexFile;Ljava/util/List;Ljava/net/Proxy;Ljava/lang/String;Ljava/lang/String;)V
public fun <init> (Lorg/fdroid/IndexFile;Ljava/util/List;Ljava/net/Proxy;Ljava/lang/String;Ljava/lang/String;Lorg/fdroid/download/Mirror;)V
public synthetic fun <init> (Lorg/fdroid/IndexFile;Ljava/util/List;Ljava/net/Proxy;Ljava/lang/String;Ljava/lang/String;Lorg/fdroid/download/Mirror;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
public final fun component1 ()Lorg/fdroid/IndexFile;
public final fun component2 ()Ljava/util/List;
public final fun component3 ()Ljava/net/Proxy;
public final fun component4 ()Ljava/lang/String;
public final fun component5 ()Ljava/lang/String;
public final fun component6 ()Lorg/fdroid/download/Mirror;
public final fun copy (Lorg/fdroid/IndexFile;Ljava/util/List;Ljava/net/Proxy;Ljava/lang/String;Ljava/lang/String;Lorg/fdroid/download/Mirror;)Lorg/fdroid/download/DownloadRequest;
public static synthetic fun copy$default (Lorg/fdroid/download/DownloadRequest;Lorg/fdroid/IndexFile;Ljava/util/List;Ljava/net/Proxy;Ljava/lang/String;Ljava/lang/String;Lorg/fdroid/download/Mirror;ILjava/lang/Object;)Lorg/fdroid/download/DownloadRequest;
public fun equals (Ljava/lang/Object;)Z
public final fun getHasCredentials ()Z
public final fun getIndexFile ()Lorg/fdroid/IndexFile;
public final fun getMirrors ()Ljava/util/List;
public final fun getPassword ()Ljava/lang/String;
public final fun getProxy ()Ljava/net/Proxy;
public final fun getTryFirstMirror ()Lorg/fdroid/download/Mirror;
public final fun getUsername ()Ljava/lang/String;
public fun hashCode ()I
public fun toString ()Ljava/lang/String;
}
public abstract class org/fdroid/download/Downloader {
public static final field Companion Lorg/fdroid/download/Downloader$Companion;
protected final field outputFile Ljava/io/File;
public fun <init> (Lorg/fdroid/IndexFile;Ljava/io/File;)V
public final fun cancelDownload ()V
public abstract fun close ()V
public abstract fun download ()V
protected final fun downloadFromBytesReceiver (ZLkotlin/coroutines/Continuation;)Ljava/lang/Object;
protected final fun downloadFromStream (Z)V
protected fun getBytes (ZLorg/fdroid/download/BytesReceiver;Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
public final fun getCacheTag ()Ljava/lang/String;
protected final fun getIndexFile ()Lorg/fdroid/IndexFile;
protected abstract fun getInputStream (Z)Ljava/io/InputStream;
public abstract fun hasChanged ()Z
public final fun setCacheTag (Ljava/lang/String;)V
public final fun setListener (Lorg/fdroid/fdroid/ProgressListener;)V
protected abstract fun totalDownloadSize ()J
public final fun wasCancelled ()Z
}
public final class org/fdroid/download/Downloader$Companion {
}
public final class org/fdroid/download/HeadInfo {
public fun <init> (ZLjava/lang/String;Ljava/lang/Long;Ljava/lang/String;)V
public final fun component1 ()Z
public final fun component2 ()Ljava/lang/String;
public final fun component3 ()Ljava/lang/Long;
public final fun component4 ()Ljava/lang/String;
public final fun copy (ZLjava/lang/String;Ljava/lang/Long;Ljava/lang/String;)Lorg/fdroid/download/HeadInfo;
public static synthetic fun copy$default (Lorg/fdroid/download/HeadInfo;ZLjava/lang/String;Ljava/lang/Long;Ljava/lang/String;ILjava/lang/Object;)Lorg/fdroid/download/HeadInfo;
public fun equals (Ljava/lang/Object;)Z
public final fun getContentLength ()Ljava/lang/Long;
public final fun getETag ()Ljava/lang/String;
public final fun getETagChanged ()Z
public final fun getLastModified ()Ljava/lang/String;
public fun hashCode ()I
public fun toString ()Ljava/lang/String;
}
public final class org/fdroid/download/HttpDownloader : org/fdroid/download/Downloader {
public fun <init> (Lorg/fdroid/download/HttpManager;Lorg/fdroid/download/DownloadRequest;Ljava/io/File;)V
public fun close ()V
public fun download ()V
public fun hasChanged ()Z
}
public final class org/fdroid/download/HttpDownloaderV2 : org/fdroid/download/Downloader {
public fun <init> (Lorg/fdroid/download/HttpManager;Lorg/fdroid/download/DownloadRequest;Ljava/io/File;)V
public fun close ()V
public fun download ()V
public fun hasChanged ()Z
}
public class org/fdroid/download/HttpManager {
public static final field Companion Lorg/fdroid/download/HttpManager$Companion;
public fun <init> (Ljava/lang/String;)V
public fun <init> (Ljava/lang/String;Ljava/lang/String;)V
public fun <init> (Ljava/lang/String;Ljava/lang/String;Ljava/net/Proxy;)V
public fun <init> (Ljava/lang/String;Ljava/lang/String;Ljava/net/Proxy;Lokhttp3/Dns;)V
public fun <init> (Ljava/lang/String;Ljava/lang/String;Ljava/net/Proxy;Lokhttp3/Dns;Lorg/fdroid/download/MirrorParameterManager;)V
public fun <init> (Ljava/lang/String;Ljava/lang/String;Ljava/net/Proxy;Lokhttp3/Dns;Lorg/fdroid/download/MirrorParameterManager;Z)V
public fun <init> (Ljava/lang/String;Ljava/lang/String;Ljava/net/Proxy;Lokhttp3/Dns;Lorg/fdroid/download/MirrorParameterManager;ZLorg/fdroid/download/MirrorChooser;)V
public fun <init> (Ljava/lang/String;Ljava/lang/String;Ljava/net/Proxy;Lokhttp3/Dns;Lorg/fdroid/download/MirrorParameterManager;ZLorg/fdroid/download/MirrorChooser;Lio/ktor/client/engine/HttpClientEngineFactory;)V
public synthetic fun <init> (Ljava/lang/String;Ljava/lang/String;Ljava/net/Proxy;Lokhttp3/Dns;Lorg/fdroid/download/MirrorParameterManager;ZLorg/fdroid/download/MirrorChooser;Lio/ktor/client/engine/HttpClientEngineFactory;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
public final fun get (Lorg/fdroid/download/DownloadRequest;Ljava/lang/Long;Lorg/fdroid/download/BytesReceiver;Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
public final fun get (Lorg/fdroid/download/DownloadRequest;Lorg/fdroid/download/BytesReceiver;Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
public static synthetic fun get$default (Lorg/fdroid/download/HttpManager;Lorg/fdroid/download/DownloadRequest;Ljava/lang/Long;Lorg/fdroid/download/BytesReceiver;Lkotlin/coroutines/Continuation;ILjava/lang/Object;)Ljava/lang/Object;
public final fun head (Lorg/fdroid/download/DownloadRequest;Ljava/lang/String;Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
public static synthetic fun head$default (Lorg/fdroid/download/HttpManager;Lorg/fdroid/download/DownloadRequest;Ljava/lang/String;Lkotlin/coroutines/Continuation;ILjava/lang/Object;)Ljava/lang/Object;
public final fun post (Ljava/lang/String;Ljava/lang/String;Ljava/net/Proxy;Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
public static synthetic fun post$default (Lorg/fdroid/download/HttpManager;Ljava/lang/String;Ljava/lang/String;Ljava/net/Proxy;Lkotlin/coroutines/Continuation;ILjava/lang/Object;)Ljava/lang/Object;
}
public final class org/fdroid/download/HttpManager$Companion {
public final fun isInvalidHttpUrl (Ljava/lang/String;)Z
}
public final class org/fdroid/download/HttpManagerKt {
public static final fun getDigestInputStream (Lorg/fdroid/download/HttpManager;Lorg/fdroid/download/DownloadRequest;Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
public static final fun getInputStream (Lorg/fdroid/download/HttpManager;Lorg/fdroid/download/DownloadRequest;Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
}
public final class org/fdroid/download/HttpPoster {
public fun <init> (Lorg/fdroid/download/HttpManager;Ljava/lang/String;)V
public final fun post (Ljava/lang/String;)V
}
public final class org/fdroid/download/Mirror {
public static final field Companion Lorg/fdroid/download/Mirror$Companion;
public fun <init> (Ljava/lang/String;)V
public fun <init> (Ljava/lang/String;Ljava/lang/String;)V
public fun <init> (Ljava/lang/String;Ljava/lang/String;Z)V
public synthetic fun <init> (Ljava/lang/String;Ljava/lang/String;ZILkotlin/jvm/internal/DefaultConstructorMarker;)V
public final fun component1 ()Ljava/lang/String;
public final fun component2 ()Ljava/lang/String;
public final fun component3 ()Z
public final fun copy (Ljava/lang/String;Ljava/lang/String;Z)Lorg/fdroid/download/Mirror;
public static synthetic fun copy$default (Lorg/fdroid/download/Mirror;Ljava/lang/String;Ljava/lang/String;ZILjava/lang/Object;)Lorg/fdroid/download/Mirror;
public fun equals (Ljava/lang/Object;)Z
public static final fun fromStrings (Ljava/util/List;)Ljava/util/List;
public final fun getBaseUrl ()Ljava/lang/String;
public final fun getCountryCode ()Ljava/lang/String;
public final fun getFDroidLinkUrl (Ljava/lang/String;)Ljava/lang/String;
public final fun getUrl ()Lio/ktor/http/Url;
public final fun getUrl (Ljava/lang/String;)Lio/ktor/http/Url;
public fun hashCode ()I
public final fun isHttp ()Z
public final fun isIpfsGateway ()Z
public final fun isLocal ()Z
public final fun isOnion ()Z
public fun toString ()Ljava/lang/String;
}
public final class org/fdroid/download/Mirror$Companion {
public final fun fromStrings (Ljava/util/List;)Ljava/util/List;
}
public abstract interface class org/fdroid/download/MirrorChooser {
public abstract fun mirrorRequest (Lorg/fdroid/download/DownloadRequest;Lkotlin/jvm/functions/Function3;Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
public abstract fun orderMirrors (Lorg/fdroid/download/DownloadRequest;)Ljava/util/List;
}
public abstract interface class org/fdroid/download/MirrorParameterManager {
public abstract fun getCurrentLocation ()Ljava/lang/String;
public abstract fun getMirrorErrorCount (Ljava/lang/String;)I
public abstract fun incrementMirrorErrorCount (Ljava/lang/String;)V
public abstract fun preferForeignMirrors ()Z
}
public final class org/fdroid/download/NoResumeException : java/lang/Exception {
public fun <init> ()V
}
public final class org/fdroid/download/NotFoundException : java/lang/Exception {
public fun <init> ()V
public fun <init> (Ljava/lang/Throwable;)V
public synthetic fun <init> (Ljava/lang/Throwable;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
}
public final class org/fdroid/download/coil/DownloadRequestFetcher : coil3/fetch/Fetcher {
public fun <init> (Lorg/fdroid/download/HttpManager;Lorg/fdroid/download/DownloadRequest;Lcoil3/request/Options;Lkotlin/Lazy;)V
public fun fetch (Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
}
public final class org/fdroid/download/coil/DownloadRequestFetcher$Factory : coil3/fetch/Fetcher$Factory {
public fun <init> (Lorg/fdroid/download/HttpManager;)V
public synthetic fun create (Ljava/lang/Object;Lcoil3/request/Options;Lcoil3/ImageLoader;)Lcoil3/fetch/Fetcher;
public fun create (Lorg/fdroid/download/DownloadRequest;Lcoil3/request/Options;Lcoil3/ImageLoader;)Lcoil3/fetch/Fetcher;
}
public final class org/fdroid/download/glide/DownloadRequestLoader : com/bumptech/glide/load/model/ModelLoader {
public fun <init> (Lorg/fdroid/download/HttpManager;)V
public synthetic fun buildLoadData (Ljava/lang/Object;IILcom/bumptech/glide/load/Options;)Lcom/bumptech/glide/load/model/ModelLoader$LoadData;
public fun buildLoadData (Lorg/fdroid/download/DownloadRequest;IILcom/bumptech/glide/load/Options;)Lcom/bumptech/glide/load/model/ModelLoader$LoadData;
public synthetic fun handles (Ljava/lang/Object;)Z
public fun handles (Lorg/fdroid/download/DownloadRequest;)Z
}
public final class org/fdroid/download/glide/DownloadRequestLoader$Factory : com/bumptech/glide/load/model/ModelLoaderFactory {
public fun <init> (Lorg/fdroid/download/HttpManager;)V
public fun build (Lcom/bumptech/glide/load/model/MultiModelLoaderFactory;)Lcom/bumptech/glide/load/model/ModelLoader;
public fun teardown ()V
}
public final class org/fdroid/fdroid/DigestInputStreamKt {
public static final fun getDigestHex (Ljava/security/DigestInputStream;)Ljava/lang/String;
}
public abstract interface class org/fdroid/fdroid/ProgressListener {
public abstract fun onProgress (JJ)V
}

138
libs/download/build.gradle
View File

@@ -1,138 +0,0 @@
plugins {
id 'org.jetbrains.kotlin.multiplatform'
id 'com.android.library'
id 'org.jetbrains.dokka'
id 'com.vanniktech.maven.publish'
}
kotlin {
androidTarget {
compilations.configureEach {
kotlinOptions.jvmTarget = '17'
}
publishLibraryVariants("release")
}
// def hostOs = System.getProperty("os.name")
// def isMingwX64 = hostOs.startsWith("Windows")
// def nativeTarget
// if (hostOs == "Mac OS X") nativeTarget = macosX64('native')
// else if (hostOs == "Linux") nativeTarget = linuxX64("native")
// else if (isMingwX64) nativeTarget = mingwX64("native")
// else throw new GradleException("Host OS is not supported in Kotlin/Native.")
sourceSets {
configureEach {
languageSettings {
optIn('kotlin.RequiresOptIn')
explicitApi('strict')
}
}
commonMain {
dependencies {
api libs.ktor.client.core
implementation libs.microutils.kotlin.logging
}
}
commonTest {
dependencies {
implementation kotlin('test')
implementation libs.ktor.client.mock
implementation libs.mockk
}
}
// JVM is disabled for now, because Android app is including it instead of Android library
jvmMain {
dependencies {
implementation libs.ktor.client.cio
}
}
jvmTest {
dependencies {
implementation libs.junit
}
}
androidMain {
dependencies {
implementation libs.ktor.client.okhttp
//noinspection UseTomlInstead
implementation("com.github.bumptech.glide:glide:4.16.0") {
transitive = false // we don't need all that it pulls in, just the basics
}
implementation libs.glide.annotations
}
}
androidUnitTest {
dependencies {
implementation kotlin('test')
implementation libs.json
implementation libs.junit
implementation libs.logback.classic
}
}
androidInstrumentedTest {
dependsOn(commonTest)
dependencies {
implementation project(":libs:sharedTest")
implementation libs.androidx.test.runner
implementation libs.androidx.test.ext.junit
implementation libs.mockk.android
}
}
nativeMain {
dependencies {
implementation libs.ktor.client.curl
}
}
nativeTest {
}
}
}
android {
namespace "org.fdroid.download"
compileSdk libs.versions.compileSdk.get().toInteger()
sourceSets {
main.manifest.srcFile('src/androidMain/AndroidManifest.xml')
getByName("androidTest").java.srcDir(file("src/androidAndroidTest/kotlin"))
}
defaultConfig {
minSdkVersion 21
targetSdkVersion 34 // needed for instrumentation tests
testInstrumentationRunner 'androidx.test.runner.AndroidJUnitRunner'
testInstrumentationRunnerArguments disableAnalytics: 'true'
}
compileOptions {
sourceCompatibility JavaVersion.VERSION_17
targetCompatibility JavaVersion.VERSION_17
}
lintOptions {
checkReleaseBuilds false
abortOnError true
htmlReport true
xmlReport false
textReport true
lintConfig file("lint.xml")
}
testOptions {
packaging {
resources.excludes.add("META-INF/*")
}
}
}
signing {
useGpgCmd()
}
import org.jetbrains.dokka.gradle.DokkaTask
tasks.withType(DokkaTask).configureEach {
pluginsMapConfiguration.set(
["org.jetbrains.dokka.base.DokkaBase": """{
"customAssets": ["${file("${rootProject.rootDir}/logo-icon.svg")}"],
"footerMessage": "© 2010-2022 F-Droid Limited and Contributors"
}"""]
)
}

124
libs/download/build.gradle.kts Normal file
View File

@@ -0,0 +1,124 @@
plugins {
alias(libs.plugins.jetbrains.kotlin.multiplatform)
alias(libs.plugins.android.library)
alias(libs.plugins.jetbrains.dokka)
alias(libs.plugins.vanniktech.maven.publish)
}
kotlin {
androidTarget {
compilerOptions {
jvmTarget = org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_17
}
publishLibraryVariants("release")
}
compilerOptions {
optIn.add("kotlin.RequiresOptIn")
}
explicitApi()
@OptIn(org.jetbrains.kotlin.gradle.dsl.abi.ExperimentalAbiValidation::class)
abiValidation {
enabled = true
}
sourceSets {
commonMain {
dependencies {
api(project(":libs:core"))
api(libs.ktor.client.core)
implementation(libs.microutils.kotlin.logging)
}
}
commonTest {
dependencies {
implementation(kotlin("test"))
implementation(libs.ktor.client.mock)
implementation(libs.mockk)
}
}
// JVM is disabled for now, because Android app is including it instead of Android library
jvmMain {
dependencies {
implementation(libs.ktor.client.cio)
}
}
jvmTest {
dependencies {
implementation(libs.junit)
}
}
androidMain {
dependencies {
implementation(libs.ktor.client.okhttp)
//noinspection UseTomlInstead
implementation("com.github.bumptech.glide:glide:4.16.0") {
isTransitive = false // we don't need all that it pulls in, just the basics
}
implementation(libs.glide.annotations)
implementation("io.coil-kt.coil3:coil-core:3.3.0") {
isTransitive = false // we don't need all that it pulls in, just the basics
}
implementation("javax.inject:javax.inject:1")
}
}
androidUnitTest {
dependencies {
implementation(kotlin("test"))
implementation(libs.json)
implementation(libs.junit)
implementation(libs.logback.classic)
}
}
val commonTest by getting
androidInstrumentedTest {
dependsOn(commonTest)
dependencies {
implementation(project(":libs:sharedTest"))
implementation(libs.androidx.test.runner)
implementation(libs.androidx.test.ext.junit)
implementation(libs.mockk.android)
}
}
}
}
android {
namespace = "org.fdroid.download"
@Suppress("ktlint:standard:chain-method-continuation")
compileSdk = libs.versions.compileSdk.get().toInt()
defaultConfig {
minSdk = 21
testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
testInstrumentationRunnerArguments["disableAnalytics"] = "true"
}
compileOptions {
sourceCompatibility = JavaVersion.VERSION_17
targetCompatibility = JavaVersion.VERSION_17
}
lint {
checkReleaseBuilds = false
abortOnError = true
htmlReport = true
xmlReport = false
textReport = true
lintConfig = file("lint.xml")
}
testOptions {
targetSdk = 34 // needed for instrumentation tests
packaging {
resources.excludes.add("META-INF/*")
}
}
}
signing {
useGpgCmd()
}
dokka {
pluginsConfiguration.html {
customAssets.from("${file("${rootProject.rootDir}/logo-icon.svg")}")
footerMessage.set("© 2010-2025 F-Droid Limited and Contributors")
}
}

2
libs/download/gradle.properties
View File

@@ -1,5 +1,5 @@
POM_ARTIFACT_ID=download
VERSION_NAME=0.1.1
VERSION_NAME=0.2.0
POM_NAME=F-Droid download library
POM_DESCRIPTION=A Kotlin multi-platform library to download F-Droid related files via HTTP.

2
libs/download/src/androidMain/kotlin/org/fdroid/download/Downloader.kt
View File

@@ -180,7 +180,7 @@ public abstract class Downloader(
private fun reportProgress(lastTimeReported: Long, bytesRead: Long, bytesTotal: Long): Long {
val now = System.currentTimeMillis()
return if (now - lastTimeReported > 100) {
return if (now - lastTimeReported > 1000) {
log.debug { "onProgress: $bytesRead/$bytesTotal" }
progressListener?.onProgress(bytesRead, bytesTotal)
now

35
libs/download/src/androidMain/kotlin/org/fdroid/download/HttpDownloaderV2.kt
View File

@@ -25,9 +25,12 @@ import io.ktor.client.plugins.ResponseException
import io.ktor.http.HttpStatusCode.Companion.NotFound
import kotlinx.coroutines.runBlocking
import mu.KotlinLogging
import org.fdroid.fdroid.toHex
import java.io.File
import java.io.IOException
import java.io.InputStream
import java.security.MessageDigest
import java.security.NoSuchAlgorithmException
/**
* Download files over HTTP, with support for proxies, `.onion` addresses, HTTP Basic Auth, etc.
@@ -62,10 +65,23 @@ public class HttpDownloaderV2 constructor(
var resumable = false
val fileLength = outputFile.length()
if (fileLength > (request.indexFile.size ?: -1)) {
// file was larger than expected, so delete and re-download
if (!outputFile.delete()) log.warn { "Warning: outputFile not deleted" }
} else if (fileLength == request.indexFile.size && outputFile.isFile) {
log.debug { "Already have outputFile, not downloading: ${outputFile.name}" }
return // already have it!
if (request.indexFile.sha256 == null) {
// no way to check file, so we trust that what we have is legit (v1 only)
return
} else {
if (hashFile(outputFile) == request.indexFile.sha256) {
// hash matched, so we already have the good file, don't download again
return
} else {
log.warn { "Hash mismatch for ${request.indexFile}" }
// delete file and continue
if (!outputFile.delete()) log.warn { "Warning: outputFile not deleted" }
}
}
} else if (fileLength > 0) {
resumable = true
}
@@ -90,4 +106,21 @@ public class HttpDownloaderV2 constructor(
override fun close() {
}
private fun hashFile(file: File): String {
val messageDigest: MessageDigest = try {
MessageDigest.getInstance("SHA-256")
} catch (e: NoSuchAlgorithmException) {
throw AssertionError(e)
}
file.inputStream().use { inputStream ->
val buffer = ByteArray(DEFAULT_BUFFER_SIZE)
var bytes = inputStream.read(buffer)
while (bytes >= 0) {
messageDigest.update(buffer, 0, bytes)
bytes = inputStream.read(buffer)
}
}
return messageDigest.digest().toHex()
}
}

150
libs/download/src/androidMain/kotlin/org/fdroid/download/coil/DownloadRequestFetcher.kt Normal file
View File

@@ -0,0 +1,150 @@
/**
* Contains disk cache related code from https://github.com/coil-kt/coil
* coil-network-core/src/commonMain/kotlin/coil3/network/NetworkFetcher.kt
* under Apache-2.0 license.
*/
package org.fdroid.download.coil
import coil3.ImageLoader
import coil3.annotation.InternalCoilApi
import coil3.decode.DataSource
import coil3.decode.ImageSource
import coil3.disk.DiskCache
import coil3.fetch.FetchResult
import coil3.fetch.Fetcher
import coil3.fetch.SourceFetchResult
import coil3.request.Options
import coil3.util.MimeTypeMap
import io.ktor.utils.io.jvm.javaio.toInputStream
import okio.BufferedSource
import okio.FileSystem
import okio.buffer
import okio.source
import org.fdroid.download.DownloadRequest
import org.fdroid.download.HttpManager
import org.fdroid.download.glide.AutoVerifyingInputStream
import org.fdroid.download.glide.getKey
import javax.inject.Inject
public class DownloadRequestFetcher(
private val httpManager: HttpManager,
private val downloadRequest: DownloadRequest,
private val options: Options,
private val diskCache: Lazy<DiskCache?>,
) : Fetcher {
private val fileSystem: FileSystem
get() = diskCache.value?.fileSystem ?: options.fileSystem
private val diskCacheKey: String
get() = options.diskCacheKey ?: downloadRequest.getKey()
@OptIn(InternalCoilApi::class)
private val mimeType: String?
get() = MimeTypeMap.getMimeTypeFromUrl(downloadRequest.indexFile.name)
override suspend fun fetch(): FetchResult? {
var snapshot = readFromDiskCache()
try {
if (snapshot != null) {
// we have the request cached, so return it right away
return SourceFetchResult(
source = snapshot.toImageSource(),
mimeType = mimeType,
dataSource = DataSource.DISK,
)
}
// TODO use channel directly and auto-verify hash without InputStream wrapper
// may need https://github.com/Kotlin/kotlinx-io/blob/master/integration/kotlinx-io-okio/Module.md
val inputStream = httpManager.getChannel(downloadRequest).toInputStream()
val sha256 = downloadRequest.indexFile.sha256
val bufferedSource = if (sha256 == null) {
inputStream
} else {
AutoVerifyingInputStream(inputStream, sha256)
}.source().buffer()
snapshot = writeToDiskCache(snapshot, bufferedSource)
if (snapshot == null) {
// we couldn't write the snapshot, so try returning directly
return SourceFetchResult(
source = ImageSource(
source = bufferedSource,
fileSystem = FileSystem.SYSTEM,
metadata = null,
),
mimeType = mimeType,
dataSource = DataSource.NETWORK,
)
}
return SourceFetchResult(
source = snapshot.toImageSource(),
mimeType = mimeType,
dataSource = DataSource.NETWORK,
)
} finally {
snapshot?.close()
}
}
private fun readFromDiskCache(): DiskCache.Snapshot? {
return if (options.diskCachePolicy.readEnabled) {
diskCache.value?.openSnapshot(downloadRequest.getKey())
} else {
null
}
}
private fun writeToDiskCache(
snapshot: DiskCache.Snapshot?,
bufferedSource: BufferedSource,
): DiskCache.Snapshot? {
// Short circuit if we're not allowed to cache this response.
if (!options.diskCachePolicy.writeEnabled) return null
// Open a new editor. Return null if we're unable to write to this entry.
val editor = if (snapshot != null) {
snapshot.closeAndOpenEditor()
} else {
diskCache.value?.openEditor(diskCacheKey)
} ?: return null
return try {
fileSystem.write(editor.data) {
writeAll(bufferedSource)
}
editor.commitAndOpenSnapshot()
} catch (e: Exception) {
try {
editor.abort()
} catch (_: Exception) {
// ignore
}
throw e
}
}
private fun DiskCache.Snapshot.toImageSource(): ImageSource {
return ImageSource(
file = data,
fileSystem = fileSystem,
diskCacheKey = diskCacheKey,
closeable = this,
)
}
public class Factory @Inject constructor(
private val httpManager: HttpManager,
) : Fetcher.Factory<DownloadRequest> {
override fun create(
data: DownloadRequest,
options: Options,
imageLoader: ImageLoader,
): Fetcher? = DownloadRequestFetcher(
httpManager = httpManager,
downloadRequest = data,
options = options,
diskCache = lazy { imageLoader.diskCache },
)
}
}

10
libs/download/src/androidMain/kotlin/org/fdroid/download/glide/DownloadRequestLoader.kt
View File

@@ -24,7 +24,7 @@ public class DownloadRequestLoader(
height: Int,
options: Options,
): LoadData<InputStream> {
return LoadData(downloadRequest.getKey(), HttpFetcher(httpManager, downloadRequest))
return LoadData(downloadRequest.getObjectKey(), HttpFetcher(httpManager, downloadRequest))
}
public class Factory(
@@ -41,6 +41,10 @@ public class DownloadRequestLoader(
}
internal fun DownloadRequest.getKey(): ObjectKey {
return ObjectKey(indexFile.sha256 ?: (mirrors[0].baseUrl + indexFile.name))
internal fun DownloadRequest.getObjectKey(): ObjectKey {
return ObjectKey(getKey())
}
internal fun DownloadRequest.getKey(): String {
return indexFile.sha256 ?: (mirrors[0].baseUrl + indexFile.name)
}

49
libs/download/src/androidUnitTest/kotlin/org/fdroid/download/HttpDownloaderTest.kt
View File

@@ -25,6 +25,7 @@ import org.fdroid.runSuspend
import org.junit.Assume.assumeTrue
import org.junit.Rule
import org.junit.rules.TemporaryFolder
import java.io.File
import java.io.IOException
import java.net.BindException
import java.net.ServerSocket
@@ -33,6 +34,7 @@ import kotlin.test.Ignore
import kotlin.test.Test
import kotlin.test.assertContentEquals
import kotlin.test.assertEquals
import kotlin.test.assertFailsWith
import kotlin.test.assertTrue
import kotlin.test.fail
@@ -214,6 +216,53 @@ internal class HttpDownloaderTest {
assertContentEquals(firstBytes + secondBytes, file.readBytes())
}
/**
* Tests re-using an already downloaded file with hash verification.
* This can fail if the hashing doesn't take the already downloaded bytes into account.
*/
@Test
fun testCompleteResumeWithHashSuccess() = runSuspend {
val sha256 = "efabb260da949061c88173c19f369b4aa0eaa82003c7c2dec08b5dfe75525368"
val file = File(folder.newFolder(), sha256).apply { createNewFile() }
val bytes = ("These are the first bytes that were already downloaded." +
"These are the last bytes that still need to be downloaded.").encodeToByteArray()
file.writeBytes(bytes)
// specifying the sha256 hash forces its validation
val indexFile = getIndexFile("foo/bar", sha256, bytes.size.toLong())
val downloadRequest = DownloadRequest(indexFile, mirrors)
val httpManager = HttpManager(userAgent, null)
val httpDownloader = HttpDownloaderV2(httpManager, downloadRequest, file)
// this throws if the hash doesn't match while downloading
httpDownloader.download()
assertContentEquals(bytes, file.readBytes())
}
@Test
fun testCompleteResumeWithHashFailure() = runSuspend {
val sha256 = "efabb260da949061c88173c19f369b4aa0eaa82003c7c2dec08b5dfe75525368"
val file = File(folder.newFolder(), sha256).apply { createNewFile() }
val bytes = ("These are the first bytes that were already downloaded." +
"These are the last bytes that still need to be downloaded.").encodeToByteArray()
file.writeBytes(bytes)
// specifying the sha256 hash forces its validation
val indexFile = getIndexFile("foo/bar", sha256.replaceFirst('e', 'f'), bytes.size.toLong())
val downloadRequest = DownloadRequest(indexFile, mirrors)
val mockEngine = MockEngine.config {
reuseHandlers = false
addHandler {
respond("", OK)
}
}
val httpManager = HttpManager(userAgent, null, httpClientEngineFactory = mockEngine)
val httpDownloader = HttpDownloaderV2(httpManager, downloadRequest, file)
val e = assertFailsWith<IOException> {
httpDownloader.download()
}
assertEquals("Hash not matching", e.message)
}
@Test
fun testResumeError() = runSuspend {
val file = folder.newFile()

5
libs/download/src/commonMain/kotlin/org/fdroid/download/Mirror.kt
View File

@@ -36,9 +36,8 @@ public data class Mirror @JvmOverloads constructor(
return URLBuilder(url).appendPathSegments(path.trimStart('/')).build()
}
public fun getFDroidLinkUrl(repoFingerprint: String?): String {
val fpr = repoFingerprint?.let { "?fingerprint=$repoFingerprint" } ?: ""
return "https://fdroid.link/#$url$fpr"
public fun getFDroidLinkUrl(repoFingerprint: String): String {
return "https://fdroid.link/#$url?fingerprint=$repoFingerprint"
}
public fun isOnion(): Boolean = url.isOnion()

1252
libs/index/api/android/index.api Normal file
View File

File diff suppressed because it is too large Load Diff

124
libs/index/build.gradle
View File

@@ -1,124 +0,0 @@
plugins {
id 'org.jetbrains.kotlin.multiplatform'
id 'org.jetbrains.kotlin.plugin.serialization'
id 'com.android.library'
id 'org.jetbrains.dokka'
id 'com.vanniktech.maven.publish'
}
kotlin {
androidTarget {
compilations.configureEach {
kotlinOptions.jvmTarget = '17'
}
publishLibraryVariants("release")
}
// def hostOs = System.getProperty("os.name")
// def isMingwX64 = hostOs.startsWith("Windows")
// def nativeTarget
// if (hostOs == "Mac OS X") nativeTarget = macosX64('native')
// else if (hostOs == "Linux") nativeTarget = linuxX64("native") {
// binaries { sharedLib { baseName = "fdroid_index" } }
// }
// else if (isMingwX64) nativeTarget = mingwX64("native")
// else throw new GradleException("Host OS is not supported in Kotlin/Native.")
sourceSets {
configureEach {
languageSettings {
optIn('kotlin.RequiresOptIn')
explicitApi('strict')
}
}
commonMain {
dependencies {
implementation libs.kotlinx.serialization.json
implementation libs.microutils.kotlin.logging
implementation project(":libs:download")
implementation libs.ktor.io
}
}
commonTest {
dependencies {
implementation project(":libs:sharedTest")
implementation kotlin('test')
implementation libs.goncalossilva.resources
}
}
// JVM is disabled for now, because Android app is including it instead of Android library
jvmMain {
dependencies {
}
}
jvmTest {
dependencies {
implementation libs.junit
}
}
androidMain {
dependencies {
implementation libs.kotlin.reflect
implementation libs.androidx.core.ktx
}
}
androidUnitTest {
dependencies {
implementation libs.junit
implementation libs.mockk
}
}
androidInstrumentedTest {
dependencies {
implementation project(":libs:sharedTest")
implementation kotlin('test')
implementation libs.androidx.test.runner
implementation libs.androidx.test.ext.junit
}
}
nativeMain {
dependencies {
}
}
nativeTest {
}
}
}
android {
namespace "org.fdroid.index"
compileSdk libs.versions.compileSdk.get().toInteger()
sourceSets {
main.manifest.srcFile('src/androidMain/AndroidManifest.xml')
getByName("androidTest").java.srcDir(file("src/androidAndroidTest/kotlin"))
}
defaultConfig {
minSdkVersion 21
consumerProguardFiles 'consumer-rules.pro'
testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
testInstrumentationRunnerArguments disableAnalytics: 'true'
}
compileOptions {
sourceCompatibility JavaVersion.VERSION_17
targetCompatibility JavaVersion.VERSION_17
}
testOptions {
unitTests.returnDefaultValues = true
}
}
signing {
useGpgCmd()
}
import org.jetbrains.dokka.gradle.DokkaTask
tasks.withType(DokkaTask).configureEach {
pluginsMapConfiguration.set(
["org.jetbrains.dokka.base.DokkaBase": """{
"customAssets": ["${file("${rootProject.rootDir}/logo-icon.svg")}"],
"footerMessage": "© 2010-2022 F-Droid Limited and Contributors"
}"""]
)
}

102
libs/index/build.gradle.kts Normal file
View File

@@ -0,0 +1,102 @@
plugins {
alias(libs.plugins.jetbrains.kotlin.multiplatform)
alias(libs.plugins.jetbrains.kotlin.plugin.serialization)
alias(libs.plugins.android.library)
alias(libs.plugins.jetbrains.dokka)
alias(libs.plugins.vanniktech.maven.publish)
}
kotlin {
androidTarget {
compilerOptions {
jvmTarget = org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_17
}
publishLibraryVariants("release")
}
compilerOptions {
optIn.add("kotlin.RequiresOptIn")
}
explicitApi()
@OptIn(org.jetbrains.kotlin.gradle.dsl.abi.ExperimentalAbiValidation::class)
abiValidation {
enabled = true
}
sourceSets {
commonMain {
dependencies {
implementation(project(":libs:core"))
implementation(libs.kotlinx.serialization.json)
implementation(libs.microutils.kotlin.logging)
}
}
commonTest {
dependencies {
implementation(project(":libs:sharedTest"))
implementation(kotlin("test"))
implementation(libs.goncalossilva.resources)
}
}
// JVM is disabled for now, because Android app is including it instead of Android library
jvmMain {
dependencies {
}
}
jvmTest {
dependencies {
implementation(libs.junit)
}
}
androidMain {
dependencies {
implementation(libs.kotlin.reflect)
implementation(libs.androidx.core.ktx)
}
}
androidUnitTest {
dependencies {
implementation(libs.junit)
implementation(libs.mockk)
}
}
androidInstrumentedTest {
dependencies {
implementation(project(":libs:sharedTest"))
implementation(kotlin("test"))
implementation(libs.androidx.test.runner)
implementation(libs.androidx.test.ext.junit)
}
}
}
}
android {
namespace = "org.fdroid.index"
@Suppress("ktlint:standard:chain-method-continuation")
compileSdk = libs.versions.compileSdk.get().toInt()
defaultConfig {
minSdk = 21
consumerProguardFiles("consumer-rules.pro")
testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
testInstrumentationRunnerArguments["disableAnalytics"] = "true"
}
compileOptions {
sourceCompatibility = JavaVersion.VERSION_17
targetCompatibility = JavaVersion.VERSION_17
}
testOptions {
unitTests {
isIncludeAndroidResources = true
}
}
}
signing {
useGpgCmd()
}
dokka {
pluginsConfiguration.html {
customAssets.from("${file("${rootProject.rootDir}/logo-icon.svg")}")
footerMessage.set("© 2010-2025 F-Droid Limited and Contributors")
}
}

2
libs/index/gradle.properties
View File

@@ -1,5 +1,5 @@
POM_ARTIFACT_ID=index
VERSION_NAME=0.1.1
VERSION_NAME=0.2.0
POM_NAME=F-Droid index library
POM_DESCRIPTION=A Kotlin multi-platform library to parse version 1 and 2 of the F-Droid index format.

9
libs/index/src/androidUnitTest/kotlin/org/fdroid/UpdateCheckerTest.kt
View File

@@ -14,9 +14,9 @@ internal class UpdateCheckerTest {
private val signer = "9f9261f0b911c60f8db722f5d430a9e9d557a3f8078ce43e1c07522ef41efedb"
private val signerV2 = SignerV2(listOf(signer))
private val betaChannels = listOf(RELEASE_CHANNEL_BETA)
private val version1 = Version(1)
private val version2 = Version(2)
private val version3 = Version(3)
private val version1 = Version(1, "1", 23, 1)
private val version2 = Version(2, "2", 23, 2)
private val version3 = Version(3, "2", 23, 3)
private val versions = listOf(version3, version2, version1)
@Test
@@ -181,6 +181,9 @@ internal class UpdateCheckerTest {
private data class Version(
override val versionCode: Long,
override val versionName: String,
override val added: Long,
override val size: Long?,
override val signer: SignerV2? = null,
override val releaseChannels: List<String>? = null,
// the manifest is only needed for compatibility checking which we can test differently

9
libs/index/src/commonMain/kotlin/org/fdroid/index/v2/PackageV2.kt
View File

@@ -2,8 +2,6 @@ package org.fdroid.index.v2
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
import kotlinx.serialization.decodeFromString
import kotlinx.serialization.encodeToString
import org.fdroid.IndexFile
import org.fdroid.index.IndexParser
@@ -78,6 +76,9 @@ public data class Screenshots(
public interface PackageVersion {
public val versionCode: Long
public val versionName: String
public val added: Long
public val size: Long?
public val signer: SignerV2?
public val releaseChannels: List<String>?
public val packageManifest: PackageManifest
@@ -88,7 +89,7 @@ public const val ANTI_FEATURE_KNOWN_VULNERABILITY: String = "KnownVuln"
@Serializable
public data class PackageVersionV2(
val added: Long,
override val added: Long,
val file: FileV1,
val src: FileV2? = null,
val manifest: ManifestV2,
@@ -97,6 +98,8 @@ public data class PackageVersionV2(
val whatsNew: LocalizedTextV2 = emptyMap(),
) : PackageVersion {
override val versionCode: Long = manifest.versionCode
override val versionName: String = manifest.versionName
override val size: Long? = file.size
override val signer: SignerV2? = manifest.signer
override val packageManifest: PackageManifest = manifest
override val hasKnownVulnerability: Boolean

29
libs/sharedTest/build.gradle
View File

@@ -1,29 +0,0 @@
plugins {
id 'kotlin-android'
id 'com.android.library'
}
// not really an Android library, but index is not publishing for JVM at the moment
android {
namespace 'org.fdroid.test'
compileSdk libs.versions.compileSdk.get().toInteger()
defaultConfig {
minSdkVersion 21
}
kotlinOptions {
jvmTarget = '17'
}
compileOptions {
sourceCompatibility JavaVersion.VERSION_17
targetCompatibility JavaVersion.VERSION_17
}
}
dependencies {
implementation project(":libs:download")
implementation project(":libs:index")
implementation libs.kotlin.test
implementation libs.kotlinx.serialization.json
}

32
libs/sharedTest/build.gradle.kts Normal file
View File

@@ -0,0 +1,32 @@
plugins {
alias(libs.plugins.jetbrains.kotlin.android)
alias(libs.plugins.android.library)
}
// not really an Android library, but index is not publishing for JVM at the moment
android {
namespace = "org.fdroid.test"
@Suppress("ktlint:standard:chain-method-continuation")
compileSdk = libs.versions.compileSdk.get().toInt()
defaultConfig {
minSdk = 21
}
compileOptions {
sourceCompatibility = JavaVersion.VERSION_17
targetCompatibility = JavaVersion.VERSION_17
}
}
kotlin {
compilerOptions {
jvmTarget = org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_17
}
}
dependencies {
implementation(project(":libs:download"))
implementation(project(":libs:index"))
implementation(libs.kotlin.test)
implementation(libs.kotlinx.serialization.json)
}

9
libs/sharedTest/src/main/assets/diff-empty-max/1337.json
View File

@@ -320,15 +320,18 @@
"metadata": {
"name": {
"en-US": "App3",
"en": "en "
"en": "en ",
"zh-CN": "自由软件仓库"
},
"summary": {
"en-US": "App3 summary",
"en": "en "
"en": "en ",
"ja": "这个仓库中的"
},
"description": {
"en-US": "App3 description",
"en": "en "
"en": "en ",
"ko-KR": "切始终是从"
},
"added": 1234567890,
"lastUpdated": 9223372036854775807,

9
libs/sharedTest/src/main/assets/diff-empty-max/23.json
View File

@@ -596,15 +596,18 @@
"metadata": {
"name": {
"en-US": "App3",
"en": "en "
"en": "en ",
"zh-CN": "自由软件仓库"
},
"summary": {
"en-US": "App3 summary",
"en": "en "
"en": "en ",
"ja": "这个仓库中的"
},
"description": {
"en-US": "App3 description",
"en": "en "
"en": "en ",
"ko-KR": "切始终是从"
},
"added": 1234567890,
"lastUpdated": 9223372036854775807,

9
libs/sharedTest/src/main/assets/diff-empty-max/42.json
View File

@@ -593,15 +593,18 @@
"metadata": {
"name": {
"en-US": "App3",
"en": "en "
"en": "en ",
"zh-CN": "自由软件仓库"
},
"summary": {
"en-US": "App3 summary",
"en": "en "
"en": "en ",
"ja": "这个仓库中的"
},
"description": {
"en-US": "App3 description",
"en": "en "
"en": "en ",
"ko-KR": "切始终是从"
},
"added": 1234567890,
"lastUpdated": 9223372036854775807,

9
libs/sharedTest/src/main/assets/index-max-v2.json
View File

@@ -603,15 +603,18 @@
"metadata": {
"name": {
"en-US": "App3",
"en": "en "
"en": "en ",
"zh-CN": "自由软件仓库"
},
"summary": {
"en-US": "App3 summary",
"en": "en "
"en": "en ",
"ja": "这个仓库中的"
},
"description": {
"en-US": "App3 description",
"en": "en "
"en": "en ",
"ko-KR": "切始终是从"
},
"added": 1234567890,
"lastUpdated": 9223372036854775807,

3
libs/sharedTest/src/main/kotlin/org/fdroid/test/TestDataV2.kt
View File

@@ -1055,14 +1055,17 @@ object TestDataMaxV2 {
name = mapOf(
LOCALE to "App3",
"en" to "en ",
"zh-CN" to "自由软件仓库",
),
summary = mapOf(
LOCALE to "App3 summary",
"en" to "en ",
"ja" to "这个仓库中的",
),
description = mapOf(
LOCALE to "App3 description",
"en" to "en ",
"ko-KR" to "切始终是从",
),
added = 1234567890,
lastUpdated = Long.MAX_VALUE,

1
settings.gradle
View File

@@ -7,6 +7,7 @@ pluginManagement {
}
include ':app'
include ':libs:core'
include ':libs:sharedTest'
include ':libs:download'
include ':libs:index'