[db] Add plumping for fetching and adding a new repo

This commit is contained in:
Torsten Grote
2023-06-23 18:14:30 -03:00
parent 937fe99062
commit f3e8a0a45b
18 changed files with 1640 additions and 1 deletions

View File

@@ -1608,6 +1608,16 @@
<sha256 value="100f793ba27f8b4e4204edb46171ebf36e54e0f94cfc02527fea07a0bb1fceb7" origin="Generated by Gradle because artifact wasn't signed"/>
</artifact>
</component>
<component group="app.cash.turbine" name="turbine" version="1.0.0">
<artifact name="Turbine-metadata.jar">
<sha256 value="a65b488d928152a91b52d1a0f783fd150ff291998f103db6592f26393d701365" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="app.cash.turbine" name="turbine-jvm" version="1.0.0">
<artifact name="Turbine-jvm.jar">
<sha256 value="41984dffaf069b84b814fa787f740bf7ebdc90bf89f69d05bc4660c69874b49e" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="backport-util-concurrent" name="backport-util-concurrent" version="3.1">
<artifact name="backport-util-concurrent-3.1.jar">
<sha256 value="f5759b7fcdfc83a525a036deedcbd32e5b536b625ebc282426f16ca137eb5902" origin="Generated by Gradle because artifact wasn't signed"/>
@@ -7258,6 +7268,11 @@
<sha256 value="965aeb2bedff369819bdde1bf7a0b3b89b8247dd69c88b86375d76163bb8c397" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.jetbrains" name="annotations" version="23.0.0">
<artifact name="annotations-23.0.0.jar">
<sha256 value="7b0f19724082cbfcbc66e5abea2b9bc92cf08a1ea11e191933ed43801eb3cd05" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.jetbrains" name="markdown-jvm" version="0.2.1">
<artifact name="markdown-jvm-0.2.1.jar">
<pgp value="e62231331bca7e1f292c9b88c1b12a5d99c0729d"/>
@@ -8459,6 +8474,11 @@
<sha256 value="042a1cd1ac976cdcfe5eb63f1d8e0b0b892c9248e15a69c8cfba495d546ea52a" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.jetbrains.kotlin" name="kotlin-stdlib" version="1.8.22">
<artifact name="kotlin-stdlib-1.8.22.jar">
<sha256 value="03a5c3965cc37051128e64e46748e394b6bd4c97fa81c6de6fc72bfd44e3421b" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.jetbrains.kotlin" name="kotlin-stdlib-common" version="1.3.71">
<artifact name="kotlin-stdlib-common-1.3.71.jar">
<sha256 value="974f8a9b7bfce3d730a86efe0eab219a72621e8530f91e30c89f400ba98092ec" origin="Generated by Gradle"/>
@@ -8544,6 +8564,11 @@
<sha256 value="6a44c9ecc9d7754d9e943fb1e3588c74d4a3f1785be51074f49d6c5723682a73" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.jetbrains.kotlin" name="kotlin-stdlib-common" version="1.8.22">
<artifact name="kotlin-stdlib-common-1.8.22.jar">
<sha256 value="d0c2365e2437ef70f34586d50f055743f79716bcfe65e4bc7239cdd2669ef7c5" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.jetbrains.kotlin" name="kotlin-stdlib-jdk7" version="1.3.71">
<artifact name="kotlin-stdlib-jdk7-1.3.71.jar">
<sha256 value="b046a5ef54c7006db852e48e547aaff525a9e7a0a5909ffe5fe2c966c1a3a72e" origin="Generated by Gradle"/>
@@ -8619,6 +8644,11 @@
<sha256 value="af1ec40c3b951afdcc0c2a0173c7b81763c5281c2d5bafbf0a8544a24c5dcc0c" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.jetbrains.kotlin" name="kotlin-stdlib-jdk7" version="1.8.22">
<artifact name="kotlin-stdlib-jdk7-1.8.22.jar">
<sha256 value="055f5cb24287fa106100995a7b47ab92126b81e832e875f5fa2cf0bd55693d0b" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.jetbrains.kotlin" name="kotlin-stdlib-jdk8" version="1.3.71">
<artifact name="kotlin-stdlib-jdk8-1.3.71.jar">
<sha256 value="a22192ac779ba8eff09d07084ae503e8be9e7c8ca1cb2b511ff8af4c68d83d66" origin="Generated by Gradle"/>
@@ -8694,6 +8724,11 @@
<sha256 value="e398b67977622718bf18ff99b739c7d9da060f33fb458a2e25203221c16af010" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.jetbrains.kotlin" name="kotlin-stdlib-jdk8" version="1.8.22">
<artifact name="kotlin-stdlib-jdk8-1.8.22.jar">
<sha256 value="4198b0eaf090a4f25b6f7e5a59581f4314ba8c9f6cd1d13ee9d348e65ed8f707" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.jetbrains.kotlin" name="kotlin-test" version="1.6.10">
<artifact name="kotlin-test-1.6.10.jar">
<sha256 value="b891453cafbf961532d2ba0fb8969e40b0f7c168c9a2fc6a8cdf7c1b0577a36a" origin="Generated by Gradle"/>
@@ -8984,6 +9019,16 @@
<sha256 value="3a0c2743b59f575affaf47e35c73ee22a2caecb6594a8948356ecd3e88564cc3" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.jetbrains.kotlinx" name="atomicfu" version="0.20.2">
<artifact name="atomicfu-metadata-0.20.2-all.jar">
<sha256 value="2a91f10b826d3febc47c2b664e268615757c311114239787f5dc37481fdec441" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.jetbrains.kotlinx" name="atomicfu" version="0.21.0">
<artifact name="atomicfu-metadata-0.21.0-all.jar">
<sha256 value="f31e5b2fd52eb73a0303cffe5c63c8390ca7cf459a02dcf06faa383dd4bb88d3" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.jetbrains.kotlinx" name="atomicfu-linuxx64" version="0.16.3">
<artifact name="atomicfu-cinterop-interop.klib">
<sha256 value="7a28fdb9f981f2802ed1a8a630c81ccaa0abe75b62cb14c4e323400866c5b286" origin="Generated by Gradle because artifact wasn't signed"/>
@@ -9033,6 +9078,16 @@
<sha256 value="3fdc0eed5bc4b83ee9622774520a2db25470370eacd1581cac1e37704f095b00" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.jetbrains.kotlinx" name="kotlinx-coroutines-android" version="1.7.1">
<artifact name="kotlinx-coroutines-android-1.7.1.jar">
<sha256 value="107313760c18f8da174e8d8103504a468e806e88f7b55a84bd1c0eaeea118e9a" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.jetbrains.kotlinx" name="kotlinx-coroutines-android" version="1.7.2">
<artifact name="kotlinx-coroutines-android-1.7.2.jar">
<sha256 value="2f98dd47d37e1f5ba23afae0e4b794c10e88cd7f69d7f5977ee6cd105b1aae0c" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.jetbrains.kotlinx" name="kotlinx-coroutines-core" version="1.3.4">
<artifact name="kotlinx-coroutines-core-1.3.4.jar">
<sha256 value="17bec6112d93f5fcb11c27ecc8a14b48e30a5689ccf42c95025b89ba2210c28f" origin="Generated by Gradle"/>
@@ -9079,6 +9134,16 @@
<sha256 value="ae24d84fd33c5aaba1564b168142d0f210b0e257e0a0077e32616acc59e67fb7" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.jetbrains.kotlinx" name="kotlinx-coroutines-core" version="1.7.1">
<artifact name="kotlinx-coroutines-core-metadata-1.7.1-all.jar">
<sha256 value="b7d5370ed0e54952003b13595a67b97ced3f873d919f5ddfbe50fcd499cbf0a7" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.jetbrains.kotlinx" name="kotlinx-coroutines-core" version="1.7.2">
<artifact name="kotlinx-coroutines-core-metadata-1.7.2-all.jar">
<sha256 value="de6c8ff818562097650230e333f98483eb855392f37228ade554f636b2fd2201" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.jetbrains.kotlinx" name="kotlinx-coroutines-core-jvm" version="1.5.0">
<artifact name="kotlinx-coroutines-core-jvm-1.5.0.jar">
<pgp value="e7dc75fc24fb3c8dfe8086ad3d5839a2262cbbfb"/>
@@ -9115,6 +9180,16 @@
<sha256 value="c24c8bb27bb320c4a93871501a7e5e0c61607638907b197aef675513d4c820be" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.jetbrains.kotlinx" name="kotlinx-coroutines-core-jvm" version="1.7.1">
<artifact name="kotlinx-coroutines-core-jvm-1.7.1.jar">
<sha256 value="7496cffdd3eb10109acdda1c3212f6ac7815789e09380dc9e2ccdec496dba3fc" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.jetbrains.kotlinx" name="kotlinx-coroutines-core-jvm" version="1.7.2">
<artifact name="kotlinx-coroutines-core-jvm-1.7.2.jar">
<sha256 value="754f3a0f11dcaa019bf17c4f67836b371eb4f37693ac8e7c3badf5c6b1308d7c" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.jetbrains.kotlinx" name="kotlinx-coroutines-core-linuxx64" version="1.5.2-native-mt">
<artifact name="kotlinx-coroutines-core.klib">
<sha256 value="bd5e3639e853cc1e8dca4db696ab29f95924453da93db96e20f30979e9463ef2" origin="Generated by Gradle because artifact wasn't signed"/>
@@ -9140,6 +9215,16 @@
<sha256 value="88c64b8eea3eb90597d2fb0fd30f3cf782fbcdad06312e5665a618f070f02119" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.jetbrains.kotlinx" name="kotlinx-coroutines-jdk8" version="1.7.1">
<artifact name="kotlinx-coroutines-jdk8-1.7.1.jar">
<sha256 value="b96f7145ba69b48f4940534cb8e19c7e159c45443e27c36821626148ba1710f2" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.jetbrains.kotlinx" name="kotlinx-coroutines-jdk8" version="1.7.2">
<artifact name="kotlinx-coroutines-jdk8-1.7.2.jar">
<sha256 value="c9a8f7f4df8d46e74a5efe0c587ecc4450b30ddf0230aa8d5ab8301ed640029e" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.jetbrains.kotlinx" name="kotlinx-coroutines-slf4j" version="1.6.0">
<artifact name="kotlinx-coroutines-slf4j-1.6.0.jar">
<sha256 value="d8a019ae7be13992867be62d97e6993afc141a956010f5f704d569f5e9677167" origin="Generated by Gradle"/>
@@ -9155,16 +9240,41 @@
<sha256 value="e8e47a5d7ff57e89e096a409486308ad58a2f392724145973f11f679e7d11d23" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.jetbrains.kotlinx" name="kotlinx-coroutines-slf4j" version="1.7.1">
<artifact name="kotlinx-coroutines-slf4j-1.7.1.jar">
<sha256 value="fbb40ca7e55a78017feac897180be64f562a2eb781c6a3f6c9bb2ff88d92223f" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.jetbrains.kotlinx" name="kotlinx-coroutines-slf4j" version="1.7.2">
<artifact name="kotlinx-coroutines-slf4j-1.7.2.jar">
<sha256 value="412a0c1129f76f8dcbcc2c1c7de6483b3f6fcd70636ba5e2a3d523342910162c" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.jetbrains.kotlinx" name="kotlinx-coroutines-test" version="1.5.2">
<artifact name="kotlinx-coroutines-test-1.5.2.jar">
<sha256 value="7f5ed01d76ecb37fc714bc0e0850b81cf753b0e968495a8db0efcd20fcb5ee60" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.jetbrains.kotlinx" name="kotlinx-coroutines-test" version="1.7.2">
<artifact name="kotlinx-coroutines-test-metadata-1.7.2-all.jar">
<sha256 value="2ddfad185b7cc7e3a2e4707c916525d37ce62cf6572ad5fcac2b9f4ba70e010a" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.jetbrains.kotlinx" name="kotlinx-coroutines-test-jvm" version="1.6.0">
<artifact name="kotlinx-coroutines-test-jvm-1.6.0.jar">
<sha256 value="bef600516dbb41b237a883609a4f7468c2ed06d437ac13082ff4471723b4e88f" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.jetbrains.kotlinx" name="kotlinx-coroutines-test-jvm" version="1.7.1">
<artifact name="kotlinx-coroutines-test-jvm-1.7.1.jar">
<sha256 value="bf88e8f0a8d88033d961d93435a132caf88fe35cee266bf71c48689eceddc72d" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.jetbrains.kotlinx" name="kotlinx-coroutines-test-jvm" version="1.7.2">
<artifact name="kotlinx-coroutines-test-jvm-1.7.2.jar">
<sha256 value="3ce50964c0f22c41be725f9294d8142c90d6e908f450bff6be38435396299a41" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.jetbrains.kotlinx" name="kotlinx-datetime-linuxx64" version="0.3.1">
<artifact name="kotlinx-datetime-cinterop-date.klib">
<sha256 value="b7a3f64fb70f8931cab585eb58ebc6471da23712e964f243e5299b51a0876ffb" origin="Generated by Gradle because artifact wasn't signed"/>

View File

@@ -87,6 +87,9 @@ dependencies {
testImplementation 'androidx.arch.core:core-testing:2.1.0'
testImplementation 'org.robolectric:robolectric:4.8.1'
testImplementation 'commons-io:commons-io:2.6'
testImplementation 'ch.qos.logback:logback-classic:1.2.11'
testImplementation 'org.jetbrains.kotlinx:kotlinx-coroutines-test:1.7.2'
testImplementation 'app.cash.turbine:turbine:1.0.0'
androidTestImplementation project(":libs:sharedTest")
androidTestImplementation 'io.mockk:mockk-android:1.12.3' // 1.12.4 has strange error

View File

@@ -152,6 +152,21 @@ internal class RepositoryDaoTest : DbTest() {
assertEquals(0, repoDao.getLiveRepositories().getOrFail().size)
}
@Test
fun testGetRepositoryByCert() {
val cert = getRandomString()
// insert repo and (required) preferences
val repo1 = getRandomRepo().toCoreRepository(version = 42L, certificate = cert)
val repoId = repoDao.insertOrReplace(repo1)
val repositoryPreferences = RepositoryPreferences(repoId, 3)
repoDao.insert(repositoryPreferences)
// repo is returned when querying for right cert
assertEquals(repo1.copy(repoId = repoId), repoDao.getRepository(cert)?.repository)
// nothing is returned when querying for non-existent cert
assertNull(repoDao.getRepository("foo bar"))
}
@Test
fun testSetRepositoryEnabled() {
// repo is enabled by default

View File

@@ -8,6 +8,7 @@ import androidx.room.RoomDatabase
import androidx.room.TypeConverters
import org.fdroid.LocaleChooser.getBestLocale
import java.util.Locale
import java.util.concurrent.Callable
@Database(
// When bumping this version, please make sure to add one (or more) migration(s) below!
@@ -101,6 +102,11 @@ public interface FDroidDatabase {
*/
public fun runInTransaction(body: Runnable)
/**
* Like [runInTransaction], but can return something.
*/
public fun <V> runInTransaction(body: Callable<V>): V
/**
* Removes all apps and associated data (such as versions) from all repositories.
* The repository data and app preferences are kept as-is.

View File

@@ -93,6 +93,7 @@ public data class Repository internal constructor(
/**
* Used to create a minimal version of a [Repository].
*/
@JvmOverloads
public constructor(
repoId: Long,
address: String,
@@ -102,6 +103,8 @@ public data class Repository internal constructor(
version: Long,
weight: Int,
lastUpdated: Long,
username: String? = null,
password: String? = null,
) : this(
repository = CoreRepository(
repoId = repoId,
@@ -121,6 +124,8 @@ public data class Repository internal constructor(
repoId = repoId,
weight = weight,
lastUpdated = lastUpdated,
username = username,
password = password,
)
)
@@ -401,3 +406,16 @@ private fun validateCertificate(certificate: String?) {
certificate.chunked(2).find { it.toIntOrNull(16) == null } == null
) { "Invalid certificate: $certificate" }
}
/**
* A reduced version of [Repository] used to add new repositories.
*/
public data class NewRepository(
val name: LocalizedTextV2,
val icon: LocalizedFileV2,
val address: String,
val formatVersion: IndexFormatVersion?,
val certificate: String,
val username: String? = null,
val password: String? = null,
)

View File

@@ -28,11 +28,17 @@ public interface RepositoryDao {
*/
public fun insert(initialRepo: InitialRepository): Long
/**
* Inserts a new repository into the database.
*/
public fun insert(newRepository: NewRepository): Long
/**
* Inserts an empty [Repository] for an initial update.
*
* @return the [Repository.repoId] of the inserted repo.
*/
@Deprecated("Use insert instead")
public fun insertEmptyRepo(
address: String,
username: String? = null,
@@ -145,6 +151,32 @@ internal interface RepositoryDaoInt : RepositoryDao {
}
@Transaction
override fun insert(newRepository: NewRepository): Long {
val repo = CoreRepository(
name = newRepository.name,
icon = newRepository.icon,
address = newRepository.address,
timestamp = -1,
version = null,
formatVersion = newRepository.formatVersion,
maxAge = null,
certificate = newRepository.certificate,
)
val repoId = insertOrReplace(repo)
val currentMaxWeight = getMaxRepositoryWeight()
val repositoryPreferences = RepositoryPreferences(
repoId = repoId,
weight = currentMaxWeight + 1,
lastUpdated = null,
username = newRepository.username,
password = newRepository.password,
)
insert(repositoryPreferences)
return repoId
}
@Transaction
@Deprecated("Use insert instead")
override fun insertEmptyRepo(
address: String,
username: String?,
@@ -191,6 +223,10 @@ internal interface RepositoryDaoInt : RepositoryDao {
@Query("SELECT * FROM ${CoreRepository.TABLE} WHERE repoId = :repoId")
override fun getRepository(repoId: Long): Repository?
@Transaction
@Query("SELECT * FROM ${CoreRepository.TABLE} WHERE certificate = :certificate COLLATE NOCASE")
fun getRepository(certificate: String): Repository?
@Transaction
@Query("SELECT * FROM ${CoreRepository.TABLE}")
override fun getRepositories(): List<Repository>

View File

@@ -1,5 +1,8 @@
package org.fdroid.index
import android.content.Context
import androidx.annotation.AnyThread
import androidx.annotation.UiThread
import androidx.annotation.WorkerThread
import androidx.lifecycle.LiveData
import androidx.lifecycle.asLiveData
@@ -13,23 +16,45 @@ import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import org.fdroid.database.FDroidDatabase
import org.fdroid.database.Repository
import org.fdroid.download.DownloaderFactory
import org.fdroid.download.HttpManager
import org.fdroid.repo.AddRepoState
import org.fdroid.repo.RepoAdder
import java.io.File
import java.net.Proxy
import java.util.concurrent.CountDownLatch
import kotlin.coroutines.CoroutineContext
@OptIn(DelicateCoroutinesApi::class)
public class RepoManager @JvmOverloads constructor(
context: Context,
db: FDroidDatabase,
downloaderFactory: DownloaderFactory,
httpManager: HttpManager,
private val coroutineContext: CoroutineContext = Dispatchers.IO,
) {
private val repositoryDao = db.getRepositoryDao()
private val tempFileProvider = TempFileProvider {
File.createTempFile("dl-", "", context.cacheDir)
}
private val repoAdder = RepoAdder(
context = context,
db = db,
tempFileProvider = tempFileProvider,
downloaderFactory = downloaderFactory,
httpManager = httpManager,
coroutineContext = coroutineContext,
)
private val _repositoriesState: MutableStateFlow<List<Repository>> =
MutableStateFlow(emptyList())
public val repositoriesState: StateFlow<List<Repository>> = _repositoriesState.asStateFlow()
public val liveRepositories: LiveData<List<Repository>> = _repositoriesState.asLiveData()
public val addRepoState: StateFlow<AddRepoState> = repoAdder.addRepoState.asStateFlow()
public val liveAddRepoState: LiveData<AddRepoState> = repoAdder.addRepoState.asLiveData()
/**
* Used internally as a mechanism to wait until repositories are loaded from the DB.
* This happens quite fast and the load is triggered at construction time.
@@ -92,4 +117,42 @@ public class RepoManager @JvmOverloads constructor(
}
}
/**
* Fetches a preview of the repository at the given [url]
* with the intention of possibly adding it to the database.
* Progress can be observed via [addRepoState] or [liveAddRepoState].
*/
@AnyThread
@JvmOverloads
public fun fetchRepositoryPreview(
url: String,
username: String? = null,
password: String? = null,
proxy: Proxy? = null,
) {
repoAdder.fetchRepository(url, username, password, proxy)
}
/**
* When [addRepoState] is in [org.fdroid.repo.Fetched],
* you can call this to actually add the repo to the DB.
* @throws IllegalStateException if [addRepoState] is currently in any other state.
*/
@AnyThread
public fun addFetchedRepository() {
GlobalScope.launch(coroutineContext) {
repoAdder.addFetchedRepository()
}
}
/**
* Aborts the process of fetching a [Repository] preview,
* e.g. when the user leaves the UI flow or wants to cancel the preview process.
* Note that this won't work after [addFetchedRepository] has already been called.
*/
@UiThread
public fun abortAddingRepository() {
repoAdder.abortAddingRepo()
}
}

View File

@@ -0,0 +1,281 @@
package org.fdroid.repo
import android.content.Context
import android.net.Uri
import android.os.Build.VERSION.SDK_INT
import android.os.UserManager
import android.os.UserManager.DISALLOW_INSTALL_UNKNOWN_SOURCES
import android.os.UserManager.DISALLOW_INSTALL_UNKNOWN_SOURCES_GLOBALLY
import androidx.annotation.VisibleForTesting
import androidx.annotation.WorkerThread
import androidx.core.content.ContextCompat.getSystemService
import kotlinx.coroutines.DelicateCoroutinesApi
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.Job
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.launch
import mu.KotlinLogging
import org.fdroid.database.AppOverviewItem
import org.fdroid.database.FDroidDatabase
import org.fdroid.database.MinimalApp
import org.fdroid.database.NewRepository
import org.fdroid.database.Repository
import org.fdroid.database.RepositoryDaoInt
import org.fdroid.download.DownloaderFactory
import org.fdroid.download.HttpManager
import org.fdroid.download.NotFoundException
import org.fdroid.index.IndexFormatVersion
import org.fdroid.index.SigningException
import org.fdroid.index.TempFileProvider
import org.fdroid.repo.AddRepoError.ErrorType.INVALID_FINGERPRINT
import org.fdroid.repo.AddRepoError.ErrorType.INVALID_INDEX
import org.fdroid.repo.AddRepoError.ErrorType.IO_ERROR
import org.fdroid.repo.AddRepoError.ErrorType.UNKNOWN_SOURCES_DISALLOWED
import java.io.IOException
import java.net.Proxy
import kotlin.coroutines.CoroutineContext
internal const val REPO_ID = 0L
public sealed class AddRepoState
public object None : AddRepoState()
public class Fetching(
public val repo: Repository?,
public val apps: List<MinimalApp>,
public val fetchResult: FetchResult?,
/**
* true if fetching is complete.
*/
public val done: Boolean = false,
) : AddRepoState() {
/**
* true if the repository can be added (be it as new [Repository] or new mirror).
*/
public val canAdd: Boolean = repo != null &&
(fetchResult != null && fetchResult !is FetchResult.IsExistingRepository)
}
public object Adding : AddRepoState()
public class Added(
public val repo: Repository,
) : AddRepoState()
public data class AddRepoError(
public val errorType: ErrorType,
public val exception: Exception? = null,
) : AddRepoState() {
public enum class ErrorType {
UNKNOWN_SOURCES_DISALLOWED,
INVALID_FINGERPRINT,
INVALID_INDEX,
IO_ERROR,
}
}
public sealed class FetchResult {
public object IsNewRepository : FetchResult()
public data class IsNewMirror(
internal val existingRepoId: Long,
internal val newMirrorUrl: String,
) : FetchResult()
public object IsExistingRepository : FetchResult()
}
@OptIn(DelicateCoroutinesApi::class)
internal class RepoAdder(
private val context: Context,
private val db: FDroidDatabase,
private val tempFileProvider: TempFileProvider,
private val downloaderFactory: DownloaderFactory,
private val httpManager: HttpManager,
private val repoUriGetter: RepoUriGetter = RepoUriGetter,
private val coroutineContext: CoroutineContext = Dispatchers.IO,
) {
private val log = KotlinLogging.logger {}
private val repositoryDao = db.getRepositoryDao() as RepositoryDaoInt
internal val addRepoState: MutableStateFlow<AddRepoState> = MutableStateFlow(None)
private var fetchJob: Job? = null
internal fun fetchRepository(url: String, username: String?, password: String?, proxy: Proxy?) {
fetchJob = GlobalScope.launch(coroutineContext) {
fetchRepositoryInt(url, username, password, proxy)
}
}
@WorkerThread
@VisibleForTesting
internal suspend fun fetchRepositoryInt(
url: String,
username: String? = null,
password: String? = null,
proxy: Proxy? = null,
) {
if (hasDisallowInstallUnknownSources(context)) {
addRepoState.value = AddRepoError(UNKNOWN_SOURCES_DISALLOWED)
return
}
// get repo url and fingerprint
val nUri = repoUriGetter.getUri(url)
log.info("Parsed URI: $nUri")
// some plumping to receive the repo preview
var receivedRepo: Repository? = null
val apps = ArrayList<AppOverviewItem>()
var fetchResult: FetchResult? = null
val receiver = object : RepoPreviewReceiver {
override fun onRepoReceived(repo: Repository) {
receivedRepo = repo
fetchResult = getFetchResult(nUri.uri.toString(), repo)
addRepoState.value = Fetching(receivedRepo, apps.toList(), fetchResult)
}
override fun onAppReceived(app: AppOverviewItem) {
apps.add(app)
addRepoState.value = Fetching(receivedRepo, apps.toList(), fetchResult)
}
}
// set a state early, so the ui can show progress animation
addRepoState.value = Fetching(receivedRepo, apps, fetchResult)
// try fetching repo with v2 format first and fallback to v1
try {
try {
val repo = getTempRepo(nUri.uri, IndexFormatVersion.TWO, username, password)
val repoFetcher =
RepoV2Fetcher(tempFileProvider, downloaderFactory, httpManager, proxy)
repoFetcher.fetchRepo(nUri.uri, repo, receiver, nUri.fingerprint)
} catch (e: NotFoundException) {
log.warn(e) { "Did not find v2 repo, trying v1 now." }
// try to fetch v1 repo
val repo = getTempRepo(nUri.uri, IndexFormatVersion.ONE, username, password)
val repoFetcher = RepoV1Fetcher(tempFileProvider, downloaderFactory)
repoFetcher.fetchRepo(nUri.uri, repo, receiver, nUri.fingerprint)
}
} catch (e: SigningException) {
log.error(e) { "Error verifying repo with given fingerprint." }
addRepoState.value = AddRepoError(INVALID_FINGERPRINT, e)
return
} catch (e: IOException) {
log.error(e) { "Error fetching repo." }
addRepoState.value = AddRepoError(IO_ERROR, e)
return
}
// set final result
val finalRepo = receivedRepo
if (finalRepo == null) {
addRepoState.value = AddRepoError(INVALID_INDEX)
} else {
addRepoState.value = Fetching(finalRepo, apps, fetchResult, done = true)
}
}
private fun getFetchResult(url: String, repo: Repository): FetchResult {
val cert = repo.certificate ?: error("Certificate was null")
val existingRepo = repositoryDao.getRepository(cert)
return if (existingRepo == null) {
FetchResult.IsNewRepository
} else {
val existingMirror = if (existingRepo.address.trimEnd('/') == url) {
url
} else {
existingRepo.mirrors.find { it.url.trimEnd('/') == url }
?: existingRepo.userMirrors.find { it.trimEnd('/') == url }
}
if (existingMirror == null) {
FetchResult.IsNewMirror(existingRepo.repoId, url)
} else {
FetchResult.IsExistingRepository
}
}
}
@WorkerThread
internal fun addFetchedRepository() {
// prevent double calls (e.g. caused by double tapping a UI button)
if (addRepoState.compareAndSet(Adding, Adding)) return
// cancel fetch preview job, so it stops emitting new states
fetchJob?.cancel()
// get current state before changing it
val state = (addRepoState.value as? Fetching)
?: throw IllegalStateException("Unexpected state: ${addRepoState.value}")
addRepoState.value = Adding
val repo = state.repo ?: throw IllegalStateException("No repo: ${addRepoState.value}")
val fetchResult = state.fetchResult
?: throw IllegalStateException("No fetchResult: ${addRepoState.value}")
val modifiedRepo: Repository = when (fetchResult) {
is FetchResult.IsExistingRepository -> error("Unexpected result: $fetchResult")
is FetchResult.IsNewRepository -> {
// reset the timestamp of the actual repo,
// so a following repo update will pick this up
val newRepo = NewRepository(
name = repo.repository.name,
icon = repo.repository.icon ?: emptyMap(),
address = repo.address,
formatVersion = repo.formatVersion,
certificate = repo.certificate ?: error("Repo had no certificate"),
username = repo.username,
password = repo.password,
)
val repoId = repositoryDao.insert(newRepo)
repositoryDao.getRepository(repoId) ?: error("New repository not found in DB")
}
is FetchResult.IsNewMirror -> {
val repoId = fetchResult.existingRepoId
db.runInTransaction<Repository> {
val existingRepo = repositoryDao.getRepository(repoId)
?: error("No repo with $repoId")
val userMirrors = existingRepo.userMirrors.toMutableList().apply {
add(fetchResult.newMirrorUrl)
}
repositoryDao.updateUserMirrors(repoId, userMirrors)
existingRepo
}
}
}
addRepoState.value = Added(modifiedRepo)
}
internal fun abortAddingRepo() {
addRepoState.value = None
fetchJob?.cancel()
}
private fun hasDisallowInstallUnknownSources(context: Context): Boolean {
val userManager = getSystemService(context, UserManager::class.java)
?: error("No UserManager available.")
return if (SDK_INT < 29) userManager.hasUserRestriction(DISALLOW_INSTALL_UNKNOWN_SOURCES)
else userManager.hasUserRestriction(DISALLOW_INSTALL_UNKNOWN_SOURCES) ||
userManager.hasUserRestriction(DISALLOW_INSTALL_UNKNOWN_SOURCES_GLOBALLY)
}
private fun getTempRepo(
uri: Uri,
indexFormatVersion: IndexFormatVersion,
username: String?,
password: String?,
) = Repository(
repoId = REPO_ID,
address = uri.toString(),
timestamp = -1L,
formatVersion = indexFormatVersion,
certificate = null,
version = 0L,
weight = 0,
lastUpdated = -1L,
username = username,
password = password,
)
}

View File

@@ -0,0 +1,23 @@
package org.fdroid.repo
import android.net.Uri
import org.fdroid.database.AppOverviewItem
import org.fdroid.database.Repository
import org.fdroid.download.NotFoundException
import org.fdroid.index.SigningException
import java.io.IOException
internal fun interface RepoFetcher {
@Throws(IOException::class, SigningException::class, NotFoundException::class)
suspend fun fetchRepo(
uri: Uri,
repo: Repository,
receiver: RepoPreviewReceiver,
fingerprint: String?,
)
}
internal interface RepoPreviewReceiver {
fun onRepoReceived(repo: Repository)
fun onAppReceived(app: AppOverviewItem)
}

View File

@@ -0,0 +1,48 @@
package org.fdroid.repo
import android.net.Uri
import org.fdroid.database.Repository
internal object RepoUriGetter {
fun getUri(url: String): NormalizedUri {
val uri = Uri.parse(url)
val fingerprint = uri.getQueryParameter("fingerprint")?.lowercase()
val pathSegments = uri.pathSegments
val normalizedUri = uri.buildUpon().apply {
clearQuery() // removes fingerprint and other query params
fragment("") // remove # hash fragment
if (pathSegments.size >= 2 &&
pathSegments[pathSegments.lastIndex - 1] == "fdroid" &&
pathSegments.last() == "repo"
) {
// path already is /fdroid/repo, use as is
} else if (pathSegments.lastOrNull() == "repo") {
// path already ends in /repo, use as is
} else if (pathSegments.size >= 1 && pathSegments.last() == "fdroid") {
// path is /fdroid with missing /repo, so add that
appendPath("repo")
} else {
// path is missing /fdroid/repo, so add it
appendPath("fdroid")
appendPath("repo")
}
}.build().let { newUri ->
// hacky way to remove trailing slash
val path = newUri.path
if (path != null && path.endsWith('/')) {
newUri.buildUpon().path(path.trimEnd('/')).build()
} else {
newUri
}
}
return NormalizedUri(normalizedUri, fingerprint)
}
/**
* A class for normalizing the [Repository] URI and holding an optional fingerprint.
*/
data class NormalizedUri(val uri: Uri, val fingerprint: String?)
}

View File

@@ -0,0 +1,66 @@
package org.fdroid.repo
import android.content.res.Resources
import android.net.Uri
import androidx.core.os.ConfigurationCompat.getLocales
import androidx.core.os.LocaleListCompat
import org.fdroid.database.Repository
import org.fdroid.download.DownloaderFactory
import org.fdroid.index.IndexConverter
import org.fdroid.index.IndexFormatVersion
import org.fdroid.index.IndexParser
import org.fdroid.index.SigningException
import org.fdroid.index.TempFileProvider
import org.fdroid.index.parseV1
import org.fdroid.index.v1.IndexV1Verifier
import org.fdroid.index.v1.SIGNED_FILE_NAME
import org.fdroid.index.v2.FileV2
internal class RepoV1Fetcher(
private val tempFileProvider: TempFileProvider,
private val downloaderFactory: DownloaderFactory,
) : RepoFetcher {
private val locales: LocaleListCompat = getLocales(Resources.getSystem().configuration)
@Throws(SigningException::class)
override suspend fun fetchRepo(
uri: Uri,
repo: Repository,
receiver: RepoPreviewReceiver,
fingerprint: String?,
) {
// download and verify index-v1.jar
val indexFile = tempFileProvider.createTempFile()
val entryDownloader = downloaderFactory.create(
repo = repo,
uri = uri.buildUpon().appendPath(SIGNED_FILE_NAME).build(),
indexFile = FileV2.fromPath("/$SIGNED_FILE_NAME"),
destFile = indexFile,
)
val (cert, indexV1) = try {
entryDownloader.download()
val verifier = IndexV1Verifier(indexFile, null, fingerprint)
verifier.getStreamAndVerify { inputStream ->
IndexParser.parseV1(inputStream)
}
} finally {
indexFile.delete()
}
val version = indexV1.repo.version
val indexV2 = IndexConverter().toIndexV2(indexV1)
val receivedRepo = RepoV2StreamReceiver.getRepository(
repo = indexV2.repo,
version = version.toLong(),
formatVersion = IndexFormatVersion.ONE,
certificate = cert,
username = repo.username,
password = repo.password,
)
receiver.onRepoReceived(receivedRepo)
indexV2.packages.forEach { (packageName, packageV2) ->
val app = RepoV2StreamReceiver.getAppOverViewItem(packageName, packageV2, locales)
receiver.onAppReceived(app)
}
}
}

View File

@@ -0,0 +1,74 @@
package org.fdroid.repo
import android.net.Uri
import mu.KotlinLogging
import org.fdroid.database.Repository
import org.fdroid.download.DownloadRequest
import org.fdroid.download.DownloaderFactory
import org.fdroid.download.HttpManager
import org.fdroid.download.getDigestInputStream
import org.fdroid.index.IndexParser
import org.fdroid.index.SigningException
import org.fdroid.index.TempFileProvider
import org.fdroid.index.parseEntry
import org.fdroid.index.v2.EntryVerifier
import org.fdroid.index.v2.FileV2
import org.fdroid.index.v2.IndexV2FullStreamProcessor
import org.fdroid.index.v2.SIGNED_FILE_NAME
import java.net.Proxy
internal class RepoV2Fetcher(
private val tempFileProvider: TempFileProvider,
private val downloaderFactory: DownloaderFactory,
private val httpManager: HttpManager,
private val proxy: Proxy? = null,
) : RepoFetcher {
private val log = KotlinLogging.logger {}
@Throws(SigningException::class)
override suspend fun fetchRepo(
uri: Uri,
repo: Repository,
receiver: RepoPreviewReceiver,
fingerprint: String?,
) {
// download and verify entry
val entryFile = tempFileProvider.createTempFile()
val entryDownloader = downloaderFactory.create(
repo = repo,
uri = uri.buildUpon().appendPath(SIGNED_FILE_NAME).build(),
indexFile = FileV2.fromPath("/$SIGNED_FILE_NAME"),
destFile = entryFile,
)
val (cert, entry) = try {
entryDownloader.download()
val verifier = EntryVerifier(entryFile, null, fingerprint)
verifier.getStreamAndVerify { inputStream ->
IndexParser.parseEntry(inputStream)
}
} finally {
entryFile.delete()
}
log.info { "Downloaded entry, now streaming index..." }
// stream index
val indexRequest = DownloadRequest(
indexFile = FileV2.fromPath(entry.index.name.trimStart('/')),
mirrors = repo.getMirrors(),
proxy = proxy,
username = repo.username,
password = repo.password,
)
val streamReceiver = RepoV2StreamReceiver(receiver, repo.username, repo.password)
val streamProcessor = IndexV2FullStreamProcessor(streamReceiver, cert)
val digestInputStream = httpManager.getDigestInputStream(indexRequest)
digestInputStream.use { inputStream ->
streamProcessor.process(entry.version, inputStream) { }
}
val hexDigest = digestInputStream.getDigestHex()
if (!hexDigest.equals(entry.index.sha256, ignoreCase = true)) {
throw SigningException("Invalid ${entry.index.name} hash: $hexDigest")
}
}
}

View File

@@ -0,0 +1,102 @@
package org.fdroid.repo
import android.content.res.Resources
import androidx.core.os.ConfigurationCompat.getLocales
import androidx.core.os.LocaleListCompat
import org.fdroid.LocaleChooser.getBestLocale
import org.fdroid.database.AppOverviewItem
import org.fdroid.database.LocalizedIcon
import org.fdroid.database.Repository
import org.fdroid.database.RepositoryPreferences
import org.fdroid.database.toCoreRepository
import org.fdroid.database.toRepoAntiFeatures
import org.fdroid.database.toRepoCategories
import org.fdroid.database.toRepoReleaseChannel
import org.fdroid.index.IndexFormatVersion
import org.fdroid.index.v2.IndexV2StreamReceiver
import org.fdroid.index.v2.PackageV2
import org.fdroid.index.v2.RepoV2
internal open class RepoV2StreamReceiver(
private val receiver: RepoPreviewReceiver,
private val username: String?,
private val password: String?,
) : IndexV2StreamReceiver {
companion object {
fun getRepository(
repo: RepoV2,
version: Long,
formatVersion: IndexFormatVersion,
certificate: String?,
username: String?,
password: String?,
) = Repository(
repository = repo.toCoreRepository(
version = version,
formatVersion = formatVersion,
certificate = certificate
),
mirrors = emptyList(),
antiFeatures = repo.antiFeatures.toRepoAntiFeatures(REPO_ID),
categories = repo.categories.toRepoCategories(REPO_ID),
releaseChannels = repo.releaseChannels.toRepoReleaseChannel(REPO_ID),
preferences = RepositoryPreferences(
repoId = REPO_ID,
weight = 0,
enabled = true,
username = username,
password = password,
),
)
fun getAppOverViewItem(
packageName: String,
p: PackageV2,
locales: LocaleListCompat,
) = AppOverviewItem(
repoId = REPO_ID,
packageName = packageName,
added = p.metadata.added,
lastUpdated = p.metadata.lastUpdated,
name = p.metadata.name.getBestLocale(locales),
summary = p.metadata.summary.getBestLocale(locales),
antiFeatures = p.versions.values.lastOrNull()?.antiFeatures,
localizedIcon = p.metadata.icon?.map { (locale, file) ->
LocalizedIcon(
repoId = 0L,
packageName = packageName,
type = "icon",
locale = locale,
name = file.name,
sha256 = file.sha256,
size = file.size,
ipfsCidV1 = file.ipfsCidV1,
)
},
)
}
private val locales: LocaleListCompat = getLocales(Resources.getSystem().configuration)
override fun receive(repo: RepoV2, version: Long, certificate: String) {
receiver.onRepoReceived(
getRepository(
repo = repo,
version = version,
formatVersion = IndexFormatVersion.TWO,
certificate = certificate,
username = username,
password = password,
)
)
}
override fun receive(packageName: String, p: PackageV2) {
receiver.onAppReceived(getAppOverViewItem(packageName, p, locales))
}
override fun onStreamEnded() {
}
}

View File

@@ -0,0 +1,32 @@
package org.fdroid.download
import android.net.Uri
import org.fdroid.IndexFile
import org.fdroid.database.Repository
import java.io.File
internal class TestDownloadFactory(private val httpManager: HttpManager) : DownloaderFactory() {
override fun create(
repo: Repository,
uri: Uri,
indexFile: IndexFile,
destFile: File,
): Downloader = HttpDownloaderV2(
httpManager = httpManager,
request = DownloadRequest(indexFile, repo.getMirrors()),
destFile = destFile
)
override fun create(
repo: Repository,
mirrors: List<Mirror>,
uri: Uri,
indexFile: IndexFile,
destFile: File,
tryFirst: Mirror?,
): Downloader = HttpDownloaderV2(
httpManager = httpManager,
request = DownloadRequest(indexFile, repo.getMirrors()),
destFile = destFile
)
}

View File

@@ -0,0 +1,135 @@
package org.fdroid.repo
import androidx.core.os.LocaleListCompat
import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.platform.app.InstrumentationRegistry
import app.cash.turbine.test
import io.mockk.every
import io.mockk.mockk
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.test.runTest
import org.fdroid.database.FDroidDatabase
import org.fdroid.database.NewRepository
import org.fdroid.database.Repository
import org.fdroid.database.RepositoryDaoInt
import org.fdroid.download.HttpManager
import org.fdroid.download.TestDownloadFactory
import org.fdroid.index.TempFileProvider
import org.fdroid.repo.AddRepoError.ErrorType.INVALID_FINGERPRINT
import org.junit.Assume.assumeTrue
import org.junit.Before
import org.junit.Rule
import org.junit.Test
import org.junit.rules.TemporaryFolder
import org.junit.runner.RunWith
import kotlin.test.assertEquals
import kotlin.test.assertNotNull
import kotlin.test.assertNull
import kotlin.test.assertTrue
@RunWith(AndroidJUnit4::class)
internal class RepoAdderIntegrationTest {
@get:Rule
var folder: TemporaryFolder = TemporaryFolder()
private val context = InstrumentationRegistry.getInstrumentation().targetContext
private val db = mockk<FDroidDatabase>()
private val repoDao = mockk<RepositoryDaoInt>()
private val tempFileProvider = TempFileProvider { folder.newFile() }
private val httpManager = HttpManager("test")
private val downloaderFactory = TestDownloadFactory(httpManager)
private val repoAdder: RepoAdder
init {
every { db.getRepositoryDao() } returns repoDao
repoAdder = RepoAdder(context, db, tempFileProvider, downloaderFactory, httpManager)
}
@Before
fun optIn() {
assumeTrue(false) // don't run integration tests with real repos all the time
}
@Test
fun testFedilabV1() = runTest {
// repo not in DB
every { repoDao.getRepository(any<String>()) } returns null
repoAdder.fetchRepository(
url = "https://fdroid.fedilab.app/repo/" +
"?fingerprint=11F0A69910A4280E2CD3CCC3146337D006BE539B18E1A9FEACE15FF757A94FEB",
username = null,
password = null,
proxy = null
)
repoAdder.addRepoState.test {
assertEquals(None, awaitItem())
val firstFetching = awaitItem()
assertTrue(firstFetching is Fetching)
assertNull(firstFetching.repo)
assertTrue(firstFetching.apps.isEmpty())
val secondFetching = awaitItem()
assertTrue(secondFetching is Fetching, "$secondFetching")
val repo = secondFetching.repo
assertNotNull(repo)
assertEquals("https://fdroid.fedilab.app/repo", repo.address)
println(repo.getName(LocaleListCompat.getDefault()) ?: "null")
println(repo.certificate)
assertEquals(1, (awaitItem() as Fetching).apps.size)
assertEquals(2, (awaitItem() as Fetching).apps.size)
assertEquals(3, (awaitItem() as Fetching).apps.size)
assertTrue(awaitItem() is Fetching)
}
val state = repoAdder.addRepoState.value
assertTrue(state is Fetching, state.toString())
assertTrue(state.apps.isNotEmpty())
state.apps.forEach { app ->
println(" ${app.packageName} ${app.summary}")
}
val newRepo: Repository = mockk()
every { repoDao.insert(any<NewRepository>()) } returns 42L
every { repoDao.getRepository(42L) } returns newRepo
repoAdder.addFetchedRepository()
repoAdder.addRepoState.test {
val addedState = awaitItem()
assertTrue(addedState is Added, addedState.toString())
assertEquals(newRepo, addedState.repo)
}
}
@Test
fun testIzzy() = runBlocking {
// repo not in DB
every { repoDao.getRepository(any<String>()) } returns null
repoAdder.fetchRepositoryInt(
url = "https://apt.izzysoft.de/fdroid/repo" +
"?fingerprint=3BF0D6ABFEAE2F401707B6D966BE743BF0EEE49C2561B9BA39073711F628937A"
)
val state = repoAdder.addRepoState.value
assertTrue(state is Fetching, state.toString())
assertTrue(state.apps.isNotEmpty())
println(state.repo?.getName(LocaleListCompat.getDefault()) ?: "null")
println(state.repo?.certificate)
state.apps.forEach { app ->
println(" ${app.packageName} ${app.summary}")
}
}
@Test
fun testIzzyWrongFingerprint() = runBlocking {
repoAdder.fetchRepositoryInt("https://apt.izzysoft.de/fdroid/repo?fingerprint=fooBar")
val state = repoAdder.addRepoState.value
assertTrue(state is AddRepoError, state.toString())
assertEquals(state.errorType, INVALID_FINGERPRINT, state.errorType.name)
}
}

View File

@@ -0,0 +1,561 @@
package org.fdroid.repo
import android.content.Context
import android.content.res.AssetManager
import android.net.Uri
import android.os.UserManager
import android.os.UserManager.DISALLOW_INSTALL_UNKNOWN_SOURCES
import androidx.core.os.LocaleListCompat
import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.platform.app.InstrumentationRegistry
import app.cash.turbine.test
import io.mockk.Runs
import io.mockk.coEvery
import io.mockk.every
import io.mockk.just
import io.mockk.mockk
import io.mockk.mockkStatic
import io.mockk.slot
import kotlinx.coroutines.test.runTest
import org.fdroid.LocaleChooser.getBestLocale
import org.fdroid.database.FDroidDatabase
import org.fdroid.database.Mirror
import org.fdroid.database.NewRepository
import org.fdroid.database.Repository
import org.fdroid.database.RepositoryDaoInt
import org.fdroid.database.RepositoryPreferences
import org.fdroid.database.toCoreRepository
import org.fdroid.download.Downloader
import org.fdroid.download.DownloaderFactory
import org.fdroid.download.HttpManager
import org.fdroid.download.NotFoundException
import org.fdroid.download.getDigestInputStream
import org.fdroid.fdroid.DigestInputStream
import org.fdroid.index.IndexFormatVersion
import org.fdroid.index.SigningException
import org.fdroid.index.TempFileProvider
import org.fdroid.repo.AddRepoError.ErrorType.INVALID_FINGERPRINT
import org.fdroid.repo.AddRepoError.ErrorType.IO_ERROR
import org.fdroid.repo.AddRepoError.ErrorType.UNKNOWN_SOURCES_DISALLOWED
import org.fdroid.test.TestDataMinV2
import org.fdroid.test.TestUtils.decodeHex
import org.fdroid.test.VerifierConstants
import org.junit.Rule
import org.junit.Test
import org.junit.rules.TemporaryFolder
import org.junit.runner.RunWith
import java.io.ByteArrayInputStream
import java.io.IOException
import java.security.MessageDigest
import java.util.concurrent.Callable
import kotlin.test.assertEquals
import kotlin.test.assertFailsWith
import kotlin.test.assertFalse
import kotlin.test.assertIs
import kotlin.test.assertNull
import kotlin.test.assertTrue
import kotlin.test.fail
@RunWith(AndroidJUnit4::class)
internal class RepoAdderTest {
@get:Rule
var folder: TemporaryFolder = TemporaryFolder()
private val context = InstrumentationRegistry.getInstrumentation().targetContext
private val db = mockk<FDroidDatabase>()
private val repoDao = mockk<RepositoryDaoInt>()
private val tempFileProvider = mockk<TempFileProvider>()
private val httpManager = mockk<HttpManager>()
private val downloaderFactory = mockk<DownloaderFactory>()
private val downloader = mockk<Downloader>()
private val digest = mockk<MessageDigest>()
private val repoAdder: RepoAdder
private val assets: AssetManager = context.resources.assets
private val localeList = LocaleListCompat.getDefault()
init {
every { db.getRepositoryDao() } returns repoDao
every { digest.update(any(), any(), any()) } just Runs
every { digest.update(any<Byte>()) } just Runs
mockkStatic("org.fdroid.download.HttpManagerKt")
repoAdder = RepoAdder(context, db, tempFileProvider, downloaderFactory, httpManager)
}
@Test
fun testDisallowInstallUnknownSources() = runTest {
val context = mockk<Context>()
val userManager = mockk<UserManager>()
val repoAdder = RepoAdder(context, db, tempFileProvider, downloaderFactory, httpManager)
every { context.getSystemService("user") } returns userManager
every {
userManager.hasUserRestriction(DISALLOW_INSTALL_UNKNOWN_SOURCES)
} returns true
repoAdder.fetchRepositoryInt("https://example.org/repo/")
repoAdder.addRepoState.test {
val state1 = awaitItem()
assertTrue(state1 is AddRepoError)
assertEquals(UNKNOWN_SOURCES_DISALLOWED, state1.errorType)
}
}
@Test
fun testAddingMinRepo() = runTest {
val url = "https://example.org/repo/"
val repoName = TestDataMinV2.repo.name.getBestLocale(localeList)
expectDownloadOfMinRepo(url)
// repo not in DB
every { repoDao.getRepository(any<String>()) } returns null
expectMinRepoPreview(repoName, FetchResult.IsNewRepository) {
repoAdder.fetchRepository(
url = url,
username = null,
password = null,
proxy = null,
)
}
val newRepo: Repository = mockk()
every {
repoDao.insert(match<NewRepository> {
// Note that we are not using the url the user used to add the repo,
// but what the repo tells us to use
it.address == TestDataMinV2.repo.address &&
it.formatVersion == IndexFormatVersion.TWO &&
it.name.getBestLocale(localeList) == repoName
})
} returns 42L
every { repoDao.getRepository(42L) } returns newRepo
repoAdder.addRepoState.test {
assertIs<Fetching>(awaitItem()) // still Fetching from last call
repoAdder.addFetchedRepository()
assertIs<Adding>(awaitItem()) // now moved to Adding
val addedState = awaitItem()
assertIs<Added>(addedState)
assertEquals(newRepo, addedState.repo)
}
}
@Test
fun testAddingMirrorForMinRepo() = runTest {
val url = "https://example.com/repo/"
val repoName = TestDataMinV2.repo.name.getBestLocale(localeList)
expectDownloadOfMinRepo(url)
// repo is already in the DB
val existingRepo = Repository(
repository = TestDataMinV2.repo.toCoreRepository(
repoId = 42L,
version = 1337L,
formatVersion = IndexFormatVersion.TWO,
certificate = "cert",
),
mirrors = emptyList(),
antiFeatures = emptyList(),
categories = emptyList(),
releaseChannels = emptyList(),
preferences = RepositoryPreferences(42L, 23),
)
every { repoDao.getRepository(any<String>()) } returns existingRepo
expectMinRepoPreview(repoName, FetchResult.IsNewMirror(42L, url.trimEnd('/'))) {
repoAdder.fetchRepository(
url = url,
username = null,
password = null,
proxy = null
)
}
val transactionSlot = slot<Callable<Repository>>()
every {
db.runInTransaction(capture(transactionSlot))
} answers { transactionSlot.captured.call() }
every { repoDao.getRepository(42L) } returns existingRepo
every { repoDao.updateUserMirrors(42L, listOf(url.trimEnd('/'))) } just Runs
repoAdder.addRepoState.test {
assertIs<Fetching>(awaitItem()) // still Fetching from last call
repoAdder.addFetchedRepository()
assertIs<Adding>(awaitItem()) // now moved to Adding
val addedState = awaitItem()
assertTrue(addedState is Added, addedState.toString())
assertEquals(existingRepo, addedState.repo)
}
}
@Test
fun testRepoAlreadyExists() = runTest {
val url = "https://min-v1.org/repo/"
// repo is already in the DB
val existingRepo = Repository(
repository = TestDataMinV2.repo.toCoreRepository(
repoId = REPO_ID,
version = 1337L,
formatVersion = IndexFormatVersion.TWO,
certificate = "cert",
).copy(address = url), // change address, because TestDataMinV2 misses /repo
mirrors = emptyList(),
antiFeatures = emptyList(),
categories = emptyList(),
releaseChannels = emptyList(),
preferences = RepositoryPreferences(REPO_ID, 23),
)
testRepoAlreadyExists(url, existingRepo)
}
@Test
fun testRepoAlreadyExistsWithMirror() = runTest {
val url = "https://example.org/repo/"
// repo is already in the DB
val existingRepo = Repository(
repository = TestDataMinV2.repo.toCoreRepository(
repoId = REPO_ID,
version = 1337L,
formatVersion = IndexFormatVersion.TWO,
certificate = "cert",
),
mirrors = listOf(Mirror(REPO_ID, url), Mirror(REPO_ID, "http://example.org")),
antiFeatures = emptyList(),
categories = emptyList(),
releaseChannels = emptyList(),
preferences = RepositoryPreferences(REPO_ID, 23),
)
testRepoAlreadyExists(url, existingRepo)
}
@Test
fun testRepoAlreadyExistsWithFingerprint() = runTest {
val url = "https://example.org/repo?fingerprint=${VerifierConstants.FINGERPRINT}"
// repo is already in the DB
val existingRepo = Repository(
repository = TestDataMinV2.repo.toCoreRepository(
repoId = REPO_ID,
version = 1337L,
formatVersion = IndexFormatVersion.TWO,
certificate = VerifierConstants.CERTIFICATE,
),
mirrors = listOf(Mirror(REPO_ID, "https://example.org/repo/")),
antiFeatures = emptyList(),
categories = emptyList(),
releaseChannels = emptyList(),
preferences = RepositoryPreferences(REPO_ID, 23),
)
testRepoAlreadyExists(url, existingRepo, "https://example.org/repo")
}
@Test
fun testRepoAlreadyExistsWithFingerprintTrailingSlash() = runTest {
val url = "https://example.org/repo/?fingerprint=${VerifierConstants.FINGERPRINT}"
// repo is already in the DB
val existingRepo = Repository(
repository = TestDataMinV2.repo.toCoreRepository(
repoId = REPO_ID,
version = 1337L,
formatVersion = IndexFormatVersion.TWO,
certificate = VerifierConstants.CERTIFICATE,
),
mirrors = listOf(
Mirror(REPO_ID, "https://example.org/repo"),
Mirror(REPO_ID, "http://example.org"),
),
antiFeatures = emptyList(),
categories = emptyList(),
releaseChannels = emptyList(),
preferences = RepositoryPreferences(REPO_ID, 23),
)
testRepoAlreadyExists(url, existingRepo, "https://example.org/repo")
}
@Test
fun testRepoAlreadyExistsUserMirror() = runTest {
val url = "https://example.net/repo/"
// repo is already in the DB
val existingRepo = Repository(
repository = TestDataMinV2.repo.toCoreRepository(
repoId = REPO_ID,
version = 1337L,
formatVersion = IndexFormatVersion.TWO,
certificate = "cert",
),
mirrors = emptyList(),
antiFeatures = emptyList(),
categories = emptyList(),
releaseChannels = emptyList(),
preferences = RepositoryPreferences(
repoId = REPO_ID,
weight = 23,
userMirrors = listOf(url, "http://example.org"),
),
)
testRepoAlreadyExists(url, existingRepo)
}
private suspend fun testRepoAlreadyExists(
url: String,
existingRepo: Repository,
downloadUrl: String = url,
) {
val repoName = TestDataMinV2.repo.name.getBestLocale(localeList)
expectDownloadOfMinRepo(downloadUrl)
// repo is already in the DB
every { repoDao.getRepository(any<String>()) } returns existingRepo
expectMinRepoPreview(repoName, FetchResult.IsExistingRepository, canAdd = false) {
repoAdder.fetchRepository(
url = url,
username = null,
password = null,
proxy = null
)
}
assertFailsWith<IllegalStateException> {
repoAdder.addFetchedRepository()
}
}
@Test
fun testDownloadEntryThrowsIoException() = runTest {
val url = "https://example.org/repo"
val jarFile = folder.newFile()
every { tempFileProvider.createTempFile() } returns jarFile
every {
downloaderFactory.create(
repo = match { it.address == url && it.formatVersion == IndexFormatVersion.TWO },
uri = Uri.parse("$url/entry.jar"),
indexFile = any(),
destFile = jarFile,
)
} returns downloader
every { downloader.download() } throws IOException()
repoAdder.addRepoState.test {
assertIs<None>(awaitItem())
repoAdder.fetchRepository(
url = url,
username = null,
password = null,
proxy = null
)
val state1 = awaitItem()
assertIs<Fetching>(state1)
assertNull(state1.repo)
assertTrue(state1.apps.isEmpty())
assertFalse(state1.canAdd)
val state2 = awaitItem()
assertTrue(state2 is AddRepoError, "$state2")
assertEquals(IO_ERROR, state2.errorType)
}
}
@Test
fun testWrongFingerprint() = runTest {
val url = "https://example.org/repo/"
val jarFile = folder.newFile()
every { tempFileProvider.createTempFile() } returns jarFile
every {
downloaderFactory.create(
repo = match {
it.address == url.trimEnd('/') && it.formatVersion == IndexFormatVersion.TWO
},
uri = Uri.parse(url + "entry.jar"),
indexFile = any(),
destFile = jarFile,
)
} returns downloader
// not actually thrown by the downloader, but mocking verifier is harder
every { downloader.download() } throws SigningException("boom!")
repoAdder.addRepoState.test {
assertIs<None>(awaitItem())
repoAdder.fetchRepository(
url = url,
username = null,
password = null,
proxy = null
)
val state1 = awaitItem()
assertIs<Fetching>(state1)
assertNull(state1.repo)
assertTrue(state1.apps.isEmpty())
assertFalse(state1.canAdd)
val state2 = awaitItem()
assertTrue(state2 is AddRepoError, "$state2")
assertEquals(INVALID_FINGERPRINT, state2.errorType)
}
}
@Test
fun testFallbackToV1() = runTest {
val url = "https://example.org/repo/"
val urlTrimmed = "https://example.org/repo"
val jarFile = folder.newFile()
every { tempFileProvider.createTempFile() } returns jarFile
every {
downloaderFactory.create(
repo = match {
it.address == urlTrimmed && it.formatVersion == IndexFormatVersion.TWO
},
uri = Uri.parse("$urlTrimmed/entry.jar"),
indexFile = any(),
destFile = jarFile,
)
} returns downloader
every { downloader.download() } throws NotFoundException()
val downloaderV1 = mockk<Downloader>()
every {
downloaderFactory.create(
repo = match {
it.address == urlTrimmed && it.formatVersion == IndexFormatVersion.ONE
},
uri = Uri.parse("$urlTrimmed/index-v1.jar"),
indexFile = any(),
destFile = jarFile,
)
} returns downloaderV1
every { downloaderV1.download() } answers {
jarFile.outputStream().use { outputStream ->
assets.open("testy.at.or.at_index-v1.jar").use { inputStream ->
inputStream.copyTo(outputStream)
}
}
}
every { repoDao.getRepository(any<String>()) } returns null
repoAdder.addRepoState.test {
assertIs<None>(awaitItem())
repoAdder.fetchRepository(
url = url,
username = null,
password = null,
proxy = null
)
val state1 = awaitItem()
assertIs<Fetching>(state1)
assertNull(state1.repo)
assertTrue(state1.apps.isEmpty())
assertFalse(state1.canAdd)
for (i in 0..64) assertIs<Fetching>(awaitItem())
}
val addRepoState = repoAdder.addRepoState.value
assertIs<Fetching>(addRepoState)
assertTrue(addRepoState.canAdd)
assertEquals(63, addRepoState.apps.size)
}
private fun expectDownloadOfMinRepo(url: String) {
val urlTrimmed = url.trimEnd('/')
val jarFile = folder.newFile()
val indexFile = assets.open("index-min-v2.json")
val index = indexFile.use { it.readBytes() }
val indexStream = DigestInputStream(ByteArrayInputStream(index), digest)
every { tempFileProvider.createTempFile() } returns jarFile
every {
downloaderFactory.create(
repo = match {
it.address == urlTrimmed && it.formatVersion == IndexFormatVersion.TWO
},
uri = Uri.parse("$urlTrimmed/entry.jar"),
indexFile = any(),
destFile = jarFile,
)
} returns downloader
every { downloader.download() } answers {
jarFile.outputStream().use { outputStream ->
assets.open("diff-empty-min/entry.jar").use { inputStream ->
inputStream.copyTo(outputStream)
}
}
}
coEvery {
httpManager.getDigestInputStream(match {
it.indexFile.name == "../index-min-v2.json" &&
it.mirrors.size == 1 && it.mirrors[0].baseUrl == urlTrimmed
})
} returns indexStream
every {
digest.digest() // sha256 from entry.json
} returns "851ecda085ed53adab25f761a9dbf4c09d59e5bff9c9d5530814d56445ae30f2".decodeHex()
}
private suspend fun expectMinRepoPreview(
repoName: String?,
fetchResult: FetchResult,
canAdd: Boolean = true,
block: suspend () -> Unit = {},
) {
repoAdder.addRepoState.test {
assertIs<None>(awaitItem())
block()
// FIXME executing this block may emit items too fast, so we might miss one
// causing flaky tests.
val state1 = awaitItem()
assertIs<Fetching>(state1)
assertNull(state1.repo)
assertEquals(emptyList(), state1.apps)
assertFalse(state1.canAdd)
assertFalse(state1.done)
val state2 = awaitItem()
assertIs<Fetching>(state2)
val repo = state2.repo ?: fail()
assertEquals(TestDataMinV2.repo.address, repo.address)
assertEquals(repoName, repo.getName(localeList))
val result = state2.fetchResult ?: fail()
assertEquals(fetchResult, result)
assertTrue(state2.apps.isEmpty())
assertEquals(canAdd, state2.canAdd)
assertFalse(state2.done)
val state3 = awaitItem()
assertIs<Fetching>(state3)
assertEquals(TestDataMinV2.packages.size, state3.apps.size)
assertEquals(TestDataMinV2.packageName, state3.apps[0].packageName)
assertEquals(canAdd, state3.canAdd)
assertFalse(state3.done)
val state4 = awaitItem()
assertIs<Fetching>(state4)
assertEquals(canAdd, state4.canAdd)
assertTrue(state4.done)
}
}
}

View File

@@ -0,0 +1,59 @@
package org.fdroid.repo
import androidx.test.ext.junit.runners.AndroidJUnit4
import org.junit.Test
import org.junit.runner.RunWith
import kotlin.test.assertEquals
import kotlin.test.assertNull
@RunWith(AndroidJUnit4::class)
internal class RepoUriGetterTest {
@Test
fun testTrailingSlash() {
val uri = RepoUriGetter.getUri("http://example.org/fdroid/repo/")
assertEquals("http://example.org/fdroid/repo", uri.uri.toString())
assertNull(uri.fingerprint)
}
@Test
fun testWithoutTrailingSlash() {
val uri = RepoUriGetter.getUri("http://example.org/fdroid/repo")
assertEquals("http://example.org/fdroid/repo", uri.uri.toString())
assertNull(uri.fingerprint)
}
@Test
fun testFingerprint() {
val uri = RepoUriGetter.getUri("https://example.org/repo?fingerprint=foobar&test=42")
assertEquals("https://example.org/repo", uri.uri.toString())
assertEquals("foobar", uri.fingerprint)
}
@Test
fun testHash() {
val uri = RepoUriGetter.getUri("https://example.org/repo?fingerprint=foobar&test=42#hash")
assertEquals("https://example.org/repo", uri.uri.toString())
assertEquals("foobar", uri.fingerprint)
}
@Test
fun testAddFdroidSlashRepo() {
val uri = RepoUriGetter.getUri("https://example.org")
assertEquals("https://example.org/fdroid/repo", uri.uri.toString())
assertNull(uri.fingerprint)
}
@Test
fun testLeaveSingleRepo() {
val uri = RepoUriGetter.getUri("https://example.org/repo")
assertEquals("https://example.org/repo", uri.uri.toString())
assertNull(uri.fingerprint)
}
@Test
fun testAddsMissingRepo() {
val uri = RepoUriGetter.getUri("https://example.org/fdroid/")
assertEquals("https://example.org/fdroid/repo", uri.uri.toString())
assertNull(uri.fingerprint)
}
}

View File

@@ -8,6 +8,13 @@ import kotlin.random.Random
object TestUtils {
fun String.decodeHex(): ByteArray {
check(length % 2 == 0) { "Must have an even length" }
return chunked(2)
.map { it.toInt(16).toByte() }
.toByteArray()
}
private val charPool: List<Char> = ('a'..'z') + ('A'..'Z') + ('0'..'9')
fun getRandomString(length: Int = Random.nextInt(1, 128)): String = (1..length)