diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index dac40edd8..7faf6174f 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -1608,6 +1608,16 @@ + + + + + + + + + + @@ -7258,6 +7268,11 @@ + + + + + @@ -8459,6 +8474,11 @@ + + + + + @@ -8544,6 +8564,11 @@ + + + + + @@ -8619,6 +8644,11 @@ + + + + + @@ -8694,6 +8724,11 @@ + + + + + @@ -8984,6 +9019,16 @@ + + + + + + + + + + @@ -9033,6 +9078,16 @@ + + + + + + + + + + @@ -9079,6 +9134,16 @@ + + + + + + + + + + @@ -9115,6 +9180,16 @@ + + + + + + + + + + @@ -9140,6 +9215,16 @@ + + + + + + + + + + @@ -9155,16 +9240,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/libs/database/build.gradle b/libs/database/build.gradle index 1a0f25202..e9858e657 100644 --- a/libs/database/build.gradle +++ b/libs/database/build.gradle @@ -87,6 +87,9 @@ dependencies { testImplementation 'androidx.arch.core:core-testing:2.1.0' testImplementation 'org.robolectric:robolectric:4.8.1' testImplementation 'commons-io:commons-io:2.6' + testImplementation 'ch.qos.logback:logback-classic:1.2.11' + testImplementation 'org.jetbrains.kotlinx:kotlinx-coroutines-test:1.7.2' + testImplementation 'app.cash.turbine:turbine:1.0.0' androidTestImplementation project(":libs:sharedTest") androidTestImplementation 'io.mockk:mockk-android:1.12.3' // 1.12.4 has strange error diff --git a/libs/database/src/dbTest/java/org/fdroid/database/RepositoryDaoTest.kt b/libs/database/src/dbTest/java/org/fdroid/database/RepositoryDaoTest.kt index 08a8dd7d3..f041fd2a7 100644 --- a/libs/database/src/dbTest/java/org/fdroid/database/RepositoryDaoTest.kt +++ b/libs/database/src/dbTest/java/org/fdroid/database/RepositoryDaoTest.kt @@ -152,6 +152,21 @@ internal class RepositoryDaoTest : DbTest() { assertEquals(0, repoDao.getLiveRepositories().getOrFail().size) } + @Test + fun testGetRepositoryByCert() { + val cert = getRandomString() + // insert repo and (required) preferences + val repo1 = getRandomRepo().toCoreRepository(version = 42L, certificate = cert) + val repoId = repoDao.insertOrReplace(repo1) + val repositoryPreferences = RepositoryPreferences(repoId, 3) + repoDao.insert(repositoryPreferences) + + // repo is returned when querying for right cert + assertEquals(repo1.copy(repoId = repoId), repoDao.getRepository(cert)?.repository) + // nothing is returned when querying for non-existent cert + assertNull(repoDao.getRepository("foo bar")) + } + @Test fun testSetRepositoryEnabled() { // repo is enabled by default diff --git a/libs/database/src/main/java/org/fdroid/database/FDroidDatabase.kt b/libs/database/src/main/java/org/fdroid/database/FDroidDatabase.kt index 5631d6a56..be228fb03 100644 --- a/libs/database/src/main/java/org/fdroid/database/FDroidDatabase.kt +++ b/libs/database/src/main/java/org/fdroid/database/FDroidDatabase.kt @@ -8,6 +8,7 @@ import androidx.room.RoomDatabase import androidx.room.TypeConverters import org.fdroid.LocaleChooser.getBestLocale import java.util.Locale +import java.util.concurrent.Callable @Database( // When bumping this version, please make sure to add one (or more) migration(s) below! @@ -101,6 +102,11 @@ public interface FDroidDatabase { */ public fun runInTransaction(body: Runnable) + /** + * Like [runInTransaction], but can return something. + */ + public fun runInTransaction(body: Callable): V + /** * Removes all apps and associated data (such as versions) from all repositories. * The repository data and app preferences are kept as-is. diff --git a/libs/database/src/main/java/org/fdroid/database/Repository.kt b/libs/database/src/main/java/org/fdroid/database/Repository.kt index f4c1b24da..668bf9905 100644 --- a/libs/database/src/main/java/org/fdroid/database/Repository.kt +++ b/libs/database/src/main/java/org/fdroid/database/Repository.kt @@ -93,6 +93,7 @@ public data class Repository internal constructor( /** * Used to create a minimal version of a [Repository]. */ + @JvmOverloads public constructor( repoId: Long, address: String, @@ -102,6 +103,8 @@ public data class Repository internal constructor( version: Long, weight: Int, lastUpdated: Long, + username: String? = null, + password: String? = null, ) : this( repository = CoreRepository( repoId = repoId, @@ -121,6 +124,8 @@ public data class Repository internal constructor( repoId = repoId, weight = weight, lastUpdated = lastUpdated, + username = username, + password = password, ) ) @@ -401,3 +406,16 @@ private fun validateCertificate(certificate: String?) { certificate.chunked(2).find { it.toIntOrNull(16) == null } == null ) { "Invalid certificate: $certificate" } } + +/** + * A reduced version of [Repository] used to add new repositories. + */ +public data class NewRepository( + val name: LocalizedTextV2, + val icon: LocalizedFileV2, + val address: String, + val formatVersion: IndexFormatVersion?, + val certificate: String, + val username: String? = null, + val password: String? = null, +) diff --git a/libs/database/src/main/java/org/fdroid/database/RepositoryDao.kt b/libs/database/src/main/java/org/fdroid/database/RepositoryDao.kt index a85559624..7fd2f2ed9 100644 --- a/libs/database/src/main/java/org/fdroid/database/RepositoryDao.kt +++ b/libs/database/src/main/java/org/fdroid/database/RepositoryDao.kt @@ -28,11 +28,17 @@ public interface RepositoryDao { */ public fun insert(initialRepo: InitialRepository): Long + /** + * Inserts a new repository into the database. + */ + public fun insert(newRepository: NewRepository): Long + /** * Inserts an empty [Repository] for an initial update. * * @return the [Repository.repoId] of the inserted repo. */ + @Deprecated("Use insert instead") public fun insertEmptyRepo( address: String, username: String? = null, @@ -145,6 +151,32 @@ internal interface RepositoryDaoInt : RepositoryDao { } @Transaction + override fun insert(newRepository: NewRepository): Long { + val repo = CoreRepository( + name = newRepository.name, + icon = newRepository.icon, + address = newRepository.address, + timestamp = -1, + version = null, + formatVersion = newRepository.formatVersion, + maxAge = null, + certificate = newRepository.certificate, + ) + val repoId = insertOrReplace(repo) + val currentMaxWeight = getMaxRepositoryWeight() + val repositoryPreferences = RepositoryPreferences( + repoId = repoId, + weight = currentMaxWeight + 1, + lastUpdated = null, + username = newRepository.username, + password = newRepository.password, + ) + insert(repositoryPreferences) + return repoId + } + + @Transaction + @Deprecated("Use insert instead") override fun insertEmptyRepo( address: String, username: String?, @@ -191,6 +223,10 @@ internal interface RepositoryDaoInt : RepositoryDao { @Query("SELECT * FROM ${CoreRepository.TABLE} WHERE repoId = :repoId") override fun getRepository(repoId: Long): Repository? + @Transaction + @Query("SELECT * FROM ${CoreRepository.TABLE} WHERE certificate = :certificate COLLATE NOCASE") + fun getRepository(certificate: String): Repository? + @Transaction @Query("SELECT * FROM ${CoreRepository.TABLE}") override fun getRepositories(): List diff --git a/libs/database/src/main/java/org/fdroid/index/RepoManager.kt b/libs/database/src/main/java/org/fdroid/index/RepoManager.kt index 635057c57..f6f6b0513 100644 --- a/libs/database/src/main/java/org/fdroid/index/RepoManager.kt +++ b/libs/database/src/main/java/org/fdroid/index/RepoManager.kt @@ -1,5 +1,8 @@ package org.fdroid.index +import android.content.Context +import androidx.annotation.AnyThread +import androidx.annotation.UiThread import androidx.annotation.WorkerThread import androidx.lifecycle.LiveData import androidx.lifecycle.asLiveData @@ -13,23 +16,45 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import org.fdroid.database.FDroidDatabase import org.fdroid.database.Repository +import org.fdroid.download.DownloaderFactory +import org.fdroid.download.HttpManager +import org.fdroid.repo.AddRepoState +import org.fdroid.repo.RepoAdder +import java.io.File +import java.net.Proxy import java.util.concurrent.CountDownLatch import kotlin.coroutines.CoroutineContext @OptIn(DelicateCoroutinesApi::class) public class RepoManager @JvmOverloads constructor( + context: Context, db: FDroidDatabase, + downloaderFactory: DownloaderFactory, + httpManager: HttpManager, private val coroutineContext: CoroutineContext = Dispatchers.IO, ) { private val repositoryDao = db.getRepositoryDao() + private val tempFileProvider = TempFileProvider { + File.createTempFile("dl-", "", context.cacheDir) + } + private val repoAdder = RepoAdder( + context = context, + db = db, + tempFileProvider = tempFileProvider, + downloaderFactory = downloaderFactory, + httpManager = httpManager, + coroutineContext = coroutineContext, + ) private val _repositoriesState: MutableStateFlow> = MutableStateFlow(emptyList()) public val repositoriesState: StateFlow> = _repositoriesState.asStateFlow() - public val liveRepositories: LiveData> = _repositoriesState.asLiveData() + public val addRepoState: StateFlow = repoAdder.addRepoState.asStateFlow() + public val liveAddRepoState: LiveData = repoAdder.addRepoState.asLiveData() + /** * Used internally as a mechanism to wait until repositories are loaded from the DB. * This happens quite fast and the load is triggered at construction time. @@ -92,4 +117,42 @@ public class RepoManager @JvmOverloads constructor( } } + /** + * Fetches a preview of the repository at the given [url] + * with the intention of possibly adding it to the database. + * Progress can be observed via [addRepoState] or [liveAddRepoState]. + */ + @AnyThread + @JvmOverloads + public fun fetchRepositoryPreview( + url: String, + username: String? = null, + password: String? = null, + proxy: Proxy? = null, + ) { + repoAdder.fetchRepository(url, username, password, proxy) + } + + /** + * When [addRepoState] is in [org.fdroid.repo.Fetched], + * you can call this to actually add the repo to the DB. + * @throws IllegalStateException if [addRepoState] is currently in any other state. + */ + @AnyThread + public fun addFetchedRepository() { + GlobalScope.launch(coroutineContext) { + repoAdder.addFetchedRepository() + } + } + + /** + * Aborts the process of fetching a [Repository] preview, + * e.g. when the user leaves the UI flow or wants to cancel the preview process. + * Note that this won't work after [addFetchedRepository] has already been called. + */ + @UiThread + public fun abortAddingRepository() { + repoAdder.abortAddingRepo() + } + } diff --git a/libs/database/src/main/java/org/fdroid/repo/RepoAdder.kt b/libs/database/src/main/java/org/fdroid/repo/RepoAdder.kt new file mode 100644 index 000000000..2e39882e2 --- /dev/null +++ b/libs/database/src/main/java/org/fdroid/repo/RepoAdder.kt @@ -0,0 +1,281 @@ +package org.fdroid.repo + +import android.content.Context +import android.net.Uri +import android.os.Build.VERSION.SDK_INT +import android.os.UserManager +import android.os.UserManager.DISALLOW_INSTALL_UNKNOWN_SOURCES +import android.os.UserManager.DISALLOW_INSTALL_UNKNOWN_SOURCES_GLOBALLY +import androidx.annotation.VisibleForTesting +import androidx.annotation.WorkerThread +import androidx.core.content.ContextCompat.getSystemService +import kotlinx.coroutines.DelicateCoroutinesApi +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.GlobalScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.launch +import mu.KotlinLogging +import org.fdroid.database.AppOverviewItem +import org.fdroid.database.FDroidDatabase +import org.fdroid.database.MinimalApp +import org.fdroid.database.NewRepository +import org.fdroid.database.Repository +import org.fdroid.database.RepositoryDaoInt +import org.fdroid.download.DownloaderFactory +import org.fdroid.download.HttpManager +import org.fdroid.download.NotFoundException +import org.fdroid.index.IndexFormatVersion +import org.fdroid.index.SigningException +import org.fdroid.index.TempFileProvider +import org.fdroid.repo.AddRepoError.ErrorType.INVALID_FINGERPRINT +import org.fdroid.repo.AddRepoError.ErrorType.INVALID_INDEX +import org.fdroid.repo.AddRepoError.ErrorType.IO_ERROR +import org.fdroid.repo.AddRepoError.ErrorType.UNKNOWN_SOURCES_DISALLOWED +import java.io.IOException +import java.net.Proxy +import kotlin.coroutines.CoroutineContext + +internal const val REPO_ID = 0L + +public sealed class AddRepoState + +public object None : AddRepoState() + +public class Fetching( + public val repo: Repository?, + public val apps: List, + public val fetchResult: FetchResult?, + /** + * true if fetching is complete. + */ + public val done: Boolean = false, +) : AddRepoState() { + /** + * true if the repository can be added (be it as new [Repository] or new mirror). + */ + public val canAdd: Boolean = repo != null && + (fetchResult != null && fetchResult !is FetchResult.IsExistingRepository) +} + +public object Adding : AddRepoState() + +public class Added( + public val repo: Repository, +) : AddRepoState() + +public data class AddRepoError( + public val errorType: ErrorType, + public val exception: Exception? = null, +) : AddRepoState() { + public enum class ErrorType { + UNKNOWN_SOURCES_DISALLOWED, + INVALID_FINGERPRINT, + INVALID_INDEX, + IO_ERROR, + } +} + +public sealed class FetchResult { + public object IsNewRepository : FetchResult() + public data class IsNewMirror( + internal val existingRepoId: Long, + internal val newMirrorUrl: String, + ) : FetchResult() + + public object IsExistingRepository : FetchResult() +} + +@OptIn(DelicateCoroutinesApi::class) +internal class RepoAdder( + private val context: Context, + private val db: FDroidDatabase, + private val tempFileProvider: TempFileProvider, + private val downloaderFactory: DownloaderFactory, + private val httpManager: HttpManager, + private val repoUriGetter: RepoUriGetter = RepoUriGetter, + private val coroutineContext: CoroutineContext = Dispatchers.IO, +) { + private val log = KotlinLogging.logger {} + private val repositoryDao = db.getRepositoryDao() as RepositoryDaoInt + + internal val addRepoState: MutableStateFlow = MutableStateFlow(None) + + private var fetchJob: Job? = null + + internal fun fetchRepository(url: String, username: String?, password: String?, proxy: Proxy?) { + fetchJob = GlobalScope.launch(coroutineContext) { + fetchRepositoryInt(url, username, password, proxy) + } + } + + @WorkerThread + @VisibleForTesting + internal suspend fun fetchRepositoryInt( + url: String, + username: String? = null, + password: String? = null, + proxy: Proxy? = null, + ) { + if (hasDisallowInstallUnknownSources(context)) { + addRepoState.value = AddRepoError(UNKNOWN_SOURCES_DISALLOWED) + return + } + // get repo url and fingerprint + val nUri = repoUriGetter.getUri(url) + log.info("Parsed URI: $nUri") + + // some plumping to receive the repo preview + var receivedRepo: Repository? = null + val apps = ArrayList() + var fetchResult: FetchResult? = null + val receiver = object : RepoPreviewReceiver { + override fun onRepoReceived(repo: Repository) { + receivedRepo = repo + fetchResult = getFetchResult(nUri.uri.toString(), repo) + addRepoState.value = Fetching(receivedRepo, apps.toList(), fetchResult) + } + + override fun onAppReceived(app: AppOverviewItem) { + apps.add(app) + addRepoState.value = Fetching(receivedRepo, apps.toList(), fetchResult) + } + } + // set a state early, so the ui can show progress animation + addRepoState.value = Fetching(receivedRepo, apps, fetchResult) + + // try fetching repo with v2 format first and fallback to v1 + try { + try { + val repo = getTempRepo(nUri.uri, IndexFormatVersion.TWO, username, password) + val repoFetcher = + RepoV2Fetcher(tempFileProvider, downloaderFactory, httpManager, proxy) + repoFetcher.fetchRepo(nUri.uri, repo, receiver, nUri.fingerprint) + } catch (e: NotFoundException) { + log.warn(e) { "Did not find v2 repo, trying v1 now." } + // try to fetch v1 repo + val repo = getTempRepo(nUri.uri, IndexFormatVersion.ONE, username, password) + val repoFetcher = RepoV1Fetcher(tempFileProvider, downloaderFactory) + repoFetcher.fetchRepo(nUri.uri, repo, receiver, nUri.fingerprint) + } + } catch (e: SigningException) { + log.error(e) { "Error verifying repo with given fingerprint." } + addRepoState.value = AddRepoError(INVALID_FINGERPRINT, e) + return + } catch (e: IOException) { + log.error(e) { "Error fetching repo." } + addRepoState.value = AddRepoError(IO_ERROR, e) + return + } + // set final result + val finalRepo = receivedRepo + if (finalRepo == null) { + addRepoState.value = AddRepoError(INVALID_INDEX) + } else { + addRepoState.value = Fetching(finalRepo, apps, fetchResult, done = true) + } + } + + private fun getFetchResult(url: String, repo: Repository): FetchResult { + val cert = repo.certificate ?: error("Certificate was null") + val existingRepo = repositoryDao.getRepository(cert) + return if (existingRepo == null) { + FetchResult.IsNewRepository + } else { + val existingMirror = if (existingRepo.address.trimEnd('/') == url) { + url + } else { + existingRepo.mirrors.find { it.url.trimEnd('/') == url } + ?: existingRepo.userMirrors.find { it.trimEnd('/') == url } + } + if (existingMirror == null) { + FetchResult.IsNewMirror(existingRepo.repoId, url) + } else { + FetchResult.IsExistingRepository + } + } + } + + @WorkerThread + internal fun addFetchedRepository() { + // prevent double calls (e.g. caused by double tapping a UI button) + if (addRepoState.compareAndSet(Adding, Adding)) return + + // cancel fetch preview job, so it stops emitting new states + fetchJob?.cancel() + + // get current state before changing it + val state = (addRepoState.value as? Fetching) + ?: throw IllegalStateException("Unexpected state: ${addRepoState.value}") + addRepoState.value = Adding + + val repo = state.repo ?: throw IllegalStateException("No repo: ${addRepoState.value}") + val fetchResult = state.fetchResult + ?: throw IllegalStateException("No fetchResult: ${addRepoState.value}") + + val modifiedRepo: Repository = when (fetchResult) { + is FetchResult.IsExistingRepository -> error("Unexpected result: $fetchResult") + is FetchResult.IsNewRepository -> { + // reset the timestamp of the actual repo, + // so a following repo update will pick this up + val newRepo = NewRepository( + name = repo.repository.name, + icon = repo.repository.icon ?: emptyMap(), + address = repo.address, + formatVersion = repo.formatVersion, + certificate = repo.certificate ?: error("Repo had no certificate"), + username = repo.username, + password = repo.password, + ) + val repoId = repositoryDao.insert(newRepo) + repositoryDao.getRepository(repoId) ?: error("New repository not found in DB") + } + + is FetchResult.IsNewMirror -> { + val repoId = fetchResult.existingRepoId + db.runInTransaction { + val existingRepo = repositoryDao.getRepository(repoId) + ?: error("No repo with $repoId") + val userMirrors = existingRepo.userMirrors.toMutableList().apply { + add(fetchResult.newMirrorUrl) + } + repositoryDao.updateUserMirrors(repoId, userMirrors) + existingRepo + } + } + } + addRepoState.value = Added(modifiedRepo) + } + + internal fun abortAddingRepo() { + addRepoState.value = None + fetchJob?.cancel() + } + + private fun hasDisallowInstallUnknownSources(context: Context): Boolean { + val userManager = getSystemService(context, UserManager::class.java) + ?: error("No UserManager available.") + return if (SDK_INT < 29) userManager.hasUserRestriction(DISALLOW_INSTALL_UNKNOWN_SOURCES) + else userManager.hasUserRestriction(DISALLOW_INSTALL_UNKNOWN_SOURCES) || + userManager.hasUserRestriction(DISALLOW_INSTALL_UNKNOWN_SOURCES_GLOBALLY) + } + + private fun getTempRepo( + uri: Uri, + indexFormatVersion: IndexFormatVersion, + username: String?, + password: String?, + ) = Repository( + repoId = REPO_ID, + address = uri.toString(), + timestamp = -1L, + formatVersion = indexFormatVersion, + certificate = null, + version = 0L, + weight = 0, + lastUpdated = -1L, + username = username, + password = password, + ) + +} diff --git a/libs/database/src/main/java/org/fdroid/repo/RepoFetcher.kt b/libs/database/src/main/java/org/fdroid/repo/RepoFetcher.kt new file mode 100644 index 000000000..a96c815d9 --- /dev/null +++ b/libs/database/src/main/java/org/fdroid/repo/RepoFetcher.kt @@ -0,0 +1,23 @@ +package org.fdroid.repo + +import android.net.Uri +import org.fdroid.database.AppOverviewItem +import org.fdroid.database.Repository +import org.fdroid.download.NotFoundException +import org.fdroid.index.SigningException +import java.io.IOException + +internal fun interface RepoFetcher { + @Throws(IOException::class, SigningException::class, NotFoundException::class) + suspend fun fetchRepo( + uri: Uri, + repo: Repository, + receiver: RepoPreviewReceiver, + fingerprint: String?, + ) +} + +internal interface RepoPreviewReceiver { + fun onRepoReceived(repo: Repository) + fun onAppReceived(app: AppOverviewItem) +} diff --git a/libs/database/src/main/java/org/fdroid/repo/RepoUriGetter.kt b/libs/database/src/main/java/org/fdroid/repo/RepoUriGetter.kt new file mode 100644 index 000000000..5a73117e0 --- /dev/null +++ b/libs/database/src/main/java/org/fdroid/repo/RepoUriGetter.kt @@ -0,0 +1,48 @@ +package org.fdroid.repo + +import android.net.Uri +import org.fdroid.database.Repository + +internal object RepoUriGetter { + + fun getUri(url: String): NormalizedUri { + val uri = Uri.parse(url) + val fingerprint = uri.getQueryParameter("fingerprint")?.lowercase() + + val pathSegments = uri.pathSegments + val normalizedUri = uri.buildUpon().apply { + clearQuery() // removes fingerprint and other query params + fragment("") // remove # hash fragment + if (pathSegments.size >= 2 && + pathSegments[pathSegments.lastIndex - 1] == "fdroid" && + pathSegments.last() == "repo" + ) { + // path already is /fdroid/repo, use as is + } else if (pathSegments.lastOrNull() == "repo") { + // path already ends in /repo, use as is + } else if (pathSegments.size >= 1 && pathSegments.last() == "fdroid") { + // path is /fdroid with missing /repo, so add that + appendPath("repo") + } else { + // path is missing /fdroid/repo, so add it + appendPath("fdroid") + appendPath("repo") + } + }.build().let { newUri -> + // hacky way to remove trailing slash + val path = newUri.path + if (path != null && path.endsWith('/')) { + newUri.buildUpon().path(path.trimEnd('/')).build() + } else { + newUri + } + } + return NormalizedUri(normalizedUri, fingerprint) + } + + /** + * A class for normalizing the [Repository] URI and holding an optional fingerprint. + */ + data class NormalizedUri(val uri: Uri, val fingerprint: String?) + +} diff --git a/libs/database/src/main/java/org/fdroid/repo/RepoV1Fetcher.kt b/libs/database/src/main/java/org/fdroid/repo/RepoV1Fetcher.kt new file mode 100644 index 000000000..cc0b9507e --- /dev/null +++ b/libs/database/src/main/java/org/fdroid/repo/RepoV1Fetcher.kt @@ -0,0 +1,66 @@ +package org.fdroid.repo + +import android.content.res.Resources +import android.net.Uri +import androidx.core.os.ConfigurationCompat.getLocales +import androidx.core.os.LocaleListCompat +import org.fdroid.database.Repository +import org.fdroid.download.DownloaderFactory +import org.fdroid.index.IndexConverter +import org.fdroid.index.IndexFormatVersion +import org.fdroid.index.IndexParser +import org.fdroid.index.SigningException +import org.fdroid.index.TempFileProvider +import org.fdroid.index.parseV1 +import org.fdroid.index.v1.IndexV1Verifier +import org.fdroid.index.v1.SIGNED_FILE_NAME +import org.fdroid.index.v2.FileV2 + +internal class RepoV1Fetcher( + private val tempFileProvider: TempFileProvider, + private val downloaderFactory: DownloaderFactory, +) : RepoFetcher { + + private val locales: LocaleListCompat = getLocales(Resources.getSystem().configuration) + + @Throws(SigningException::class) + override suspend fun fetchRepo( + uri: Uri, + repo: Repository, + receiver: RepoPreviewReceiver, + fingerprint: String?, + ) { + // download and verify index-v1.jar + val indexFile = tempFileProvider.createTempFile() + val entryDownloader = downloaderFactory.create( + repo = repo, + uri = uri.buildUpon().appendPath(SIGNED_FILE_NAME).build(), + indexFile = FileV2.fromPath("/$SIGNED_FILE_NAME"), + destFile = indexFile, + ) + val (cert, indexV1) = try { + entryDownloader.download() + val verifier = IndexV1Verifier(indexFile, null, fingerprint) + verifier.getStreamAndVerify { inputStream -> + IndexParser.parseV1(inputStream) + } + } finally { + indexFile.delete() + } + val version = indexV1.repo.version + val indexV2 = IndexConverter().toIndexV2(indexV1) + val receivedRepo = RepoV2StreamReceiver.getRepository( + repo = indexV2.repo, + version = version.toLong(), + formatVersion = IndexFormatVersion.ONE, + certificate = cert, + username = repo.username, + password = repo.password, + ) + receiver.onRepoReceived(receivedRepo) + indexV2.packages.forEach { (packageName, packageV2) -> + val app = RepoV2StreamReceiver.getAppOverViewItem(packageName, packageV2, locales) + receiver.onAppReceived(app) + } + } +} diff --git a/libs/database/src/main/java/org/fdroid/repo/RepoV2Fetcher.kt b/libs/database/src/main/java/org/fdroid/repo/RepoV2Fetcher.kt new file mode 100644 index 000000000..7803b33e5 --- /dev/null +++ b/libs/database/src/main/java/org/fdroid/repo/RepoV2Fetcher.kt @@ -0,0 +1,74 @@ +package org.fdroid.repo + +import android.net.Uri +import mu.KotlinLogging +import org.fdroid.database.Repository +import org.fdroid.download.DownloadRequest +import org.fdroid.download.DownloaderFactory +import org.fdroid.download.HttpManager +import org.fdroid.download.getDigestInputStream +import org.fdroid.index.IndexParser +import org.fdroid.index.SigningException +import org.fdroid.index.TempFileProvider +import org.fdroid.index.parseEntry +import org.fdroid.index.v2.EntryVerifier +import org.fdroid.index.v2.FileV2 +import org.fdroid.index.v2.IndexV2FullStreamProcessor +import org.fdroid.index.v2.SIGNED_FILE_NAME +import java.net.Proxy + +internal class RepoV2Fetcher( + private val tempFileProvider: TempFileProvider, + private val downloaderFactory: DownloaderFactory, + private val httpManager: HttpManager, + private val proxy: Proxy? = null, +) : RepoFetcher { + private val log = KotlinLogging.logger {} + + @Throws(SigningException::class) + override suspend fun fetchRepo( + uri: Uri, + repo: Repository, + receiver: RepoPreviewReceiver, + fingerprint: String?, + ) { + // download and verify entry + val entryFile = tempFileProvider.createTempFile() + val entryDownloader = downloaderFactory.create( + repo = repo, + uri = uri.buildUpon().appendPath(SIGNED_FILE_NAME).build(), + indexFile = FileV2.fromPath("/$SIGNED_FILE_NAME"), + destFile = entryFile, + ) + val (cert, entry) = try { + entryDownloader.download() + val verifier = EntryVerifier(entryFile, null, fingerprint) + verifier.getStreamAndVerify { inputStream -> + IndexParser.parseEntry(inputStream) + } + } finally { + entryFile.delete() + } + + log.info { "Downloaded entry, now streaming index..." } + + // stream index + val indexRequest = DownloadRequest( + indexFile = FileV2.fromPath(entry.index.name.trimStart('/')), + mirrors = repo.getMirrors(), + proxy = proxy, + username = repo.username, + password = repo.password, + ) + val streamReceiver = RepoV2StreamReceiver(receiver, repo.username, repo.password) + val streamProcessor = IndexV2FullStreamProcessor(streamReceiver, cert) + val digestInputStream = httpManager.getDigestInputStream(indexRequest) + digestInputStream.use { inputStream -> + streamProcessor.process(entry.version, inputStream) { } + } + val hexDigest = digestInputStream.getDigestHex() + if (!hexDigest.equals(entry.index.sha256, ignoreCase = true)) { + throw SigningException("Invalid ${entry.index.name} hash: $hexDigest") + } + } +} diff --git a/libs/database/src/main/java/org/fdroid/repo/RepoV2StreamReceiver.kt b/libs/database/src/main/java/org/fdroid/repo/RepoV2StreamReceiver.kt new file mode 100644 index 000000000..8f0683721 --- /dev/null +++ b/libs/database/src/main/java/org/fdroid/repo/RepoV2StreamReceiver.kt @@ -0,0 +1,102 @@ +package org.fdroid.repo + +import android.content.res.Resources +import androidx.core.os.ConfigurationCompat.getLocales +import androidx.core.os.LocaleListCompat +import org.fdroid.LocaleChooser.getBestLocale +import org.fdroid.database.AppOverviewItem +import org.fdroid.database.LocalizedIcon +import org.fdroid.database.Repository +import org.fdroid.database.RepositoryPreferences +import org.fdroid.database.toCoreRepository +import org.fdroid.database.toRepoAntiFeatures +import org.fdroid.database.toRepoCategories +import org.fdroid.database.toRepoReleaseChannel +import org.fdroid.index.IndexFormatVersion +import org.fdroid.index.v2.IndexV2StreamReceiver +import org.fdroid.index.v2.PackageV2 +import org.fdroid.index.v2.RepoV2 + +internal open class RepoV2StreamReceiver( + private val receiver: RepoPreviewReceiver, + private val username: String?, + private val password: String?, +) : IndexV2StreamReceiver { + + companion object { + fun getRepository( + repo: RepoV2, + version: Long, + formatVersion: IndexFormatVersion, + certificate: String?, + username: String?, + password: String?, + ) = Repository( + repository = repo.toCoreRepository( + version = version, + formatVersion = formatVersion, + certificate = certificate + ), + mirrors = emptyList(), + antiFeatures = repo.antiFeatures.toRepoAntiFeatures(REPO_ID), + categories = repo.categories.toRepoCategories(REPO_ID), + releaseChannels = repo.releaseChannels.toRepoReleaseChannel(REPO_ID), + preferences = RepositoryPreferences( + repoId = REPO_ID, + weight = 0, + enabled = true, + username = username, + password = password, + ), + ) + + fun getAppOverViewItem( + packageName: String, + p: PackageV2, + locales: LocaleListCompat, + ) = AppOverviewItem( + repoId = REPO_ID, + packageName = packageName, + added = p.metadata.added, + lastUpdated = p.metadata.lastUpdated, + name = p.metadata.name.getBestLocale(locales), + summary = p.metadata.summary.getBestLocale(locales), + antiFeatures = p.versions.values.lastOrNull()?.antiFeatures, + localizedIcon = p.metadata.icon?.map { (locale, file) -> + LocalizedIcon( + repoId = 0L, + packageName = packageName, + type = "icon", + locale = locale, + name = file.name, + sha256 = file.sha256, + size = file.size, + ipfsCidV1 = file.ipfsCidV1, + ) + }, + ) + } + + private val locales: LocaleListCompat = getLocales(Resources.getSystem().configuration) + + override fun receive(repo: RepoV2, version: Long, certificate: String) { + receiver.onRepoReceived( + getRepository( + repo = repo, + version = version, + formatVersion = IndexFormatVersion.TWO, + certificate = certificate, + username = username, + password = password, + ) + ) + } + + override fun receive(packageName: String, p: PackageV2) { + receiver.onAppReceived(getAppOverViewItem(packageName, p, locales)) + } + + override fun onStreamEnded() { + } + +} diff --git a/libs/database/src/test/java/org/fdroid/download/TestDownloadFactory.kt b/libs/database/src/test/java/org/fdroid/download/TestDownloadFactory.kt new file mode 100644 index 000000000..ee328fe5d --- /dev/null +++ b/libs/database/src/test/java/org/fdroid/download/TestDownloadFactory.kt @@ -0,0 +1,32 @@ +package org.fdroid.download + +import android.net.Uri +import org.fdroid.IndexFile +import org.fdroid.database.Repository +import java.io.File + +internal class TestDownloadFactory(private val httpManager: HttpManager) : DownloaderFactory() { + override fun create( + repo: Repository, + uri: Uri, + indexFile: IndexFile, + destFile: File, + ): Downloader = HttpDownloaderV2( + httpManager = httpManager, + request = DownloadRequest(indexFile, repo.getMirrors()), + destFile = destFile + ) + + override fun create( + repo: Repository, + mirrors: List, + uri: Uri, + indexFile: IndexFile, + destFile: File, + tryFirst: Mirror?, + ): Downloader = HttpDownloaderV2( + httpManager = httpManager, + request = DownloadRequest(indexFile, repo.getMirrors()), + destFile = destFile + ) +} diff --git a/libs/database/src/test/java/org/fdroid/repo/RepoAdderIntegrationTest.kt b/libs/database/src/test/java/org/fdroid/repo/RepoAdderIntegrationTest.kt new file mode 100644 index 000000000..d2f7c8f5c --- /dev/null +++ b/libs/database/src/test/java/org/fdroid/repo/RepoAdderIntegrationTest.kt @@ -0,0 +1,135 @@ +package org.fdroid.repo + +import androidx.core.os.LocaleListCompat +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import app.cash.turbine.test +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.test.runTest +import org.fdroid.database.FDroidDatabase +import org.fdroid.database.NewRepository +import org.fdroid.database.Repository +import org.fdroid.database.RepositoryDaoInt +import org.fdroid.download.HttpManager +import org.fdroid.download.TestDownloadFactory +import org.fdroid.index.TempFileProvider +import org.fdroid.repo.AddRepoError.ErrorType.INVALID_FINGERPRINT +import org.junit.Assume.assumeTrue +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue + +@RunWith(AndroidJUnit4::class) +internal class RepoAdderIntegrationTest { + + @get:Rule + var folder: TemporaryFolder = TemporaryFolder() + + private val context = InstrumentationRegistry.getInstrumentation().targetContext + private val db = mockk() + private val repoDao = mockk() + private val tempFileProvider = TempFileProvider { folder.newFile() } + private val httpManager = HttpManager("test") + private val downloaderFactory = TestDownloadFactory(httpManager) + + private val repoAdder: RepoAdder + + init { + every { db.getRepositoryDao() } returns repoDao + repoAdder = RepoAdder(context, db, tempFileProvider, downloaderFactory, httpManager) + } + + @Before + fun optIn() { + assumeTrue(false) // don't run integration tests with real repos all the time + } + + @Test + fun testFedilabV1() = runTest { + // repo not in DB + every { repoDao.getRepository(any()) } returns null + + repoAdder.fetchRepository( + url = "https://fdroid.fedilab.app/repo/" + + "?fingerprint=11F0A69910A4280E2CD3CCC3146337D006BE539B18E1A9FEACE15FF757A94FEB", + username = null, + password = null, + proxy = null + ) + repoAdder.addRepoState.test { + assertEquals(None, awaitItem()) + val firstFetching = awaitItem() + assertTrue(firstFetching is Fetching) + assertNull(firstFetching.repo) + assertTrue(firstFetching.apps.isEmpty()) + + val secondFetching = awaitItem() + assertTrue(secondFetching is Fetching, "$secondFetching") + val repo = secondFetching.repo + assertNotNull(repo) + assertEquals("https://fdroid.fedilab.app/repo", repo.address) + println(repo.getName(LocaleListCompat.getDefault()) ?: "null") + println(repo.certificate) + + assertEquals(1, (awaitItem() as Fetching).apps.size) + assertEquals(2, (awaitItem() as Fetching).apps.size) + assertEquals(3, (awaitItem() as Fetching).apps.size) + assertTrue(awaitItem() is Fetching) + } + + val state = repoAdder.addRepoState.value + assertTrue(state is Fetching, state.toString()) + assertTrue(state.apps.isNotEmpty()) + state.apps.forEach { app -> + println(" ${app.packageName} ${app.summary}") + } + + val newRepo: Repository = mockk() + every { repoDao.insert(any()) } returns 42L + every { repoDao.getRepository(42L) } returns newRepo + + repoAdder.addFetchedRepository() + repoAdder.addRepoState.test { + val addedState = awaitItem() + assertTrue(addedState is Added, addedState.toString()) + assertEquals(newRepo, addedState.repo) + } + } + + @Test + fun testIzzy() = runBlocking { + // repo not in DB + every { repoDao.getRepository(any()) } returns null + + repoAdder.fetchRepositoryInt( + url = "https://apt.izzysoft.de/fdroid/repo" + + "?fingerprint=3BF0D6ABFEAE2F401707B6D966BE743BF0EEE49C2561B9BA39073711F628937A" + ) + val state = repoAdder.addRepoState.value + assertTrue(state is Fetching, state.toString()) + assertTrue(state.apps.isNotEmpty()) + + println(state.repo?.getName(LocaleListCompat.getDefault()) ?: "null") + println(state.repo?.certificate) + state.apps.forEach { app -> + println(" ${app.packageName} ${app.summary}") + } + } + + @Test + fun testIzzyWrongFingerprint() = runBlocking { + repoAdder.fetchRepositoryInt("https://apt.izzysoft.de/fdroid/repo?fingerprint=fooBar") + val state = repoAdder.addRepoState.value + assertTrue(state is AddRepoError, state.toString()) + assertEquals(state.errorType, INVALID_FINGERPRINT, state.errorType.name) + } + +} diff --git a/libs/database/src/test/java/org/fdroid/repo/RepoAdderTest.kt b/libs/database/src/test/java/org/fdroid/repo/RepoAdderTest.kt new file mode 100644 index 000000000..f5bc50880 --- /dev/null +++ b/libs/database/src/test/java/org/fdroid/repo/RepoAdderTest.kt @@ -0,0 +1,561 @@ +package org.fdroid.repo + +import android.content.Context +import android.content.res.AssetManager +import android.net.Uri +import android.os.UserManager +import android.os.UserManager.DISALLOW_INSTALL_UNKNOWN_SOURCES +import androidx.core.os.LocaleListCompat +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import app.cash.turbine.test +import io.mockk.Runs +import io.mockk.coEvery +import io.mockk.every +import io.mockk.just +import io.mockk.mockk +import io.mockk.mockkStatic +import io.mockk.slot +import kotlinx.coroutines.test.runTest +import org.fdroid.LocaleChooser.getBestLocale +import org.fdroid.database.FDroidDatabase +import org.fdroid.database.Mirror +import org.fdroid.database.NewRepository +import org.fdroid.database.Repository +import org.fdroid.database.RepositoryDaoInt +import org.fdroid.database.RepositoryPreferences +import org.fdroid.database.toCoreRepository +import org.fdroid.download.Downloader +import org.fdroid.download.DownloaderFactory +import org.fdroid.download.HttpManager +import org.fdroid.download.NotFoundException +import org.fdroid.download.getDigestInputStream +import org.fdroid.fdroid.DigestInputStream +import org.fdroid.index.IndexFormatVersion +import org.fdroid.index.SigningException +import org.fdroid.index.TempFileProvider +import org.fdroid.repo.AddRepoError.ErrorType.INVALID_FINGERPRINT +import org.fdroid.repo.AddRepoError.ErrorType.IO_ERROR +import org.fdroid.repo.AddRepoError.ErrorType.UNKNOWN_SOURCES_DISALLOWED +import org.fdroid.test.TestDataMinV2 +import org.fdroid.test.TestUtils.decodeHex +import org.fdroid.test.VerifierConstants +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import java.io.ByteArrayInputStream +import java.io.IOException +import java.security.MessageDigest +import java.util.concurrent.Callable +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertIs +import kotlin.test.assertNull +import kotlin.test.assertTrue +import kotlin.test.fail + +@RunWith(AndroidJUnit4::class) +internal class RepoAdderTest { + + @get:Rule + var folder: TemporaryFolder = TemporaryFolder() + + private val context = InstrumentationRegistry.getInstrumentation().targetContext + private val db = mockk() + private val repoDao = mockk() + private val tempFileProvider = mockk() + private val httpManager = mockk() + private val downloaderFactory = mockk() + private val downloader = mockk() + private val digest = mockk() + + private val repoAdder: RepoAdder + private val assets: AssetManager = context.resources.assets + private val localeList = LocaleListCompat.getDefault() + + init { + every { db.getRepositoryDao() } returns repoDao + every { digest.update(any(), any(), any()) } just Runs + every { digest.update(any()) } just Runs + + mockkStatic("org.fdroid.download.HttpManagerKt") + + repoAdder = RepoAdder(context, db, tempFileProvider, downloaderFactory, httpManager) + } + + @Test + fun testDisallowInstallUnknownSources() = runTest { + val context = mockk() + val userManager = mockk() + val repoAdder = RepoAdder(context, db, tempFileProvider, downloaderFactory, httpManager) + + every { context.getSystemService("user") } returns userManager + every { + userManager.hasUserRestriction(DISALLOW_INSTALL_UNKNOWN_SOURCES) + } returns true + + repoAdder.fetchRepositoryInt("https://example.org/repo/") + repoAdder.addRepoState.test { + val state1 = awaitItem() + assertTrue(state1 is AddRepoError) + assertEquals(UNKNOWN_SOURCES_DISALLOWED, state1.errorType) + } + } + + @Test + fun testAddingMinRepo() = runTest { + val url = "https://example.org/repo/" + val repoName = TestDataMinV2.repo.name.getBestLocale(localeList) + + expectDownloadOfMinRepo(url) + + // repo not in DB + every { repoDao.getRepository(any()) } returns null + + expectMinRepoPreview(repoName, FetchResult.IsNewRepository) { + repoAdder.fetchRepository( + url = url, + username = null, + password = null, + proxy = null, + ) + } + + val newRepo: Repository = mockk() + every { + repoDao.insert(match { + // Note that we are not using the url the user used to add the repo, + // but what the repo tells us to use + it.address == TestDataMinV2.repo.address && + it.formatVersion == IndexFormatVersion.TWO && + it.name.getBestLocale(localeList) == repoName + }) + } returns 42L + every { repoDao.getRepository(42L) } returns newRepo + + repoAdder.addRepoState.test { + assertIs(awaitItem()) // still Fetching from last call + + repoAdder.addFetchedRepository() + + assertIs(awaitItem()) // now moved to Adding + + val addedState = awaitItem() + assertIs(addedState) + assertEquals(newRepo, addedState.repo) + } + } + + @Test + fun testAddingMirrorForMinRepo() = runTest { + val url = "https://example.com/repo/" + val repoName = TestDataMinV2.repo.name.getBestLocale(localeList) + + expectDownloadOfMinRepo(url) + + // repo is already in the DB + val existingRepo = Repository( + repository = TestDataMinV2.repo.toCoreRepository( + repoId = 42L, + version = 1337L, + formatVersion = IndexFormatVersion.TWO, + certificate = "cert", + ), + mirrors = emptyList(), + antiFeatures = emptyList(), + categories = emptyList(), + releaseChannels = emptyList(), + preferences = RepositoryPreferences(42L, 23), + ) + every { repoDao.getRepository(any()) } returns existingRepo + + expectMinRepoPreview(repoName, FetchResult.IsNewMirror(42L, url.trimEnd('/'))) { + repoAdder.fetchRepository( + url = url, + username = null, + password = null, + proxy = null + ) + } + + val transactionSlot = slot>() + every { + db.runInTransaction(capture(transactionSlot)) + } answers { transactionSlot.captured.call() } + every { repoDao.getRepository(42L) } returns existingRepo + every { repoDao.updateUserMirrors(42L, listOf(url.trimEnd('/'))) } just Runs + + repoAdder.addRepoState.test { + assertIs(awaitItem()) // still Fetching from last call + + repoAdder.addFetchedRepository() + + assertIs(awaitItem()) // now moved to Adding + + val addedState = awaitItem() + assertTrue(addedState is Added, addedState.toString()) + assertEquals(existingRepo, addedState.repo) + } + } + + @Test + fun testRepoAlreadyExists() = runTest { + val url = "https://min-v1.org/repo/" + + // repo is already in the DB + val existingRepo = Repository( + repository = TestDataMinV2.repo.toCoreRepository( + repoId = REPO_ID, + version = 1337L, + formatVersion = IndexFormatVersion.TWO, + certificate = "cert", + ).copy(address = url), // change address, because TestDataMinV2 misses /repo + mirrors = emptyList(), + antiFeatures = emptyList(), + categories = emptyList(), + releaseChannels = emptyList(), + preferences = RepositoryPreferences(REPO_ID, 23), + ) + testRepoAlreadyExists(url, existingRepo) + } + + @Test + fun testRepoAlreadyExistsWithMirror() = runTest { + val url = "https://example.org/repo/" + + // repo is already in the DB + val existingRepo = Repository( + repository = TestDataMinV2.repo.toCoreRepository( + repoId = REPO_ID, + version = 1337L, + formatVersion = IndexFormatVersion.TWO, + certificate = "cert", + ), + mirrors = listOf(Mirror(REPO_ID, url), Mirror(REPO_ID, "http://example.org")), + antiFeatures = emptyList(), + categories = emptyList(), + releaseChannels = emptyList(), + preferences = RepositoryPreferences(REPO_ID, 23), + ) + testRepoAlreadyExists(url, existingRepo) + } + + @Test + fun testRepoAlreadyExistsWithFingerprint() = runTest { + val url = "https://example.org/repo?fingerprint=${VerifierConstants.FINGERPRINT}" + + // repo is already in the DB + val existingRepo = Repository( + repository = TestDataMinV2.repo.toCoreRepository( + repoId = REPO_ID, + version = 1337L, + formatVersion = IndexFormatVersion.TWO, + certificate = VerifierConstants.CERTIFICATE, + ), + mirrors = listOf(Mirror(REPO_ID, "https://example.org/repo/")), + antiFeatures = emptyList(), + categories = emptyList(), + releaseChannels = emptyList(), + preferences = RepositoryPreferences(REPO_ID, 23), + ) + testRepoAlreadyExists(url, existingRepo, "https://example.org/repo") + } + + @Test + fun testRepoAlreadyExistsWithFingerprintTrailingSlash() = runTest { + val url = "https://example.org/repo/?fingerprint=${VerifierConstants.FINGERPRINT}" + + // repo is already in the DB + val existingRepo = Repository( + repository = TestDataMinV2.repo.toCoreRepository( + repoId = REPO_ID, + version = 1337L, + formatVersion = IndexFormatVersion.TWO, + certificate = VerifierConstants.CERTIFICATE, + ), + mirrors = listOf( + Mirror(REPO_ID, "https://example.org/repo"), + Mirror(REPO_ID, "http://example.org"), + ), + antiFeatures = emptyList(), + categories = emptyList(), + releaseChannels = emptyList(), + preferences = RepositoryPreferences(REPO_ID, 23), + ) + testRepoAlreadyExists(url, existingRepo, "https://example.org/repo") + } + + @Test + fun testRepoAlreadyExistsUserMirror() = runTest { + val url = "https://example.net/repo/" + + // repo is already in the DB + val existingRepo = Repository( + repository = TestDataMinV2.repo.toCoreRepository( + repoId = REPO_ID, + version = 1337L, + formatVersion = IndexFormatVersion.TWO, + certificate = "cert", + ), + mirrors = emptyList(), + antiFeatures = emptyList(), + categories = emptyList(), + releaseChannels = emptyList(), + preferences = RepositoryPreferences( + repoId = REPO_ID, + weight = 23, + userMirrors = listOf(url, "http://example.org"), + ), + ) + testRepoAlreadyExists(url, existingRepo) + } + + private suspend fun testRepoAlreadyExists( + url: String, + existingRepo: Repository, + downloadUrl: String = url, + ) { + val repoName = TestDataMinV2.repo.name.getBestLocale(localeList) + + expectDownloadOfMinRepo(downloadUrl) + + // repo is already in the DB + every { repoDao.getRepository(any()) } returns existingRepo + + expectMinRepoPreview(repoName, FetchResult.IsExistingRepository, canAdd = false) { + repoAdder.fetchRepository( + url = url, + username = null, + password = null, + proxy = null + ) + } + assertFailsWith { + repoAdder.addFetchedRepository() + } + } + + @Test + fun testDownloadEntryThrowsIoException() = runTest { + val url = "https://example.org/repo" + val jarFile = folder.newFile() + + every { tempFileProvider.createTempFile() } returns jarFile + every { + downloaderFactory.create( + repo = match { it.address == url && it.formatVersion == IndexFormatVersion.TWO }, + uri = Uri.parse("$url/entry.jar"), + indexFile = any(), + destFile = jarFile, + ) + } returns downloader + every { downloader.download() } throws IOException() + + repoAdder.addRepoState.test { + assertIs(awaitItem()) + + repoAdder.fetchRepository( + url = url, + username = null, + password = null, + proxy = null + ) + + val state1 = awaitItem() + assertIs(state1) + assertNull(state1.repo) + assertTrue(state1.apps.isEmpty()) + assertFalse(state1.canAdd) + + val state2 = awaitItem() + assertTrue(state2 is AddRepoError, "$state2") + assertEquals(IO_ERROR, state2.errorType) + } + } + + @Test + fun testWrongFingerprint() = runTest { + val url = "https://example.org/repo/" + val jarFile = folder.newFile() + + every { tempFileProvider.createTempFile() } returns jarFile + every { + downloaderFactory.create( + repo = match { + it.address == url.trimEnd('/') && it.formatVersion == IndexFormatVersion.TWO + }, + uri = Uri.parse(url + "entry.jar"), + indexFile = any(), + destFile = jarFile, + ) + } returns downloader + // not actually thrown by the downloader, but mocking verifier is harder + every { downloader.download() } throws SigningException("boom!") + + repoAdder.addRepoState.test { + assertIs(awaitItem()) + + repoAdder.fetchRepository( + url = url, + username = null, + password = null, + proxy = null + ) + + val state1 = awaitItem() + assertIs(state1) + assertNull(state1.repo) + assertTrue(state1.apps.isEmpty()) + assertFalse(state1.canAdd) + + val state2 = awaitItem() + assertTrue(state2 is AddRepoError, "$state2") + assertEquals(INVALID_FINGERPRINT, state2.errorType) + } + } + + @Test + fun testFallbackToV1() = runTest { + val url = "https://example.org/repo/" + val urlTrimmed = "https://example.org/repo" + val jarFile = folder.newFile() + + every { tempFileProvider.createTempFile() } returns jarFile + every { + downloaderFactory.create( + repo = match { + it.address == urlTrimmed && it.formatVersion == IndexFormatVersion.TWO + }, + uri = Uri.parse("$urlTrimmed/entry.jar"), + indexFile = any(), + destFile = jarFile, + ) + } returns downloader + every { downloader.download() } throws NotFoundException() + val downloaderV1 = mockk() + every { + downloaderFactory.create( + repo = match { + it.address == urlTrimmed && it.formatVersion == IndexFormatVersion.ONE + }, + uri = Uri.parse("$urlTrimmed/index-v1.jar"), + indexFile = any(), + destFile = jarFile, + ) + } returns downloaderV1 + every { downloaderV1.download() } answers { + jarFile.outputStream().use { outputStream -> + assets.open("testy.at.or.at_index-v1.jar").use { inputStream -> + inputStream.copyTo(outputStream) + } + } + } + every { repoDao.getRepository(any()) } returns null + + repoAdder.addRepoState.test { + assertIs(awaitItem()) + + repoAdder.fetchRepository( + url = url, + username = null, + password = null, + proxy = null + ) + + val state1 = awaitItem() + assertIs(state1) + assertNull(state1.repo) + assertTrue(state1.apps.isEmpty()) + assertFalse(state1.canAdd) + + for (i in 0..64) assertIs(awaitItem()) + } + val addRepoState = repoAdder.addRepoState.value + assertIs(addRepoState) + assertTrue(addRepoState.canAdd) + assertEquals(63, addRepoState.apps.size) + } + + private fun expectDownloadOfMinRepo(url: String) { + val urlTrimmed = url.trimEnd('/') + val jarFile = folder.newFile() + val indexFile = assets.open("index-min-v2.json") + val index = indexFile.use { it.readBytes() } + val indexStream = DigestInputStream(ByteArrayInputStream(index), digest) + + every { tempFileProvider.createTempFile() } returns jarFile + every { + downloaderFactory.create( + repo = match { + it.address == urlTrimmed && it.formatVersion == IndexFormatVersion.TWO + }, + uri = Uri.parse("$urlTrimmed/entry.jar"), + indexFile = any(), + destFile = jarFile, + ) + } returns downloader + every { downloader.download() } answers { + jarFile.outputStream().use { outputStream -> + assets.open("diff-empty-min/entry.jar").use { inputStream -> + inputStream.copyTo(outputStream) + } + } + } + coEvery { + httpManager.getDigestInputStream(match { + it.indexFile.name == "../index-min-v2.json" && + it.mirrors.size == 1 && it.mirrors[0].baseUrl == urlTrimmed + }) + } returns indexStream + every { + digest.digest() // sha256 from entry.json + } returns "851ecda085ed53adab25f761a9dbf4c09d59e5bff9c9d5530814d56445ae30f2".decodeHex() + } + + private suspend fun expectMinRepoPreview( + repoName: String?, + fetchResult: FetchResult, + canAdd: Boolean = true, + block: suspend () -> Unit = {}, + ) { + repoAdder.addRepoState.test { + assertIs(awaitItem()) + + block() + // FIXME executing this block may emit items too fast, so we might miss one + // causing flaky tests. + + val state1 = awaitItem() + assertIs(state1) + assertNull(state1.repo) + assertEquals(emptyList(), state1.apps) + assertFalse(state1.canAdd) + assertFalse(state1.done) + + val state2 = awaitItem() + assertIs(state2) + val repo = state2.repo ?: fail() + assertEquals(TestDataMinV2.repo.address, repo.address) + assertEquals(repoName, repo.getName(localeList)) + val result = state2.fetchResult ?: fail() + assertEquals(fetchResult, result) + assertTrue(state2.apps.isEmpty()) + assertEquals(canAdd, state2.canAdd) + assertFalse(state2.done) + + val state3 = awaitItem() + assertIs(state3) + assertEquals(TestDataMinV2.packages.size, state3.apps.size) + assertEquals(TestDataMinV2.packageName, state3.apps[0].packageName) + assertEquals(canAdd, state3.canAdd) + assertFalse(state3.done) + + val state4 = awaitItem() + assertIs(state4) + assertEquals(canAdd, state4.canAdd) + assertTrue(state4.done) + } + } +} diff --git a/libs/database/src/test/java/org/fdroid/repo/RepoUriGetterTest.kt b/libs/database/src/test/java/org/fdroid/repo/RepoUriGetterTest.kt new file mode 100644 index 000000000..e8ac3f21d --- /dev/null +++ b/libs/database/src/test/java/org/fdroid/repo/RepoUriGetterTest.kt @@ -0,0 +1,59 @@ +package org.fdroid.repo + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import org.junit.Test +import org.junit.runner.RunWith +import kotlin.test.assertEquals +import kotlin.test.assertNull + +@RunWith(AndroidJUnit4::class) +internal class RepoUriGetterTest { + @Test + fun testTrailingSlash() { + val uri = RepoUriGetter.getUri("http://example.org/fdroid/repo/") + assertEquals("http://example.org/fdroid/repo", uri.uri.toString()) + assertNull(uri.fingerprint) + } + + @Test + fun testWithoutTrailingSlash() { + val uri = RepoUriGetter.getUri("http://example.org/fdroid/repo") + assertEquals("http://example.org/fdroid/repo", uri.uri.toString()) + assertNull(uri.fingerprint) + } + + @Test + fun testFingerprint() { + val uri = RepoUriGetter.getUri("https://example.org/repo?fingerprint=foobar&test=42") + assertEquals("https://example.org/repo", uri.uri.toString()) + assertEquals("foobar", uri.fingerprint) + } + + @Test + fun testHash() { + val uri = RepoUriGetter.getUri("https://example.org/repo?fingerprint=foobar&test=42#hash") + assertEquals("https://example.org/repo", uri.uri.toString()) + assertEquals("foobar", uri.fingerprint) + } + + @Test + fun testAddFdroidSlashRepo() { + val uri = RepoUriGetter.getUri("https://example.org") + assertEquals("https://example.org/fdroid/repo", uri.uri.toString()) + assertNull(uri.fingerprint) + } + + @Test + fun testLeaveSingleRepo() { + val uri = RepoUriGetter.getUri("https://example.org/repo") + assertEquals("https://example.org/repo", uri.uri.toString()) + assertNull(uri.fingerprint) + } + + @Test + fun testAddsMissingRepo() { + val uri = RepoUriGetter.getUri("https://example.org/fdroid/") + assertEquals("https://example.org/fdroid/repo", uri.uri.toString()) + assertNull(uri.fingerprint) + } +} diff --git a/libs/sharedTest/src/main/kotlin/org/fdroid/test/TestUtils.kt b/libs/sharedTest/src/main/kotlin/org/fdroid/test/TestUtils.kt index af79d6d8c..3c1a24bc6 100644 --- a/libs/sharedTest/src/main/kotlin/org/fdroid/test/TestUtils.kt +++ b/libs/sharedTest/src/main/kotlin/org/fdroid/test/TestUtils.kt @@ -8,6 +8,13 @@ import kotlin.random.Random object TestUtils { + fun String.decodeHex(): ByteArray { + check(length % 2 == 0) { "Must have an even length" } + return chunked(2) + .map { it.toInt(16).toByte() } + .toByteArray() + } + private val charPool: List = ('a'..'z') + ('A'..'Z') + ('0'..'9') fun getRandomString(length: Int = Random.nextInt(1, 128)): String = (1..length)