From 62aa00b5b45cf6eafbb8600896b4c269d2155b00 Mon Sep 17 00:00:00 2001 From: Torsten Grote Date: Thu, 28 Sep 2023 14:29:46 +0200 Subject: [PATCH] [db] add support for adding repos on content:// and file:// URIs This complicates the code for this niche use-case, but is wanted by F-Droid to support communities with bad or no internet connectivity. --- .../main/java/org/fdroid/index/RepoManager.kt | 2 + .../main/java/org/fdroid/repo/RepoAdder.kt | 19 ++++++++-- .../java/org/fdroid/repo/RepoUriGetter.kt | 31 ++++++++------- .../java/org/fdroid/repo/RepoV1Fetcher.kt | 4 +- .../java/org/fdroid/repo/RepoV2Fetcher.kt | 38 ++++++++++++++----- .../java/org/fdroid/repo/RepoAdderTest.kt | 5 +++ 6 files changed, 70 insertions(+), 29 deletions(-) diff --git a/libs/database/src/main/java/org/fdroid/index/RepoManager.kt b/libs/database/src/main/java/org/fdroid/index/RepoManager.kt index 7acdd1986..5cd554525 100644 --- a/libs/database/src/main/java/org/fdroid/index/RepoManager.kt +++ b/libs/database/src/main/java/org/fdroid/index/RepoManager.kt @@ -33,6 +33,7 @@ public class RepoManager @JvmOverloads constructor( db: FDroidDatabase, downloaderFactory: DownloaderFactory, httpManager: HttpManager, + private val repoUriBuilder: RepoUriBuilder = defaultRepoUriBuilder, private val coroutineContext: CoroutineContext = Dispatchers.IO, ) { @@ -46,6 +47,7 @@ public class RepoManager @JvmOverloads constructor( tempFileProvider = tempFileProvider, downloaderFactory = downloaderFactory, httpManager = httpManager, + repoUriBuilder = repoUriBuilder, coroutineContext = coroutineContext, ) diff --git a/libs/database/src/main/java/org/fdroid/repo/RepoAdder.kt b/libs/database/src/main/java/org/fdroid/repo/RepoAdder.kt index 317dd5226..8e6bdcd4f 100644 --- a/libs/database/src/main/java/org/fdroid/repo/RepoAdder.kt +++ b/libs/database/src/main/java/org/fdroid/repo/RepoAdder.kt @@ -28,6 +28,7 @@ import org.fdroid.download.HttpManager import org.fdroid.download.HttpManager.Companion.isInvalidHttpUrl import org.fdroid.download.NotFoundException import org.fdroid.index.IndexFormatVersion +import org.fdroid.index.RepoUriBuilder import org.fdroid.index.SigningException import org.fdroid.index.TempFileProvider import org.fdroid.repo.AddRepoError.ErrorType.INVALID_FINGERPRINT @@ -101,6 +102,7 @@ internal class RepoAdder( private val downloaderFactory: DownloaderFactory, private val httpManager: HttpManager, private val repoUriGetter: RepoUriGetter = RepoUriGetter, + private val repoUriBuilder: RepoUriBuilder = defaultRepoUriBuilder, private val coroutineContext: CoroutineContext = Dispatchers.IO, ) { private val log = KotlinLogging.logger {} @@ -129,7 +131,9 @@ internal class RepoAdder( // get repo url and fingerprint val nUri = repoUriGetter.getUri(url) log.info("Parsed URI: $nUri") - if (isInvalidHttpUrl(nUri.uri.toString())) { + if (nUri.uri.scheme !in listOf("content", "file") && + isInvalidHttpUrl(nUri.uri.toString()) + ) { val e = IllegalArgumentException("Unsupported URI: ${nUri.uri}") addRepoState.value = AddRepoError(INVALID_INDEX, e) return @@ -159,15 +163,16 @@ internal class RepoAdder( try { val repo = getTempRepo(nUri.uri, IndexFormatVersion.TWO, nUri.username, nUri.password) - val repoFetcher = - RepoV2Fetcher(tempFileProvider, downloaderFactory, httpManager, proxy) + val repoFetcher = RepoV2Fetcher( + tempFileProvider, downloaderFactory, httpManager, repoUriBuilder, proxy + ) repoFetcher.fetchRepo(nUri.uri, repo, receiver, nUri.fingerprint) } catch (e: NotFoundException) { log.warn(e) { "Did not find v2 repo, trying v1 now." } // try to fetch v1 repo val repo = getTempRepo(nUri.uri, IndexFormatVersion.ONE, nUri.username, nUri.password) - val repoFetcher = RepoV1Fetcher(tempFileProvider, downloaderFactory) + val repoFetcher = RepoV1Fetcher(tempFileProvider, downloaderFactory, repoUriBuilder) repoFetcher.fetchRepo(nUri.uri, repo, receiver, nUri.fingerprint) } } catch (e: SigningException) { @@ -300,3 +305,9 @@ internal class RepoAdder( ) } + +internal val defaultRepoUriBuilder = RepoUriBuilder { repo, pathElements -> + val builder = Uri.parse(repo.address).buildUpon() + pathElements.forEach { builder.appendEncodedPath(it) } + builder.build() +} diff --git a/libs/database/src/main/java/org/fdroid/repo/RepoUriGetter.kt b/libs/database/src/main/java/org/fdroid/repo/RepoUriGetter.kt index 823908b77..0048a24a2 100644 --- a/libs/database/src/main/java/org/fdroid/repo/RepoUriGetter.kt +++ b/libs/database/src/main/java/org/fdroid/repo/RepoUriGetter.kt @@ -45,20 +45,23 @@ internal object RepoUriGetter { } clearQuery() // removes fingerprint and other query params fragment("") // remove # hash fragment - if (pathSegments.size >= 2 && - pathSegments[pathSegments.lastIndex - 1] == "fdroid" && - pathSegments.last() == "repo" - ) { - // path already is /fdroid/repo, use as is - } else if (pathSegments.lastOrNull() == "repo") { - // path already ends in /repo, use as is - } else if (pathSegments.size >= 1 && pathSegments.last() == "fdroid") { - // path is /fdroid with missing /repo, so add that - appendPath("repo") - } else { - // path is missing /fdroid/repo, so add it - appendPath("fdroid") - appendPath("repo") + if (uri.scheme != "content" && uri.scheme != "file") { + // do some path auto-adding, if it is missing + if (pathSegments.size >= 2 && + pathSegments[pathSegments.lastIndex - 1] == "fdroid" && + pathSegments.last() == "repo" + ) { + // path already is /fdroid/repo, use as is + } else if (pathSegments.lastOrNull() == "repo") { + // path already ends in /repo, use as is + } else if (pathSegments.size >= 1 && pathSegments.last() == "fdroid") { + // path is /fdroid with missing /repo, so add that + appendPath("repo") + } else { + // path is missing /fdroid/repo, so add it + appendPath("fdroid") + appendPath("repo") + } } }.build().let { newUri -> // hacky way to remove trailing slash diff --git a/libs/database/src/main/java/org/fdroid/repo/RepoV1Fetcher.kt b/libs/database/src/main/java/org/fdroid/repo/RepoV1Fetcher.kt index 0d6e7fafd..2db4669d5 100644 --- a/libs/database/src/main/java/org/fdroid/repo/RepoV1Fetcher.kt +++ b/libs/database/src/main/java/org/fdroid/repo/RepoV1Fetcher.kt @@ -10,6 +10,7 @@ import org.fdroid.download.DownloaderFactory import org.fdroid.index.IndexConverter import org.fdroid.index.IndexFormatVersion import org.fdroid.index.IndexParser +import org.fdroid.index.RepoUriBuilder import org.fdroid.index.SigningException import org.fdroid.index.TempFileProvider import org.fdroid.index.parseV1 @@ -20,6 +21,7 @@ import org.fdroid.index.v2.FileV2 internal class RepoV1Fetcher( private val tempFileProvider: TempFileProvider, private val downloaderFactory: DownloaderFactory, + private val repoUriBuilder: RepoUriBuilder, ) : RepoFetcher { private val locales: LocaleListCompat = getLocales(Resources.getSystem().configuration) @@ -35,7 +37,7 @@ internal class RepoV1Fetcher( val indexFile = tempFileProvider.createTempFile() val entryDownloader = downloaderFactory.create( repo = repo, - uri = uri.buildUpon().appendPath(SIGNED_FILE_NAME).build(), + uri = repoUriBuilder.getUri(repo, SIGNED_FILE_NAME), indexFile = FileV2.fromPath("/$SIGNED_FILE_NAME"), destFile = indexFile, ) diff --git a/libs/database/src/main/java/org/fdroid/repo/RepoV2Fetcher.kt b/libs/database/src/main/java/org/fdroid/repo/RepoV2Fetcher.kt index 7803b33e5..ca2360f65 100644 --- a/libs/database/src/main/java/org/fdroid/repo/RepoV2Fetcher.kt +++ b/libs/database/src/main/java/org/fdroid/repo/RepoV2Fetcher.kt @@ -7,7 +7,9 @@ import org.fdroid.download.DownloadRequest import org.fdroid.download.DownloaderFactory import org.fdroid.download.HttpManager import org.fdroid.download.getDigestInputStream +import org.fdroid.fdroid.DigestInputStream import org.fdroid.index.IndexParser +import org.fdroid.index.RepoUriBuilder import org.fdroid.index.SigningException import org.fdroid.index.TempFileProvider import org.fdroid.index.parseEntry @@ -16,11 +18,13 @@ import org.fdroid.index.v2.FileV2 import org.fdroid.index.v2.IndexV2FullStreamProcessor import org.fdroid.index.v2.SIGNED_FILE_NAME import java.net.Proxy +import java.security.MessageDigest internal class RepoV2Fetcher( private val tempFileProvider: TempFileProvider, private val downloaderFactory: DownloaderFactory, private val httpManager: HttpManager, + private val repoUriBuilder: RepoUriBuilder, private val proxy: Proxy? = null, ) : RepoFetcher { private val log = KotlinLogging.logger {} @@ -36,7 +40,7 @@ internal class RepoV2Fetcher( val entryFile = tempFileProvider.createTempFile() val entryDownloader = downloaderFactory.create( repo = repo, - uri = uri.buildUpon().appendPath(SIGNED_FILE_NAME).build(), + uri = repoUriBuilder.getUri(repo, SIGNED_FILE_NAME), indexFile = FileV2.fromPath("/$SIGNED_FILE_NAME"), destFile = entryFile, ) @@ -52,17 +56,31 @@ internal class RepoV2Fetcher( log.info { "Downloaded entry, now streaming index..." } - // stream index - val indexRequest = DownloadRequest( - indexFile = FileV2.fromPath(entry.index.name.trimStart('/')), - mirrors = repo.getMirrors(), - proxy = proxy, - username = repo.username, - password = repo.password, - ) val streamReceiver = RepoV2StreamReceiver(receiver, repo.username, repo.password) val streamProcessor = IndexV2FullStreamProcessor(streamReceiver, cert) - val digestInputStream = httpManager.getDigestInputStream(indexRequest) + val digestInputStream = if (uri.scheme?.startsWith("http") == true) { + // stream index for http(s) downloads + val indexRequest = DownloadRequest( + indexFile = entry.index, + mirrors = repo.getMirrors(), + proxy = proxy, + username = repo.username, + password = repo.password, + ) + httpManager.getDigestInputStream(indexRequest) + } else { + // no streaming supported, download file first + val indexFile = tempFileProvider.createTempFile() + val indexDownloader = downloaderFactory.create( + repo = repo, + uri = repoUriBuilder.getUri(repo, entry.index.name.trimStart('/')), + indexFile = entry.index, + destFile = indexFile, + ) + indexDownloader.download() + val digest = MessageDigest.getInstance("SHA-256") + DigestInputStream(indexFile.inputStream(), digest) + } digestInputStream.use { inputStream -> streamProcessor.process(entry.version, inputStream) { } } diff --git a/libs/database/src/test/java/org/fdroid/repo/RepoAdderTest.kt b/libs/database/src/test/java/org/fdroid/repo/RepoAdderTest.kt index 154d57c69..e552a68d6 100644 --- a/libs/database/src/test/java/org/fdroid/repo/RepoAdderTest.kt +++ b/libs/database/src/test/java/org/fdroid/repo/RepoAdderTest.kt @@ -16,6 +16,7 @@ import io.mockk.just import io.mockk.mockk import io.mockk.mockkStatic import io.mockk.slot +import io.mockk.verify import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.delay import kotlinx.coroutines.launch @@ -213,6 +214,10 @@ internal class RepoAdderTest { assertTrue(addedState is Added, addedState.toString()) assertEquals(existingRepo, addedState.repo) } + + verify(exactly = 1) { + repoDao.updateUserMirrors(42L, listOf(url.trimEnd('/'))) + } } @Test