[db] add support for adding repos on content:// and file:// URIs

This complicates the code for this niche use-case, but is wanted by F-Droid to support communities with bad or no internet connectivity.

libs/database/src/main/java/org/fdroid/index/RepoManager.kt

@@ -33,6 +33,7 @@ public class RepoManager @JvmOverloads constructor(
     db: FDroidDatabase,
     downloaderFactory: DownloaderFactory,
     httpManager: HttpManager,
+    private val repoUriBuilder: RepoUriBuilder = defaultRepoUriBuilder,
     private val coroutineContext: CoroutineContext = Dispatchers.IO,
 ) {
 
@@ -46,6 +47,7 @@ public class RepoManager @JvmOverloads constructor(
         tempFileProvider = tempFileProvider,
         downloaderFactory = downloaderFactory,
         httpManager = httpManager,
+        repoUriBuilder = repoUriBuilder,
         coroutineContext = coroutineContext,
     )
 

libs/database/src/main/java/org/fdroid/repo/RepoAdder.kt

@@ -28,6 +28,7 @@ import org.fdroid.download.HttpManager
 import org.fdroid.download.HttpManager.Companion.isInvalidHttpUrl
 import org.fdroid.download.NotFoundException
 import org.fdroid.index.IndexFormatVersion
+import org.fdroid.index.RepoUriBuilder
 import org.fdroid.index.SigningException
 import org.fdroid.index.TempFileProvider
 import org.fdroid.repo.AddRepoError.ErrorType.INVALID_FINGERPRINT
@@ -101,6 +102,7 @@ internal class RepoAdder(
     private val downloaderFactory: DownloaderFactory,
     private val httpManager: HttpManager,
     private val repoUriGetter: RepoUriGetter = RepoUriGetter,
+    private val repoUriBuilder: RepoUriBuilder = defaultRepoUriBuilder,
     private val coroutineContext: CoroutineContext = Dispatchers.IO,
 ) {
     private val log = KotlinLogging.logger {}
@@ -129,7 +131,9 @@ internal class RepoAdder(
         // get repo url and fingerprint
         val nUri = repoUriGetter.getUri(url)
         log.info("Parsed URI: $nUri")
-        if (isInvalidHttpUrl(nUri.uri.toString())) {
+        if (nUri.uri.scheme !in listOf("content", "file") &&
+            isInvalidHttpUrl(nUri.uri.toString())
+        ) {
             val e = IllegalArgumentException("Unsupported URI: ${nUri.uri}")
             addRepoState.value = AddRepoError(INVALID_INDEX, e)
             return
@@ -159,15 +163,16 @@ internal class RepoAdder(
             try {
                 val repo =
                     getTempRepo(nUri.uri, IndexFormatVersion.TWO, nUri.username, nUri.password)
-                val repoFetcher =
-                    RepoV2Fetcher(tempFileProvider, downloaderFactory, httpManager, proxy)
+                val repoFetcher = RepoV2Fetcher(
+                    tempFileProvider, downloaderFactory, httpManager, repoUriBuilder, proxy
+                )
                 repoFetcher.fetchRepo(nUri.uri, repo, receiver, nUri.fingerprint)
             } catch (e: NotFoundException) {
                 log.warn(e) { "Did not find v2 repo, trying v1 now." }
                 // try to fetch v1 repo
                 val repo =
                     getTempRepo(nUri.uri, IndexFormatVersion.ONE, nUri.username, nUri.password)
-                val repoFetcher = RepoV1Fetcher(tempFileProvider, downloaderFactory)
+                val repoFetcher = RepoV1Fetcher(tempFileProvider, downloaderFactory, repoUriBuilder)
                 repoFetcher.fetchRepo(nUri.uri, repo, receiver, nUri.fingerprint)
             }
         } catch (e: SigningException) {
@@ -300,3 +305,9 @@ internal class RepoAdder(
     )
 
 }
+
+internal val defaultRepoUriBuilder = RepoUriBuilder { repo, pathElements ->
+    val builder = Uri.parse(repo.address).buildUpon()
+    pathElements.forEach { builder.appendEncodedPath(it) }
+    builder.build()
+}

libs/database/src/main/java/org/fdroid/repo/RepoUriGetter.kt

@@ -45,20 +45,23 @@ internal object RepoUriGetter {
             }
             clearQuery() // removes fingerprint and other query params
             fragment("") // remove # hash fragment
-            if (pathSegments.size >= 2 &&
-                pathSegments[pathSegments.lastIndex - 1] == "fdroid" &&
-                pathSegments.last() == "repo"
-            ) {
-                // path already is /fdroid/repo, use as is
-            } else if (pathSegments.lastOrNull() == "repo") {
-                // path already ends in /repo, use as is
-            } else if (pathSegments.size >= 1 && pathSegments.last() == "fdroid") {
-                // path is /fdroid with missing /repo, so add that
-                appendPath("repo")
-            } else {
-                // path is missing /fdroid/repo, so add it
-                appendPath("fdroid")
-                appendPath("repo")
+            if (uri.scheme != "content" && uri.scheme != "file") {
+                // do some path auto-adding, if it is missing
+                if (pathSegments.size >= 2 &&
+                    pathSegments[pathSegments.lastIndex - 1] == "fdroid" &&
+                    pathSegments.last() == "repo"
+                ) {
+                    // path already is /fdroid/repo, use as is
+                } else if (pathSegments.lastOrNull() == "repo") {
+                    // path already ends in /repo, use as is
+                } else if (pathSegments.size >= 1 && pathSegments.last() == "fdroid") {
+                    // path is /fdroid with missing /repo, so add that
+                    appendPath("repo")
+                } else {
+                    // path is missing /fdroid/repo, so add it
+                    appendPath("fdroid")
+                    appendPath("repo")
+                }
             }
         }.build().let { newUri ->
             // hacky way to remove trailing slash

libs/database/src/main/java/org/fdroid/repo/RepoV1Fetcher.kt

@@ -10,6 +10,7 @@ import org.fdroid.download.DownloaderFactory
 import org.fdroid.index.IndexConverter
 import org.fdroid.index.IndexFormatVersion
 import org.fdroid.index.IndexParser
+import org.fdroid.index.RepoUriBuilder
 import org.fdroid.index.SigningException
 import org.fdroid.index.TempFileProvider
 import org.fdroid.index.parseV1
@@ -20,6 +21,7 @@ import org.fdroid.index.v2.FileV2
 internal class RepoV1Fetcher(
     private val tempFileProvider: TempFileProvider,
     private val downloaderFactory: DownloaderFactory,
+    private val repoUriBuilder: RepoUriBuilder,
 ) : RepoFetcher {
 
     private val locales: LocaleListCompat = getLocales(Resources.getSystem().configuration)
@@ -35,7 +37,7 @@ internal class RepoV1Fetcher(
         val indexFile = tempFileProvider.createTempFile()
         val entryDownloader = downloaderFactory.create(
             repo = repo,
-            uri = uri.buildUpon().appendPath(SIGNED_FILE_NAME).build(),
+            uri = repoUriBuilder.getUri(repo, SIGNED_FILE_NAME),
             indexFile = FileV2.fromPath("/$SIGNED_FILE_NAME"),
             destFile = indexFile,
         )

libs/database/src/main/java/org/fdroid/repo/RepoV2Fetcher.kt

@@ -7,7 +7,9 @@ import org.fdroid.download.DownloadRequest
 import org.fdroid.download.DownloaderFactory
 import org.fdroid.download.HttpManager
 import org.fdroid.download.getDigestInputStream
+import org.fdroid.fdroid.DigestInputStream
 import org.fdroid.index.IndexParser
+import org.fdroid.index.RepoUriBuilder
 import org.fdroid.index.SigningException
 import org.fdroid.index.TempFileProvider
 import org.fdroid.index.parseEntry
@@ -16,11 +18,13 @@ import org.fdroid.index.v2.FileV2
 import org.fdroid.index.v2.IndexV2FullStreamProcessor
 import org.fdroid.index.v2.SIGNED_FILE_NAME
 import java.net.Proxy
+import java.security.MessageDigest
 
 internal class RepoV2Fetcher(
     private val tempFileProvider: TempFileProvider,
     private val downloaderFactory: DownloaderFactory,
     private val httpManager: HttpManager,
+    private val repoUriBuilder: RepoUriBuilder,
     private val proxy: Proxy? = null,
 ) : RepoFetcher {
     private val log = KotlinLogging.logger {}
@@ -36,7 +40,7 @@ internal class RepoV2Fetcher(
         val entryFile = tempFileProvider.createTempFile()
         val entryDownloader = downloaderFactory.create(
             repo = repo,
-            uri = uri.buildUpon().appendPath(SIGNED_FILE_NAME).build(),
+            uri = repoUriBuilder.getUri(repo, SIGNED_FILE_NAME),
             indexFile = FileV2.fromPath("/$SIGNED_FILE_NAME"),
             destFile = entryFile,
         )
@@ -52,17 +56,31 @@ internal class RepoV2Fetcher(
 
         log.info { "Downloaded entry, now streaming index..." }
 
-        // stream index
-        val indexRequest = DownloadRequest(
-            indexFile = FileV2.fromPath(entry.index.name.trimStart('/')),
-            mirrors = repo.getMirrors(),
-            proxy = proxy,
-            username = repo.username,
-            password = repo.password,
-        )
         val streamReceiver = RepoV2StreamReceiver(receiver, repo.username, repo.password)
         val streamProcessor = IndexV2FullStreamProcessor(streamReceiver, cert)
-        val digestInputStream = httpManager.getDigestInputStream(indexRequest)
+        val digestInputStream = if (uri.scheme?.startsWith("http") == true) {
+            // stream index for http(s) downloads
+            val indexRequest = DownloadRequest(
+                indexFile = entry.index,
+                mirrors = repo.getMirrors(),
+                proxy = proxy,
+                username = repo.username,
+                password = repo.password,
+            )
+            httpManager.getDigestInputStream(indexRequest)
+        } else {
+            // no streaming supported, download file first
+            val indexFile = tempFileProvider.createTempFile()
+            val indexDownloader = downloaderFactory.create(
+                repo = repo,
+                uri = repoUriBuilder.getUri(repo, entry.index.name.trimStart('/')),
+                indexFile = entry.index,
+                destFile = indexFile,
+            )
+            indexDownloader.download()
+            val digest = MessageDigest.getInstance("SHA-256")
+            DigestInputStream(indexFile.inputStream(), digest)
+        }
         digestInputStream.use { inputStream ->
             streamProcessor.process(entry.version, inputStream) { }
         }