From 7d64492d92856ab0595700f25f0bb6c3a007f16f Mon Sep 17 00:00:00 2001 From: Torsten Grote Date: Wed, 27 Sep 2023 11:39:08 +0200 Subject: [PATCH] [db] support adding repos protected with basic auth --- .../repo/RepoManagerAddAllIntegrationTest.kt | 7 +- .../main/java/org/fdroid/index/RepoManager.kt | 11 +- .../main/java/org/fdroid/repo/RepoAdder.kt | 12 +- .../java/org/fdroid/repo/RepoUriGetter.kt | 24 +++- .../fdroid/repo/RepoAdderIntegrationTest.kt | 2 - .../java/org/fdroid/repo/RepoAdderTest.kt | 131 +++++++++++------- .../java/org/fdroid/repo/RepoUriGetterTest.kt | 62 +++++++-- 7 files changed, 163 insertions(+), 86 deletions(-) diff --git a/app/src/androidTest/java/org/fdroid/repo/RepoManagerAddAllIntegrationTest.kt b/app/src/androidTest/java/org/fdroid/repo/RepoManagerAddAllIntegrationTest.kt index a7f164e1a..fedd84594 100644 --- a/app/src/androidTest/java/org/fdroid/repo/RepoManagerAddAllIntegrationTest.kt +++ b/app/src/androidTest/java/org/fdroid/repo/RepoManagerAddAllIntegrationTest.kt @@ -132,12 +132,7 @@ internal class RepoManagerAddAllIntegrationTest { private suspend fun addRepo(url: String) { log.info("Fetching $url") - repoManager.fetchRepositoryPreview( - url = url, - username = null, - password = null, - proxy = null, - ) + repoManager.fetchRepositoryPreview(url = url, proxy = null) repoManager.addRepoState.test(timeout = 15.seconds) { val fetchState = awaitFinalFetchState() if (fetchState is Fetching && fetchState.canAdd) { diff --git a/libs/database/src/main/java/org/fdroid/index/RepoManager.kt b/libs/database/src/main/java/org/fdroid/index/RepoManager.kt index 4f12b0ce9..7acdd1986 100644 --- a/libs/database/src/main/java/org/fdroid/index/RepoManager.kt +++ b/libs/database/src/main/java/org/fdroid/index/RepoManager.kt @@ -126,17 +126,12 @@ public class RepoManager @JvmOverloads constructor( */ @AnyThread @JvmOverloads - public fun fetchRepositoryPreview( - url: String, - username: String? = null, - password: String? = null, - proxy: Proxy? = null, - ) { - repoAdder.fetchRepository(url, username, password, proxy) + public fun fetchRepositoryPreview(url: String, proxy: Proxy? = null) { + repoAdder.fetchRepository(url, proxy) } /** - * When [addRepoState] is in [org.fdroid.repo.Fetched], + * When [addRepoState] is in [org.fdroid.repo.Fetching.done], * you can call this to actually add the repo to the DB. * @throws IllegalStateException if [addRepoState] is currently in any other state. */ diff --git a/libs/database/src/main/java/org/fdroid/repo/RepoAdder.kt b/libs/database/src/main/java/org/fdroid/repo/RepoAdder.kt index b361dc3a4..317dd5226 100644 --- a/libs/database/src/main/java/org/fdroid/repo/RepoAdder.kt +++ b/libs/database/src/main/java/org/fdroid/repo/RepoAdder.kt @@ -110,9 +110,9 @@ internal class RepoAdder( private var fetchJob: Job? = null - internal fun fetchRepository(url: String, username: String?, password: String?, proxy: Proxy?) { + internal fun fetchRepository(url: String, proxy: Proxy?) { fetchJob = GlobalScope.launch(coroutineContext) { - fetchRepositoryInt(url, username, password, proxy) + fetchRepositoryInt(url, proxy) } } @@ -120,8 +120,6 @@ internal class RepoAdder( @VisibleForTesting internal suspend fun fetchRepositoryInt( url: String, - username: String? = null, - password: String? = null, proxy: Proxy? = null, ) { if (hasDisallowInstallUnknownSources(context)) { @@ -159,14 +157,16 @@ internal class RepoAdder( // try fetching repo with v2 format first and fallback to v1 try { try { - val repo = getTempRepo(nUri.uri, IndexFormatVersion.TWO, username, password) + val repo = + getTempRepo(nUri.uri, IndexFormatVersion.TWO, nUri.username, nUri.password) val repoFetcher = RepoV2Fetcher(tempFileProvider, downloaderFactory, httpManager, proxy) repoFetcher.fetchRepo(nUri.uri, repo, receiver, nUri.fingerprint) } catch (e: NotFoundException) { log.warn(e) { "Did not find v2 repo, trying v1 now." } // try to fetch v1 repo - val repo = getTempRepo(nUri.uri, IndexFormatVersion.ONE, username, password) + val repo = + getTempRepo(nUri.uri, IndexFormatVersion.ONE, nUri.username, nUri.password) val repoFetcher = RepoV1Fetcher(tempFileProvider, downloaderFactory) repoFetcher.fetchRepo(nUri.uri, repo, receiver, nUri.fingerprint) } diff --git a/libs/database/src/main/java/org/fdroid/repo/RepoUriGetter.kt b/libs/database/src/main/java/org/fdroid/repo/RepoUriGetter.kt index 3c0f20d1f..823908b77 100644 --- a/libs/database/src/main/java/org/fdroid/repo/RepoUriGetter.kt +++ b/libs/database/src/main/java/org/fdroid/repo/RepoUriGetter.kt @@ -30,7 +30,19 @@ internal object RepoUriGetter { ?: uri.getQueryParameter("FINGERPRINT")?.lowercase() val pathSegments = uri.pathSegments + var username: String? = null + var password: String? = null val normalizedUri = uri.buildUpon().apply { + // extract and remove userInfo, if available + val userInfo = uri.userInfo + val authority = uri.authority + if (userInfo != null && authority != null) { + val host = authority.split('@')[1] + val usernamePassword = userInfo.split(':') + if (usernamePassword.isNotEmpty()) username = usernamePassword[0] + if (usernamePassword.size > 1) password = usernamePassword[1] + authority(host) // remove userInfo from URI + } clearQuery() // removes fingerprint and other query params fragment("") // remove # hash fragment if (pathSegments.size >= 2 && @@ -57,7 +69,7 @@ internal object RepoUriGetter { newUri } } - return NormalizedUri(normalizedUri, fingerprint) + return NormalizedUri(normalizedUri, fingerprint, username, password) } fun isSwapUri(uri: Uri): Boolean { @@ -71,8 +83,14 @@ internal object RepoUriGetter { } /** - * A class for normalizing the [Repository] URI and holding an optional fingerprint. + * A class for normalizing the [Repository] URI and holding an optional fingerprint + * as well as username/password for basic authentication. */ - data class NormalizedUri(val uri: Uri, val fingerprint: String?) + data class NormalizedUri( + val uri: Uri, + val fingerprint: String?, + val username: String? = null, + val password: String? = null, + ) } diff --git a/libs/database/src/test/java/org/fdroid/repo/RepoAdderIntegrationTest.kt b/libs/database/src/test/java/org/fdroid/repo/RepoAdderIntegrationTest.kt index d2f7c8f5c..bfb4509dd 100644 --- a/libs/database/src/test/java/org/fdroid/repo/RepoAdderIntegrationTest.kt +++ b/libs/database/src/test/java/org/fdroid/repo/RepoAdderIntegrationTest.kt @@ -60,8 +60,6 @@ internal class RepoAdderIntegrationTest { repoAdder.fetchRepository( url = "https://fdroid.fedilab.app/repo/" + "?fingerprint=11F0A69910A4280E2CD3CCC3146337D006BE539B18E1A9FEACE15FF757A94FEB", - username = null, - password = null, proxy = null ) repoAdder.addRepoState.test { diff --git a/libs/database/src/test/java/org/fdroid/repo/RepoAdderTest.kt b/libs/database/src/test/java/org/fdroid/repo/RepoAdderTest.kt index cbf126665..154d57c69 100644 --- a/libs/database/src/test/java/org/fdroid/repo/RepoAdderTest.kt +++ b/libs/database/src/test/java/org/fdroid/repo/RepoAdderTest.kt @@ -43,6 +43,7 @@ import org.fdroid.repo.AddRepoError.ErrorType.IO_ERROR import org.fdroid.repo.AddRepoError.ErrorType.UNKNOWN_SOURCES_DISALLOWED import org.fdroid.test.TestDataMinV2 import org.fdroid.test.TestUtils.decodeHex +import org.fdroid.test.TestUtils.getRandomString import org.fdroid.test.VerifierConstants import org.junit.Rule import org.junit.Test @@ -139,12 +140,7 @@ internal class RepoAdderTest { every { repoDao.getRepository(any()) } returns null expectMinRepoPreview(repoName, FetchResult.IsNewRepository) { - repoAdder.fetchRepository( - url = url, - username = null, - password = null, - proxy = null, - ) + repoAdder.fetchRepository(url = url, proxy = null) } val newRepo: Repository = mockk() @@ -196,12 +192,7 @@ internal class RepoAdderTest { every { repoDao.getRepository(any()) } returns existingRepo expectMinRepoPreview(repoName, FetchResult.IsNewMirror(42L, url.trimEnd('/'))) { - repoAdder.fetchRepository( - url = url, - username = null, - password = null, - proxy = null - ) + repoAdder.fetchRepository(url = url, proxy = null) } val transactionSlot = slot>() @@ -349,12 +340,7 @@ internal class RepoAdderTest { every { repoDao.getRepository(any()) } returns existingRepo expectMinRepoPreview(repoName, FetchResult.IsExistingRepository, canAdd = false) { - repoAdder.fetchRepository( - url = url, - username = null, - password = null, - proxy = null - ) + repoAdder.fetchRepository(url = url, proxy = null) } assertFailsWith { repoAdder.addFetchedRepository() @@ -380,12 +366,7 @@ internal class RepoAdderTest { repoAdder.addRepoState.test { assertIs(awaitItem()) - repoAdder.fetchRepository( - url = url, - username = null, - password = null, - proxy = null - ) + repoAdder.fetchRepository(url = url, proxy = null) val state1 = awaitItem() assertIs(state1) @@ -438,12 +419,7 @@ internal class RepoAdderTest { repoAdder.addRepoState.test { assertIs(awaitItem()) - repoAdder.fetchRepository( - url = url, - username = null, - password = null, - proxy = null - ) + repoAdder.fetchRepository(url = url, proxy = null) val state1 = awaitItem() assertIs(state1) @@ -479,12 +455,7 @@ internal class RepoAdderTest { repoAdder.addRepoState.test { assertIs(awaitItem()) - repoAdder.fetchRepository( - url = url, - username = null, - password = null, - proxy = null - ) + repoAdder.fetchRepository(url = url, proxy = null) val state1 = awaitItem() assertIs(state1) @@ -539,12 +510,7 @@ internal class RepoAdderTest { repoAdder.addRepoState.test { assertIs(awaitItem()) - repoAdder.fetchRepository( - url = url, - username = null, - password = null, - proxy = null - ) + repoAdder.fetchRepository(url = url, proxy = null) val state1 = awaitItem() assertIs(state1) @@ -589,12 +555,7 @@ internal class RepoAdderTest { repoAdder.addRepoState.test { assertIs(awaitItem()) - repoAdder.fetchRepository( - url = url, - username = null, - password = null, - proxy = null - ) + repoAdder.fetchRepository(url = url, proxy = null) val state1 = awaitItem() assertIs(state1) @@ -608,6 +569,80 @@ internal class RepoAdderTest { } } + @Test + fun testAddingMinRepoWithBasicAuth() = runTest { + val username = getRandomString() + val password = getRandomString() + val url = "https://$username:$password@example.org/repo/" + val repoName = TestDataMinV2.repo.name.getBestLocale(localeList) + + val urlTrimmed = "https://example.org/repo" + val jarFile = folder.newFile() + val indexFile = assets.open("index-min-v2.json") + val index = indexFile.use { it.readBytes() } + val indexStream = DigestInputStream(ByteArrayInputStream(index), digest) + + every { tempFileProvider.createTempFile() } returns jarFile + every { + downloaderFactory.create( + repo = match { + it.address == urlTrimmed && it.formatVersion == IndexFormatVersion.TWO + }, + uri = Uri.parse("$urlTrimmed/entry.jar"), + indexFile = any(), + destFile = jarFile, + ) + } returns downloader + every { downloader.download() } answers { + jarFile.outputStream().use { outputStream -> + assets.open("diff-empty-min/entry.jar").use { inputStream -> + inputStream.copyTo(outputStream) + } + } + } + coEvery { + httpManager.getDigestInputStream(match { + it.indexFile.name == "../index-min-v2.json" && + it.mirrors.size == 1 && it.mirrors[0].baseUrl == urlTrimmed + }) + } returns indexStream + every { + digest.digest() // sha256 from entry.json + } returns "851ecda085ed53adab25f761a9dbf4c09d59e5bff9c9d5530814d56445ae30f2".decodeHex() + + // repo not in DB + every { repoDao.getRepository(any()) } returns null + + expectMinRepoPreview(repoName, FetchResult.IsNewRepository) { + repoAdder.fetchRepository(url = url, proxy = null) + } + + val newRepo: Repository = mockk() + every { + repoDao.insert(match { + // Note that we are not using the url the user used to add the repo, + // but what the repo tells us to use + it.address == TestDataMinV2.repo.address && + it.formatVersion == IndexFormatVersion.TWO && + it.name.getBestLocale(localeList) == repoName && + it.username == username && it.password == password // this is the important bit + }) + } returns 42L + every { repoDao.getRepository(42L) } returns newRepo + + repoAdder.addRepoState.test { + assertIs(awaitItem()) // still Fetching from last call + + repoAdder.addFetchedRepository() + + assertIs(awaitItem()) // now moved to Adding + + val addedState = awaitItem() + assertIs(addedState) + assertEquals(newRepo, addedState.repo) + } + } + private fun expectDownloadOfMinRepo(url: String) { val urlTrimmed = url.trimEnd('/') val jarFile = folder.newFile() diff --git a/libs/database/src/test/java/org/fdroid/repo/RepoUriGetterTest.kt b/libs/database/src/test/java/org/fdroid/repo/RepoUriGetterTest.kt index f1755b604..2d5848b00 100644 --- a/libs/database/src/test/java/org/fdroid/repo/RepoUriGetterTest.kt +++ b/libs/database/src/test/java/org/fdroid/repo/RepoUriGetterTest.kt @@ -62,9 +62,10 @@ internal class RepoUriGetterTest { @Test fun testFDroidLink() { - val uri1 = - RepoUriGetter.getUri("https://fdroid.link/index.html#repo=https://f-droid.org/repo?" + - "fingerprint=43238d512c1e5eb2d6569f4a3afbf5523418b82e0a3ed1552770abb9a9c9ccab") + val uri1 = RepoUriGetter.getUri( + "https://fdroid.link/index.html#repo=https://f-droid.org/repo?" + + "fingerprint=43238d512c1e5eb2d6569f4a3afbf5523418b82e0a3ed1552770abb9a9c9ccab" + ) assertEquals("https://f-droid.org/repo", uri1.uri.toString()) assertEquals( "43238d512c1e5eb2d6569f4a3afbf5523418b82e0a3ed1552770abb9a9c9ccab", @@ -82,8 +83,7 @@ internal class RepoUriGetterTest { @Test fun testAddScheme() { - val uri1 = - RepoUriGetter.getUri("example.com/repo") + val uri1 = RepoUriGetter.getUri("example.com/repo") assertEquals("https://example.com/repo", uri1.uri.toString()) assertNull(uri1.fingerprint) @@ -100,9 +100,10 @@ internal class RepoUriGetterTest { @Test fun testFDroidRepoUriScheme() { - val uri1 = - RepoUriGetter.getUri("fdroidrepos://grobox.de/fdroid/repo?fingerprint=" + - "28e14fb3b280bce8ff1e0f8e82726ff46923662cecff2a0689108ce19e8b347c") + val uri1 = RepoUriGetter.getUri( + "fdroidrepos://grobox.de/fdroid/repo?fingerprint=" + + "28e14fb3b280bce8ff1e0f8e82726ff46923662cecff2a0689108ce19e8b347c" + ) assertEquals("https://grobox.de/fdroid/repo", uri1.uri.toString()) assertEquals( "28e14fb3b280bce8ff1e0f8e82726ff46923662cecff2a0689108ce19e8b347c", @@ -113,9 +114,10 @@ internal class RepoUriGetterTest { assertEquals("http://grobox.de/fdroid/repo", uri2.uri.toString()) assertNull(uri2.fingerprint) - val uri3 = - RepoUriGetter.getUri("FDROIDREPOS://grobox.de/fdroid/repo?fingerprint=" + - "28e14fb3b280bce8ff1e0f8e82726ff46923662cecff2a0689108ce19e8b347c") + val uri3 = RepoUriGetter.getUri( + "FDROIDREPOS://grobox.de/fdroid/repo?fingerprint=" + + "28e14fb3b280bce8ff1e0f8e82726ff46923662cecff2a0689108ce19e8b347c" + ) assertEquals("https://grobox.de/fdroid/repo", uri3.uri.toString()) assertEquals( "28e14fb3b280bce8ff1e0f8e82726ff46923662cecff2a0689108ce19e8b347c", @@ -127,6 +129,38 @@ internal class RepoUriGetterTest { assertNull(uri4.fingerprint) } + @Test + fun testUsernamePassword() { + val uri1 = RepoUriGetter.getUri( + "https://username:password@example.org/repo?fingerprint=foobar&test=42" + ) + assertEquals("https://example.org/repo", uri1.uri.toString()) + assertEquals("foobar", uri1.fingerprint) + assertEquals("username", uri1.username) + assertEquals("password", uri1.password) + + // no password + val uri2 = + RepoUriGetter.getUri("https://username@example.org/repo?fingerprint=foobar&test=42") + assertEquals("https://example.org/repo", uri2.uri.toString()) + assertEquals("foobar", uri2.fingerprint) + assertEquals("username", uri2.username) + assertNull(uri2.password) + + // empty host + val uri3 = RepoUriGetter.getUri("https://foo:bar@/repo?fingerprint=foobar&test=42") + assertEquals("https:///repo", uri3.uri.toString()) + assertEquals("foobar", uri3.fingerprint) + assertEquals("foo", uri3.username) + assertEquals("bar", uri3.password) + + // empty everything doesn't crash + RepoUriGetter.getUri(":@/") + RepoUriGetter.getUri(":@") + RepoUriGetter.getUri("@") + RepoUriGetter.getUri("") + } + @Test fun testSwapUri() { val uri = @@ -151,8 +185,10 @@ internal class RepoUriGetterTest { ) assertTrue(RepoUriGetter.isSwapUri(uri1)) - val uri2 = Uri.parse("http://192.168.3.159:8888/fdroid/repo?" + - "swap=1&BSSID=44:FE:3B:7F:7F:EE") + val uri2 = Uri.parse( + "http://192.168.3.159:8888/fdroid/repo?" + + "swap=1&BSSID=44:FE:3B:7F:7F:EE" + ) assertTrue(RepoUriGetter.isSwapUri(uri2)) val uri3 = Uri.parse("http://192.168.3.159:8888/fdroid/repo?BSSID=44:FE:3B:7F:7F:EE")