From d5ceb0b202dcdb7e9f8c75ffc118f761d33af857 Mon Sep 17 00:00:00 2001 From: Torsten Grote Date: Mon, 17 Jan 2022 15:33:07 -0300 Subject: [PATCH] Add proxy support to download library --- .../fdroid/fdroid/net/HttpDownloaderTest.java | 29 ++++---- .../fdroid/fdroid/net/DownloaderFactory.java | 17 +++-- .../org/fdroid/download/DownloadManager.kt | 10 --- .../org/fdroid/download/HttpDownloader.kt | 14 ++-- .../kotlin/org/fdroid/download/HttpManager.kt | 31 ++++++++ .../org/fdroid/download/HttpDownloaderTest.kt | 51 ++++++++++--- .../org/fdroid/download/HttpPosterTest.kt | 5 +- .../org/fdroid/download/DownloadRequest.kt | 4 +- .../kotlin/org/fdroid/download/HttpManager.kt | 59 ++++++++++----- .../org/fdroid/download/MirrorChooser.kt | 4 +- .../kotlin/org/fdroid/download/Proxy.kt | 20 ++++++ .../commonTest/kotlin/org/fdroid/TestUtils.kt | 10 +++ .../download/HttpManagerIntegrationTest.kt | 22 +++++- .../org/fdroid/download/HttpManagerTest.kt | 71 +++++++++++++------ .../org/fdroid/download/DownloadManager.kt | 10 --- .../kotlin/org/fdroid/download/HttpManager.kt | 8 +++ .../org/fdroid/download/DownloadManager.kt | 10 --- .../kotlin/org/fdroid/download/HttpManager.kt | 8 +++ 18 files changed, 267 insertions(+), 116 deletions(-) delete mode 100644 download/src/androidMain/kotlin/org/fdroid/download/DownloadManager.kt create mode 100644 download/src/androidMain/kotlin/org/fdroid/download/HttpManager.kt create mode 100644 download/src/commonMain/kotlin/org/fdroid/download/Proxy.kt delete mode 100644 download/src/jvmMain/kotlin/org/fdroid/download/DownloadManager.kt create mode 100644 download/src/jvmMain/kotlin/org/fdroid/download/HttpManager.kt delete mode 100644 download/src/nativeMain/kotlin/org/fdroid/download/DownloadManager.kt create mode 100644 download/src/nativeMain/kotlin/org/fdroid/download/HttpManager.kt diff --git a/app/src/androidTest/java/org/fdroid/fdroid/net/HttpDownloaderTest.java b/app/src/androidTest/java/org/fdroid/fdroid/net/HttpDownloaderTest.java index f4c39770e..429c77873 100644 --- a/app/src/androidTest/java/org/fdroid/fdroid/net/HttpDownloaderTest.java +++ b/app/src/androidTest/java/org/fdroid/fdroid/net/HttpDownloaderTest.java @@ -10,8 +10,9 @@ import android.util.Log; import androidx.core.util.Pair; -import org.fdroid.download.HttpManager; +import org.fdroid.download.DownloadRequest; import org.fdroid.download.HttpDownloader; +import org.fdroid.download.HttpManager; import org.fdroid.download.Mirror; import org.fdroid.fdroid.FDroidApp; import org.fdroid.fdroid.ProgressListener; @@ -31,7 +32,7 @@ import java.util.concurrent.TimeUnit; public class HttpDownloaderTest { private static final String TAG = "HttpDownloaderTest"; - private final HttpManager httpManager = new HttpManager(Utils.getUserAgent(), FDroidApp.queryString); + private final HttpManager httpManager = new HttpManager(Utils.getUserAgent(), FDroidApp.queryString, null); private static final Collection> URLS; // https://developer.android.com/reference/javax/net/ssl/SSLContext @@ -63,8 +64,8 @@ public class HttpDownloaderTest { Log.i(TAG, "URL: " + pair.first + pair.second); File destFile = File.createTempFile("dl-", ""); List mirrors = Mirror.fromStrings(Collections.singletonList(pair.first)); - HttpDownloader httpDownloader = - new HttpDownloader(httpManager, pair.second, destFile, mirrors, null, null); + DownloadRequest request = new DownloadRequest(pair.second, mirrors, null, null, null); + HttpDownloader httpDownloader = new HttpDownloader(httpManager, request, destFile); httpDownloader.download(); assertTrue(destFile.exists()); assertTrue(destFile.canRead()); @@ -79,8 +80,8 @@ public class HttpDownloaderTest { List mirrors = Mirror.fromStrings(Collections.singletonList("https://ftp.fau.de/fdroid/repo/")); receivedProgress = false; File destFile = File.createTempFile("dl-", ""); - final HttpDownloader httpDownloader = - new HttpDownloader(httpManager, path, destFile, mirrors, null, null); + final DownloadRequest request = new DownloadRequest(path, mirrors, null, null, null); + final HttpDownloader httpDownloader = new HttpDownloader(httpManager, request, destFile); httpDownloader.setListener(new ProgressListener() { @Override public void onProgress(long bytesRead, long totalBytes) { @@ -111,8 +112,8 @@ public class HttpDownloaderTest { String path = "myusername/supersecretpassword"; List mirrors = Mirror.fromStrings(Collections.singletonList("https://httpbin.org/basic-auth/")); File destFile = File.createTempFile("dl-", ""); - HttpDownloader httpDownloader = - new HttpDownloader(httpManager, path, destFile, mirrors, "myusername", "supersecretpassword"); + final DownloadRequest request = new DownloadRequest(path, mirrors, null, "myusername", "supersecretpassword"); + HttpDownloader httpDownloader = new HttpDownloader(httpManager, request, destFile); httpDownloader.download(); assertTrue(destFile.exists()); assertTrue(destFile.canRead()); @@ -124,8 +125,8 @@ public class HttpDownloaderTest { String path = "myusername/supersecretpassword"; List mirrors = Mirror.fromStrings(Collections.singletonList("https://httpbin.org/basic-auth/")); File destFile = File.createTempFile("dl-", ""); - HttpDownloader httpDownloader = - new HttpDownloader(httpManager, path, destFile, mirrors, "myusername", "wrongpassword"); + final DownloadRequest request = new DownloadRequest(path, mirrors, null,"myusername", "wrongpassword"); + HttpDownloader httpDownloader = new HttpDownloader(httpManager, request, destFile); httpDownloader.download(); assertFalse(destFile.exists()); destFile.deleteOnExit(); @@ -136,8 +137,8 @@ public class HttpDownloaderTest { String path = "myusername/supersecretpassword"; List mirrors = Mirror.fromStrings(Collections.singletonList("https://httpbin.org/basic-auth/")); File destFile = File.createTempFile("dl-", ""); - HttpDownloader httpDownloader = - new HttpDownloader(httpManager, path, destFile, mirrors, "wrongusername", "supersecretpassword"); + final DownloadRequest request = new DownloadRequest(path, mirrors, null, "wrongusername", "supersecretpassword"); + HttpDownloader httpDownloader = new HttpDownloader(httpManager, request, destFile); httpDownloader.download(); assertFalse(destFile.exists()); destFile.deleteOnExit(); @@ -149,8 +150,8 @@ public class HttpDownloaderTest { String path = "index.jar"; List mirrors = Mirror.fromStrings(Collections.singletonList("https://f-droid.org/repo/")); File destFile = File.createTempFile("dl-", ""); - final HttpDownloader httpDownloader = - new HttpDownloader(httpManager, path, destFile, mirrors, null, null); + final DownloadRequest request = new DownloadRequest(path, mirrors, null, null, null); + final HttpDownloader httpDownloader = new HttpDownloader(httpManager, request, destFile); httpDownloader.setListener(new ProgressListener() { @Override public void onProgress(long bytesRead, long totalBytes) { diff --git a/app/src/main/java/org/fdroid/fdroid/net/DownloaderFactory.java b/app/src/main/java/org/fdroid/fdroid/net/DownloaderFactory.java index 7426dd87d..27a1a6ab4 100644 --- a/app/src/main/java/org/fdroid/fdroid/net/DownloaderFactory.java +++ b/app/src/main/java/org/fdroid/fdroid/net/DownloaderFactory.java @@ -3,9 +3,10 @@ package org.fdroid.fdroid.net; import android.content.ContentResolver; import android.net.Uri; -import org.fdroid.download.HttpManager; +import org.fdroid.download.DownloadRequest; import org.fdroid.download.Downloader; import org.fdroid.download.HttpDownloader; +import org.fdroid.download.HttpManager; import org.fdroid.download.Mirror; import org.fdroid.fdroid.FDroidApp; import org.fdroid.fdroid.Utils; @@ -13,15 +14,18 @@ import org.fdroid.fdroid.data.Repo; import java.io.File; import java.io.IOException; +import java.net.Proxy; import java.util.Collections; import java.util.List; +import info.guardianproject.netcipher.NetCipher; + public class DownloaderFactory { private static final String TAG = "DownloaderFactory"; // TODO move to application object or inject where needed public static final HttpManager HTTP_MANAGER = - new HttpManager(Utils.getUserAgent(), FDroidApp.queryString); + new HttpManager(Utils.getUserAgent(), FDroidApp.queryString, NetCipher.getProxy()); /** * Same as {@link #create(Repo, Uri, File)}, but not using mirrors for download. @@ -50,10 +54,11 @@ public class DownloaderFactory { } else if (ContentResolver.SCHEME_FILE.equals(scheme)) { downloader = new LocalFileDownloader(uri, destFile); } else { - String urlSuffix = uri.toString().replace(repo.address, ""); - Utils.debugLog(TAG, "Using suffix " + urlSuffix + " with mirrors " + mirrors); - downloader = - new HttpDownloader(HTTP_MANAGER, urlSuffix, destFile, mirrors, repo.username, repo.password); + String path = uri.toString().replace(repo.address, ""); + Utils.debugLog(TAG, "Using suffix " + path + " with mirrors " + mirrors); + Proxy proxy = NetCipher.getProxy(); + DownloadRequest request = new DownloadRequest(path, mirrors, proxy, repo.username, repo.password); + downloader = new HttpDownloader(HTTP_MANAGER, request, destFile); } return downloader; } diff --git a/download/src/androidMain/kotlin/org/fdroid/download/DownloadManager.kt b/download/src/androidMain/kotlin/org/fdroid/download/DownloadManager.kt deleted file mode 100644 index 73187db48..000000000 --- a/download/src/androidMain/kotlin/org/fdroid/download/DownloadManager.kt +++ /dev/null @@ -1,10 +0,0 @@ -package org.fdroid.download - -import io.ktor.client.engine.HttpClientEngine -import io.ktor.client.engine.okhttp.OkHttp - -actual fun getHttpClientEngine(): HttpClientEngine { - return OkHttp.create { - // we could add special OkHttp config options here - } -} diff --git a/download/src/androidMain/kotlin/org/fdroid/download/HttpDownloader.kt b/download/src/androidMain/kotlin/org/fdroid/download/HttpDownloader.kt index 6c901bb76..3c604b7ec 100644 --- a/download/src/androidMain/kotlin/org/fdroid/download/HttpDownloader.kt +++ b/download/src/androidMain/kotlin/org/fdroid/download/HttpDownloader.kt @@ -36,13 +36,10 @@ import java.util.Date /** * Download files over HTTP, with support for proxies, `.onion` addresses, HTTP Basic Auth, etc. */ -class HttpDownloader @JvmOverloads constructor( +class HttpDownloader constructor( private val httpManager: HttpManager, - private val path: String, + private val request: DownloadRequest, destFile: File, - private val mirrors: List, - private val username: String? = null, - private val password: String? = null, ) : Downloader(destFile) { companion object { @@ -70,7 +67,6 @@ class HttpDownloader @JvmOverloads constructor( @Throws(IOException::class) override suspend fun getBytes(resumable: Boolean, receiver: (ByteArray) -> Unit) { - val request = DownloadRequest(path, mirrors, username, password) val skipBytes = if (resumable) outputFile.length() else null return try { httpManager.get(request, skipBytes, receiver) @@ -118,8 +114,6 @@ class HttpDownloader @JvmOverloads constructor( @OptIn(DelicateCoroutinesApi::class) @Throws(IOException::class, InterruptedException::class) override fun download() { - // boolean isSwap = isSwapUrl(sourceUrl); - val request = DownloadRequest(path, mirrors, username, password) val headInfo = runBlocking { httpManager.head(request, cacheTag) ?: throw IOException() } @@ -150,7 +144,7 @@ class HttpDownloader @JvmOverloads constructor( // calculatedEtag == expectedETag (ETag calculated from server response matches expected ETag) if (!headInfo.eTagChanged || calculatedEtag == expectedETag) { // ETag has not changed, don't download again - log.debug { "$path cached, not downloading." } + log.debug { "${request.path} cached, not downloading." } hasChanged = false return } @@ -166,7 +160,7 @@ class HttpDownloader @JvmOverloads constructor( } else if (fileLength > 0) { resumable = true } - log.debug { "downloading $path (is resumable: $resumable)" } + log.debug { "downloading ${request.path} (is resumable: $resumable)" } runBlocking { downloadFromBytesReceiver(resumable) } } diff --git a/download/src/androidMain/kotlin/org/fdroid/download/HttpManager.kt b/download/src/androidMain/kotlin/org/fdroid/download/HttpManager.kt new file mode 100644 index 000000000..250a27f26 --- /dev/null +++ b/download/src/androidMain/kotlin/org/fdroid/download/HttpManager.kt @@ -0,0 +1,31 @@ +package org.fdroid.download + +import io.ktor.client.engine.HttpClientEngine +import io.ktor.client.engine.HttpClientEngineFactory +import io.ktor.client.engine.okhttp.OkHttp +import io.ktor.client.engine.okhttp.OkHttpConfig +import okhttp3.Dns +import java.net.InetAddress + +actual fun getHttpClientEngineFactory(): HttpClientEngineFactory<*> { + return object : HttpClientEngineFactory { + override fun create(block: OkHttpConfig.() -> Unit): HttpClientEngine = OkHttp.create { + block() + if (proxy.isTor()) { // don't allow DNS requests when using Tor + config { + dns(NoDns()) + } + } + } + } +} + +/** + * Prevent DNS requests. + * Important when proxying all requests over Tor to not leak DNS queries. + */ +private class NoDns : Dns { + override fun lookup(hostname: String): List { + return listOf(InetAddress.getByAddress(hostname, ByteArray(4))) + } +} diff --git a/download/src/androidTest/kotlin/org/fdroid/download/HttpDownloaderTest.kt b/download/src/androidTest/kotlin/org/fdroid/download/HttpDownloaderTest.kt index 776011b60..0def799a7 100644 --- a/download/src/androidTest/kotlin/org/fdroid/download/HttpDownloaderTest.kt +++ b/download/src/androidTest/kotlin/org/fdroid/download/HttpDownloaderTest.kt @@ -1,5 +1,6 @@ package org.fdroid.download +import io.ktor.client.engine.ProxyBuilder import io.ktor.client.engine.mock.MockEngine import io.ktor.client.engine.mock.respond import io.ktor.client.engine.mock.respondOk @@ -12,15 +13,22 @@ import io.ktor.http.HttpMethod.Companion.Head import io.ktor.http.HttpStatusCode.Companion.OK import io.ktor.http.HttpStatusCode.Companion.PartialContent import io.ktor.http.headersOf +import org.fdroid.get import org.fdroid.getRandomString import org.fdroid.runSuspend +import org.junit.Assume.assumeTrue import org.junit.Rule import org.junit.rules.TemporaryFolder +import java.net.BindException +import java.net.ServerSocket import kotlin.random.Random import kotlin.test.Ignore import kotlin.test.Test import kotlin.test.assertContentEquals import kotlin.test.assertEquals +import kotlin.test.assertTrue + +private const val TOR_SOCKS_PORT = 9050 @Suppress("BlockingMethodInNonBlockingContext") class HttpDownloaderTest { @@ -31,6 +39,7 @@ class HttpDownloaderTest { private val userAgent = getRandomString() private val mirror1 = Mirror("http://example.org") private val mirrors = listOf(mirror1) + private val downloadRequest = DownloadRequest("foo/bar", mirrors) @Test fun testDownload() = runSuspend { @@ -38,8 +47,8 @@ class HttpDownloaderTest { val bytes = Random.nextBytes(1024) val mockEngine = MockEngine { respond(bytes) } - val httpManager = HttpManager(userAgent, null, httpClientEngine = mockEngine) - val httpDownloader = HttpDownloader(httpManager, "foo/bar", file, mirrors) + val httpManager = HttpManager(userAgent, null, httpClientEngineFactory = get(mockEngine)) + val httpDownloader = HttpDownloader(httpManager, downloadRequest, file) httpDownloader.download() assertContentEquals(bytes, file.readBytes()) @@ -57,8 +66,8 @@ class HttpDownloaderTest { if (numRequest++ == 1) respond("", OK, headers = headersOf(ContentLength, "2048")) else respond(secondBytes, PartialContent) } - val httpManager = HttpManager(userAgent, null, httpClientEngine = mockEngine) - val httpDownloader = HttpDownloader(httpManager, "foo/bar", file, mirrors) + val httpManager = HttpManager(userAgent, null, httpClientEngineFactory = get(mockEngine)) + val httpDownloader = HttpDownloader(httpManager, downloadRequest, file) httpDownloader.download() assertContentEquals(firstBytes + secondBytes, file.readBytes()) @@ -67,8 +76,8 @@ class HttpDownloaderTest { @Test fun testNoETagNotTreatedAsNoChange() = runSuspend { val mockEngine = MockEngine { respondOk() } - val httpManager = HttpManager(userAgent, null, httpClientEngine = mockEngine) - val httpDownloader = HttpDownloader(httpManager, "foo/bar", folder.newFile(), mirrors) + val httpManager = HttpManager(userAgent, null, httpClientEngineFactory = get(mockEngine)) + val httpDownloader = HttpDownloader(httpManager, downloadRequest, folder.newFile()) httpDownloader.cacheTag = null httpDownloader.download() @@ -84,8 +93,8 @@ class HttpDownloaderTest { val eTag = getRandomString() val mockEngine = MockEngine { respond("", OK, headers = headersOf(ETag, eTag)) } - val httpManager = HttpManager(userAgent, null, httpClientEngine = mockEngine) - val httpDownloader = HttpDownloader(httpManager, "foo/bar", folder.newFile(), mirrors) + val httpManager = HttpManager(userAgent, null, httpClientEngineFactory = get(mockEngine)) + val httpDownloader = HttpDownloader(httpManager, downloadRequest, folder.newFile()) httpDownloader.cacheTag = eTag httpDownloader.download() @@ -105,8 +114,8 @@ class HttpDownloaderTest { } val mockEngine = MockEngine { respond("", OK, headers = headers) } - val httpManager = HttpManager(userAgent, null, httpClientEngine = mockEngine) - val httpDownloader = HttpDownloader(httpManager, "foo/bar", folder.newFile(), mirrors) + val httpManager = HttpManager(userAgent, null, httpClientEngineFactory = get(mockEngine)) + val httpDownloader = HttpDownloader(httpManager, downloadRequest, folder.newFile()) // the ETag is calculated, but we expect a real ETag httpDownloader.cacheTag = "60a29a-5d55d390de574" httpDownloader.download() @@ -116,5 +125,27 @@ class HttpDownloaderTest { assertEquals(Head, mockEngine.requestHistory[0].method) } + @Test + fun testTorProxy() = runSuspend { + assumeTrue(isTorRunning()) + + val file = folder.newFile() + + val httpManager = HttpManager(userAgent, null) + val torHost = "http://2gzyxa5ihm7nsggfxnu52rck2vv4rvmdlkiu3zzui5du4xyclen53wid.onion" // tor-project.org + val proxy = ProxyBuilder.socks("localhost", TOR_SOCKS_PORT) + val downloadRequest = DownloadRequest("index.html", listOf(Mirror(torHost)), proxy) + val httpDownloader = HttpDownloader(httpManager, downloadRequest, file) + httpDownloader.download() + + assertTrue { file.length() > 1024 } + } + + private fun isTorRunning(): Boolean = try { + ServerSocket(TOR_SOCKS_PORT) + false + } catch (e: BindException) { + true + } } diff --git a/download/src/androidTest/kotlin/org/fdroid/download/HttpPosterTest.kt b/download/src/androidTest/kotlin/org/fdroid/download/HttpPosterTest.kt index 17c3e409d..4b5c46d49 100644 --- a/download/src/androidTest/kotlin/org/fdroid/download/HttpPosterTest.kt +++ b/download/src/androidTest/kotlin/org/fdroid/download/HttpPosterTest.kt @@ -5,6 +5,7 @@ import io.ktor.client.engine.mock.respondError import io.ktor.client.engine.mock.respondOk import io.ktor.client.engine.mock.toByteArray import io.ktor.http.HttpStatusCode.Companion.BadRequest +import org.fdroid.get import org.fdroid.getRandomString import org.fdroid.runSuspend import java.io.IOException @@ -21,7 +22,7 @@ class HttpPosterTest { fun testPostSucceeds() = runSuspend { val body = """{ "foo": "bar" }""" val mockEngine = MockEngine { respondOk() } - val httpManager = HttpManager(userAgent, null, httpClientEngine = mockEngine) + val httpManager = HttpManager(userAgent, null, httpClientEngineFactory = get(mockEngine)) val httpPoster = HttpPoster(httpManager, "http://example.org") httpPoster.post(body) @@ -35,7 +36,7 @@ class HttpPosterTest { fun testPostThrowsIOExceptionOnError() = runSuspend { val body = """{ "foo": "bar" }""" val mockEngine = MockEngine { respondError(BadRequest) } - val httpManager = HttpManager(userAgent, null, httpClientEngine = mockEngine) + val httpManager = HttpManager(userAgent, null, httpClientEngineFactory = get(mockEngine)) val httpPoster = HttpPoster(httpManager, "http://example.org") assertFailsWith { diff --git a/download/src/commonMain/kotlin/org/fdroid/download/DownloadRequest.kt b/download/src/commonMain/kotlin/org/fdroid/download/DownloadRequest.kt index 6a175da70..8fdf5fb7b 100644 --- a/download/src/commonMain/kotlin/org/fdroid/download/DownloadRequest.kt +++ b/download/src/commonMain/kotlin/org/fdroid/download/DownloadRequest.kt @@ -1,12 +1,12 @@ package org.fdroid.download +import io.ktor.client.engine.ProxyConfig import kotlin.jvm.JvmOverloads data class DownloadRequest @JvmOverloads constructor( val path: String, val mirrors: List, + val proxy: ProxyConfig? = null, val username: String? = null, val password: String? = null, - @Deprecated("One of the mirrors might be swap, we should check when/after selecting the mirror.") - val isSwap: Boolean = false, ) diff --git a/download/src/commonMain/kotlin/org/fdroid/download/HttpManager.kt b/download/src/commonMain/kotlin/org/fdroid/download/HttpManager.kt index 33464e558..b4f245d14 100644 --- a/download/src/commonMain/kotlin/org/fdroid/download/HttpManager.kt +++ b/download/src/commonMain/kotlin/org/fdroid/download/HttpManager.kt @@ -2,7 +2,8 @@ package org.fdroid.download import io.ktor.client.HttpClient import io.ktor.client.call.receive -import io.ktor.client.engine.HttpClientEngine +import io.ktor.client.engine.HttpClientEngineFactory +import io.ktor.client.engine.ProxyConfig import io.ktor.client.features.ResponseException import io.ktor.client.features.ServerResponseException import io.ktor.client.features.UserAgent @@ -15,7 +16,6 @@ import io.ktor.client.request.post import io.ktor.client.statement.HttpResponse import io.ktor.client.statement.HttpStatement import io.ktor.http.HttpHeaders.Authorization -import io.ktor.http.HttpHeaders.Connection import io.ktor.http.HttpHeaders.ContentType import io.ktor.http.HttpHeaders.ETag import io.ktor.http.HttpHeaders.LastModified @@ -37,27 +37,43 @@ import io.ktor.utils.io.writeFully import mu.KotlinLogging import kotlin.jvm.JvmOverloads -internal expect fun getHttpClientEngine(): HttpClientEngine +internal expect fun getHttpClientEngineFactory(): HttpClientEngineFactory<*> public open class HttpManager @JvmOverloads constructor( private val userAgent: String, queryString: String? = null, + proxyConfig: ProxyConfig? = null, private val mirrorChooser: MirrorChooser = MirrorChooser(), - httpClientEngine: HttpClientEngine = getHttpClientEngine(), + private val httpClientEngineFactory: HttpClientEngineFactory<*> = getHttpClientEngineFactory(), ) { companion object { val log = KotlinLogging.logger {} } - private val httpClient by lazy { - HttpClient(httpClientEngine) { + private var httpClient = getNewHttpClient(proxyConfig) + + /** + * Only exists because KTor doesn't keep a reference to the proxy its client uses. + * Should only get set in [getNewHttpClient]. + */ + internal var currentProxy: ProxyConfig? = null + private set + + private val parameters = queryString?.split('&')?.map { p -> + val (key, value) = p.split('=') + Pair(key, value) + } + + private fun getNewHttpClient(proxyConfig: ProxyConfig? = null): HttpClient { + currentProxy = proxyConfig + return HttpClient(httpClientEngineFactory) { followRedirects = false expectSuccess = true engine { - proxy = null // TODO use proxy except when swap threadsCount = 4 pipelining = true + proxy = proxyConfig } install(UserAgent) { agent = userAgent @@ -70,12 +86,6 @@ public open class HttpManager @JvmOverloads constructor( } } } - private val parameters = queryString?.split('&')?.map { p -> - val (key, value) = p.split('=') - Pair(key, value) - } - - // TODO try to force onion addresses over proxy like NetCipher.getHttpURLConnection() /** * Performs a HEAD request and returns [HeadInfo]. @@ -86,8 +96,9 @@ public open class HttpManager @JvmOverloads constructor( suspend fun head(request: DownloadRequest, eTag: String? = null): HeadInfo? { val authString = constructBasicAuthValue(request) val response: HttpResponse = try { - mirrorChooser.mirrorRequest(request) { url -> - log.debug { "URL: $url" } + mirrorChooser.mirrorRequest(request) { mirror, url -> + log.debug { "HEAD $url" } + resetProxyIfNeeded(request.proxy, mirror) httpClient.head(url) { // add authorization header from username / password if set if (authString != null) header(Authorization, authString) @@ -112,7 +123,9 @@ public open class HttpManager @JvmOverloads constructor( receiver: suspend (ByteArray) -> Unit, ) { val authString = constructBasicAuthValue(request) - mirrorChooser.mirrorRequest(request) { url -> + mirrorChooser.mirrorRequest(request) { mirror, url -> + log.debug { "GET $url" } + resetProxyIfNeeded(request.proxy, mirror) httpClient.get(url) { // add authorization header from username / password if set if (authString != null) header(Authorization, authString) @@ -120,7 +133,7 @@ public open class HttpManager @JvmOverloads constructor( if (skipFirstBytes != null) header(Range, "bytes=${skipFirstBytes}-") // avoid keep-alive for swap due to strange errors observed in the past // TODO still needed? - if (request.isSwap) header(Connection, "Close") +// if (request.isSwap) header(Connection, "Close") } }.execute { response -> if (skipFirstBytes != null && response.status != PartialContent) { @@ -151,13 +164,23 @@ public open class HttpManager @JvmOverloads constructor( return channel.toByteArray() } - suspend fun post(url: String, json: String) { + suspend fun post(url: String, json: String, proxy: ProxyConfig? = null) { + resetProxyIfNeeded(proxy) httpClient.post(url) { header(ContentType, "application/json; utf-8") body = json } } + private fun resetProxyIfNeeded(proxyConfig: ProxyConfig?, mirror: Mirror? = null) { + // TODO based on mirror: disable on swap, + if (currentProxy != proxyConfig) { + log.info { "Switching proxy from [$currentProxy] to [$proxyConfig]"} + httpClient.close() + httpClient = getNewHttpClient(proxyConfig) + } + } + @OptIn(InternalAPI::class) // ktor 2.0 remove private fun constructBasicAuthValue(request: DownloadRequest): String? { if (request.username == null || request.password == null) return null diff --git a/download/src/commonMain/kotlin/org/fdroid/download/MirrorChooser.kt b/download/src/commonMain/kotlin/org/fdroid/download/MirrorChooser.kt index 0e1e2e468..46bbdbd8d 100644 --- a/download/src/commonMain/kotlin/org/fdroid/download/MirrorChooser.kt +++ b/download/src/commonMain/kotlin/org/fdroid/download/MirrorChooser.kt @@ -24,11 +24,11 @@ class MirrorChooser { */ internal suspend fun mirrorRequest( downloadRequest: DownloadRequest, - request: suspend (url: Url) -> T, + request: suspend (mirror: Mirror, url: Url) -> T, ): T { orderMirrors(downloadRequest.mirrors).forEachIndexed { index, mirror -> try { - return request(mirror.getUrl(downloadRequest.path)) + return request(mirror, mirror.getUrl(downloadRequest.path)) } catch (e: ResponseException) { val wasLastMirror = index == downloadRequest.mirrors.size - 1 log.warn(e) { if (wasLastMirror) "Last mirror, rethrowing..." else "Trying other mirror now..." } diff --git a/download/src/commonMain/kotlin/org/fdroid/download/Proxy.kt b/download/src/commonMain/kotlin/org/fdroid/download/Proxy.kt new file mode 100644 index 000000000..ae34e1581 --- /dev/null +++ b/download/src/commonMain/kotlin/org/fdroid/download/Proxy.kt @@ -0,0 +1,20 @@ +package org.fdroid.download + +import io.ktor.client.engine.ProxyConfig +import io.ktor.client.engine.ProxyType.HTTP +import io.ktor.client.engine.ProxyType.SOCKS +import io.ktor.client.engine.resolveAddress +import io.ktor.client.engine.type +import io.ktor.http.hostIsIp +import io.ktor.util.network.port + +private const val DEFAULT_PROXY_HOST = "127.0.0.1" +private const val DEFAULT_PROXY_HTTP_PORT = 8118 +private const val DEFAULT_PROXY_SOCKS_PORT = 9050 + +internal fun ProxyConfig?.isTor(): Boolean { + if (this == null || !hostIsIp(DEFAULT_PROXY_HOST)) return false + val address = resolveAddress() + return (type == HTTP && address.port == DEFAULT_PROXY_HTTP_PORT) || + (type == SOCKS && address.port == DEFAULT_PROXY_SOCKS_PORT) +} diff --git a/download/src/commonTest/kotlin/org/fdroid/TestUtils.kt b/download/src/commonTest/kotlin/org/fdroid/TestUtils.kt index 7a202e74a..d8be6be71 100644 --- a/download/src/commonTest/kotlin/org/fdroid/TestUtils.kt +++ b/download/src/commonTest/kotlin/org/fdroid/TestUtils.kt @@ -1,5 +1,9 @@ package org.fdroid +import io.ktor.client.engine.HttpClientEngine +import io.ktor.client.engine.HttpClientEngineFactory +import io.ktor.client.engine.mock.MockEngine +import io.ktor.client.engine.mock.MockEngineConfig import kotlinx.coroutines.runBlocking import kotlin.random.Random @@ -13,3 +17,9 @@ fun getRandomString(length: Int = Random.nextInt(4, 16)): String { fun runSuspend(block: suspend () -> Unit) = runBlocking { block() } + +fun get(mockEngine: MockEngine) = object : HttpClientEngineFactory { + override fun create(block: MockEngineConfig.() -> Unit): HttpClientEngine { + return mockEngine + } +} diff --git a/download/src/commonTest/kotlin/org/fdroid/download/HttpManagerIntegrationTest.kt b/download/src/commonTest/kotlin/org/fdroid/download/HttpManagerIntegrationTest.kt index f3439002e..a19fb439a 100644 --- a/download/src/commonTest/kotlin/org/fdroid/download/HttpManagerIntegrationTest.kt +++ b/download/src/commonTest/kotlin/org/fdroid/download/HttpManagerIntegrationTest.kt @@ -1,9 +1,13 @@ package org.fdroid.download -import kotlinx.coroutines.runBlocking +import io.ktor.client.engine.ProxyBuilder +import io.ktor.http.Url import org.fdroid.getRandomString +import org.fdroid.runSuspend +import java.net.ConnectException import kotlin.test.Test import kotlin.test.assertEquals +import kotlin.test.assertFailsWith class HttpManagerIntegrationTest { @@ -12,10 +16,24 @@ class HttpManagerIntegrationTest { private val downloadRequest = DownloadRequest("", mirrors) @Test - fun testResumeOnExample() = runBlocking { + fun testResumeOnExample() = runSuspend { val httpManager = HttpManager(userAgent, null) val lastLine = httpManager.getBytes(downloadRequest, 1248).decodeToString() assertEquals("\n", lastLine) } + + @Test + fun testProxy() = runSuspend { + val proxyRequest = downloadRequest.copy(proxy = ProxyBuilder.http(Url("http://127.0.0.1"))) + val httpManager = HttpManager(userAgent, null) + + val e = assertFailsWith { + httpManager.getBytes(proxyRequest) + } + assertEquals("Failed to connect to /127.0.0.1:80", e.message) + + val lastLine = httpManager.getBytes(downloadRequest, 1248).decodeToString() + assertEquals("\n", lastLine) + } } diff --git a/download/src/commonTest/kotlin/org/fdroid/download/HttpManagerTest.kt b/download/src/commonTest/kotlin/org/fdroid/download/HttpManagerTest.kt index 8a30abbb0..fb3242d63 100644 --- a/download/src/commonTest/kotlin/org/fdroid/download/HttpManagerTest.kt +++ b/download/src/commonTest/kotlin/org/fdroid/download/HttpManagerTest.kt @@ -1,6 +1,10 @@ package org.fdroid.download +import io.ktor.client.engine.HttpClientEngine +import io.ktor.client.engine.HttpClientEngineFactory +import io.ktor.client.engine.ProxyBuilder import io.ktor.client.engine.mock.MockEngine +import io.ktor.client.engine.mock.MockEngineConfig import io.ktor.client.engine.mock.respond import io.ktor.client.engine.mock.respondError import io.ktor.client.engine.mock.respondOk @@ -15,7 +19,9 @@ import io.ktor.http.HttpStatusCode.Companion.InternalServerError import io.ktor.http.HttpStatusCode.Companion.OK import io.ktor.http.HttpStatusCode.Companion.PartialContent import io.ktor.http.HttpStatusCode.Companion.TemporaryRedirect +import io.ktor.http.Url import io.ktor.http.headersOf +import org.fdroid.get import org.fdroid.getRandomString import org.fdroid.runSuspend import kotlin.random.Random @@ -38,7 +44,7 @@ class HttpManagerTest { @Test fun testUserAgent() = runSuspend { val mockEngine = MockEngine { respondOk() } - val httpManager = HttpManager(userAgent, null, httpClientEngine = mockEngine) + val httpManager = HttpManager(userAgent, null, httpClientEngineFactory = get(mockEngine)) httpManager.head(downloadRequest) httpManager.getBytes(downloadRequest) @@ -54,7 +60,7 @@ class HttpManagerTest { val version = getRandomString() val queryString = "id=$id&client_version=$version" val mockEngine = MockEngine { respondOk() } - val httpManager = HttpManager(userAgent, queryString, httpClientEngine = mockEngine) + val httpManager = HttpManager(userAgent, queryString, httpClientEngineFactory = get(mockEngine)) httpManager.head(downloadRequest) httpManager.getBytes(downloadRequest) @@ -67,10 +73,10 @@ class HttpManagerTest { @Test fun testBasicAuth() = runSuspend { - val downloadRequest = DownloadRequest("foo", mirrors, "Foo", "Bar") + val downloadRequest = DownloadRequest("foo", mirrors, null, "Foo", "Bar") val mockEngine = MockEngine { respondOk() } - val httpManager = HttpManager(userAgent, null, httpClientEngine = mockEngine) + val httpManager = HttpManager(userAgent, null, httpClientEngineFactory = get(mockEngine)) httpManager.head(downloadRequest) httpManager.getBytes(downloadRequest) @@ -82,12 +88,10 @@ class HttpManagerTest { @Test fun testHeadETagCheck() = runSuspend { - val downloadRequest = DownloadRequest("foo", mirrors, "Foo", "Bar") - val eTag = getRandomString() val headers = headersOf(ETag, eTag) val mockEngine = MockEngine { respond("", headers = headers) } - val httpManager = HttpManager(userAgent, null, httpClientEngine = mockEngine) + val httpManager = HttpManager(userAgent, null, httpClientEngineFactory = get(mockEngine)) // ETag is considered changed when none (null) passed into the request assertTrue(httpManager.head(downloadRequest)!!.eTagChanged) @@ -100,10 +104,9 @@ class HttpManagerTest { @Test fun testDownload() = runSuspend { val content = Random.nextBytes(1024) - val downloadRequest = DownloadRequest("foo", mirrors, "Foo", "Bar") val mockEngine = MockEngine { respond(content) } - val httpManager = HttpManager(userAgent, null, httpClientEngine = mockEngine) + val httpManager = HttpManager(userAgent, null, httpClientEngineFactory = get(mockEngine)) assertContentEquals(content, httpManager.getBytes(downloadRequest)) } @@ -112,7 +115,6 @@ class HttpManagerTest { fun testResumeDownload() = runSuspend { val skipBytes = Random.nextInt(0, 1024) val content = Random.nextBytes(1024) - val downloadRequest = DownloadRequest("foo", mirrors, "Foo", "Bar") var requestNum = 1 val mockEngine = MockEngine { request -> @@ -124,7 +126,7 @@ class HttpManagerTest { if (requestNum++ == 1) respond(content.copyOfRange(from, content.size), PartialContent) else respond(content, OK) } - val httpManager = HttpManager(userAgent, null, httpClientEngine = mockEngine) + val httpManager = HttpManager(userAgent, null, httpClientEngineFactory = get(mockEngine)) // first request gets only the skipped bytes assertContentEquals(content.copyOfRange(skipBytes, content.size), @@ -139,10 +141,8 @@ class HttpManagerTest { @Test fun testMirrorFallback() = runSuspend { - val downloadRequest = DownloadRequest("foo", mirrors, "Foo", "Bar") - val mockEngine = MockEngine { respondError(InternalServerError) } - val httpManager = HttpManager(userAgent, null, httpClientEngine = mockEngine) + val httpManager = HttpManager(userAgent, null, httpClientEngineFactory = get(mockEngine)) assertNull(httpManager.head(downloadRequest)) assertFailsWith { @@ -156,10 +156,8 @@ class HttpManagerTest { @Test fun testFirstMirrorSuccess() = runSuspend { - val downloadRequest = DownloadRequest("foo", mirrors, "Foo", "Bar") - val mockEngine = MockEngine { respondOk() } - val httpManager = HttpManager(userAgent, null, httpClientEngine = mockEngine) + val httpManager = HttpManager(userAgent, null, httpClientEngineFactory = get(mockEngine)) assertNotNull(httpManager.head(downloadRequest)) httpManager.getBytes(downloadRequest) @@ -174,10 +172,8 @@ class HttpManagerTest { @Test fun testNoRedirect() = runSuspend { - val downloadRequest = DownloadRequest("foo", mirrors, "Foo", "Bar") - val mockEngine = MockEngine { respondRedirect("http://example.com") } - val httpManager = HttpManager(userAgent, null, httpClientEngine = mockEngine) + val httpManager = HttpManager(userAgent, null, httpClientEngineFactory = get(mockEngine)) assertNull(httpManager.head(downloadRequest)) assertFailsWith { @@ -191,4 +187,39 @@ class HttpManagerTest { } } + @Test + fun testProxyGetsApplied() = runSuspend { + val proxyConfig = ProxyBuilder.http(Url("http://127.0.0.1:5050")) + val proxyRequest = DownloadRequest("foo", mirrors, proxyConfig) + val noProxyRequest = DownloadRequest("foo", mirrors) + + var numRequests = 0 + val factory = object : HttpClientEngineFactory { + override fun create(block: MockEngineConfig.() -> Unit): HttpClientEngine { + return when (++numRequests) { + 1 -> MockEngine { respondOk() } + 2 -> MockEngine { respondOk() } + 3 -> MockEngine { respondOk() } + else -> fail("Too many engine creations") + } + } + } + val httpManager = HttpManager(userAgent, null, httpClientEngineFactory = factory) + assertNull(httpManager.currentProxy) + + // does not need a new engine, because also doesn't use a proxy + assertNotNull(httpManager.head(noProxyRequest)) + assertNull(httpManager.currentProxy) + + // now wants proxy, creates new engine (2) + assertNotNull(httpManager.head(proxyRequest)) + assertEquals(proxyConfig, httpManager.currentProxy) + + // no more proxy, creates new engine (3) + httpManager.getBytes(noProxyRequest) + assertNull(httpManager.currentProxy) + + assertEquals(3, numRequests) + } + } diff --git a/download/src/jvmMain/kotlin/org/fdroid/download/DownloadManager.kt b/download/src/jvmMain/kotlin/org/fdroid/download/DownloadManager.kt deleted file mode 100644 index 6f5177a54..000000000 --- a/download/src/jvmMain/kotlin/org/fdroid/download/DownloadManager.kt +++ /dev/null @@ -1,10 +0,0 @@ -package org.fdroid.download - -import io.ktor.client.engine.HttpClientEngine -import io.ktor.client.engine.cio.CIO - -actual fun getHttpClientEngine(): HttpClientEngine { - return CIO.create { - // we could add special OkHttp config options here - } -} diff --git a/download/src/jvmMain/kotlin/org/fdroid/download/HttpManager.kt b/download/src/jvmMain/kotlin/org/fdroid/download/HttpManager.kt new file mode 100644 index 000000000..9a5cdfad8 --- /dev/null +++ b/download/src/jvmMain/kotlin/org/fdroid/download/HttpManager.kt @@ -0,0 +1,8 @@ +package org.fdroid.download + +import io.ktor.client.engine.HttpClientEngineFactory +import io.ktor.client.engine.cio.CIO + +actual fun getHttpClientEngineFactory(): HttpClientEngineFactory<*> { + return CIO +} diff --git a/download/src/nativeMain/kotlin/org/fdroid/download/DownloadManager.kt b/download/src/nativeMain/kotlin/org/fdroid/download/DownloadManager.kt deleted file mode 100644 index b77a96aba..000000000 --- a/download/src/nativeMain/kotlin/org/fdroid/download/DownloadManager.kt +++ /dev/null @@ -1,10 +0,0 @@ -package org.fdroid.download - -import io.ktor.client.engine.HttpClientEngine -import io.ktor.client.engine.curl.Curl - -actual fun getHttpClientEngine(): HttpClientEngine { - return Curl.create { - // we could add special curl config options here - } -} diff --git a/download/src/nativeMain/kotlin/org/fdroid/download/HttpManager.kt b/download/src/nativeMain/kotlin/org/fdroid/download/HttpManager.kt new file mode 100644 index 000000000..db1ed0993 --- /dev/null +++ b/download/src/nativeMain/kotlin/org/fdroid/download/HttpManager.kt @@ -0,0 +1,8 @@ +package org.fdroid.download + +import io.ktor.client.engine.HttpClientEngineFactory +import io.ktor.client.engine.curl.Curl + +actual fun getHttpClientEngineFactory(): HttpClientEngineFactory<*> { + return Curl +}