From 9ca7bebc88345d53ad01c01b3c2eaf1ba14d301a Mon Sep 17 00:00:00 2001 From: Torsten Grote Date: Tue, 11 Jan 2022 14:34:51 -0300 Subject: [PATCH] Add mirror support in downloads library --- .../fdroid/fdroid/net/HttpDownloaderTest.java | 76 ++++---- .../fdroid/fdroid/net/DownloaderFactory.java | 16 +- .../org/fdroid/fdroid/net/HttpDownloader.java | 37 ++-- .../org/fdroid/fdroid/net/HttpPoster.java | 11 +- download/build.gradle | 5 +- .../org/fdroid/download/DownloadManager.kt | 10 + .../org/fdroid/download/JvmDownloadManager.kt | 1 + .../org/fdroid/download/DownloadManager.kt | 56 ++++-- .../org/fdroid/download/DownloadRequest.kt | 5 +- .../kotlin/org/fdroid/download/Mirror.kt | 21 +++ .../org/fdroid/download/MirrorChooser.kt | 36 ++++ .../commonTest/kotlin/org/fdroid/TestUtils.kt | 15 ++ .../DownloadManagerIntegrationTest.kt | 22 +++ .../fdroid/download/DownloadManagerTest.kt | 172 ++++++++++++++++++ .../org/fdroid/download/DownloadManager.kt | 10 + .../org/fdroid/download/JvmDownloadManager.kt | 10 + gradle/verification-metadata.xml | 24 +++ 17 files changed, 447 insertions(+), 80 deletions(-) create mode 100644 download/src/androidMain/kotlin/org/fdroid/download/DownloadManager.kt create mode 100644 download/src/commonMain/kotlin/org/fdroid/download/Mirror.kt create mode 100644 download/src/commonMain/kotlin/org/fdroid/download/MirrorChooser.kt create mode 100644 download/src/commonTest/kotlin/org/fdroid/TestUtils.kt create mode 100644 download/src/commonTest/kotlin/org/fdroid/download/DownloadManagerIntegrationTest.kt create mode 100644 download/src/commonTest/kotlin/org/fdroid/download/DownloadManagerTest.kt create mode 100644 download/src/jvmMain/kotlin/org/fdroid/download/DownloadManager.kt create mode 100644 download/src/nativeMain/kotlin/org/fdroid/download/JvmDownloadManager.kt diff --git a/app/src/androidTest/java/org/fdroid/fdroid/net/HttpDownloaderTest.java b/app/src/androidTest/java/org/fdroid/fdroid/net/HttpDownloaderTest.java index 8f0422076..7d815b819 100644 --- a/app/src/androidTest/java/org/fdroid/fdroid/net/HttpDownloaderTest.java +++ b/app/src/androidTest/java/org/fdroid/fdroid/net/HttpDownloaderTest.java @@ -1,9 +1,16 @@ package org.fdroid.fdroid.net; -import android.net.Uri; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + import android.os.Build; import android.util.Log; + +import androidx.core.util.Pair; + +import org.fdroid.download.Mirror; import org.fdroid.fdroid.ProgressListener; import org.junit.Test; @@ -11,48 +18,47 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - public class HttpDownloaderTest { private static final String TAG = "HttpDownloaderTest"; - static final String[] URLS; + private static final Collection> URLS; // https://developer.android.com/reference/javax/net/ssl/SSLContext static { - ArrayList tempUrls = new ArrayList<>(Arrays.asList( - "https://f-droid.org/repo/index-v1.jar", + ArrayList> tempUrls = new ArrayList<>(Arrays.asList( + new Pair<>("https://f-droid.org/repo/", "index-v1.jar"), // sites that use SNI for HTTPS - "https://mirrors.edge.kernel.org/debian/dists/stable/Release", - "https://fdroid.tetaneutral.net/fdroid/repo/index-v1.jar", - "https://ftp.fau.de/fdroid/repo/index-v1.jar", - //"https://microg.org/fdroid/repo/index-v1.jar", - //"https://grobox.de/fdroid/repo/index.jar", - "https://guardianproject.info/fdroid/repo/index-v1.jar" + new Pair<>("https://mirrors.edge.kernel.org/", "debian/dists/stable/Release"), + new Pair<>("https://fdroid.tetaneutral.net/fdroid/repo/", "index-v1.jar"), + new Pair<>("https://ftp.fau.de/fdroid/repo/", "index-v1.jar"), + //new Pair<>("https://microg.org/fdroid/repo/index-v1.jar"), + //new Pair<>("https://grobox.de/fdroid/repo/index.jar"), + new Pair<>("https://guardianproject.info/fdroid/repo/", "index-v1.jar") )); if (Build.VERSION.SDK_INT >= 22) { tempUrls.addAll(Arrays.asList( - "https://en.wikipedia.org/wiki/Index.html", // no SNI but weird ipv6 lookup issues - "https://mirror.cyberbits.eu/fdroid/repo/index-v1.jar" // TLSv1.2 only and SNI + new Pair<>("https://en.wikipedia.org/wiki/", "Index.html"), // no SNI but weird ipv6 lookup issues + new Pair<>("https://mirror.cyberbits.eu/fdroid/repo/", "index-v1.jar") // TLSv1.2 only and SNI )); } - URLS = tempUrls.toArray(new String[tempUrls.size()]); + URLS = tempUrls; } private boolean receivedProgress; @Test public void downloadUninterruptedTest() throws IOException, InterruptedException { - for (String urlString : URLS) { - Log.i(TAG, "URL: " + urlString); - Uri uri = Uri.parse(urlString); + for (Pair pair : URLS) { + Log.i(TAG, "URL: " + pair.first + pair.second); File destFile = File.createTempFile("dl-", ""); - HttpDownloader httpDownloader = new HttpDownloader(uri, destFile); + List mirrors = Mirror.fromStrings(Collections.singletonList(pair.first)); + HttpDownloader httpDownloader = new HttpDownloader(pair.second, destFile, mirrors); httpDownloader.download(); assertTrue(destFile.exists()); assertTrue(destFile.canRead()); @@ -63,11 +69,11 @@ public class HttpDownloaderTest { @Test public void downloadUninterruptedTestWithProgress() throws IOException, InterruptedException { final CountDownLatch latch = new CountDownLatch(1); - String urlString = "https://f-droid.org/repo/index.jar"; + String path = "index.jar"; + List mirrors = Mirror.fromStrings(Collections.singletonList("https://f-droid.org/repo/")); receivedProgress = false; - Uri uri = Uri.parse(urlString); File destFile = File.createTempFile("dl-", ""); - final HttpDownloader httpDownloader = new HttpDownloader(uri, destFile); + final HttpDownloader httpDownloader = new HttpDownloader(path, destFile, mirrors); httpDownloader.setListener(new ProgressListener() { @Override public void onProgress(long bytesRead, long totalBytes) { @@ -95,9 +101,11 @@ public class HttpDownloaderTest { @Test public void downloadHttpBasicAuth() throws IOException, InterruptedException { - Uri uri = Uri.parse("https://httpbin.org/basic-auth/myusername/supersecretpassword"); + String path = "myusername/supersecretpassword"; + List mirrors = Mirror.fromStrings(Collections.singletonList("https://httpbin.org/basic-auth/")); File destFile = File.createTempFile("dl-", ""); - HttpDownloader httpDownloader = new HttpDownloader(uri, destFile, "myusername", "supersecretpassword"); + HttpDownloader httpDownloader = + new HttpDownloader(path, destFile, mirrors, "myusername", "supersecretpassword"); httpDownloader.download(); assertTrue(destFile.exists()); assertTrue(destFile.canRead()); @@ -106,9 +114,10 @@ public class HttpDownloaderTest { @Test(expected = IOException.class) public void downloadHttpBasicAuthWrongPassword() throws IOException, InterruptedException { - Uri uri = Uri.parse("https://httpbin.org/basic-auth/myusername/supersecretpassword"); + String path = "myusername/supersecretpassword"; + List mirrors = Mirror.fromStrings(Collections.singletonList("https://httpbin.org/basic-auth/")); File destFile = File.createTempFile("dl-", ""); - HttpDownloader httpDownloader = new HttpDownloader(uri, destFile, "myusername", "wrongpassword"); + HttpDownloader httpDownloader = new HttpDownloader(path, destFile, mirrors, "myusername", "wrongpassword"); httpDownloader.download(); assertFalse(destFile.exists()); destFile.deleteOnExit(); @@ -116,9 +125,11 @@ public class HttpDownloaderTest { @Test(expected = IOException.class) public void downloadHttpBasicAuthWrongUsername() throws IOException, InterruptedException { - Uri uri = Uri.parse("https://httpbin.org/basic-auth/myusername/supersecretpassword"); + String path = "myusername/supersecretpassword"; + List mirrors = Mirror.fromStrings(Collections.singletonList("https://httpbin.org/basic-auth/")); File destFile = File.createTempFile("dl-", ""); - HttpDownloader httpDownloader = new HttpDownloader(uri, destFile, "wrongusername", "supersecretpassword"); + HttpDownloader httpDownloader = + new HttpDownloader(path, destFile, mirrors, "wrongusername", "supersecretpassword"); httpDownloader.download(); assertFalse(destFile.exists()); destFile.deleteOnExit(); @@ -127,9 +138,10 @@ public class HttpDownloaderTest { @Test public void downloadThenCancel() throws IOException, InterruptedException { final CountDownLatch latch = new CountDownLatch(2); - Uri uri = Uri.parse("https://f-droid.org/repo/index.jar"); + String path = "index.jar"; + List mirrors = Mirror.fromStrings(Collections.singletonList("https://f-droid.org/repo/")); File destFile = File.createTempFile("dl-", ""); - final HttpDownloader httpDownloader = new HttpDownloader(uri, destFile); + final HttpDownloader httpDownloader = new HttpDownloader(path, destFile, mirrors); httpDownloader.setListener(new ProgressListener() { @Override public void onProgress(long bytesRead, long totalBytes) { diff --git a/app/src/main/java/org/fdroid/fdroid/net/DownloaderFactory.java b/app/src/main/java/org/fdroid/fdroid/net/DownloaderFactory.java index 6f5a21ada..d821bc495 100644 --- a/app/src/main/java/org/fdroid/fdroid/net/DownloaderFactory.java +++ b/app/src/main/java/org/fdroid/fdroid/net/DownloaderFactory.java @@ -4,15 +4,21 @@ import android.content.ContentResolver; import android.content.Context; import android.net.Uri; +import org.fdroid.download.Mirror; +import org.fdroid.fdroid.Utils; import org.fdroid.fdroid.data.Repo; import org.fdroid.fdroid.data.RepoProvider; import org.fdroid.fdroid.data.Schema; import java.io.File; import java.io.IOException; +import java.util.Collections; +import java.util.List; public class DownloaderFactory { + private static final String TAG = "DownloaderFactory"; + /** * Downloads to a temporary file, which *you must delete yourself when * you are done. It is stored in {@link Context#getCacheDir()} and starts @@ -41,9 +47,15 @@ public class DownloaderFactory { final String[] projection = {Schema.RepoTable.Cols.USERNAME, Schema.RepoTable.Cols.PASSWORD}; Repo repo = RepoProvider.Helper.findByUrl(context, uri, projection); if (repo == null) { - downloader = new HttpDownloader(uri, destFile); + Utils.debugLog(TAG, "Warning: no repo found for " + uri); + Mirror mirror = new Mirror(uri.toString()); + downloader = new HttpDownloader("", destFile, Collections.singletonList(mirror)); } else { - downloader = new HttpDownloader(uri, destFile, repo.username, repo.password); + String urlSuffix = uri.toString().replace(repo.address, ""); + List mirrors = Mirror.fromStrings(repo.getMirrorList()); + Utils.debugLog(TAG, "Using suffix " + urlSuffix + " with mirrors " + mirrors); + downloader = + new HttpDownloader(urlSuffix, destFile, mirrors, repo.username, repo.password); } } return downloader; diff --git a/app/src/main/java/org/fdroid/fdroid/net/HttpDownloader.java b/app/src/main/java/org/fdroid/fdroid/net/HttpDownloader.java index 1a2ba2322..544823c12 100644 --- a/app/src/main/java/org/fdroid/fdroid/net/HttpDownloader.java +++ b/app/src/main/java/org/fdroid/fdroid/net/HttpDownloader.java @@ -25,10 +25,13 @@ import android.annotation.TargetApi; import android.net.Uri; import android.os.Build; +import androidx.annotation.Nullable; + import org.apache.commons.io.FileUtils; import org.fdroid.download.DownloadRequest; import org.fdroid.download.HeadInfo; import org.fdroid.download.JvmDownloadManager; +import org.fdroid.download.Mirror; import org.fdroid.fdroid.FDroidApp; import org.fdroid.fdroid.Utils; @@ -36,9 +39,7 @@ import java.io.BufferedInputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.net.MalformedURLException; import java.net.URL; -import java.util.Collections; import java.util.List; /** @@ -53,39 +54,40 @@ public class HttpDownloader extends Downloader { private final JvmDownloadManager downloadManager = new JvmDownloadManager(Utils.getUserAgent(), FDroidApp.queryString); + private final String path; private final String username; private final String password; - private final URL sourceUrl; + private final List mirrors; private boolean newFileAvailableOnServer; private long fileFullSize = -1L; - HttpDownloader(Uri uri, File destFile) throws MalformedURLException { - this(uri, destFile, null, null); + HttpDownloader(String path, File destFile, List mirrors) { + this(path, destFile, mirrors, null, null); } /** * Create a downloader that can authenticate via HTTP Basic Auth using the supplied * {@code username} and {@code password}. * - * @param uri The file to download + * @param path The path to the file to download * @param destFile Where the download is saved + * @param mirrors The repo base URLs where the file can be found * @param username Username for HTTP Basic Auth, use {@code null} to ignore * @param password Password for HTTP Basic Auth, use {@code null} to ignore - * @throws MalformedURLException */ - HttpDownloader(Uri uri, File destFile, String username, String password) - throws MalformedURLException { - super(uri, destFile); - this.sourceUrl = new URL(urlString); + HttpDownloader(String path, File destFile, List mirrors, @Nullable String username, + @Nullable String password) { + super(Uri.EMPTY, destFile); + this.path = path; + this.mirrors = mirrors; this.username = username; this.password = password; } @Override protected InputStream getDownloadersInputStream() throws IOException { - List mirrors = Collections.singletonList(""); // TODO get real mirrors here - DownloadRequest request = new DownloadRequest(urlString, mirrors, username, password, isSwapUrl(sourceUrl)); + DownloadRequest request = new DownloadRequest(path, mirrors, username, password); // TODO why do we need to wrap this in a BufferedInputStream here? return new BufferedInputStream(downloadManager.getBlocking(request)); } @@ -124,14 +126,13 @@ public class HttpDownloader extends Downloader { */ @Override public void download() throws IOException, InterruptedException { - boolean isSwap = isSwapUrl(sourceUrl); - DownloadRequest request = - new DownloadRequest(urlString, Collections.singletonList(""), username, password, isSwap); + // boolean isSwap = isSwapUrl(sourceUrl); + DownloadRequest request = new DownloadRequest(path, mirrors, username, password); HeadInfo headInfo = downloadManager.headBlocking(request, cacheTag); fileFullSize = headInfo.getContentLength() == null ? -1 : headInfo.getContentLength(); if (!headInfo.getETagChanged()) { // ETag has not changed, don't download again - Utils.debugLog(TAG, urlString + " cached, not downloading."); + Utils.debugLog(TAG, path + " cached, not downloading."); newFileAvailableOnServer = false; return; } @@ -147,7 +148,7 @@ public class HttpDownloader extends Downloader { } else if (fileLength > 0) { resumable = true; } - Utils.debugLog(TAG, "downloading " + urlString + " (is resumable: " + resumable + ")"); + Utils.debugLog(TAG, "downloading " + path + " (is resumable: " + resumable + ")"); downloadFromStream(resumable); } diff --git a/app/src/main/java/org/fdroid/fdroid/net/HttpPoster.java b/app/src/main/java/org/fdroid/fdroid/net/HttpPoster.java index 226922f04..dd6d5c632 100644 --- a/app/src/main/java/org/fdroid/fdroid/net/HttpPoster.java +++ b/app/src/main/java/org/fdroid/fdroid/net/HttpPoster.java @@ -2,32 +2,33 @@ package org.fdroid.fdroid.net; import android.net.Uri; +import org.fdroid.download.Mirror; import org.fdroid.fdroid.FDroidApp; import org.fdroid.fdroid.Utils; import java.io.BufferedWriter; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.net.HttpURLConnection; -import java.net.MalformedURLException; import java.net.URL; +import java.util.Collections; import info.guardianproject.netcipher.NetCipher; /** * HTTP POST a JSON string to the URL configured in the constructor. */ +// TODO don't extend HttpDownloader public class HttpPoster extends HttpDownloader { - public HttpPoster(String url) throws FileNotFoundException, MalformedURLException { + public HttpPoster(String url) { this(Uri.parse(url), null); } - HttpPoster(Uri uri, File destFile) throws FileNotFoundException, MalformedURLException { - super(uri, destFile); + private HttpPoster(Uri uri, File destFile) { + super("", destFile, Collections.singletonList(new Mirror(uri.toString()))); } /** diff --git a/download/build.gradle b/download/build.gradle index 90981028e..ff504a4b3 100644 --- a/download/build.gradle +++ b/download/build.gradle @@ -37,6 +37,7 @@ kotlin { commonTest { dependencies { implementation kotlin('test') + implementation "io.ktor:ktor-client-mock:$ktor_version" } } jvmMain { @@ -46,7 +47,9 @@ kotlin { } } jvmTest { - + dependencies { + implementation 'junit:junit:4.13.2' + } } androidMain { kotlin.srcDir('src/commonJvmAndroid/kotlin') diff --git a/download/src/androidMain/kotlin/org/fdroid/download/DownloadManager.kt b/download/src/androidMain/kotlin/org/fdroid/download/DownloadManager.kt new file mode 100644 index 000000000..73187db48 --- /dev/null +++ b/download/src/androidMain/kotlin/org/fdroid/download/DownloadManager.kt @@ -0,0 +1,10 @@ +package org.fdroid.download + +import io.ktor.client.engine.HttpClientEngine +import io.ktor.client.engine.okhttp.OkHttp + +actual fun getHttpClientEngine(): HttpClientEngine { + return OkHttp.create { + // we could add special OkHttp config options here + } +} diff --git a/download/src/commonJvmAndroid/kotlin/org/fdroid/download/JvmDownloadManager.kt b/download/src/commonJvmAndroid/kotlin/org/fdroid/download/JvmDownloadManager.kt index 7541413f2..2c28be5a0 100644 --- a/download/src/commonJvmAndroid/kotlin/org/fdroid/download/JvmDownloadManager.kt +++ b/download/src/commonJvmAndroid/kotlin/org/fdroid/download/JvmDownloadManager.kt @@ -7,6 +7,7 @@ import java.io.IOException import java.io.InputStream import java.util.Date +// FIXME ideally we can get rid of this wrapper, only need it for Java 7 right now (SDK < 24) public class JvmDownloadManager( userAgent: String, queryString: String?, diff --git a/download/src/commonMain/kotlin/org/fdroid/download/DownloadManager.kt b/download/src/commonMain/kotlin/org/fdroid/download/DownloadManager.kt index 5be8b8eb3..1753c55d6 100644 --- a/download/src/commonMain/kotlin/org/fdroid/download/DownloadManager.kt +++ b/download/src/commonMain/kotlin/org/fdroid/download/DownloadManager.kt @@ -2,8 +2,11 @@ package org.fdroid.download import io.ktor.client.HttpClient import io.ktor.client.call.receive +import io.ktor.client.engine.HttpClientEngine import io.ktor.client.features.ResponseException +import io.ktor.client.features.ServerResponseException import io.ktor.client.features.UserAgent +import io.ktor.client.features.defaultRequest import io.ktor.client.features.onDownload import io.ktor.client.request.get import io.ktor.client.request.head @@ -14,6 +17,7 @@ import io.ktor.http.HttpHeaders.Authorization import io.ktor.http.HttpHeaders.Connection import io.ktor.http.HttpHeaders.ETag import io.ktor.http.HttpHeaders.LastModified +import io.ktor.http.HttpStatusCode.Companion.PartialContent import io.ktor.http.contentLength import io.ktor.util.InternalAPI import io.ktor.util.encodeBase64 @@ -22,16 +26,19 @@ import io.ktor.utils.io.charsets.Charsets import io.ktor.utils.io.core.toByteArray import kotlin.jvm.JvmOverloads -public open class DownloadManager( +internal expect fun getHttpClientEngine(): HttpClientEngine + +public open class DownloadManager @JvmOverloads constructor( private val userAgent: String, queryString: String? = null, + private val mirrorChooser: MirrorChooser = MirrorChooser(), + httpClientEngine: HttpClientEngine = getHttpClientEngine(), ) { private val httpClient by lazy { - HttpClient { + HttpClient(httpClientEngine) { followRedirects = false expectSuccess = true - developmentMode = true // TODO remove engine { proxy = null // TODO use proxy except when swap threadsCount = 4 @@ -40,6 +47,12 @@ public open class DownloadManager( install(UserAgent) { agent = userAgent } + defaultRequest { + // add query string parameters if existing + parameters?.forEach { (key, value) -> + parameter(key, value) + } + } } } private val parameters = queryString?.split('&')?.map { p -> @@ -55,12 +68,14 @@ public open class DownloadManager( * This is useful for checking if the repository index has changed before downloading it again. * However, due to non-standard ETags on mirrors, change detection is unreliable. */ - suspend fun head(request: DownloadRequest, eTag: String?): HeadInfo? { + suspend fun head(request: DownloadRequest, eTag: String? = null): HeadInfo? { val authString = constructBasicAuthValue(request) val response: HttpResponse = try { - httpClient.head(request.url) { - // add authorization header from username / password if set - if (authString != null) header(Authorization, authString) + mirrorChooser.mirrorRequest(request) { url -> + httpClient.head(url) { + // add authorization header from username / password if set + if (authString != null) header(Authorization, authString) + } } } catch (e: ResponseException) { println(e) @@ -77,22 +92,23 @@ public open class DownloadManager( @JvmOverloads suspend fun get(request: DownloadRequest, skipFirstBytes: Long? = null): ByteReadChannel { val authString = constructBasicAuthValue(request) - val response: HttpResponse = httpClient.get(request.url) { - // add query string parameters if existing - parameters?.forEach { (key, value) -> - parameter(key, value) - } - // add authorization header from username / password if set - if (authString != null) header(Authorization, authString) - // add range header if set - if (skipFirstBytes != null) header("Range", "bytes=${skipFirstBytes}-") - // avoid keep-alive for swap due to strange errors observed in the past - if (request.isSwap) header(Connection, "Close") + val response: HttpResponse = mirrorChooser.mirrorRequest(request) { url -> + httpClient.get(url) { + // add authorization header from username / password if set + if (authString != null) header(Authorization, authString) + // add range header if set + if (skipFirstBytes != null) header("Range", "bytes=${skipFirstBytes}-") + // avoid keep-alive for swap due to strange errors observed in the past + if (request.isSwap) header(Connection, "Close") - onDownload { bytesSentTotal, contentLength -> - println("Received $bytesSentTotal bytes from $contentLength") + onDownload { bytesSentTotal, contentLength -> + println("Received $bytesSentTotal bytes from $contentLength") + } } } + if (skipFirstBytes != null && response.status != PartialContent) { + throw ServerResponseException(response, "expected 206") + } return response.receive() // 2.0 .bodyAsChannel() } diff --git a/download/src/commonMain/kotlin/org/fdroid/download/DownloadRequest.kt b/download/src/commonMain/kotlin/org/fdroid/download/DownloadRequest.kt index 7a881fe42..6a175da70 100644 --- a/download/src/commonMain/kotlin/org/fdroid/download/DownloadRequest.kt +++ b/download/src/commonMain/kotlin/org/fdroid/download/DownloadRequest.kt @@ -3,9 +3,10 @@ package org.fdroid.download import kotlin.jvm.JvmOverloads data class DownloadRequest @JvmOverloads constructor( - val url: String, - val mirrors: List, + val path: String, + val mirrors: List, val username: String? = null, val password: String? = null, + @Deprecated("One of the mirrors might be swap, we should check when/after selecting the mirror.") val isSwap: Boolean = false, ) diff --git a/download/src/commonMain/kotlin/org/fdroid/download/Mirror.kt b/download/src/commonMain/kotlin/org/fdroid/download/Mirror.kt new file mode 100644 index 000000000..4aac8b23f --- /dev/null +++ b/download/src/commonMain/kotlin/org/fdroid/download/Mirror.kt @@ -0,0 +1,21 @@ +package org.fdroid.download + +import io.ktor.http.URLBuilder +import io.ktor.http.Url +import io.ktor.http.pathComponents +import kotlin.jvm.JvmOverloads +import kotlin.jvm.JvmStatic + +data class Mirror @JvmOverloads constructor( + val baseUrl: String, + val location: String? = null, +) { + fun getUrl(path: String): Url { + return URLBuilder(baseUrl).pathComponents(path).build() + } + + companion object { + @JvmStatic + fun fromStrings(list: List): List = list.map { Mirror(it) } + } +} diff --git a/download/src/commonMain/kotlin/org/fdroid/download/MirrorChooser.kt b/download/src/commonMain/kotlin/org/fdroid/download/MirrorChooser.kt new file mode 100644 index 000000000..4d5f018b2 --- /dev/null +++ b/download/src/commonMain/kotlin/org/fdroid/download/MirrorChooser.kt @@ -0,0 +1,36 @@ +package org.fdroid.download + +import io.ktor.client.features.ResponseException +import io.ktor.client.statement.HttpResponse +import io.ktor.http.Url + +class MirrorChooser { + + /** + * Returns a list of mirrors with the best mirrors first. + */ + private fun orderMirrors(mirrors: List): List { + // simple random selection for now + // TODO Filter-out onion mirrors for non-tor connections + return mirrors.toMutableList().apply { shuffle() } + } + + /** + * Executes the given request on the best mirror and tries the next best ones if that fails. + */ + internal suspend fun mirrorRequest( + downloadRequest: DownloadRequest, + request: suspend (url: Url) -> HttpResponse, + ): HttpResponse { + orderMirrors(downloadRequest.mirrors).forEachIndexed { index, mirror -> + try { + return request(mirror.getUrl(downloadRequest.path)) + } catch (e: ResponseException) { + println(e) + if (index == downloadRequest.mirrors.size - 1) throw e + } + } + error("Reached code that was thought to be unreachable.") + } + +} diff --git a/download/src/commonTest/kotlin/org/fdroid/TestUtils.kt b/download/src/commonTest/kotlin/org/fdroid/TestUtils.kt new file mode 100644 index 000000000..7a202e74a --- /dev/null +++ b/download/src/commonTest/kotlin/org/fdroid/TestUtils.kt @@ -0,0 +1,15 @@ +package org.fdroid + +import kotlinx.coroutines.runBlocking +import kotlin.random.Random + +fun getRandomString(length: Int = Random.nextInt(4, 16)): String { + val allowedChars = ('A'..'Z') + ('a'..'z') + ('0'..'9') + return (1..length) + .map { allowedChars.random() } + .joinToString("") +} + +fun runSuspend(block: suspend () -> Unit) = runBlocking { + block() +} diff --git a/download/src/commonTest/kotlin/org/fdroid/download/DownloadManagerIntegrationTest.kt b/download/src/commonTest/kotlin/org/fdroid/download/DownloadManagerIntegrationTest.kt new file mode 100644 index 000000000..a427ea531 --- /dev/null +++ b/download/src/commonTest/kotlin/org/fdroid/download/DownloadManagerIntegrationTest.kt @@ -0,0 +1,22 @@ +package org.fdroid.download + +import io.ktor.util.toByteArray +import kotlinx.coroutines.runBlocking +import org.fdroid.getRandomString +import kotlin.test.Test +import kotlin.test.assertEquals + +class DownloadManagerIntegrationTest { + + private val userAgent = getRandomString() + private val mirrors = listOf(Mirror("http://example.org"), Mirror("http://example.net/")) + private val downloadRequest = DownloadRequest("", mirrors) + + @Test + fun testResumeOnExample() = runBlocking { + val downloadManager = DownloadManager(userAgent, null) + + val lastLine = downloadManager.get(downloadRequest, 1248).toByteArray().decodeToString() + assertEquals("\n", lastLine) + } +} diff --git a/download/src/commonTest/kotlin/org/fdroid/download/DownloadManagerTest.kt b/download/src/commonTest/kotlin/org/fdroid/download/DownloadManagerTest.kt new file mode 100644 index 000000000..193fefa1a --- /dev/null +++ b/download/src/commonTest/kotlin/org/fdroid/download/DownloadManagerTest.kt @@ -0,0 +1,172 @@ +package org.fdroid.download + +import io.ktor.client.engine.mock.MockEngine +import io.ktor.client.engine.mock.respond +import io.ktor.client.engine.mock.respondError +import io.ktor.client.engine.mock.respondOk +import io.ktor.client.features.ServerResponseException +import io.ktor.http.HttpHeaders.Authorization +import io.ktor.http.HttpHeaders.ETag +import io.ktor.http.HttpHeaders.Range +import io.ktor.http.HttpHeaders.UserAgent +import io.ktor.http.HttpStatusCode.Companion.InternalServerError +import io.ktor.http.HttpStatusCode.Companion.OK +import io.ktor.http.HttpStatusCode.Companion.PartialContent +import io.ktor.http.headersOf +import io.ktor.util.toByteArray +import org.fdroid.getRandomString +import org.fdroid.runSuspend +import kotlin.random.Random +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue +import kotlin.test.fail + +class DownloadManagerTest { + + private val userAgent = getRandomString() + private val mirrors = listOf(Mirror("http://example.org"), Mirror("http://example.net/")) + private val downloadRequest = DownloadRequest("foo", mirrors) + + @Test + fun testUserAgent() = runSuspend { + val mockEngine = MockEngine { respondOk() } + val downloadManager = DownloadManager(userAgent, null, httpClientEngine = mockEngine) + + downloadManager.head(downloadRequest) + downloadManager.get(downloadRequest) + + mockEngine.requestHistory.forEach { request -> + assertEquals(userAgent, request.headers[UserAgent]) + } + } + + @Test + fun testQueryString() = runSuspend { + val id = getRandomString() + val version = getRandomString() + val queryString = "id=$id&client_version=$version" + val mockEngine = MockEngine { respondOk() } + val downloadManager = DownloadManager(userAgent, queryString, httpClientEngine = mockEngine) + + downloadManager.head(downloadRequest) + downloadManager.get(downloadRequest) + + mockEngine.requestHistory.forEach { request -> + assertEquals(id, request.url.parameters["id"]) + assertEquals(version, request.url.parameters["client_version"]) + } + } + + @Test + fun testBasicAuth() = runSuspend { + val downloadRequest = DownloadRequest("foo", mirrors, "Foo", "Bar") + + val mockEngine = MockEngine { respondOk() } + val downloadManager = DownloadManager(userAgent, null, httpClientEngine = mockEngine) + + downloadManager.head(downloadRequest) + downloadManager.get(downloadRequest) + + mockEngine.requestHistory.forEach { request -> + assertEquals("Basic Rm9vOkJhcg==", request.headers[Authorization]) + } + } + + @Test + fun testHeadETagCheck() = runSuspend { + val downloadRequest = DownloadRequest("foo", mirrors, "Foo", "Bar") + + val eTag = getRandomString() + val headers = headersOf(ETag, eTag) + val mockEngine = MockEngine { respond("", headers = headers) } + val downloadManager = DownloadManager(userAgent, null, httpClientEngine = mockEngine) + + // ETag is considered changed when none (null) passed into the request + assertTrue(downloadManager.head(downloadRequest)!!.eTagChanged) + // Random ETag will be different than what we expect + assertTrue(downloadManager.head(downloadRequest, getRandomString())!!.eTagChanged) + // Expected ETag should match response, so it hasn't changed + assertFalse(downloadManager.head(downloadRequest, eTag)!!.eTagChanged) + } + + @Test + fun testDownload() = runSuspend { + val content = Random.nextBytes(1024) + val downloadRequest = DownloadRequest("foo", mirrors, "Foo", "Bar") + + val mockEngine = MockEngine { respond(content) } + val downloadManager = DownloadManager(userAgent, null, httpClientEngine = mockEngine) + + assertContentEquals(content, downloadManager.get(downloadRequest).toByteArray()) + } + + @Test + fun testResumeDownload() = runSuspend { + val skipBytes = Random.nextInt(0, 1024) + val content = Random.nextBytes(1024) + val downloadRequest = DownloadRequest("foo", mirrors, "Foo", "Bar") + + var requestNum = 1 + val mockEngine = MockEngine { request -> + assertNotNull(request.headers[Range]) + val (fromStr, endStr) = request.headers[Range]!!.replace("bytes=", "").split('-') + val from = fromStr.toIntOrNull() ?: fail("No valid content range ${request.headers[Range]}") + assertEquals("", endStr) + assertEquals(skipBytes, from) + if (requestNum++ == 1) respond(content.copyOfRange(from, content.size), PartialContent) + else respond(content, OK) + } + val downloadManager = DownloadManager(userAgent, null, httpClientEngine = mockEngine) + + // first request gets only the skipped bytes + assertContentEquals(content.copyOfRange(skipBytes, content.size), + downloadManager.get(downloadRequest, skipBytes.toLong()).toByteArray()) + // second request fails, because it responds with OK and full content + val exception = assertFailsWith { + downloadManager.get(downloadRequest, skipBytes.toLong()) + } + assertEquals("Server error(http://example.net/foo: 200 OK. Text: \"expected 206\"", exception.message) + } + + @Test + fun testMirrorFallback() = runSuspend { + val downloadRequest = DownloadRequest("foo", mirrors, "Foo", "Bar") + + val mockEngine = MockEngine { respondError(InternalServerError) } + val downloadManager = DownloadManager(userAgent, null, httpClientEngine = mockEngine) + + assertNull(downloadManager.head(downloadRequest)) + assertFailsWith { + downloadManager.get(downloadRequest) + } + + // assert that URLs for each mirror get tried + val urls = mockEngine.requestHistory.map { request -> request.url.toString() }.toSet() + assertEquals(setOf("http://example.org/foo", "http://example.net/foo"), urls) + } + + @Test + fun testFirstMirrorSuccess() = runSuspend { + val downloadRequest = DownloadRequest("foo", mirrors, "Foo", "Bar") + + val mockEngine = MockEngine { respondOk() } + val downloadManager = DownloadManager(userAgent, null, httpClientEngine = mockEngine) + + assertNotNull(downloadManager.head(downloadRequest)) + downloadManager.get(downloadRequest) + + // assert there is only one request per API call using one of the mirrors + assertEquals(2, mockEngine.requestHistory.size) + mockEngine.requestHistory.forEach { request -> + val url = request.url.toString() + assertTrue(url == "http://example.org/foo" || url == "http://example.net/foo") + } + } + +} diff --git a/download/src/jvmMain/kotlin/org/fdroid/download/DownloadManager.kt b/download/src/jvmMain/kotlin/org/fdroid/download/DownloadManager.kt new file mode 100644 index 000000000..6f5177a54 --- /dev/null +++ b/download/src/jvmMain/kotlin/org/fdroid/download/DownloadManager.kt @@ -0,0 +1,10 @@ +package org.fdroid.download + +import io.ktor.client.engine.HttpClientEngine +import io.ktor.client.engine.cio.CIO + +actual fun getHttpClientEngine(): HttpClientEngine { + return CIO.create { + // we could add special OkHttp config options here + } +} diff --git a/download/src/nativeMain/kotlin/org/fdroid/download/JvmDownloadManager.kt b/download/src/nativeMain/kotlin/org/fdroid/download/JvmDownloadManager.kt new file mode 100644 index 000000000..b77a96aba --- /dev/null +++ b/download/src/nativeMain/kotlin/org/fdroid/download/JvmDownloadManager.kt @@ -0,0 +1,10 @@ +package org.fdroid.download + +import io.ktor.client.engine.HttpClientEngine +import io.ktor.client.engine.curl.Curl + +actual fun getHttpClientEngine(): HttpClientEngine { + return Curl.create { + // we could add special curl config options here + } +} diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index d4fabd6f7..02d42a65f 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -1964,6 +1964,7 @@ + @@ -2239,6 +2240,21 @@ + + + + + + + + + + + + + + + @@ -2612,6 +2628,11 @@ + + + + + @@ -2760,6 +2781,7 @@ + @@ -3293,6 +3315,7 @@ + @@ -3340,6 +3363,7 @@ +