mirror/fdroidclient
1
0
Fork 0
mirror of https://github.com/f-droid/fdroidclient.git synced 2026-04-24 08:47:11 -04:00
Code Issues Packages Projects Releases Wiki Activity

[download] add support for IPFS HTTP gateways

Browse Source 
This refactors the library so that Downloaders receive the IndexFile directly so that they get access to the IPFS CID, but also to the SHA256 hash and the file size. Mirrors can now be marked as IPFS gateways.
This commit is contained in:
Torsten Grote
2022-09-01 15:30:42 -03:00
committed by Hans-Christoph Steiner
parent 9035649f52
commit 066a4e265f
14 changed files with 259 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
libs/download/src/androidMain/kotlin/org/fdroid/download/Downloader.kt
View File

@@ -1,6 +1,7 @@
package org.fdroid.download
import mu.KotlinLogging
import org.fdroid.IndexFile
import org.fdroid.fdroid.ProgressListener
import org.fdroid.fdroid.isMatching
import java.io.File
@@ -11,6 +12,7 @@ import java.io.OutputStream
import java.security.MessageDigest
public abstract class Downloader constructor(
protected val indexFile: IndexFile,
@JvmField
protected val outputFile: File,
) {
@@ -19,13 +21,6 @@ public abstract class Downloader constructor(
private val log = KotlinLogging.logger {}
}
protected var fileSize: Long? = null
/**
* If not null, this is the expected sha256 hash of the [outputFile] after download.
*/
protected var sha256: String? = null
/**
* If you ask for the cacheTag before calling download(), you will get the
* same one you passed in (if any). If you call it after download(), you
@@ -46,19 +41,8 @@ public abstract class Downloader constructor(
/**
* Call this to start the download.
* Never call this more than once. Create a new [Downloader], if you need to download again!
*
* @totalSize must be set to what the index tells us the size will be
* @sha256 must be set to the sha256 hash from the index and only be null for `entry.jar`.
*/
@Throws(IOException::class, InterruptedException::class)
public abstract fun download(totalSize: Long, sha256: String? = null)
/**
* Call this to start the download.
* Never call this more than once. Create a new [Downloader], if you need to download again!
*/
@Deprecated("Use only for v1 repos")
@Throws(IOException::class, InterruptedException::class)
public abstract fun download()
@Throws(IOException::class, NotFoundException::class)
@@ -110,7 +94,7 @@ public abstract class Downloader constructor(
@Throws(InterruptedException::class, IOException::class, NoResumeException::class)
protected suspend fun downloadFromBytesReceiver(isResume: Boolean) {
try {
val messageDigest: MessageDigest? = if (sha256 == null) null else {
val messageDigest: MessageDigest? = if (indexFile.sha256 == null) null else {
MessageDigest.getInstance("SHA-256")
}
FileOutputStream(outputFile, isResume).use { outputStream ->
@@ -128,7 +112,7 @@ public abstract class Downloader constructor(
lastTimeReported = reportProgress(lastTimeReported, bytesCopied, total)
}
// check if expected sha256 hash matches
sha256?.let { expectedHash ->
indexFile.sha256?.let { expectedHash ->
if (!messageDigest.isMatching(expectedHash)) {
throw IOException("Hash not matching")
}
@@ -152,7 +136,7 @@ public abstract class Downloader constructor(
*/
@Throws(IOException::class, InterruptedException::class)
private fun copyInputToOutputStream(input: InputStream, output: OutputStream) {
val messageDigest: MessageDigest? = if (sha256 == null) null else {
val messageDigest: MessageDigest? = if (indexFile.sha256 == null) null else {
MessageDigest.getInstance("SHA-256")
}
try {
@@ -170,7 +154,7 @@ public abstract class Downloader constructor(
numBytes = input.read(buffer)
}
// check if expected sha256 hash matches
sha256?.let { expectedHash ->
indexFile.sha256?.let { expectedHash ->
if (!messageDigest.isMatching(expectedHash)) {
throw IOException("Hash not matching")
}

19
libs/download/src/androidMain/kotlin/org/fdroid/download/HttpDownloader.kt
View File

@@ -33,18 +33,19 @@ import java.util.Date
/**
* Download files over HTTP, with support for proxies, `.onion` addresses, HTTP Basic Auth, etc.
*/
@Deprecated("Only for v1 repos")
public class HttpDownloader constructor(
private val httpManager: HttpManager,
private val request: DownloadRequest,
destFile: File,
) : Downloader(destFile) {
) : Downloader(request.indexFile, destFile) {
private companion object {
val log = KotlinLogging.logger {}
}
@Deprecated("Only for v1 repos")
private var hasChanged = false
private var fileSize: Long? = request.indexFile.size
override fun getInputStream(resumable: Boolean): InputStream {
throw NotImplementedError("Use getInputStreamSuspend instead.")
@@ -61,12 +62,6 @@ public class HttpDownloader constructor(
}
}
public override fun download(totalSize: Long, sha256: String?) {
this.fileSize = totalSize
this.sha256 = sha256
downloadToFile()
}
/**
* Get a remote file, checking the HTTP response code, if it has changed since
* the last time a download was tried.
@@ -112,7 +107,7 @@ public class HttpDownloader constructor(
}
val expectedETag = cacheTag
cacheTag = headInfo.eTag
fileSize = headInfo.contentLength ?: -1
fileSize = headInfo.contentLength ?: request.indexFile.size ?: -1
// If the ETag does not match, it could be because the file is on a mirror
// running a different webserver, e.g. Apache vs Nginx.
@@ -137,7 +132,7 @@ public class HttpDownloader constructor(
// calculatedEtag == expectedETag (ETag calculated from server response matches expected ETag)
if (!headInfo.eTagChanged || calculatedEtag == expectedETag) {
// ETag has not changed, don't download again
log.info { "${request.path} cached, not downloading." }
log.info { "${request.indexFile.name} cached, not downloading." }
hasChanged = false
return
}
@@ -149,7 +144,7 @@ public class HttpDownloader constructor(
private fun downloadToFile() {
var resumable = false
val fileLength = outputFile.length()
if (fileLength > fileSize ?: -1) {
if (fileLength > (fileSize ?: -1)) {
if (!outputFile.delete()) log.warn {
"Warning: " + outputFile.absolutePath + " not deleted"
}
@@ -159,7 +154,7 @@ public class HttpDownloader constructor(
} else if (fileLength > 0) {
resumable = true
}
log.info { "downloading ${request.path} (is resumable: $resumable)" }
log.info { "downloading ${request.indexFile.name} (is resumable: $resumable)" }
runBlocking {
try {
downloadFromBytesReceiver(resumable)

98
libs/download/src/androidMain/kotlin/org/fdroid/download/HttpDownloaderV2.kt Normal file
View File

@@ -0,0 +1,98 @@
/*
* Copyright (C) 2014-2017 Peter Serwylo <peter@serwylo.com>
* Copyright (C) 2014-2018 Hans-Christoph Steiner <hans@eds.org>
* Copyright (C) 2015-2016 Daniel Martí <mvdan@mvdan.cc>
* Copyright (c) 2018 Senecto Limited
* Copyright (C) 2022 Torsten Grote <t at grobox.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 3
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.fdroid.download
import io.ktor.client.plugins.ResponseException
import io.ktor.http.HttpStatusCode.Companion.NotFound
import kotlinx.coroutines.runBlocking
import mu.KotlinLogging
import java.io.File
import java.io.IOException
import java.io.InputStream
/**
* Download files over HTTP, with support for proxies, `.onion` addresses, HTTP Basic Auth, etc.
*/
public class HttpDownloaderV2 constructor(
private val httpManager: HttpManager,
private val request: DownloadRequest,
destFile: File,
) : Downloader(request.indexFile, destFile) {
private companion object {
val log = KotlinLogging.logger {}
}
override fun getInputStream(resumable: Boolean): InputStream {
throw NotImplementedError("Use getInputStreamSuspend instead.")
}
@Throws(IOException::class, NoResumeException::class)
protected override suspend fun getBytes(resumable: Boolean, receiver: BytesReceiver) {
val skipBytes = if (resumable) outputFile.length() else null
return try {
httpManager.get(request, skipBytes, receiver)
} catch (e: ResponseException) {
if (e.response.status == NotFound) throw NotFoundException(e)
else throw IOException(e)
}
}
@Throws(IOException::class, InterruptedException::class)
public override fun download() {
var resumable = false
val fileLength = outputFile.length()
if (fileLength > (request.indexFile.size ?: -1)) {
if (!outputFile.delete()) log.warn {
"Warning: " + outputFile.absolutePath + " not deleted"
}
} else if (fileLength == request.indexFile.size && outputFile.isFile) {
log.info { "Already have outputFile, not download. ${outputFile.absolutePath}" }
return // already have it!
} else if (fileLength > 0) {
resumable = true
}
log.info { "downloading ${request.indexFile.name} (is resumable: $resumable)" }
runBlocking {
try {
downloadFromBytesReceiver(resumable)
} catch (e: NoResumeException) {
require(resumable) { "Got $e even though download was not resumable" }
if (!outputFile.delete()) log.warn {
"Warning: " + outputFile.absolutePath + " not deleted"
}
downloadFromBytesReceiver(false)
}
}
}
protected override fun totalDownloadSize(): Long = request.indexFile.size ?: -1L
@Deprecated("Only for v1 repos")
override fun hasChanged(): Boolean {
error("hasChanged() was called for V2 where it should not be needed.")
}
override fun close() {
}
}

5
libs/download/src/androidMain/kotlin/org/fdroid/download/glide/DownloadRequestLoader.kt
View File

@@ -42,6 +42,7 @@ public class DownloadRequestLoader(
}
internal fun DownloadRequest.getKey(): ObjectKey {
// TODO should we choose a unique key or is it ok for this to work cross-repo based on file path only?
return ObjectKey(path)
// TODO should we always choose a unique key
// or is it ok for this to work cross-repo based on file path only?
return ObjectKey(indexFile.sha256 ?: indexFile.name)
}

20
libs/download/src/androidTest/kotlin/org/fdroid/download/HttpDownloaderTest.kt
View File

@@ -14,6 +14,7 @@ import io.ktor.http.HttpStatusCode.Companion.OK
import io.ktor.http.HttpStatusCode.Companion.PartialContent
import io.ktor.http.headersOf
import org.fdroid.get
import org.fdroid.getIndexFile
import org.fdroid.getRandomString
import org.fdroid.runSuspend
import org.junit.Assume.assumeTrue
@@ -41,7 +42,7 @@ internal class HttpDownloaderTest {
private val userAgent = getRandomString()
private val mirror1 = Mirror("http://example.org")
private val mirrors = listOf(mirror1)
private val downloadRequest = DownloadRequest("foo/bar", mirrors)
private val downloadRequest = DownloadRequest(getIndexFile("foo/bar"), mirrors)
@Test
fun testDownload() = runSuspend {
@@ -60,6 +61,12 @@ internal class HttpDownloaderTest {
fun testDownloadWithCorrectHash() = runSuspend {
val file = folder.newFile()
val bytes = "We know the hash for this string".encodeToByteArray()
val indexFile = getIndexFile(
name = "/foo/bar",
sha256 = "e3802e5f8ae3dc7bbf5f1f4f7fb825d9bce9d1ddce50ac564fcbcfdeb31f1b90",
size = bytes.size.toLong(),
)
val downloadRequest = DownloadRequest(indexFile, mirrors = mirrors)
var progressReported = false
val mockEngine = MockEngine { respond(bytes) }
@@ -69,8 +76,7 @@ internal class HttpDownloaderTest {
assertEquals(bytes.size.toLong(), totalBytes)
progressReported = true
}
httpDownloader.download(bytes.size.toLong(),
"e3802e5f8ae3dc7bbf5f1f4f7fb825d9bce9d1ddce50ac564fcbcfdeb31f1b90")
httpDownloader.download()
assertContentEquals(bytes, file.readBytes())
assertTrue(progressReported)
@@ -80,11 +86,17 @@ internal class HttpDownloaderTest {
fun testDownloadWithWrongHash() = runSuspend {
val file = folder.newFile()
val bytes = "We know the hash for this string".encodeToByteArray()
val indexFile = getIndexFile(
name = "/foo/bar",
sha256 = "This is not the right hash",
size = bytes.size.toLong(),
)
val downloadRequest = DownloadRequest(indexFile, mirrors = mirrors)
val mockEngine = MockEngine { respond(bytes) }
val httpManager = HttpManager(userAgent, null, httpClientEngineFactory = get(mockEngine))
val httpDownloader = HttpDownloader(httpManager, downloadRequest, file)
httpDownloader.download(bytes.size.toLong(), "This is not the right hash")
httpDownloader.download()
assertContentEquals(bytes, file.readBytes())
}

10
libs/download/src/commonMain/kotlin/org/fdroid/IndexFile.kt Normal file
View File

@@ -0,0 +1,10 @@
package org.fdroid
public interface IndexFile {
public val name: String
public val sha256: String?
public val size: Long?
public val ipfsCidV1: String?
public fun serialize(): String
}

23
libs/download/src/commonMain/kotlin/org/fdroid/download/DownloadRequest.kt
View File

@@ -1,10 +1,10 @@
package org.fdroid.download
import io.ktor.client.engine.ProxyConfig
import kotlin.jvm.JvmOverloads
import org.fdroid.IndexFile
public data class DownloadRequest @JvmOverloads constructor(
val path: String,
val indexFile: IndexFile,
val mirrors: List<Mirror>,
val proxy: ProxyConfig? = null,
val username: String? = null,
@@ -19,5 +19,24 @@ public data class DownloadRequest @JvmOverloads constructor(
*/
val tryFirstMirror: Mirror? = null,
) {
@JvmOverloads
@Deprecated("Use other constructor instead")
public constructor(
path: String,
mirrors: List<Mirror>,
proxy: ProxyConfig? = null,
username: String? = null,
password: String? = null,
tryFirstMirror: Mirror? = null,
) : this(object : IndexFile {
override val name = path
override val sha256: String? = null
override val size = 0L
override val ipfsCidV1: String? = null
override fun serialize(): String {
throw NotImplementedError("Serialization is not implemented.")
}
}, mirrors, proxy, username, password, tryFirstMirror)
val hasCredentials: Boolean = username != null && password != null
}

2
libs/download/src/commonMain/kotlin/org/fdroid/download/HttpManager.kt
View File

@@ -25,6 +25,7 @@ import io.ktor.http.HttpHeaders.ETag
import io.ktor.http.HttpHeaders.LastModified
import io.ktor.http.HttpHeaders.Range
import io.ktor.http.HttpMessageBuilder
import io.ktor.http.HttpStatusCode.Companion.NotFound
import io.ktor.http.HttpStatusCode.Companion.PartialContent
import io.ktor.http.Url
import io.ktor.http.contentLength
@@ -104,6 +105,7 @@ public open class HttpManager @JvmOverloads constructor(
}
} catch (e: ResponseException) {
log.warn(e) { "Error getting HEAD" }
if (e.response.status == NotFound) throw NotFoundException()
return null
}
val contentLength = response.contentLength()

5
libs/download/src/commonMain/kotlin/org/fdroid/download/Mirror.kt
View File

@@ -9,6 +9,11 @@ import mu.KotlinLogging
public data class Mirror @JvmOverloads constructor(
val baseUrl: String,
val location: String? = null,
/**
* If this is true, this as an IPFS HTTP gateway that only accepts CIDv1 and not regular paths.
* So use this mirror only, if you have a CIDv1 available for supplying it to [getUrl].
*/
val isIpfsGateway: Boolean = false,
) {
public val url: Url by lazy {
try {

18
libs/download/src/commonMain/kotlin/org/fdroid/download/MirrorChooser.kt
View File

@@ -1,5 +1,6 @@
package org.fdroid.download
import io.ktor.client.network.sockets.SocketTimeoutException
import io.ktor.client.plugins.ResponseException
import io.ktor.http.HttpStatusCode.Companion.Forbidden
import io.ktor.http.HttpStatusCode.Companion.NotFound
@@ -38,7 +39,16 @@ internal abstract class MirrorChooserImpl : MirrorChooser {
orderMirrors(downloadRequest)
}
mirrors.forEachIndexed { index, mirror ->
val url = mirror.getUrl(downloadRequest.path)
val ipfsCidV1 = downloadRequest.indexFile.ipfsCidV1
val url = if (mirror.isIpfsGateway) {
if (ipfsCidV1 == null) {
val e = IOException("Got IPFS gateway without CID")
throwOnLastMirror(e, index == mirrors.size - 1)
return@forEachIndexed
} else mirror.getUrl(ipfsCidV1)
} else {
mirror.getUrl(downloadRequest.indexFile.name)
}
try {
return request(mirror, url)
} catch (e: ResponseException) {
@@ -47,9 +57,11 @@ internal abstract class MirrorChooserImpl : MirrorChooser {
// don't try other mirrors if we got NotFount response and downloaded a repo
if (downloadRequest.tryFirstMirror != null && e.response.status == NotFound) throw e
// also throw if this is the last mirror to try, otherwise try next
throwOnLastMirror(e, index == downloadRequest.mirrors.size - 1)
throwOnLastMirror(e, index == mirrors.size - 1)
} catch (e: IOException) {
throwOnLastMirror(e, index == downloadRequest.mirrors.size - 1)
throwOnLastMirror(e, index == mirrors.size - 1)
} catch (e: SocketTimeoutException) {
throwOnLastMirror(e, index == mirrors.size - 1)
}
}
error("Reached code that was thought to be unreachable.")

15
libs/download/src/commonTest/kotlin/org/fdroid/TestUtils.kt
View File

@@ -23,3 +23,18 @@ fun get(mockEngine: MockEngine) = object : HttpClientEngineFactory<MockEngineCon
return mockEngine
}
}
internal fun getIndexFile(
name: String,
sha256: String? = null,
size: Long? = null,
ipfsCidV1: String? = null,
): IndexFile {
return object : IndexFile {
override val name: String = name
override val sha256: String? = sha256
override val size: Long? = size
override val ipfsCidV1: String? = ipfsCidV1
override fun serialize(): String = error("Not yet implemented")
}
}

4
libs/download/src/commonTest/kotlin/org/fdroid/download/HttpManagerTest.kt
View File

@@ -209,7 +209,9 @@ internal class HttpManagerTest {
assertTrue(downloadRequest.tryFirstMirror != null)
assertNull(httpManager.head(downloadRequest))
assertFailsWith<NotFoundException> {
httpManager.head(downloadRequest)
}
val e = assertFailsWith<ClientRequestException> {
httpManager.getBytes(downloadRequest)
}

60
libs/download/src/commonTest/kotlin/org/fdroid/download/MirrorChooserTest.kt
View File

@@ -1,10 +1,13 @@
package org.fdroid.download
import io.ktor.client.network.sockets.SocketTimeoutException
import io.ktor.utils.io.errors.IOException
import org.fdroid.getIndexFile
import org.fdroid.runSuspend
import kotlin.random.Random
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertFailsWith
import kotlin.test.assertTrue
class MirrorChooserTest {
@@ -12,6 +15,8 @@ class MirrorChooserTest {
private val mirrors = listOf(Mirror("foo"), Mirror("bar"), Mirror("42"), Mirror("1337"))
private val downloadRequest = DownloadRequest("foo", mirrors)
private val ipfsIndexFile = getIndexFile(name = "foo", ipfsCidV1 = "CIDv1")
@Test
fun testMirrorChooserDefaultImpl() = runSuspend {
val mirrorChooser = MirrorChooserRandom()
@@ -19,19 +24,19 @@ class MirrorChooserTest {
val result = mirrorChooser.mirrorRequest(downloadRequest) { mirror, url ->
assertTrue { mirrors.contains(mirror) }
assertEquals(mirror.getUrl(downloadRequest.path), url)
assertEquals(mirror.getUrl(downloadRequest.indexFile.name), url)
expectedResult
}
assertEquals(expectedResult, result)
}
@Test
fun testFallbackToNextMirror() = runSuspend {
fun testFallbackToNextMirrorWithIOException() = runSuspend {
val mirrorChooser = MirrorChooserRandom()
val expectedResult = Random.nextInt()
val result = mirrorChooser.mirrorRequest(downloadRequest) { mirror, url ->
assertEquals(mirror.getUrl(downloadRequest.path), url)
assertEquals(mirror.getUrl(downloadRequest.indexFile.name), url)
// fails with all except last mirror
if (mirror != downloadRequest.mirrors.last()) throw IOException("foo")
expectedResult
@@ -39,6 +44,20 @@ class MirrorChooserTest {
assertEquals(expectedResult, result)
}
@Test
fun testFallbackToNextMirrorWithSocketTimeoutException() = runSuspend {
val mirrorChooser = MirrorChooserRandom()
val expectedResult = Random.nextInt()
val result = mirrorChooser.mirrorRequest(downloadRequest) { mirror, url ->
assertEquals(mirror.getUrl(downloadRequest.indexFile.name), url)
// fails with all except last mirror
if (mirror != downloadRequest.mirrors.last()) throw SocketTimeoutException("foo")
expectedResult
}
assertEquals(expectedResult, result)
}
@Test
fun testMirrorChooserRandom() {
val mirrorChooser = MirrorChooserRandom()
@@ -73,4 +92,39 @@ class MirrorChooserTest {
assertEquals(mirrors.toSet(), orderedMirrors.toSet())
}
@Test
fun testMirrorChooserIgnoresIpfsGatewayIfNoCid() = runSuspend {
val mirrorChooser = object : MirrorChooserImpl() {
override fun orderMirrors(downloadRequest: DownloadRequest): List<Mirror> {
return downloadRequest.mirrors // keep mirror list stable, no random please
}
}
val mirrors = listOf(
Mirror("http://ipfs.com", isIpfsGateway = true),
Mirror("http://example.com", isIpfsGateway = false),
)
val ipfsRequest = downloadRequest.copy(mirrors = mirrors)
val result = mirrorChooser.mirrorRequest(ipfsRequest) { _, url ->
url.toString()
}
assertEquals("http://example.com/foo", result)
}
@Test
fun testMirrorChooserThrowsIfOnlyIpfsGateways() = runSuspend {
val mirrorChooser = MirrorChooserRandom()
val mirrors = listOf(
Mirror("foo/bar", isIpfsGateway = true),
Mirror("bar/foo", isIpfsGateway = true),
)
val ipfsRequest = downloadRequest.copy(mirrors = mirrors)
val e = assertFailsWith<IOException> {
mirrorChooser.mirrorRequest(ipfsRequest) { _, _ ->
}
}
assertEquals("Got IPFS gateway without CID", e.message)
}
}

1
libs/sharedTest/build.gradle
View File

@@ -18,6 +18,7 @@ android {
}
dependencies {
implementation project(":libs:download")
implementation project(":libs:index")
implementation 'org.jetbrains.kotlin:kotlin-test'