From 1c9570575f1b1dcc7a3258095679dbde5c8cc6ef Mon Sep 17 00:00:00 2001 From: Matthew Bogner Date: Thu, 2 Apr 2026 13:11:58 +0000 Subject: [PATCH] Support for dnsA/dnsAAAA data added to v2 index --- .../java/org/fdroid/download/DnsCacheTest.kt | 35 +- .../download/DownloadRequestInterceptor.kt | 3 +- .../kotlin/org/fdroid/download/DnsCache.kt | 4 + .../org/fdroid/download/DnsWithCache.kt | 43 +- .../download/FDroidMirrorParameterManager.kt | 8 + .../fdroid/fdroid/net/DownloaderFactory.java | 11 +- .../net/FDroidMirrorParameterManager.java | 10 + .../12.json | 1112 +++++++++++++++++ .../org/fdroid/database/IndexV2DiffTest.kt | 212 ++++ .../org/fdroid/database/RepositoryDiffTest.kt | 2 +- .../org/fdroid/database/FDroidDatabase.kt | 3 +- .../java/org/fdroid/database/Repository.kt | 8 + libs/download/api/jvm/download.api | 17 +- .../kotlin/org/fdroid/download/Mirror.kt | 2 + .../org/fdroid/download/MirrorChooser.kt | 8 + .../fdroid/download/MirrorParameterManager.kt | 10 + libs/index/api/jvm/index.api | 12 +- .../kotlin/org/fdroid/index/v2/IndexV2.kt | 8 +- .../kotlin/org/fdroid/test/TestDataV2.kt | 19 +- .../resources/diff-empty-max/1337.json | 16 +- .../resources/diff-empty-max/23.json | 16 +- .../resources/diff-empty-max/42.json | 16 +- .../resources/index-base-dns-v2.json | 27 + .../commonMain/resources/index-max-v2.json | 16 +- 24 files changed, 1590 insertions(+), 28 deletions(-) create mode 100644 libs/database/schemas/org.fdroid.database.FDroidDatabaseInt/12.json create mode 100644 libs/sharedTest/src/commonMain/resources/index-base-dns-v2.json diff --git a/app/src/androidTest/java/org/fdroid/download/DnsCacheTest.kt b/app/src/androidTest/java/org/fdroid/download/DnsCacheTest.kt index 137e396b1..ddad92633 100644 --- a/app/src/androidTest/java/org/fdroid/download/DnsCacheTest.kt +++ b/app/src/androidTest/java/org/fdroid/download/DnsCacheTest.kt @@ -22,9 +22,16 @@ class DnsCacheTest { private val url2 = "fdroid.org" private val url3 = "fdroid.net" - private val ip1 = InetAddress.getByName("127.0.0.1") - private val ip2 = InetAddress.getByName("127.0.0.2") - private val ip3 = InetAddress.getByName("127.0.0.3") + + private val ip1String = "127.0.0.1" + private val ip2String = "127.0.0.2" + private val ip3String = "1a00:2b00:0:0:0::1" + private val ip4String = "1a00:2b00:0:0:0::2" + + private val ip1 = InetAddress.getByName(ip1String) + private val ip2 = InetAddress.getByName(ip2String) + private val ip3 = InetAddress.getByName(ip3String) + private val ip4 = InetAddress.getByName(ip4String) private val list1 = listOf(ip1, ip2, ip3) private val list2 = listOf(ip2) @@ -73,6 +80,28 @@ class DnsCacheTest { assertNull(testList4) } + @Test + fun preloadCacheTest() { + // test setup + settings.useDnsCache = true + val testCache = DnsCache(settings) + val testObject = DnsWithCache(settings, testCache) + + val ipv4Strings = listOf(ip1String, ip2String) + val ipv6Strings = listOf(ip3String, ip4String) + + testObject.populateCacheWithStrings(url1,ipv4Strings, ipv6Strings) + val resultList1 = testObject.lookup(url1) + assertEquals(4, resultList1.size) + + val ipv4 = listOf(ip1, ip2) + val ipv6 = listOf(ip3, ip4) + + testObject.populateCacheWithIps(url2, ipv4,ipv6) + val resultList2 = testObject.lookup(url2) + assertEquals(4, resultList2.size) + } + @Test fun dnsRetryTest() { // test setup diff --git a/app/src/full/kotlin/org/fdroid/download/DownloadRequestInterceptor.kt b/app/src/full/kotlin/org/fdroid/download/DownloadRequestInterceptor.kt index 798182504..2c81b5ecc 100644 --- a/app/src/full/kotlin/org/fdroid/download/DownloadRequestInterceptor.kt +++ b/app/src/full/kotlin/org/fdroid/download/DownloadRequestInterceptor.kt @@ -12,7 +12,8 @@ class DownloadRequestInterceptor @Inject constructor(private val ipfsManager: Ip // because have a CIDv1 and IPFS is enabled in preferences val newMirrors = request.mirrors.toMutableList().apply { - val gatewayMirrors = ipfsManager.activeGateways.map { Mirror(it, null, true) } + val gatewayMirrors = + ipfsManager.activeGateways.map { Mirror(baseUrl = it, isIpfsGateway = true) } addAll(gatewayMirrors) } request.copy(mirrors = newMirrors) diff --git a/app/src/main/kotlin/org/fdroid/download/DnsCache.kt b/app/src/main/kotlin/org/fdroid/download/DnsCache.kt index 513e52e3a..cc06cec7f 100644 --- a/app/src/main/kotlin/org/fdroid/download/DnsCache.kt +++ b/app/src/main/kotlin/org/fdroid/download/DnsCache.kt @@ -45,6 +45,10 @@ class DnsCache @Inject constructor(private val settingsManager: SettingsManager) } } + fun keys(): Set { + return cache.keys + } + private fun cacheWrite() { if (writeScheduled.compareAndSet(expectedValue = false, newValue = true)) { MainScope().launch { diff --git a/app/src/main/kotlin/org/fdroid/download/DnsWithCache.kt b/app/src/main/kotlin/org/fdroid/download/DnsWithCache.kt index 7e3642566..04aaa068b 100644 --- a/app/src/main/kotlin/org/fdroid/download/DnsWithCache.kt +++ b/app/src/main/kotlin/org/fdroid/download/DnsWithCache.kt @@ -4,6 +4,7 @@ import java.net.InetAddress import java.net.UnknownHostException import javax.inject.Inject import javax.inject.Singleton +import mu.KotlinLogging import okhttp3.Dns import org.fdroid.settings.SettingsManager @@ -17,13 +18,53 @@ constructor(private val settingsManager: SettingsManager, private val cache: Dns return Dns.SYSTEM.lookup(hostname) } var ipList = cache.lookup(hostname) - if (ipList == null) { + if (ipList.isNullOrEmpty()) { ipList = Dns.SYSTEM.lookup(hostname) cache.insert(hostname, ipList) } return ipList } + /** + * these methods provide a way to pre-load the cache with values from other sources such as + * the mirror info from the index, further reducing the needs to do DNS lookups + */ + fun populateCacheWithStrings(hostname: String, ipv4List: List, ipv6List: List) { + val ipv4ConvertedList = stringListToIpList(ipv4List) + val ipv6ConvertedList = stringListToIpList(ipv6List) + populateCacheWithIps(hostname, ipv4ConvertedList, ipv6ConvertedList) + } + + fun populateCacheWithIps(hostname: String, ipv4List: List, ipv6List: List) { + // at this time, the DNS cache only supports a single collection because that's what the DNS + // lookup method returns. since these values are being inserted manually, it's possible that + // the cache might end up with values that wouldn't have been returned by the lookup method. + // if that becomes an issue, logic could be added here to filter the values that are inserted. + val mergedList = mutableListOf() + mergedList.addAll(ipv4List) + mergedList.addAll(ipv6List) + if (!cache.keys().contains(hostname) && !mergedList.isEmpty()) { + cache.insert(hostname, mergedList) + } + } + + private fun stringListToIpList(ipList: List): List { + val log = KotlinLogging.logger {} + try { + return ipList.mapNotNull { + try { + InetAddress.getByName(it) + } catch (e: UnknownHostException) { + log.warn { "Unexpected format for IP address: $it" } + null + } + } + } catch (e: Exception) { + log.warn { "Failed to parse list of IP addresses, returning empty list" } + return emptyList() + } + } + /** * in case a host is unreachable, check whether the cached dns result is different from the * current result. if the cached result is different, remove that result from the cache. returns diff --git a/app/src/main/kotlin/org/fdroid/download/FDroidMirrorParameterManager.kt b/app/src/main/kotlin/org/fdroid/download/FDroidMirrorParameterManager.kt index 01c0df4e3..5fd8dfcf8 100644 --- a/app/src/main/kotlin/org/fdroid/download/FDroidMirrorParameterManager.kt +++ b/app/src/main/kotlin/org/fdroid/download/FDroidMirrorParameterManager.kt @@ -21,6 +21,14 @@ constructor( private val dnsWithCache: DnsWithCache, ) : MirrorParameterManager { + override fun cacheMirrorIpAddresses( + mirrorUrl: String, + ipv4Addresses: List, + ipv6Addresses: List + ) { + dnsWithCache.populateCacheWithStrings(mirrorUrl, ipv4Addresses, ipv6Addresses) + } + override fun shouldRetryRequest(mirrorUrl: String): Boolean { return dnsWithCache.shouldRetryRequest(mirrorUrl) } diff --git a/legacy/src/main/java/org/fdroid/fdroid/net/DownloaderFactory.java b/legacy/src/main/java/org/fdroid/fdroid/net/DownloaderFactory.java index 606685f33..32a5aa8d2 100644 --- a/legacy/src/main/java/org/fdroid/fdroid/net/DownloaderFactory.java +++ b/legacy/src/main/java/org/fdroid/fdroid/net/DownloaderFactory.java @@ -24,6 +24,7 @@ import java.io.File; import java.io.IOException; import java.net.Proxy; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import info.guardianproject.netcipher.NetCipher; @@ -50,7 +51,8 @@ public class DownloaderFactory extends org.fdroid.download.DownloaderFactory { @NonNull @Override - @SuppressLint("MissingPermission") // we'd need to ask for Bluetooth permission, but code unmaintained + @SuppressLint("MissingPermission") + // we'd need to ask for Bluetooth permission, but code unmaintained protected Downloader create(@NonNull Repository repo, @NonNull List mirrors, @NonNull Uri uri, @NonNull IndexFile indexFile, @NonNull File destFile, @Nullable Mirror tryFirst) throws IOException { @@ -97,7 +99,12 @@ public class DownloaderFactory extends org.fdroid.download.DownloaderFactory { private static List loadIpfsGateways(Preferences prefs) { List mirrorList = new ArrayList<>(); for (String gatewayUrl : prefs.getActiveIpfsGateways()) { - mirrorList.add(new Mirror(gatewayUrl, null, true)); + mirrorList.add(new Mirror(gatewayUrl, + null, + Collections.emptyList(), + Collections.emptyList(), + true + )); } return mirrorList; } diff --git a/legacy/src/main/java/org/fdroid/fdroid/net/FDroidMirrorParameterManager.java b/legacy/src/main/java/org/fdroid/fdroid/net/FDroidMirrorParameterManager.java index eb60f2ee2..069fb23ff 100644 --- a/legacy/src/main/java/org/fdroid/fdroid/net/FDroidMirrorParameterManager.java +++ b/legacy/src/main/java/org/fdroid/fdroid/net/FDroidMirrorParameterManager.java @@ -14,6 +14,7 @@ import org.jetbrains.annotations.NotNull; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; @@ -67,6 +68,15 @@ public class FDroidMirrorParameterManager implements MirrorParameterManager { } } + @Override + public void cacheMirrorIpAddresses( + @NotNull String mirrorUrl, + @NotNull List<@NotNull String> ipv4Addresses, + @NotNull List<@NotNull String> ipv6Addresses + ) { + // TODO: is it necessary to implement this in the legacy code? + } + /** * Returns true or false depending on whether a particular mirror should be retried before * moving on to the next one (based on checking dns results) diff --git a/libs/database/schemas/org.fdroid.database.FDroidDatabaseInt/12.json b/libs/database/schemas/org.fdroid.database.FDroidDatabaseInt/12.json new file mode 100644 index 000000000..bc04dbb9c --- /dev/null +++ b/libs/database/schemas/org.fdroid.database.FDroidDatabaseInt/12.json @@ -0,0 +1,1112 @@ +{ + "formatVersion": 1, + "database": { + "version": 12, + "identityHash": "fc2ce55ef5cf95f92794ff728a36abc1", + "entities": [ + { + "tableName": "CoreRepository", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`repoId` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `name` TEXT NOT NULL, `icon` TEXT, `address` TEXT NOT NULL, `webBaseUrl` TEXT, `timestamp` INTEGER NOT NULL, `version` INTEGER, `formatVersion` TEXT, `maxAge` INTEGER, `description` TEXT NOT NULL, `certificate` TEXT NOT NULL)", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "icon", + "columnName": "icon", + "affinity": "TEXT" + }, + { + "fieldPath": "address", + "columnName": "address", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "webBaseUrl", + "columnName": "webBaseUrl", + "affinity": "TEXT" + }, + { + "fieldPath": "timestamp", + "columnName": "timestamp", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "version", + "columnName": "version", + "affinity": "INTEGER" + }, + { + "fieldPath": "formatVersion", + "columnName": "formatVersion", + "affinity": "TEXT" + }, + { + "fieldPath": "maxAge", + "columnName": "maxAge", + "affinity": "INTEGER" + }, + { + "fieldPath": "description", + "columnName": "description", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "certificate", + "columnName": "certificate", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "repoId" + ] + } + }, + { + "tableName": "Mirror", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`repoId` INTEGER NOT NULL, `url` TEXT NOT NULL, `countryCode` TEXT, `dnsA` TEXT, `dnsAAAA` TEXT, `isPrimary` INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(`repoId`, `url`), FOREIGN KEY(`repoId`) REFERENCES `CoreRepository`(`repoId`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "url", + "columnName": "url", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "countryCode", + "columnName": "countryCode", + "affinity": "TEXT" + }, + { + "fieldPath": "dnsA", + "columnName": "dnsA", + "affinity": "TEXT" + }, + { + "fieldPath": "dnsAAAA", + "columnName": "dnsAAAA", + "affinity": "TEXT" + }, + { + "fieldPath": "isPrimary", + "columnName": "isPrimary", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "repoId", + "url" + ] + }, + "foreignKeys": [ + { + "table": "CoreRepository", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "repoId" + ], + "referencedColumns": [ + "repoId" + ] + } + ] + }, + { + "tableName": "AntiFeature", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`repoId` INTEGER NOT NULL, `id` TEXT NOT NULL, `icon` TEXT NOT NULL, `name` TEXT NOT NULL, `description` TEXT NOT NULL, PRIMARY KEY(`repoId`, `id`), FOREIGN KEY(`repoId`) REFERENCES `CoreRepository`(`repoId`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "id", + "columnName": "id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "icon", + "columnName": "icon", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "description", + "columnName": "description", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "repoId", + "id" + ] + }, + "foreignKeys": [ + { + "table": "CoreRepository", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "repoId" + ], + "referencedColumns": [ + "repoId" + ] + } + ] + }, + { + "tableName": "Category", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`repoId` INTEGER NOT NULL, `id` TEXT NOT NULL, `icon` TEXT NOT NULL, `name` TEXT NOT NULL, `description` TEXT NOT NULL, PRIMARY KEY(`repoId`, `id`), FOREIGN KEY(`repoId`) REFERENCES `CoreRepository`(`repoId`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "id", + "columnName": "id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "icon", + "columnName": "icon", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "description", + "columnName": "description", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "repoId", + "id" + ] + }, + "foreignKeys": [ + { + "table": "CoreRepository", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "repoId" + ], + "referencedColumns": [ + "repoId" + ] + } + ] + }, + { + "tableName": "ReleaseChannel", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`repoId` INTEGER NOT NULL, `id` TEXT NOT NULL, `icon` TEXT NOT NULL, `name` TEXT NOT NULL, `description` TEXT NOT NULL, PRIMARY KEY(`repoId`, `id`), FOREIGN KEY(`repoId`) REFERENCES `CoreRepository`(`repoId`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "id", + "columnName": "id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "icon", + "columnName": "icon", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "description", + "columnName": "description", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "repoId", + "id" + ] + }, + "foreignKeys": [ + { + "table": "CoreRepository", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "repoId" + ], + "referencedColumns": [ + "repoId" + ] + } + ] + }, + { + "tableName": "RepositoryPreferences", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`repoId` INTEGER NOT NULL, `weight` INTEGER NOT NULL, `enabled` INTEGER NOT NULL, `lastUpdated` INTEGER, `lastETag` TEXT, `userMirrors` TEXT, `disabledMirrors` TEXT, `username` TEXT, `password` TEXT, `errorCount` INTEGER NOT NULL DEFAULT 0, `lastError` TEXT, PRIMARY KEY(`repoId`))", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "weight", + "columnName": "weight", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "enabled", + "columnName": "enabled", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "lastUpdated", + "columnName": "lastUpdated", + "affinity": "INTEGER" + }, + { + "fieldPath": "lastETag", + "columnName": "lastETag", + "affinity": "TEXT" + }, + { + "fieldPath": "userMirrors", + "columnName": "userMirrors", + "affinity": "TEXT" + }, + { + "fieldPath": "disabledMirrors", + "columnName": "disabledMirrors", + "affinity": "TEXT" + }, + { + "fieldPath": "username", + "columnName": "username", + "affinity": "TEXT" + }, + { + "fieldPath": "password", + "columnName": "password", + "affinity": "TEXT" + }, + { + "fieldPath": "errorCount", + "columnName": "errorCount", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "lastError", + "columnName": "lastError", + "affinity": "TEXT" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "repoId" + ] + } + }, + { + "tableName": "AppMetadata", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`repoId` INTEGER NOT NULL, `packageName` TEXT NOT NULL, `added` INTEGER NOT NULL, `lastUpdated` INTEGER NOT NULL, `name` TEXT, `summary` TEXT, `description` TEXT, `localizedName` TEXT, `localizedSummary` TEXT, `webSite` TEXT, `changelog` TEXT, `license` TEXT, `sourceCode` TEXT, `issueTracker` TEXT, `translation` TEXT, `preferredSigner` TEXT, `video` TEXT, `authorName` TEXT, `authorEmail` TEXT, `authorWebSite` TEXT, `authorPhone` TEXT, `donate` TEXT, `liberapayID` TEXT, `liberapay` TEXT, `openCollective` TEXT, `bitcoin` TEXT, `litecoin` TEXT, `flattrID` TEXT, `categories` TEXT, `isCompatible` INTEGER NOT NULL, PRIMARY KEY(`repoId`, `packageName`), FOREIGN KEY(`repoId`) REFERENCES `CoreRepository`(`repoId`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "packageName", + "columnName": "packageName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "added", + "columnName": "added", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "lastUpdated", + "columnName": "lastUpdated", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT" + }, + { + "fieldPath": "summary", + "columnName": "summary", + "affinity": "TEXT" + }, + { + "fieldPath": "description", + "columnName": "description", + "affinity": "TEXT" + }, + { + "fieldPath": "localizedName", + "columnName": "localizedName", + "affinity": "TEXT" + }, + { + "fieldPath": "localizedSummary", + "columnName": "localizedSummary", + "affinity": "TEXT" + }, + { + "fieldPath": "webSite", + "columnName": "webSite", + "affinity": "TEXT" + }, + { + "fieldPath": "changelog", + "columnName": "changelog", + "affinity": "TEXT" + }, + { + "fieldPath": "license", + "columnName": "license", + "affinity": "TEXT" + }, + { + "fieldPath": "sourceCode", + "columnName": "sourceCode", + "affinity": "TEXT" + }, + { + "fieldPath": "issueTracker", + "columnName": "issueTracker", + "affinity": "TEXT" + }, + { + "fieldPath": "translation", + "columnName": "translation", + "affinity": "TEXT" + }, + { + "fieldPath": "preferredSigner", + "columnName": "preferredSigner", + "affinity": "TEXT" + }, + { + "fieldPath": "video", + "columnName": "video", + "affinity": "TEXT" + }, + { + "fieldPath": "authorName", + "columnName": "authorName", + "affinity": "TEXT" + }, + { + "fieldPath": "authorEmail", + "columnName": "authorEmail", + "affinity": "TEXT" + }, + { + "fieldPath": "authorWebSite", + "columnName": "authorWebSite", + "affinity": "TEXT" + }, + { + "fieldPath": "authorPhone", + "columnName": "authorPhone", + "affinity": "TEXT" + }, + { + "fieldPath": "donate", + "columnName": "donate", + "affinity": "TEXT" + }, + { + "fieldPath": "liberapayID", + "columnName": "liberapayID", + "affinity": "TEXT" + }, + { + "fieldPath": "liberapay", + "columnName": "liberapay", + "affinity": "TEXT" + }, + { + "fieldPath": "openCollective", + "columnName": "openCollective", + "affinity": "TEXT" + }, + { + "fieldPath": "bitcoin", + "columnName": "bitcoin", + "affinity": "TEXT" + }, + { + "fieldPath": "litecoin", + "columnName": "litecoin", + "affinity": "TEXT" + }, + { + "fieldPath": "flattrID", + "columnName": "flattrID", + "affinity": "TEXT" + }, + { + "fieldPath": "categories", + "columnName": "categories", + "affinity": "TEXT" + }, + { + "fieldPath": "isCompatible", + "columnName": "isCompatible", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "repoId", + "packageName" + ] + }, + "indices": [ + { + "name": "index_AppMetadata_authorName", + "unique": false, + "columnNames": [ + "authorName" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_AppMetadata_authorName` ON `${TABLE_NAME}` (`authorName`)" + } + ], + "foreignKeys": [ + { + "table": "CoreRepository", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "repoId" + ], + "referencedColumns": [ + "repoId" + ] + } + ] + }, + { + "tableName": "AppMetadataFts", + "createSql": "CREATE VIRTUAL TABLE IF NOT EXISTS `${TABLE_NAME}` USING FTS4(`repoId` INTEGER NOT NULL, `name` TEXT, `summary` TEXT, `description` TEXT, `authorName` TEXT, `packageName` TEXT NOT NULL, tokenize=unicode61 `remove_diacritics=1` `separators=.` `tokenchars=-`, content=`AppMetadata`, notindexed=`repoId`)", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT" + }, + { + "fieldPath": "summary", + "columnName": "summary", + "affinity": "TEXT" + }, + { + "fieldPath": "description", + "columnName": "description", + "affinity": "TEXT" + }, + { + "fieldPath": "authorName", + "columnName": "authorName", + "affinity": "TEXT" + }, + { + "fieldPath": "packageName", + "columnName": "packageName", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [] + }, + "ftsVersion": "FTS4", + "ftsOptions": { + "tokenizer": "unicode61", + "tokenizerArgs": [ + "remove_diacritics=1", + "separators=.", + "tokenchars=-" + ], + "contentTable": "AppMetadata", + "languageIdColumnName": "", + "matchInfo": "FTS4", + "notIndexedColumns": [ + "repoId" + ], + "prefixSizes": [], + "preferredOrder": "ASC" + }, + "contentSyncTriggers": [ + "CREATE TRIGGER IF NOT EXISTS room_fts_content_sync_AppMetadataFts_BEFORE_UPDATE BEFORE UPDATE ON `AppMetadata` BEGIN DELETE FROM `AppMetadataFts` WHERE `docid`=OLD.`rowid`; END", + "CREATE TRIGGER IF NOT EXISTS room_fts_content_sync_AppMetadataFts_BEFORE_DELETE BEFORE DELETE ON `AppMetadata` BEGIN DELETE FROM `AppMetadataFts` WHERE `docid`=OLD.`rowid`; END", + "CREATE TRIGGER IF NOT EXISTS room_fts_content_sync_AppMetadataFts_AFTER_UPDATE AFTER UPDATE ON `AppMetadata` BEGIN INSERT INTO `AppMetadataFts`(`docid`, `repoId`, `name`, `summary`, `description`, `authorName`, `packageName`) VALUES (NEW.`rowid`, NEW.`repoId`, NEW.`name`, NEW.`summary`, NEW.`description`, NEW.`authorName`, NEW.`packageName`); END", + "CREATE TRIGGER IF NOT EXISTS room_fts_content_sync_AppMetadataFts_AFTER_INSERT AFTER INSERT ON `AppMetadata` BEGIN INSERT INTO `AppMetadataFts`(`docid`, `repoId`, `name`, `summary`, `description`, `authorName`, `packageName`) VALUES (NEW.`rowid`, NEW.`repoId`, NEW.`name`, NEW.`summary`, NEW.`description`, NEW.`authorName`, NEW.`packageName`); END" + ] + }, + { + "tableName": "LocalizedFile", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`repoId` INTEGER NOT NULL, `packageName` TEXT NOT NULL, `type` TEXT NOT NULL, `locale` TEXT NOT NULL, `name` TEXT NOT NULL, `sha256` TEXT, `size` INTEGER, `ipfsCidV1` TEXT, PRIMARY KEY(`repoId`, `packageName`, `type`, `locale`), FOREIGN KEY(`repoId`, `packageName`) REFERENCES `AppMetadata`(`repoId`, `packageName`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "packageName", + "columnName": "packageName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "type", + "columnName": "type", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "locale", + "columnName": "locale", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "sha256", + "columnName": "sha256", + "affinity": "TEXT" + }, + { + "fieldPath": "size", + "columnName": "size", + "affinity": "INTEGER" + }, + { + "fieldPath": "ipfsCidV1", + "columnName": "ipfsCidV1", + "affinity": "TEXT" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "repoId", + "packageName", + "type", + "locale" + ] + }, + "indices": [ + { + "name": "index_LocalizedFile_packageName", + "unique": false, + "columnNames": [ + "packageName" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_LocalizedFile_packageName` ON `${TABLE_NAME}` (`packageName`)" + } + ], + "foreignKeys": [ + { + "table": "AppMetadata", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "repoId", + "packageName" + ], + "referencedColumns": [ + "repoId", + "packageName" + ] + } + ] + }, + { + "tableName": "LocalizedFileList", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`repoId` INTEGER NOT NULL, `packageName` TEXT NOT NULL, `type` TEXT NOT NULL, `locale` TEXT NOT NULL, `name` TEXT NOT NULL, `sha256` TEXT, `size` INTEGER, `ipfsCidV1` TEXT, PRIMARY KEY(`repoId`, `packageName`, `type`, `locale`, `name`), FOREIGN KEY(`repoId`, `packageName`) REFERENCES `AppMetadata`(`repoId`, `packageName`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "packageName", + "columnName": "packageName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "type", + "columnName": "type", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "locale", + "columnName": "locale", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "sha256", + "columnName": "sha256", + "affinity": "TEXT" + }, + { + "fieldPath": "size", + "columnName": "size", + "affinity": "INTEGER" + }, + { + "fieldPath": "ipfsCidV1", + "columnName": "ipfsCidV1", + "affinity": "TEXT" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "repoId", + "packageName", + "type", + "locale", + "name" + ] + }, + "indices": [ + { + "name": "index_LocalizedFileList_packageName_repoId", + "unique": false, + "columnNames": [ + "packageName", + "repoId" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_LocalizedFileList_packageName_repoId` ON `${TABLE_NAME}` (`packageName`, `repoId`)" + } + ], + "foreignKeys": [ + { + "table": "AppMetadata", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "repoId", + "packageName" + ], + "referencedColumns": [ + "repoId", + "packageName" + ] + } + ] + }, + { + "tableName": "Version", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`repoId` INTEGER NOT NULL, `packageName` TEXT NOT NULL, `versionId` TEXT NOT NULL, `added` INTEGER NOT NULL, `releaseChannels` TEXT, `antiFeatures` TEXT, `whatsNew` TEXT, `appLabel` TEXT, `isCompatible` INTEGER NOT NULL, `file_name` TEXT NOT NULL, `file_sha256` TEXT NOT NULL, `file_size` INTEGER, `file_ipfsCidV1` TEXT, `src_name` TEXT, `src_sha256` TEXT, `src_size` INTEGER, `src_ipfsCidV1` TEXT, `manifest_versionName` TEXT NOT NULL, `manifest_versionCode` INTEGER NOT NULL, `manifest_maxSdkVersion` INTEGER, `manifest_nativecode` TEXT, `manifest_features` TEXT, `manifest_usesSdk_minSdkVersion` INTEGER, `manifest_usesSdk_targetSdkVersion` INTEGER, `manifest_signer_sha256` TEXT, `manifest_signer_hasMultipleSigners` INTEGER, PRIMARY KEY(`repoId`, `packageName`, `versionId`), FOREIGN KEY(`repoId`, `packageName`) REFERENCES `AppMetadata`(`repoId`, `packageName`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "packageName", + "columnName": "packageName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "versionId", + "columnName": "versionId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "added", + "columnName": "added", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "releaseChannels", + "columnName": "releaseChannels", + "affinity": "TEXT" + }, + { + "fieldPath": "antiFeatures", + "columnName": "antiFeatures", + "affinity": "TEXT" + }, + { + "fieldPath": "whatsNew", + "columnName": "whatsNew", + "affinity": "TEXT" + }, + { + "fieldPath": "appLabel", + "columnName": "appLabel", + "affinity": "TEXT" + }, + { + "fieldPath": "isCompatible", + "columnName": "isCompatible", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "file.name", + "columnName": "file_name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "file.sha256", + "columnName": "file_sha256", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "file.size", + "columnName": "file_size", + "affinity": "INTEGER" + }, + { + "fieldPath": "file.ipfsCidV1", + "columnName": "file_ipfsCidV1", + "affinity": "TEXT" + }, + { + "fieldPath": "src.name", + "columnName": "src_name", + "affinity": "TEXT" + }, + { + "fieldPath": "src.sha256", + "columnName": "src_sha256", + "affinity": "TEXT" + }, + { + "fieldPath": "src.size", + "columnName": "src_size", + "affinity": "INTEGER" + }, + { + "fieldPath": "src.ipfsCidV1", + "columnName": "src_ipfsCidV1", + "affinity": "TEXT" + }, + { + "fieldPath": "manifest.versionName", + "columnName": "manifest_versionName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "manifest.versionCode", + "columnName": "manifest_versionCode", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "manifest.maxSdkVersion", + "columnName": "manifest_maxSdkVersion", + "affinity": "INTEGER" + }, + { + "fieldPath": "manifest.nativecode", + "columnName": "manifest_nativecode", + "affinity": "TEXT" + }, + { + "fieldPath": "manifest.features", + "columnName": "manifest_features", + "affinity": "TEXT" + }, + { + "fieldPath": "manifest.usesSdk.minSdkVersion", + "columnName": "manifest_usesSdk_minSdkVersion", + "affinity": "INTEGER" + }, + { + "fieldPath": "manifest.usesSdk.targetSdkVersion", + "columnName": "manifest_usesSdk_targetSdkVersion", + "affinity": "INTEGER" + }, + { + "fieldPath": "manifest.signer.sha256", + "columnName": "manifest_signer_sha256", + "affinity": "TEXT" + }, + { + "fieldPath": "manifest.signer.hasMultipleSigners", + "columnName": "manifest_signer_hasMultipleSigners", + "affinity": "INTEGER" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "repoId", + "packageName", + "versionId" + ] + }, + "foreignKeys": [ + { + "table": "AppMetadata", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "repoId", + "packageName" + ], + "referencedColumns": [ + "repoId", + "packageName" + ] + } + ] + }, + { + "tableName": "VersionedString", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`repoId` INTEGER NOT NULL, `packageName` TEXT NOT NULL, `versionId` TEXT NOT NULL, `type` TEXT NOT NULL, `name` TEXT NOT NULL, `version` INTEGER, PRIMARY KEY(`repoId`, `packageName`, `versionId`, `type`, `name`), FOREIGN KEY(`repoId`, `packageName`, `versionId`) REFERENCES `Version`(`repoId`, `packageName`, `versionId`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "packageName", + "columnName": "packageName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "versionId", + "columnName": "versionId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "type", + "columnName": "type", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "version", + "columnName": "version", + "affinity": "INTEGER" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "repoId", + "packageName", + "versionId", + "type", + "name" + ] + }, + "indices": [ + { + "name": "index_VersionedString_versionId", + "unique": false, + "columnNames": [ + "versionId" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_VersionedString_versionId` ON `${TABLE_NAME}` (`versionId`)" + } + ], + "foreignKeys": [ + { + "table": "Version", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "repoId", + "packageName", + "versionId" + ], + "referencedColumns": [ + "repoId", + "packageName", + "versionId" + ] + } + ] + }, + { + "tableName": "AppPrefs", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`packageName` TEXT NOT NULL, `ignoreVersionCodeUpdate` INTEGER NOT NULL, `preferredRepoId` INTEGER, `appPrefReleaseChannels` TEXT, PRIMARY KEY(`packageName`))", + "fields": [ + { + "fieldPath": "packageName", + "columnName": "packageName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "ignoreVersionCodeUpdate", + "columnName": "ignoreVersionCodeUpdate", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "preferredRepoId", + "columnName": "preferredRepoId", + "affinity": "INTEGER" + }, + { + "fieldPath": "appPrefReleaseChannels", + "columnName": "appPrefReleaseChannels", + "affinity": "TEXT" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "packageName" + ] + } + } + ], + "views": [ + { + "viewName": "LocalizedIcon", + "createSql": "CREATE VIEW `${VIEW_NAME}` AS SELECT * FROM LocalizedFile WHERE type='icon'" + }, + { + "viewName": "HighestVersion", + "createSql": "CREATE VIEW `${VIEW_NAME}` AS SELECT repoId, packageName, antiFeatures FROM Version\n GROUP BY repoId, packageName HAVING MAX(manifest_versionCode)" + }, + { + "viewName": "PreferredRepo", + "createSql": "CREATE VIEW `${VIEW_NAME}` AS SELECT packageName, repoId AS preferredRepoId FROM AppMetadata\n JOIN RepositoryPreferences AS pref USING (repoId)\n LEFT JOIN AppPrefs USING (packageName)\n WHERE pref.enabled = 1 AND (repoId = COALESCE(preferredRepoId, repoId) OR\n NOT EXISTS (SELECT 1 FROM AppMetadata WHERE repoId=AppPrefs.preferredRepoId AND packageName=AppPrefs.packageName)\n )\n GROUP BY packageName HAVING MAX(pref.weight)" + } + ], + "setupQueries": [ + "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", + "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, 'fc2ce55ef5cf95f92794ff728a36abc1')" + ] + } +} \ No newline at end of file diff --git a/libs/database/src/dbTest/java/org/fdroid/database/IndexV2DiffTest.kt b/libs/database/src/dbTest/java/org/fdroid/database/IndexV2DiffTest.kt index 3b12f347b..8e3ade64e 100644 --- a/libs/database/src/dbTest/java/org/fdroid/database/IndexV2DiffTest.kt +++ b/libs/database/src/dbTest/java/org/fdroid/database/IndexV2DiffTest.kt @@ -9,6 +9,9 @@ import org.fdroid.index.IndexParser import org.fdroid.index.parseV2 import org.fdroid.index.v2.IndexV2 import org.fdroid.index.v2.IndexV2DiffStreamProcessor +import org.fdroid.index.v2.MirrorV2 +import org.fdroid.index.v2.RepoV2 +import org.fdroid.test.LOCALE import org.fdroid.test.TestDataMaxV2 import org.fdroid.test.TestDataMaxV2.PACKAGE_NAME_3 import org.fdroid.test.TestDataMaxV2.app3 @@ -85,6 +88,142 @@ internal class IndexV2DiffTest : DbTest() { ) } + // add more dnsA/dnsAAAA results to mirrors + @Test + fun testAddDns() { + val diffJson = + """ + { + "repo": { + "mirrors": [ + { + "url": "https://dns-test.org/repo", + "countryCode": "us", + "dnsA": [ + "16.15.191.37", + "16.15.191.44", + "16.15.199.90", + "16.15.219.121" + ] + }, + { + "url": "https://dns-test.com/repo", + "countryCode": "nl", + "dnsAAAA": [ + "2600:1f60:80a0::100f:db9b", + "2600:1f60:80a0::100f:df70", + "2600:1f60:80a0::100f:df7b", + "2600:1f60:80c0::100f:b9f4" + ] + } + ] + } + } + """ + .trimIndent() + testJsonDiff( + startPath = "index-base-dns-v2.json", + diff = diffJson, + endIndex = TestDataAddDnsV2.index, + ) + } + + // remove dnsA/dnsAAAA results from mirrors + @Test + fun testRemoveDns() { + val diffJson = + """ + { + "repo": { + "mirrors": [ + { + "url": "https://dns-test.org/repo", + "countryCode": "us", + "dnsA": [] + }, + { + "url": "https://dns-test.com/repo", + "countryCode": "nl", + "dnsAAAA": [] + } + ] + } + } + """ + .trimIndent() + testJsonDiff( + startPath = "index-base-dns-v2.json", + diff = diffJson, + endIndex = TestDataRemoveDnsV2.index, + ) + } + + // set dnsA/dnsAAAA results from mirrors to null + @Test + fun testNullDns() { + val diffJson = + """ + { + "repo": { + "mirrors": [ + { + "url": "https://dns-test.org/repo", + "countryCode": "us", + "dnsA": null + }, + { + "url": "https://dns-test.com/repo", + "countryCode": "nl", + "dnsAAAA": null + } + ] + } + } + """ + .trimIndent() + testJsonDiff( + startPath = "index-base-dns-v2.json", + diff = diffJson, + endIndex = TestDataRemoveDnsV2.index, + ) + } + + // replace dnsA results with dnsAAAA results and vice versa + @Test + fun testSwapDns() { + val diffJson = + """ + { + "repo": { + "mirrors": [ + { + "url": "https://dns-test.org/repo", + "countryCode": "us", + "dnsAAAA": [ + "2600:1f60:80a0::100f:db9b", + "2600:1f60:80a0::100f:df70" + ] + }, + { + "url": "https://dns-test.com/repo", + "countryCode": "nl", + "dnsA": [ + "16.15.191.37", + "16.15.191.44" + ] + } + ] + } + } + """ + .trimIndent() + testJsonDiff( + startPath = "index-base-dns-v2.json", + diff = diffJson, + endIndex = TestDataSwapDnsV2.index, + ) + } + @Test fun testMinRemoveApp() { val diffJson = @@ -533,4 +672,77 @@ internal class IndexV2DiffTest : DbTest() { assertDbEquals(repoId, endIndex) return repoId } + + object TestDataAddDnsV2 { + + val repo = + RepoV2( + timestamp = 99, + name = mapOf(LOCALE to "DnsTest"), + address = "https://dns-test.org/repo", + mirrors = + listOf( + MirrorV2( + "https://dns-test.org/repo", + "us", + dnsA = listOf("16.15.191.37", "16.15.191.44", "16.15.199.90", "16.15.219.121"), + ), + MirrorV2( + "https://dns-test.com/repo", + "nl", + dnsAAAA = + listOf( + "2600:1f60:80a0::100f:db9b", + "2600:1f60:80a0::100f:df70", + "2600:1f60:80a0::100f:df7b", + "2600:1f60:80c0::100f:b9f4", + ), + ), + ), + ) + + val index = IndexV2(repo = repo) + } + + object TestDataRemoveDnsV2 { + + val repo = + RepoV2( + timestamp = 99, + name = mapOf(LOCALE to "DnsTest"), + address = "https://dns-test.org/repo", + mirrors = + listOf( + MirrorV2("https://dns-test.org/repo", "us", dnsA = emptyList()), + MirrorV2("https://dns-test.com/repo", "nl", dnsAAAA = emptyList()), + ), + ) + + val index = IndexV2(repo = repo) + } + + object TestDataSwapDnsV2 { + + val repo = + RepoV2( + timestamp = 99, + name = mapOf(LOCALE to "DnsTest"), + address = "https://dns-test.org/repo", + mirrors = + listOf( + MirrorV2( + "https://dns-test.org/repo", + "us", + dnsAAAA = listOf("2600:1f60:80a0::100f:db9b", "2600:1f60:80a0::100f:df70"), + ), + MirrorV2( + "https://dns-test.com/repo", + "nl", + dnsA = listOf("16.15.191.37", "16.15.191.44"), + ), + ), + ) + + val index = IndexV2(repo = repo) + } } diff --git a/libs/database/src/dbTest/java/org/fdroid/database/RepositoryDiffTest.kt b/libs/database/src/dbTest/java/org/fdroid/database/RepositoryDiffTest.kt index d7d501f2a..0ed1ce7ec 100644 --- a/libs/database/src/dbTest/java/org/fdroid/database/RepositoryDiffTest.kt +++ b/libs/database/src/dbTest/java/org/fdroid/database/RepositoryDiffTest.kt @@ -116,7 +116,7 @@ internal class RepositoryDiffTest : DbTest() { """ .trimIndent() testDiff(repo, json) { repos -> - val expectedMirrors = setOf(Mirror(repos[0].repoId, "foo", "bar")) + val expectedMirrors = setOf(Mirror(repos[0].repoId, "foo", "bar", emptyList(), emptyList())) assertEquals(expectedMirrors, repos[0].mirrors.toSet()) assertRepoEquals(repo.copy(mirrors = listOf(MirrorV2("foo", "bar"))), repos[0]) } diff --git a/libs/database/src/main/java/org/fdroid/database/FDroidDatabase.kt b/libs/database/src/main/java/org/fdroid/database/FDroidDatabase.kt index fb430740f..343bbcdca 100644 --- a/libs/database/src/main/java/org/fdroid/database/FDroidDatabase.kt +++ b/libs/database/src/main/java/org/fdroid/database/FDroidDatabase.kt @@ -18,7 +18,7 @@ import org.fdroid.LocaleChooser.getBestLocale // When bumping this version, please make sure to add one (or more) migration(s) below! // Consider also providing tests for that migration. // Don't forget to commit the new schema to the git repo as well. - version = 11, + version = 12, entities = [ // repo @@ -53,6 +53,7 @@ import org.fdroid.LocaleChooser.getBestLocale // 8 to 9 is a manual migration AutoMigration(9, 10), AutoMigration(10, 11), + AutoMigration(11, 12), // add future migrations above! ], ) diff --git a/libs/database/src/main/java/org/fdroid/database/Repository.kt b/libs/database/src/main/java/org/fdroid/database/Repository.kt index 55d6f2ba1..85237e97e 100644 --- a/libs/database/src/main/java/org/fdroid/database/Repository.kt +++ b/libs/database/src/main/java/org/fdroid/database/Repository.kt @@ -10,6 +10,8 @@ import androidx.room.Ignore import androidx.room.PrimaryKey import androidx.room.Relation import java.util.concurrent.TimeUnit +import kotlin.String +import kotlin.collections.List import org.fdroid.LocaleChooser.getBestLocale import org.fdroid.index.IndexFormatVersion import org.fdroid.index.IndexUtils.getFingerprint @@ -275,6 +277,8 @@ internal data class Mirror( val repoId: Long, val url: String, val countryCode: String? = null, + val dnsA: List? = null, + val dnsAAAA: List? = null, @ColumnInfo(defaultValue = "0") val isPrimary: Boolean = false, ) { internal companion object { @@ -285,6 +289,8 @@ internal data class Mirror( org.fdroid.download.Mirror( baseUrl = url, countryCode = countryCode, + ipv4Addresses = dnsA ?: emptyList(), + ipv6Addresses = dnsAAAA ?: emptyList(), // TODO add isPrimary = isPrimary, ) } @@ -294,6 +300,8 @@ internal fun MirrorV2.toMirror(repoId: Long) = repoId = repoId, url = url, countryCode = countryCode, + dnsA = dnsA ?: emptyList(), + dnsAAAA = dnsAAAA ?: emptyList(), // TODO add isPrimary = isPrimary, ) diff --git a/libs/download/api/jvm/download.api b/libs/download/api/jvm/download.api index 3460c4c11..eb927a309 100644 --- a/libs/download/api/jvm/download.api +++ b/libs/download/api/jvm/download.api @@ -82,18 +82,24 @@ public final class org/fdroid/download/Mirror { public static final field Companion Lorg/fdroid/download/Mirror$Companion; public fun (Ljava/lang/String;)V public fun (Ljava/lang/String;Ljava/lang/String;)V - public fun (Ljava/lang/String;Ljava/lang/String;Z)V - public synthetic fun (Ljava/lang/String;Ljava/lang/String;ZILkotlin/jvm/internal/DefaultConstructorMarker;)V + public fun (Ljava/lang/String;Ljava/lang/String;Ljava/util/List;)V + public fun (Ljava/lang/String;Ljava/lang/String;Ljava/util/List;Ljava/util/List;)V + public fun (Ljava/lang/String;Ljava/lang/String;Ljava/util/List;Ljava/util/List;Z)V + public synthetic fun (Ljava/lang/String;Ljava/lang/String;Ljava/util/List;Ljava/util/List;ZILkotlin/jvm/internal/DefaultConstructorMarker;)V public final fun component1 ()Ljava/lang/String; public final fun component2 ()Ljava/lang/String; - public final fun component3 ()Z - public final fun copy (Ljava/lang/String;Ljava/lang/String;Z)Lorg/fdroid/download/Mirror; - public static synthetic fun copy$default (Lorg/fdroid/download/Mirror;Ljava/lang/String;Ljava/lang/String;ZILjava/lang/Object;)Lorg/fdroid/download/Mirror; + public final fun component3 ()Ljava/util/List; + public final fun component4 ()Ljava/util/List; + public final fun component5 ()Z + public final fun copy (Ljava/lang/String;Ljava/lang/String;Ljava/util/List;Ljava/util/List;Z)Lorg/fdroid/download/Mirror; + public static synthetic fun copy$default (Lorg/fdroid/download/Mirror;Ljava/lang/String;Ljava/lang/String;Ljava/util/List;Ljava/util/List;ZILjava/lang/Object;)Lorg/fdroid/download/Mirror; public fun equals (Ljava/lang/Object;)Z public static final fun fromStrings (Ljava/util/List;)Ljava/util/List; public final fun getBaseUrl ()Ljava/lang/String; public final fun getCountryCode ()Ljava/lang/String; public final fun getFDroidLinkUrl (Ljava/lang/String;)Ljava/lang/String; + public final fun getIpv4Addresses ()Ljava/util/List; + public final fun getIpv6Addresses ()Ljava/util/List; public final fun getUrl ()Lio/ktor/http/Url; public final fun getUrl (Ljava/lang/String;)Lio/ktor/http/Url; public fun hashCode ()I @@ -114,6 +120,7 @@ public abstract interface class org/fdroid/download/MirrorChooser { } public abstract interface class org/fdroid/download/MirrorParameterManager { + public abstract fun cacheMirrorIpAddresses (Ljava/lang/String;Ljava/util/List;Ljava/util/List;)V public abstract fun getCurrentLocation ()Ljava/lang/String; public abstract fun getMirrorErrorCount (Ljava/lang/String;)I public abstract fun incrementMirrorErrorCount (Ljava/lang/String;)V diff --git a/libs/download/src/commonMain/kotlin/org/fdroid/download/Mirror.kt b/libs/download/src/commonMain/kotlin/org/fdroid/download/Mirror.kt index 4b2a6999a..3265f0545 100644 --- a/libs/download/src/commonMain/kotlin/org/fdroid/download/Mirror.kt +++ b/libs/download/src/commonMain/kotlin/org/fdroid/download/Mirror.kt @@ -11,6 +11,8 @@ public data class Mirror constructor( val baseUrl: String, val countryCode: String? = null, + val ipv4Addresses: List = emptyList(), + val ipv6Addresses: List = emptyList(), /** * If this is true, this as an IPFS HTTP gateway that only accepts CIDv1 and not regular paths. So * use this mirror only, if you have a CIDv1 available for supplying it to [getUrl]. diff --git a/libs/download/src/commonMain/kotlin/org/fdroid/download/MirrorChooser.kt b/libs/download/src/commonMain/kotlin/org/fdroid/download/MirrorChooser.kt index ca4ed32c3..99d8f605a 100644 --- a/libs/download/src/commonMain/kotlin/org/fdroid/download/MirrorChooser.kt +++ b/libs/download/src/commonMain/kotlin/org/fdroid/download/MirrorChooser.kt @@ -198,6 +198,14 @@ internal class MirrorChooserWithParameters( url: Url, request: suspend (mirror: Mirror, url: Url) -> T, ): T { + // check mirror for ips and pre-load dns cache + if (!mirror.ipv4Addresses.isEmpty() || !mirror.ipv6Addresses.isEmpty()) { + mirrorParameterManager?.cacheMirrorIpAddresses( + url.host, + mirror.ipv4Addresses, + mirror.ipv6Addresses, + ) + } return try { request(mirror, url) } catch (e: Exception) { diff --git a/libs/download/src/commonMain/kotlin/org/fdroid/download/MirrorParameterManager.kt b/libs/download/src/commonMain/kotlin/org/fdroid/download/MirrorParameterManager.kt index 41dcb8f6f..96d96def2 100644 --- a/libs/download/src/commonMain/kotlin/org/fdroid/download/MirrorParameterManager.kt +++ b/libs/download/src/commonMain/kotlin/org/fdroid/download/MirrorParameterManager.kt @@ -17,6 +17,16 @@ public interface MirrorParameterManager { public fun getMirrorErrorCount(mirrorUrl: String): Int + /** + * Cache the ip addresses for a mirror to bypass DNS lookups. The interface assumes the mirror + * will include the ip addresses in string format so they will need to be converted. + */ + public fun cacheMirrorIpAddresses( + mirrorUrl: String, + ipv4Addresses: List, + ipv6Addresses: List, + ) + /** * Returns true or false depending on whether a particular mirror should be retried before moving * on to the next one (typically based on checking dns results) diff --git a/libs/index/api/jvm/index.api b/libs/index/api/jvm/index.api index 09ead77a6..9356deeb6 100644 --- a/libs/index/api/jvm/index.api +++ b/libs/index/api/jvm/index.api @@ -784,14 +784,18 @@ public final class org/fdroid/index/v2/MetadataV2$Companion { public final class org/fdroid/index/v2/MirrorV2 { public static final field Companion Lorg/fdroid/index/v2/MirrorV2$Companion; - public fun (Ljava/lang/String;Ljava/lang/String;)V - public synthetic fun (Ljava/lang/String;Ljava/lang/String;ILkotlin/jvm/internal/DefaultConstructorMarker;)V + public fun (Ljava/lang/String;Ljava/lang/String;Ljava/util/List;Ljava/util/List;)V + public synthetic fun (Ljava/lang/String;Ljava/lang/String;Ljava/util/List;Ljava/util/List;ILkotlin/jvm/internal/DefaultConstructorMarker;)V public final fun component1 ()Ljava/lang/String; public final fun component2 ()Ljava/lang/String; - public final fun copy (Ljava/lang/String;Ljava/lang/String;)Lorg/fdroid/index/v2/MirrorV2; - public static synthetic fun copy$default (Lorg/fdroid/index/v2/MirrorV2;Ljava/lang/String;Ljava/lang/String;ILjava/lang/Object;)Lorg/fdroid/index/v2/MirrorV2; + public final fun component3 ()Ljava/util/List; + public final fun component4 ()Ljava/util/List; + public final fun copy (Ljava/lang/String;Ljava/lang/String;Ljava/util/List;Ljava/util/List;)Lorg/fdroid/index/v2/MirrorV2; + public static synthetic fun copy$default (Lorg/fdroid/index/v2/MirrorV2;Ljava/lang/String;Ljava/lang/String;Ljava/util/List;Ljava/util/List;ILjava/lang/Object;)Lorg/fdroid/index/v2/MirrorV2; public fun equals (Ljava/lang/Object;)Z public final fun getCountryCode ()Ljava/lang/String; + public final fun getDnsA ()Ljava/util/List; + public final fun getDnsAAAA ()Ljava/util/List; public final fun getUrl ()Ljava/lang/String; public fun hashCode ()I public fun toString ()Ljava/lang/String; diff --git a/libs/index/src/commonMain/kotlin/org/fdroid/index/v2/IndexV2.kt b/libs/index/src/commonMain/kotlin/org/fdroid/index/v2/IndexV2.kt index caee1b45c..ea8af80f7 100644 --- a/libs/index/src/commonMain/kotlin/org/fdroid/index/v2/IndexV2.kt +++ b/libs/index/src/commonMain/kotlin/org/fdroid/index/v2/IndexV2.kt @@ -101,7 +101,13 @@ public typealias LocalizedFileV2 = Map public typealias LocalizedFileListV2 = Map> -@Serializable public data class MirrorV2(val url: String, val countryCode: String? = null) +@Serializable +public data class MirrorV2( + val url: String, + val countryCode: String? = null, + val dnsA: List? = null, + val dnsAAAA: List? = null, +) @Serializable public data class AntiFeatureV2( diff --git a/libs/sharedTest/src/commonMain/kotlin/org/fdroid/test/TestDataV2.kt b/libs/sharedTest/src/commonMain/kotlin/org/fdroid/test/TestDataV2.kt index 550cbd4da..216689dcb 100644 --- a/libs/sharedTest/src/commonMain/kotlin/org/fdroid/test/TestDataV2.kt +++ b/libs/sharedTest/src/commonMain/kotlin/org/fdroid/test/TestDataV2.kt @@ -764,7 +764,24 @@ object TestDataMaxV2 { "de" to "Dies ist ein Repo mit maximaler Datendichte.", ), mirrors = - listOf(MirrorV2("https://max-v1.com", "us"), MirrorV2("https://max-v1.org/repo", "nl")), + listOf( + MirrorV2( + "https://max-v1.com", + "us", + dnsA = listOf("16.15.191.37", "16.15.191.44", "16.15.199.90", "16.15.219.121"), + ), + MirrorV2( + "https://max-v1.org/repo", + "nl", + dnsAAAA = + listOf( + "2600:1f60:80a0::100f:db9b", + "2600:1f60:80a0::100f:df70", + "2600:1f60:80a0::100f:df7b", + "2600:1f60:80c0::100f:b9f4", + ), + ), + ), antiFeatures = mapOf( "VeryBad" to AntiFeatureV2(name = emptyMap()), diff --git a/libs/sharedTest/src/commonMain/resources/diff-empty-max/1337.json b/libs/sharedTest/src/commonMain/resources/diff-empty-max/1337.json index 9a76377dd..3745766c3 100644 --- a/libs/sharedTest/src/commonMain/resources/diff-empty-max/1337.json +++ b/libs/sharedTest/src/commonMain/resources/diff-empty-max/1337.json @@ -27,11 +27,23 @@ "mirrors": [ { "url": "https://max-v1.com", - "countryCode": "us" + "countryCode": "us", + "dnsA": [ + "16.15.191.37", + "16.15.191.44", + "16.15.199.90", + "16.15.219.121" + ] }, { "url": "https://max-v1.org/repo", - "countryCode": "nl" + "countryCode": "nl", + "dnsAAAA": [ + "2600:1f60:80a0::100f:db9b", + "2600:1f60:80a0::100f:df70", + "2600:1f60:80a0::100f:df7b", + "2600:1f60:80c0::100f:b9f4" + ] } ], "timestamp": 9223372036854775807, diff --git a/libs/sharedTest/src/commonMain/resources/diff-empty-max/23.json b/libs/sharedTest/src/commonMain/resources/diff-empty-max/23.json index 314a6844b..164ff59e7 100644 --- a/libs/sharedTest/src/commonMain/resources/diff-empty-max/23.json +++ b/libs/sharedTest/src/commonMain/resources/diff-empty-max/23.json @@ -25,11 +25,23 @@ "mirrors": [ { "url": "https://max-v1.com", - "countryCode": "us" + "countryCode": "us", + "dnsA": [ + "16.15.191.37", + "16.15.191.44", + "16.15.199.90", + "16.15.219.121" + ] }, { "url": "https://max-v1.org/repo", - "countryCode": "nl" + "countryCode": "nl", + "dnsAAAA": [ + "2600:1f60:80a0::100f:db9b", + "2600:1f60:80a0::100f:df70", + "2600:1f60:80a0::100f:df7b", + "2600:1f60:80c0::100f:b9f4" + ] } ], "timestamp": 9223372036854775807, diff --git a/libs/sharedTest/src/commonMain/resources/diff-empty-max/42.json b/libs/sharedTest/src/commonMain/resources/diff-empty-max/42.json index ce3679463..8ead3e73b 100644 --- a/libs/sharedTest/src/commonMain/resources/diff-empty-max/42.json +++ b/libs/sharedTest/src/commonMain/resources/diff-empty-max/42.json @@ -25,11 +25,23 @@ "mirrors": [ { "url": "https://max-v1.com", - "countryCode": "us" + "countryCode": "us", + "dnsA": [ + "16.15.191.37", + "16.15.191.44", + "16.15.199.90", + "16.15.219.121" + ] }, { "url": "https://max-v1.org/repo", - "countryCode": "nl" + "countryCode": "nl", + "dnsAAAA": [ + "2600:1f60:80a0::100f:db9b", + "2600:1f60:80a0::100f:df70", + "2600:1f60:80a0::100f:df7b", + "2600:1f60:80c0::100f:b9f4" + ] } ], "timestamp": 9223372036854775807, diff --git a/libs/sharedTest/src/commonMain/resources/index-base-dns-v2.json b/libs/sharedTest/src/commonMain/resources/index-base-dns-v2.json new file mode 100644 index 000000000..548b614ff --- /dev/null +++ b/libs/sharedTest/src/commonMain/resources/index-base-dns-v2.json @@ -0,0 +1,27 @@ +{ + "repo": { + "name": { + "en-US": "DnsTest" + }, + "address": "https://dns-test.org/repo", + "mirrors": [ + { + "url": "https://dns-test.org/repo", + "countryCode": "us", + "dnsA": [ + "16.15.191.37", + "16.15.191.44" + ] + }, + { + "url": "https://dns-test.com/repo", + "countryCode": "nl", + "dnsAAAA": [ + "2600:1f60:80a0::100f:db9b", + "2600:1f60:80a0::100f:df70" + ] + } + ], + "timestamp": 99 + } +} diff --git a/libs/sharedTest/src/commonMain/resources/index-max-v2.json b/libs/sharedTest/src/commonMain/resources/index-max-v2.json index 4da00689b..3bc5d8d21 100644 --- a/libs/sharedTest/src/commonMain/resources/index-max-v2.json +++ b/libs/sharedTest/src/commonMain/resources/index-max-v2.json @@ -26,11 +26,23 @@ "mirrors": [ { "url": "https://max-v1.com", - "countryCode": "us" + "countryCode": "us", + "dnsA": [ + "16.15.191.37", + "16.15.191.44", + "16.15.199.90", + "16.15.219.121" + ] }, { "url": "https://max-v1.org/repo", - "countryCode": "nl" + "countryCode": "nl", + "dnsAAAA": [ + "2600:1f60:80a0::100f:db9b", + "2600:1f60:80a0::100f:df70", + "2600:1f60:80a0::100f:df7b", + "2600:1f60:80c0::100f:b9f4" + ] } ], "timestamp": 9223372036854775807,