[db] make repo certs non-null and remove repos without cert

historically, repos were added to the DB without much information and could stay in a broken state until manually removed. If a repo is updating, it should have a cert. So only repos that never did a single index update don't have a cert. Nowadays, this can not happen anymore as we get the repo and its cert before adding it to the DB. So whenever we update a repo, we know its certificate and fingerprint. Thus, this includes a DB migration removing all broken repos and making the certificate for repos in the DB non-null.

libs/database/src/main/java/org/fdroid/database/DbV1StreamReceiver.kt

@@ -26,9 +26,9 @@ internal class DbV1StreamReceiver(
 
     private val locales: LocaleListCompat = getLocales(Resources.getSystem().configuration)
 
-    override fun receive(repo: RepoV2, version: Long, certificate: String?) {
+    override fun receive(repo: RepoV2, version: Long) {
         db.getRepositoryDao().clear(repoId)
-        db.getRepositoryDao().update(repoId, repo, version, ONE, certificate)
+        db.getRepositoryDao().update(repoId, repo, version, ONE)
     }
 
     override fun receive(packageName: String, m: MetadataV2) {

libs/database/src/main/java/org/fdroid/database/DbV2StreamReceiver.kt

@@ -36,10 +36,10 @@ internal class DbV2StreamReceiver(
     }
 
     @Synchronized
-    override fun receive(repo: RepoV2, version: Long, certificate: String) {
+    override fun receive(repo: RepoV2, version: Long) {
         repo.walkFiles(nonNullFileV2)
         clearRepoDataIfNeeded()
-        db.getRepositoryDao().update(repoId, repo, version, TWO, certificate)
+        db.getRepositoryDao().update(repoId, repo, version, TWO)
     }
 
     @Synchronized

libs/database/src/main/java/org/fdroid/database/FDroidDatabase.kt

@@ -16,7 +16,7 @@ import java.util.concurrent.Callable
     // When bumping this version, please make sure to add one (or more) migration(s) below!
     // Consider also providing tests for that migration.
     // Don't forget to commit the new schema to the git repo as well.
-    version = 2,
+    version = 4,
     entities = [
         // repo
         CoreRepository::class,
@@ -44,6 +44,8 @@ import java.util.concurrent.Callable
     exportSchema = true,
     autoMigrations = [
         AutoMigration(1, 2, MultiRepoMigration::class),
+        // 2 to 3 is a manual migration
+        AutoMigration(3, 4),
         // add future migrations here (if they are easy enough to be done automatically)
     ],
 )

libs/database/src/main/java/org/fdroid/database/FDroidDatabaseHolder.kt

@@ -58,6 +58,7 @@ public object FDroidDatabaseHolder {
                 FDroidDatabaseInt::class.java,
                 name,
             ).apply {
+                addMigrations(MIGRATION_2_3)
                 // We allow destructive migration (if no real migration was provided),
                 // so we have the option to nuke the DB in production (if that will ever be needed).
                 fallbackToDestructiveMigration()

libs/database/src/main/java/org/fdroid/database/Migrations.kt

@@ -4,6 +4,7 @@ import android.content.ContentValues
 import android.database.Cursor
 import android.database.sqlite.SQLiteDatabase.CONFLICT_FAIL
 import androidx.room.migration.AutoMigrationSpec
+import androidx.room.migration.Migration
 import androidx.sqlite.db.SupportSQLiteDatabase
 import mu.KotlinLogging
 
@@ -104,3 +105,13 @@ internal class MultiRepoMigration : AutoMigrationSpec {
         fun isArchive(): Boolean = address.trimEnd('/').endsWith("/archive")
     }
 }
+
+/**
+ * Removes all repos without a certificate as those are broken anyway
+ * and force us to handle repos without certs.
+ */
+internal val MIGRATION_2_3 = object : Migration(2, 3) {
+    override fun migrate(db: SupportSQLiteDatabase) {
+        db.delete(CoreRepository.TABLE, "certificate IS NULL", null)
+    }
+}

libs/database/src/main/java/org/fdroid/database/Repository.kt

@@ -31,7 +31,7 @@ internal data class CoreRepository(
     val formatVersion: IndexFormatVersion?,
     val maxAge: Int?,
     val description: LocalizedTextV2 = emptyMap(),
-    val certificate: String?,
+    val certificate: String,
 ) {
     internal companion object {
         const val TABLE = "CoreRepository"
@@ -47,7 +47,7 @@ internal fun RepoV2.toCoreRepository(
     repoId: Long = 0,
     version: Long,
     formatVersion: IndexFormatVersion? = null,
-    certificate: String? = null,
+    certificate: String,
 ) = CoreRepository(
     repoId = repoId,
     name = name,
@@ -99,7 +99,7 @@ public data class Repository internal constructor(
         address: String,
         timestamp: Long,
         formatVersion: IndexFormatVersion,
-        certificate: String?,
+        certificate: String,
         version: Long,
         weight: Int,
         lastUpdated: Long,
@@ -135,7 +135,7 @@ public data class Repository internal constructor(
     public val timestamp: Long get() = repository.timestamp
     public val version: Long get() = repository.version ?: 0
     public val formatVersion: IndexFormatVersion? get() = repository.formatVersion
-    public val certificate: String? get() = repository.certificate
+    public val certificate: String get() = repository.certificate
 
     /**
      * True if this repository is an archive repo.

libs/database/src/main/java/org/fdroid/database/RepositoryDao.kt

@@ -165,11 +165,13 @@ internal interface RepositoryDaoInt : RepositoryDao {
     }
 
     @Transaction
+    @VisibleForTesting
     @Deprecated("Use insert instead")
     fun insertEmptyRepo(
         address: String,
         username: String? = null,
         password: String? = null,
+        certificate: String = "6789" // just used for testing
     ): Long {
         val repo = CoreRepository(
             name = mapOf("en-US" to address),
@@ -179,7 +181,7 @@ internal interface RepositoryDaoInt : RepositoryDao {
             version = null,
             formatVersion = null,
             maxAge = null,
-            certificate = null,
+            certificate = certificate,
         )
         val repoId = insertOrReplace(repo)
         val currentMinWeight = getMinRepositoryWeight()
@@ -197,7 +199,12 @@ internal interface RepositoryDaoInt : RepositoryDao {
     @Transaction
     @VisibleForTesting
     fun insertOrReplace(repository: RepoV2, version: Long = 0): Long {
-        val repoId = insertOrReplace(repository.toCoreRepository(version = version))
+        val repoId = insertOrReplace(
+            repository.toCoreRepository(
+                version = version,
+                certificate = "0123", // just for testing
+            )
+        )
         val currentMinWeight = getMinRepositoryWeight()
         val repositoryPreferences = RepositoryPreferences(repoId, currentMinWeight - 2)
         insert(repositoryPreferences)
@@ -258,9 +265,9 @@ internal interface RepositoryDaoInt : RepositoryDao {
         repository: RepoV2,
         version: Long,
         formatVersion: IndexFormatVersion,
-        certificate: String?,
     ) {
-        update(repository.toCoreRepository(repoId, version, formatVersion, certificate))
+        val repo = getRepository(repoId) ?: error("Repo with id $repoId did not exist")
+        update(repository.toCoreRepository(repoId, version, formatVersion, repo.certificate))
         insertRepoTables(repoId, repository)
     }
 
@@ -274,16 +281,6 @@ internal interface RepositoryDaoInt : RepositoryDao {
     @Update
     fun updateRepository(repo: CoreRepository): Int
 
-    /**
-     * Updates the certificate for the [Repository] with the given [repoId].
-     * This should be used for V1 index updating where we only get the full cert
-     * after reading the entire index file.
-     * V2 index should use [update] instead as there the certificate is known
-     * before reading full index.
-     */
-    @Query("UPDATE ${CoreRepository.TABLE} SET certificate = :certificate WHERE repoId = :repoId")
-    fun updateRepository(repoId: Long, certificate: String)
-
     @Update
     fun updateRepositoryPreferences(preferences: RepositoryPreferences)
 

libs/database/src/main/java/org/fdroid/index/IndexUpdater.kt

@@ -56,24 +56,11 @@ public abstract class IndexUpdater {
      */
     public abstract val formatVersion: IndexFormatVersion
 
-    /**
-     * Updates a new [repo] for the first time.
-     */
-    public fun updateNewRepo(
-        repo: Repository,
-        expectedSigningFingerprint: String?,
-    ): IndexUpdateResult = catchExceptions {
-        update(repo, null, expectedSigningFingerprint)
-    }
-
     /**
      * Updates an existing [repo] with a known [Repository.certificate].
      */
-    public fun update(
-        repo: Repository,
-    ): IndexUpdateResult = catchExceptions {
-        require(repo.certificate != null) { "Repo ${repo.address} had no certificate" }
-        update(repo, repo.certificate, null)
+    public fun update(repo: Repository): IndexUpdateResult = catchExceptions {
+        updateRepo(repo)
     }
 
     private fun catchExceptions(block: () -> IndexUpdateResult): IndexUpdateResult {
@@ -86,11 +73,7 @@ public abstract class IndexUpdater {
         }
     }
 
-    protected abstract fun update(
-        repo: Repository,
-        certificate: String?,
-        fingerprint: String?,
-    ): IndexUpdateResult
+    protected abstract fun updateRepo(repo: Repository): IndexUpdateResult
 }
 
 internal fun Downloader.setIndexUpdateListener(

libs/database/src/main/java/org/fdroid/index/RepoUpdater.kt

@@ -51,29 +51,8 @@ public class RepoUpdater(
 
     /**
      * Updates the given [repo].
-     * If [Repository.certificate] is null,
-     * the repo is considered to be new this being the first update.
      */
-    public fun update(
-        repo: Repository,
-        fingerprint: String? = null,
-    ): IndexUpdateResult {
-        return if (repo.certificate == null) {
-            // This is a new repo without a certificate
-            updateNewRepo(repo, fingerprint)
-        } else {
-            update(repo)
-        }
-    }
-
-    private fun updateNewRepo(
-        repo: Repository,
-        expectedSigningFingerprint: String?,
-    ): IndexUpdateResult = update(repo) { updater ->
-        updater.updateNewRepo(repo, expectedSigningFingerprint)
-    }
-
-    private fun update(repo: Repository): IndexUpdateResult = update(repo) { updater ->
+    public fun update(repo: Repository): IndexUpdateResult = update(repo) { updater ->
         updater.update(repo)
     }
 

libs/database/src/main/java/org/fdroid/index/v1/IndexV1Updater.kt

@@ -35,11 +35,7 @@ public class IndexV1Updater(
     public override val formatVersion: IndexFormatVersion = ONE
     private val db: FDroidDatabaseInt = database as FDroidDatabaseInt
 
-    override fun update(
-        repo: Repository,
-        certificate: String?,
-        fingerprint: String?,
-    ): IndexUpdateResult {
+    override fun updateRepo(repo: Repository): IndexUpdateResult {
         // Normally, we shouldn't allow repository downgrades and assert the condition below.
         // However, F-Droid is concerned that late v2 bugs will require users to downgrade to v1,
         // as it happened already with the migration from v0 to v1.
@@ -61,21 +57,16 @@ public class IndexV1Updater(
             if (!downloader.hasChanged()) return IndexUpdateResult.Unchanged
             val eTag = downloader.cacheTag
 
-            val verifier = IndexV1Verifier(file, certificate, fingerprint)
+            val verifier = IndexV1Verifier(file, repo.certificate, null)
             db.runInTransaction {
-                val (cert, _) = verifier.getStreamAndVerify { inputStream ->
+                verifier.getStreamAndVerify { inputStream ->
                     listener?.onUpdateProgress(repo, 0, 0)
                     val streamReceiver = DbV1StreamReceiver(db, repo.repoId, compatibilityChecker)
-                    val streamProcessor =
-                        IndexV1StreamProcessor(streamReceiver, certificate, repo.timestamp)
+                    val streamProcessor = IndexV1StreamProcessor(streamReceiver, repo.timestamp)
                     streamProcessor.process(inputStream)
                 }
-                // update certificate, if we didn't have any before
-                val repoDao = db.getRepositoryDao()
-                if (certificate == null) {
-                    repoDao.updateRepository(repo.repoId, cert)
-                }
                 // update RepositoryPreferences with timestamp and ETag (for v1)
+                val repoDao = db.getRepositoryDao()
                 val updatedPrefs = repo.preferences.copy(
                     lastUpdated = System.currentTimeMillis(),
                     lastETag = eTag,

libs/database/src/main/java/org/fdroid/index/v2/IndexV2Updater.kt

@@ -34,12 +34,8 @@ public class IndexV2Updater(
     public override val formatVersion: IndexFormatVersion = TWO
     private val db: FDroidDatabaseInt = database as FDroidDatabaseInt
 
-    override fun update(
-        repo: Repository,
-        certificate: String?,
-        fingerprint: String?,
-    ): IndexUpdateResult {
-        val (cert, entry) = getCertAndEntry(repo, certificate, fingerprint)
+    override fun updateRepo(repo: Repository): IndexUpdateResult {
+        val (_, entry) = getCertAndEntry(repo, repo.certificate)
         // don't process repos that we already did process in the past
         if (entry.timestamp <= repo.timestamp) return IndexUpdateResult.Unchanged
         // get diff, if available
@@ -47,7 +43,7 @@ public class IndexV2Updater(
         return if (diff == null || repo.formatVersion == ONE) {
             // no diff found (or this is upgrade from v1 repo), so do full index update
             val streamReceiver = DbV2StreamReceiver(db, repo.repoId, compatibilityChecker)
-            val streamProcessor = IndexV2FullStreamProcessor(streamReceiver, cert)
+            val streamProcessor = IndexV2FullStreamProcessor(streamReceiver)
             processStream(repo, entry.index, entry.version, streamProcessor)
         } else {
             // use available diff
@@ -57,11 +53,7 @@ public class IndexV2Updater(
         }
     }
 
-    private fun getCertAndEntry(
-        repo: Repository,
-        certificate: String?,
-        fingerprint: String?,
-    ): Pair<String, Entry> {
+    private fun getCertAndEntry(repo: Repository, certificate: String): Pair<String, Entry> {
         val file = tempFileProvider.createTempFile()
         val downloader = downloaderFactory.createWithTryFirstMirror(
             repo = repo,
@@ -73,7 +65,7 @@ public class IndexV2Updater(
         }
         try {
             downloader.download()
-            val verifier = EntryVerifier(file, certificate, fingerprint)
+            val verifier = EntryVerifier(file, certificate, null)
             return verifier.getStreamAndVerify { inputStream ->
                 IndexParser.parseEntry(inputStream)
             }

libs/database/src/main/java/org/fdroid/repo/RepoAdder.kt

@@ -366,7 +366,7 @@ internal class RepoAdder(
         address = uri.toString(),
         timestamp = -1L,
         formatVersion = indexFormatVersion,
-        certificate = null,
+        certificate = "This is fake and will be replaced by real cert before saving in DB.",
         version = 0L,
         weight = 0,
         lastUpdated = -1L,

libs/database/src/main/java/org/fdroid/repo/RepoV2Fetcher.kt

@@ -56,8 +56,8 @@ internal class RepoV2Fetcher(
 
         log.info { "Downloaded entry, now streaming index..." }
 
-        val streamReceiver = RepoV2StreamReceiver(receiver, repo.username, repo.password)
-        val streamProcessor = IndexV2FullStreamProcessor(streamReceiver, cert)
+        val streamReceiver = RepoV2StreamReceiver(receiver, cert, repo.username, repo.password)
+        val streamProcessor = IndexV2FullStreamProcessor(streamReceiver)
         val digestInputStream = if (uri.scheme?.startsWith("http") == true) {
             // stream index for http(s) downloads
             val indexRequest = DownloadRequest(

libs/database/src/main/java/org/fdroid/repo/RepoV2StreamReceiver.kt

@@ -19,6 +19,7 @@ import org.fdroid.index.v2.RepoV2
 
 internal open class RepoV2StreamReceiver(
     private val receiver: RepoPreviewReceiver,
+    private val certificate: String,
     private val username: String?,
     private val password: String?,
 ) : IndexV2StreamReceiver {
@@ -28,7 +29,7 @@ internal open class RepoV2StreamReceiver(
             repo: RepoV2,
             version: Long,
             formatVersion: IndexFormatVersion,
-            certificate: String?,
+            certificate: String,
             username: String?,
             password: String?,
         ) = Repository(
@@ -80,7 +81,7 @@ internal open class RepoV2StreamReceiver(
 
     private val locales: LocaleListCompat = getLocales(Resources.getSystem().configuration)
 
-    override fun receive(repo: RepoV2, version: Long, certificate: String) {
+    override fun receive(repo: RepoV2, version: Long) {
         receiver.onRepoReceived(
             getRepository(
                 repo = repo,