[db] Implement diffing via DbV2DiffStreamReceiver

2026-04-20 14:57:15 -04:00 · 2022-05-05 17:43:52 -03:00
parent aecff91dda
commit 93ff390f07
17 changed files with 1000 additions and 292 deletions
--- a/database/src/androidTest/java/org/fdroid/database/DbTest.kt
+++ b/database/src/androidTest/java/org/fdroid/database/DbTest.kt
@@ -1,6 +1,7 @@
 package org.fdroid.database

 import android.content.Context
+import android.content.res.AssetManager
 import androidx.core.os.LocaleListCompat
 import androidx.room.Room
 import androidx.test.core.app.ApplicationProvider
@@ -10,11 +11,19 @@ import io.mockk.mockkObject
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.ExperimentalCoroutinesApi
 import kotlinx.coroutines.test.setMain
+import org.fdroid.database.test.TestUtils.assertRepoEquals
+import org.fdroid.database.test.TestUtils.toMetadataV2
+import org.fdroid.database.test.TestUtils.toPackageVersionV2
+import org.fdroid.index.v2.IndexV2
+import org.fdroid.test.TestUtils.sort
+import org.fdroid.test.TestUtils.sorted
 import org.junit.After
 import org.junit.Before
 import org.junit.runner.RunWith
 import java.io.IOException
 import java.util.Locale
+import kotlin.test.assertEquals
+import kotlin.test.fail

@RunWith(AndroidJUnit4::class)
@OptIn(ExperimentalCoroutinesApi::class)
@@ -26,11 +35,12 @@ internal abstract class DbTest {
    internal lateinit var db: FDroidDatabaseInt
    private val testCoroutineDispatcher = Dispatchers.Unconfined

+    protected val context: Context = ApplicationProvider.getApplicationContext()
+    protected val assets: AssetManager = context.resources.assets
    protected val locales = LocaleListCompat.create(Locale.US)

    @Before
    open fun createDb() {
-        val context = ApplicationProvider.getApplicationContext<Context>()
        db = Room.inMemoryDatabaseBuilder(context, FDroidDatabaseInt::class.java).build()
        repoDao = db.getRepositoryDao()
        appDao = db.getAppDao()
@@ -48,4 +58,28 @@ internal abstract class DbTest {
        db.close()
    }

+    /**
+     * Asserts that data associated with the given [repoId] is equal to the given [index].
+     */
+    protected fun assertDbEquals(repoId: Long, index: IndexV2) {
+        val repo = repoDao.getRepository(repoId) ?: fail()
+        val sortedIndex = index.sorted()
+        assertRepoEquals(sortedIndex.repo, repo)
+        assertEquals(sortedIndex.packages.size, appDao.countApps(), "number of packages")
+        sortedIndex.packages.forEach { (packageName, packageV2) ->
+            assertEquals(
+                packageV2.metadata,
+                appDao.getApp(repoId, packageName)?.toMetadataV2()?.sort()
+            )
+            val versions = versionDao.getAppVersions(repoId, packageName).map {
+                it.toPackageVersionV2()
+            }.associateBy { it.file.sha256 }
+            assertEquals(packageV2.versions.size, versions.size, "number of versions")
+            packageV2.versions.forEach { (versionId, packageVersionV2) ->
+                val version = versions[versionId] ?: fail()
+                assertEquals(packageVersionV2, version)
+            }
+        }
+    }
+
 }
--- a/database/src/androidTest/java/org/fdroid/database/IndexV1InsertTest.kt
+++ b/database/src/androidTest/java/org/fdroid/database/IndexV1InsertTest.kt
@@ -1,149 +1,88 @@
 package org.fdroid.database

-import android.content.Context
-import android.util.Log
-import androidx.test.core.app.ApplicationProvider.getApplicationContext
 import androidx.test.ext.junit.runners.AndroidJUnit4
 import kotlinx.serialization.SerializationException
 import org.apache.commons.io.input.CountingInputStream
-import org.fdroid.CompatibilityChecker
+import org.fdroid.index.IndexConverter
 import org.fdroid.index.v1.IndexV1StreamProcessor
 import org.fdroid.index.v1.IndexV1StreamReceiver
 import org.fdroid.index.v2.AntiFeatureV2
 import org.fdroid.index.v2.CategoryV2
-import org.fdroid.index.v2.IndexV2StreamProcessor
 import org.fdroid.index.v2.MetadataV2
 import org.fdroid.index.v2.PackageVersionV2
 import org.fdroid.index.v2.ReleaseChannelV2
 import org.fdroid.index.v2.RepoV2
+import org.fdroid.test.TestDataEmptyV1
+import org.fdroid.test.TestDataMaxV1
+import org.fdroid.test.TestDataMidV1
+import org.fdroid.test.TestDataMinV1
 import org.junit.Test
 import org.junit.runner.RunWith
-import kotlin.math.roundToInt
 import kotlin.test.assertEquals
 import kotlin.test.assertFailsWith
 import kotlin.test.assertTrue
-import kotlin.test.fail

@RunWith(AndroidJUnit4::class)
 internal class IndexV1InsertTest : DbTest() {

-    @Test
-    fun testStreamIndexV1IntoDb() {
-        val c = getApplicationContext<Context>()
-        val fileSize = c.resources.assets.openFd("index-v1.json").use { it.length }
-        val inputStream = CountingInputStream(c.resources.assets.open("index-v1.json"))
-        var currentByteCount: Long = 0
-        val repoId = db.getRepositoryDao().insertEmptyRepo("https://f-droid.org/repo")
-        val streamReceiver = TestStreamReceiver(repoId) {
-            val bytesRead = inputStream.byteCount
-            val bytesSinceLastCall = bytesRead - currentByteCount
-            if (bytesSinceLastCall > 0) {
-                val percent = ((bytesRead.toDouble() / fileSize) * 100).roundToInt()
-                Log.e("IndexV1InsertTest",
-                    "Stream bytes read: $bytesRead ($percent%) +$bytesSinceLastCall")
-            }
-            // the stream gets read in big chunks, but ensure they are not too big, e.g. entire file
-            assertTrue(bytesSinceLastCall < 600_000, "$bytesSinceLastCall")
-            currentByteCount = bytesRead
-        }
-        val indexProcessor = IndexV1StreamProcessor(streamReceiver, null)
+    private val indexConverter = IndexConverter()

+    @Test
+    fun testStreamEmptyIntoDb() {
+        val repoId = streamIndex("resources/index-empty-v1.json")
+        assertEquals(1, repoDao.getRepositories().size)
+        val index = indexConverter.toIndexV2(TestDataEmptyV1.index)
+        assertDbEquals(repoId, index)
+    }
+
+    @Test
+    fun testStreamMinIntoDb() {
+        val repoId = streamIndex("resources/index-min-v1.json")
+        assertTrue(repoDao.getRepositories().size == 1)
+        val index = indexConverter.toIndexV2(TestDataMinV1.index)
+        assertDbEquals(repoId, index)
+    }
+
+    @Test
+    fun testStreamMidIntoDb() {
+        val repoId = streamIndex("resources/index-mid-v1.json")
+        assertTrue(repoDao.getRepositories().size == 1)
+        val index = indexConverter.toIndexV2(TestDataMidV1.index)
+        assertDbEquals(repoId, index)
+    }
+
+    @Test
+    fun testStreamMaxIntoDb() {
+        val repoId = streamIndex("resources/index-max-v1.json")
+        assertTrue(repoDao.getRepositories().size == 1)
+        val index = indexConverter.toIndexV2(TestDataMaxV1.index)
+        assertDbEquals(repoId, index)
+    }
+
+    private fun streamIndex(path: String): Long {
+        val repoId = db.getRepositoryDao().insertEmptyRepo("https://f-droid.org/repo")
+        val streamReceiver = TestStreamReceiver(repoId)
+        val indexProcessor = IndexV1StreamProcessor(streamReceiver, null)
        db.runInTransaction {
-            inputStream.use { indexStream ->
+            assets.open(path).use { indexStream ->
                indexProcessor.process(indexStream)
            }
        }
-        assertTrue(repoDao.getRepositories().size == 1)
-        assertTrue(appDao.countApps() > 0)
-        assertTrue(appDao.countLocalizedFiles() > 0)
-        assertTrue(appDao.countLocalizedFileLists() > 0)
-        assertTrue(versionDao.countAppVersions() > 0)
-        assertTrue(versionDao.countVersionedStrings() > 0)
-
-        println("Apps: " + appDao.countApps())
-        println("LocalizedFiles: " + appDao.countLocalizedFiles())
-        println("LocalizedFileLists: " + appDao.countLocalizedFileLists())
-        println("Versions: " + versionDao.countAppVersions())
-        println("Perms/Features: " + versionDao.countVersionedStrings())
-
-        val version = repoDao.getRepositories()[0].repository.version ?: fail()
-        insertV2ForComparison(version)
-
-        val repo1 = repoDao.getRepository(1) ?: fail()
-        val repo2 = repoDao.getRepository(2) ?: fail()
-        assertEquals(repo1.repository, repo2.repository.copy(repoId = 1))
-        assertEquals(repo1.mirrors, repo2.mirrors.map { it.copy(repoId = 1) })
-        // TODO enable when better test data
-//        assertEquals(repo1.antiFeatures, repo2.antiFeatures)
-//        assertEquals(repo1.categories, repo2.categories)
-//        assertEquals(repo1.releaseChannels, repo2.releaseChannels)
-
-        val appMetadata = appDao.getAppMetadata()
-        val appMetadata1 = appMetadata.count { it.repoId == 1L }
-        val appMetadata2 = appMetadata.count { it.repoId == 2L }
-        assertEquals(appMetadata1, appMetadata2)
-
-        val localizedFiles = appDao.getLocalizedFiles()
-        val localizedFiles1 = localizedFiles.count { it.repoId == 1L }
-        val localizedFiles2 = localizedFiles.count { it.repoId == 2L }
-        assertEquals(localizedFiles1, localizedFiles2)
-
-        val localizedFileLists = appDao.getLocalizedFileLists()
-        val localizedFileLists1 = localizedFileLists.count { it.repoId == 1L }
-        val localizedFileLists2 = localizedFileLists.count { it.repoId == 2L }
-        assertEquals(localizedFileLists1, localizedFileLists2)
-
-        appMetadata.filter { it.repoId == 2L }.forEach { m ->
-            val metadata1 = appDao.getAppMetadata(1, m.packageId)
-            val metadata2 = appDao.getAppMetadata(2, m.packageId)
-            assertEquals(metadata1, metadata2.copy(repoId = 1, isCompatible = true))
-
-            val lFiles1 = appDao.getLocalizedFiles(1, m.packageId).toSet()
-            val lFiles2 = appDao.getLocalizedFiles(2, m.packageId)
-            assertEquals(lFiles1, lFiles2.map { it.copy(repoId = 1) }.toSet())
-
-            val lFileLists1 = appDao.getLocalizedFileLists(1, m.packageId).toSet()
-            val lFileLists2 = appDao.getLocalizedFileLists(2, m.packageId)
-            assertEquals(lFileLists1, lFileLists2.map { it.copy(repoId = 1) }.toSet())
-
-            val version1 = versionDao.getVersions(1, m.packageId).toSet()
-            val version2 = versionDao.getVersions(2, m.packageId)
-            assertEquals(version1, version2.map { it.copy(repoId = 1) }.toSet())
-
-            val vStrings1 = versionDao.getVersionedStrings(1, m.packageId).toSet()
-            val vStrings2 = versionDao.getVersionedStrings(2, m.packageId)
-            assertEquals(vStrings1, vStrings2.map { it.copy(repoId = 1) }.toSet())
-        }
-    }
-
-    @Suppress("SameParameterValue")
-    private fun insertV2ForComparison(version: Int) {
-        val c = getApplicationContext<Context>()
-        val inputStream = CountingInputStream(c.resources.assets.open("index-v2.json"))
-        val repoId = db.getRepositoryDao().insertEmptyRepo("https://f-droid.org/repo")
-        val indexProcessor = IndexV2StreamProcessor(DbV2StreamReceiver(db, { true }, repoId), null)
-        db.runInTransaction {
-            inputStream.use { indexStream ->
-                indexProcessor.process(version, indexStream)
-            }
-        }
+        return repoId
    }

    @Test
    fun testExceptionWhileStreamingDoesNotSaveIntoDb() {
-        val c = getApplicationContext<Context>()
-        val cIn = CountingInputStream(c.resources.assets.open("index-v1.json"))
-        val compatibilityChecker = CompatibilityChecker {
-            if (cIn.byteCount > 824096) throw SerializationException()
-            true
-        }
-        val indexProcessor =
-            IndexV2StreamProcessor(DbV2StreamReceiver(db, compatibilityChecker, 1), null)
-
+        val cIn = CountingInputStream(assets.open("resources/index-max-v1.json"))
        assertFailsWith<SerializationException> {
            db.runInTransaction {
+                val repoId = db.getRepositoryDao().insertEmptyRepo("https://f-droid.org/repo")
+                val streamReceiver = TestStreamReceiver(repoId) {
+                    if (cIn.byteCount > 0) throw SerializationException()
+                }
+                val indexProcessor = IndexV1StreamProcessor(streamReceiver, null)
                cIn.use { indexStream ->
-                    indexProcessor.process(42, indexStream)
+                    indexProcessor.process(indexStream)
                }
            }
        }
@@ -155,11 +94,12 @@ internal class IndexV1InsertTest : DbTest() {
        assertTrue(versionDao.countVersionedStrings() == 0)
    }

+    @Suppress("DEPRECATION")
    inner class TestStreamReceiver(
        repoId: Long,
-        private val callback: () -> Unit,
+        private val callback: () -> Unit = {},
    ) : IndexV1StreamReceiver {
-        private val streamReceiver = DbV1StreamReceiver(db, { true }, repoId)
+        private val streamReceiver = DbV1StreamReceiver(db, repoId) { true }
        override fun receive(repo: RepoV2, version: Int, certificate: String?) {
            streamReceiver.receive(repo, version, certificate)
            callback()
--- a/database/src/androidTest/java/org/fdroid/database/IndexV2DiffTest.kt
+++ b/database/src/androidTest/java/org/fdroid/database/IndexV2DiffTest.kt
@@ -0,0 +1,327 @@
+package org.fdroid.database
+
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import kotlinx.serialization.SerializationException
+import org.fdroid.index.IndexParser
+import org.fdroid.index.parseV2
+import org.fdroid.index.v2.IndexV2
+import org.fdroid.index.v2.IndexV2DiffStreamProcessor
+import org.fdroid.index.v2.IndexV2StreamProcessor
+import org.fdroid.test.TestDataMaxV2
+import org.fdroid.test.TestDataMidV2
+import org.fdroid.test.TestDataMinV2
+import org.junit.Ignore
+import org.junit.Test
+import org.junit.runner.RunWith
+import java.io.ByteArrayInputStream
+import java.io.InputStream
+import kotlin.test.assertFailsWith
+
+@RunWith(AndroidJUnit4::class)
+internal class IndexV2DiffTest : DbTest() {
+
+    @Test
+    @Ignore("use for testing specific index on demand")
+    fun testBrokenIndexDiff() {
+        val endPath = "resources/tmp/index-end.json"
+        val endIndex = IndexParser.parseV2(assets.open(endPath))
+        testDiff(
+            startPath = "resources/tmp/index-start.json",
+            diffPath = "resources/tmp/diff.json",
+            endIndex = endIndex,
+        )
+    }
+
+    @Test
+    fun testEmptyToMin() = testDiff(
+        startPath = "resources/index-empty-v2.json",
+        diffPath = "resources/diff-empty-min/23.json",
+        endIndex = TestDataMinV2.index,
+    )
+
+    @Test
+    fun testEmptyToMid() = testDiff(
+        startPath = "resources/index-empty-v2.json",
+        diffPath = "resources/diff-empty-mid/23.json",
+        endIndex = TestDataMidV2.index,
+    )
+
+    @Test
+    fun testEmptyToMax() = testDiff(
+        startPath = "resources/index-empty-v2.json",
+        diffPath = "resources/diff-empty-max/23.json",
+        endIndex = TestDataMaxV2.index,
+    )
+
+    @Test
+    fun testMinToMid() = testDiff(
+        startPath = "resources/index-min-v2.json",
+        diffPath = "resources/diff-empty-mid/42.json",
+        endIndex = TestDataMidV2.index,
+    )
+
+    @Test
+    fun testMinToMax() = testDiff(
+        startPath = "resources/index-min-v2.json",
+        diffPath = "resources/diff-empty-max/42.json",
+        endIndex = TestDataMaxV2.index,
+    )
+
+    @Test
+    fun testMidToMax() = testDiff(
+        startPath = "resources/index-mid-v2.json",
+        diffPath = "resources/diff-empty-max/1337.json",
+        endIndex = TestDataMaxV2.index,
+    )
+
+    @Test
+    fun testMinRemoveApp() {
+        val diffJson = """{
+          "packages": {
+            "org.fdroid.min1": null
+          }
+        }""".trimIndent()
+        testJsonDiff(
+            startPath = "resources/index-min-v2.json",
+            diff = diffJson,
+            endIndex = TestDataMinV2.index.copy(packages = emptyMap()),
+        )
+    }
+
+    @Test
+    fun testMinNoMetadataRemoveVersion() {
+        val diffJson = """{
+          "packages": {
+            "org.fdroid.min1": {
+              "metadata": {
+                "added": 0
+              },
+              "versions": {
+                "824a109b2352138c3699760e1683385d0ed50ce526fc7982f8d65757743374bf": null
+              }
+            }
+          }
+        }""".trimIndent()
+        testJsonDiff(
+            startPath = "resources/index-min-v2.json",
+            diff = diffJson,
+            endIndex = TestDataMinV2.index.copy(
+                packages = TestDataMinV2.index.packages.mapValues {
+                    it.value.copy(versions = emptyMap())
+                }
+            ),
+        )
+    }
+
+    @Test
+    fun testMinNoVersionsUnknownKey() {
+        val diffJson = """{
+          "packages": {
+            "org.fdroid.min1": {
+              "metadata": {
+                "added": 42
+              },
+              "unknownKey": "should get ignored" 
+            }
+          }
+        }""".trimIndent()
+        testJsonDiff(
+            startPath = "resources/index-min-v2.json",
+            diff = diffJson,
+            endIndex = TestDataMinV2.index.copy(
+                packages = TestDataMinV2.index.packages.mapValues {
+                    it.value.copy(metadata = it.value.metadata.copy(added = 42))
+                }
+            ),
+        )
+    }
+
+    @Test
+    fun testMinRemoveMetadata() {
+        val diffJson = """{
+          "packages": {
+            "org.fdroid.min1": {
+              "metadata": null
+            }
+          },
+          "unknownKey": "should get ignored" 
+        }""".trimIndent()
+        testJsonDiff(
+            startPath = "resources/index-min-v2.json",
+            diff = diffJson,
+            endIndex = TestDataMinV2.index.copy(
+                packages = emptyMap()
+            ),
+        )
+    }
+
+    @Test
+    fun testMinRemoveVersions() {
+        val diffJson = """{
+          "packages": {
+            "org.fdroid.min1": {
+              "versions": null
+            }
+          }
+        }""".trimIndent()
+        testJsonDiff(
+            startPath = "resources/index-min-v2.json",
+            diff = diffJson,
+            endIndex = TestDataMinV2.index.copy(
+                packages = TestDataMinV2.index.packages.mapValues {
+                    it.value.copy(versions = emptyMap())
+                }
+            ),
+        )
+    }
+
+    @Test
+    fun testMinNoMetadataNoVersion() {
+        val diffJson = """{
+          "packages": {
+            "org.fdroid.min1": {
+            }
+          }
+        }""".trimIndent()
+        testJsonDiff(
+            startPath = "resources/index-min-v2.json",
+            diff = diffJson,
+            endIndex = TestDataMinV2.index,
+        )
+    }
+
+    @Test
+    fun testAppDenyKeyList() {
+        val diffRepoIdJson = """{
+          "packages": {
+            "org.fdroid.min1": {
+              "metadata": {
+                "repoId": 1
+              }
+            }
+          }
+        }""".trimIndent()
+        assertFailsWith<SerializationException> {
+            testJsonDiff(
+                startPath = "resources/index-min-v2.json",
+                diff = diffRepoIdJson,
+                endIndex = TestDataMinV2.index,
+            )
+        }
+        val diffPackageIdJson = """{
+          "packages": {
+            "org.fdroid.min1": {
+              "metadata": {
+                "packageId": "foo"
+              }
+            }
+          }
+        }""".trimIndent()
+        assertFailsWith<SerializationException> {
+            testJsonDiff(
+                startPath = "resources/index-min-v2.json",
+                diff = diffPackageIdJson,
+                endIndex = TestDataMinV2.index,
+            )
+        }
+    }
+
+    @Test
+    fun testVersionsDenyKeyList() {
+        assertFailsWith<SerializationException> {
+            testJsonDiff(
+                startPath = "resources/index-min-v2.json",
+                diff = getMinVersionJson(""""packageId": "foo""""),
+                endIndex = TestDataMinV2.index,
+            )
+        }
+        assertFailsWith<SerializationException> {
+            testJsonDiff(
+                startPath = "resources/index-min-v2.json",
+                diff = getMinVersionJson(""""repoId": 1"""),
+                endIndex = TestDataMinV2.index,
+            )
+        }
+        assertFailsWith<SerializationException> {
+            testJsonDiff(
+                startPath = "resources/index-min-v2.json",
+                diff = getMinVersionJson(""""versionId": "bar""""),
+                endIndex = TestDataMinV2.index,
+            )
+        }
+    }
+
+    private fun getMinVersionJson(insert: String) = """{
+      "packages": {
+        "org.fdroid.min1": {
+          "versions": {
+            "824a109b2352138c3699760e1683385d0ed50ce526fc7982f8d65757743374bf": {
+              $insert
+            }
+        }
+      }
+    }""".trimIndent()
+
+    @Test
+    fun testMidRemoveScreenshots() {
+        val diffRepoIdJson = """{
+          "packages": {
+            "org.fdroid.fdroid": {
+              "metadata": {
+                "screenshots": null
+              }
+            }
+          }
+        }""".trimIndent()
+        val fdroidPackage = TestDataMidV2.packages["org.fdroid.fdroid"]!!.copy(
+            metadata = TestDataMidV2.packages["org.fdroid.fdroid"]!!.metadata.copy(
+                screenshots = null,
+            )
+        )
+        testJsonDiff(
+            startPath = "resources/index-mid-v2.json",
+            diff = diffRepoIdJson,
+            endIndex = TestDataMidV2.index.copy(
+                packages = mapOf(
+                    TestDataMidV2.packageName1 to TestDataMidV2.app1,
+                    TestDataMidV2.packageName2 to fdroidPackage,
+                )
+            ),
+        )
+    }
+
+    private fun testJsonDiff(startPath: String, diff: String, endIndex: IndexV2) {
+        testDiff(startPath, ByteArrayInputStream(diff.toByteArray()), endIndex)
+    }
+
+    private fun testDiff(startPath: String, diffPath: String, endIndex: IndexV2) {
+        testDiff(startPath, assets.open(diffPath), endIndex)
+    }
+
+    private fun testDiff(startPath: String, diffStream: InputStream, endIndex: IndexV2) {
+        // stream start index into the DB
+        val repoId = streamIndex(startPath)
+
+        // apply diff stream to the DB
+        val streamReceiver = DbV2DiffStreamReceiver(db, repoId) { true }
+        val streamProcessor = IndexV2DiffStreamProcessor(streamReceiver)
+        db.runInTransaction {
+            streamProcessor.process(diffStream)
+        }
+        // assert that changed DB data is equal to given endIndex
+        assertDbEquals(repoId, endIndex)
+    }
+
+    private fun streamIndex(path: String): Long {
+        val repoId = db.getRepositoryDao().insertEmptyRepo("https://f-droid.org/repo")
+        val streamReceiver = DbV2StreamReceiver(db, repoId) { true }
+        val indexProcessor = IndexV2StreamProcessor(streamReceiver, null)
+        db.runInTransaction {
+            assets.open(path).use { indexStream ->
+                indexProcessor.process(42, indexStream)
+            }
+        }
+        return repoId
+    }
+
+}
--- a/database/src/androidTest/java/org/fdroid/database/IndexV2InsertTest.kt
+++ b/database/src/androidTest/java/org/fdroid/database/IndexV2InsertTest.kt
@@ -1,16 +1,17 @@
 package org.fdroid.database

-import android.content.Context
-import android.util.Log
-import androidx.test.core.app.ApplicationProvider.getApplicationContext
 import androidx.test.ext.junit.runners.AndroidJUnit4
 import kotlinx.serialization.SerializationException
 import org.apache.commons.io.input.CountingInputStream
 import org.fdroid.CompatibilityChecker
 import org.fdroid.index.v2.IndexV2StreamProcessor
+import org.fdroid.test.TestDataEmptyV2
+import org.fdroid.test.TestDataMaxV2
+import org.fdroid.test.TestDataMidV2
+import org.fdroid.test.TestDataMinV2
 import org.junit.Test
 import org.junit.runner.RunWith
-import kotlin.math.roundToInt
+import kotlin.test.assertEquals
 import kotlin.test.assertFailsWith
 import kotlin.test.assertTrue

@@ -18,59 +19,63 @@ import kotlin.test.assertTrue
 internal class IndexV2InsertTest : DbTest() {

    @Test
-    fun testStreamIndexV2IntoDb() {
-        val c = getApplicationContext<Context>()
-        val fileSize = c.resources.assets.openFd("index-v2.json").use { it.length }
-        val inputStream = CountingInputStream(c.resources.assets.open("index-v2.json"))
-        var currentByteCount: Long = 0
-        val compatibilityChecker = CompatibilityChecker {
-            val bytesRead = inputStream.byteCount
-            val bytesSinceLastCall = bytesRead - currentByteCount
-            if (bytesSinceLastCall > 0) {
-                val percent = ((bytesRead.toDouble() / fileSize) * 100).roundToInt()
-                Log.e("IndexV2InsertTest",
-                    "Stream bytes read: $bytesRead ($percent%) +$bytesSinceLastCall")
-            }
-            // the stream gets read in big chunks, but ensure they are not too big, e.g. entire file
-            assertTrue(bytesSinceLastCall < 400_000, "$bytesSinceLastCall")
-            currentByteCount = bytesRead
-            true
-        }
-        val repoId = db.getRepositoryDao().insertEmptyRepo("https://f-droid.org/repo")
-        val streamReceiver = DbV2StreamReceiver(db, compatibilityChecker, repoId)
-        val indexProcessor = IndexV2StreamProcessor(streamReceiver, null)
+    fun testStreamEmptyIntoDb() {
+        val repoId = streamIndex("resources/index-empty-v2.json")
+        assertEquals(1, repoDao.getRepositories().size)
+        assertDbEquals(repoId, TestDataEmptyV2.index)
+    }

+    @Test
+    fun testStreamMinIntoDb() {
+        val repoId = streamIndex("resources/index-min-v2.json")
+        assertEquals(1, repoDao.getRepositories().size)
+        assertDbEquals(repoId, TestDataMinV2.index)
+    }
+
+    @Test
+    fun testStreamMinReorderedIntoDb() {
+        val repoId = streamIndex("resources/index-min-reordered-v2.json")
+        assertEquals(1, repoDao.getRepositories().size)
+        assertDbEquals(repoId, TestDataMinV2.index)
+    }
+
+    @Test
+    fun testStreamMidIntoDb() {
+        val repoId = streamIndex("resources/index-mid-v2.json")
+        assertEquals(1, repoDao.getRepositories().size)
+        assertDbEquals(repoId, TestDataMidV2.index)
+    }
+
+    @Test
+    fun testStreamMaxIntoDb() {
+        val repoId = streamIndex("resources/index-max-v2.json")
+        assertEquals(1, repoDao.getRepositories().size)
+        assertDbEquals(repoId, TestDataMaxV2.index)
+    }
+
+    private fun streamIndex(path: String): Long {
+        val repoId = db.getRepositoryDao().insertEmptyRepo("https://f-droid.org/repo")
+        val streamReceiver = DbV2StreamReceiver(db, repoId) { true }
+        val indexProcessor = IndexV2StreamProcessor(streamReceiver, null)
        db.runInTransaction {
-            inputStream.use { indexStream ->
+            assets.open(path).use { indexStream ->
                indexProcessor.process(42, indexStream)
            }
        }
-        assertTrue(repoDao.getRepositories().size == 1)
-        assertTrue(appDao.countApps() > 0)
-        assertTrue(appDao.countLocalizedFiles() > 0)
-        assertTrue(appDao.countLocalizedFileLists() > 0)
-        assertTrue(versionDao.countAppVersions() > 0)
-        assertTrue(versionDao.countVersionedStrings() > 0)
-
-        println("Apps: " + appDao.countApps())
-        println("LocalizedFiles: " + appDao.countLocalizedFiles())
-        println("LocalizedFileLists: " + appDao.countLocalizedFileLists())
-        println("Versions: " + versionDao.countAppVersions())
-        println("Perms/Features: " + versionDao.countVersionedStrings())
+        return repoId
    }

    @Test
    fun testExceptionWhileStreamingDoesNotSaveIntoDb() {
-        val c = getApplicationContext<Context>()
-        val cIn = CountingInputStream(c.resources.assets.open("index-v2.json"))
+        val cIn = CountingInputStream(assets.open("resources/index-max-v2.json"))
        val compatibilityChecker = CompatibilityChecker {
-            if (cIn.byteCount > 824096) throw SerializationException()
+            if (cIn.byteCount > 0) throw SerializationException()
            true
        }
        assertFailsWith<SerializationException> {
            db.runInTransaction {
                val repoId = db.getRepositoryDao().insertEmptyRepo("http://example.org")
-                val streamReceiver = DbV2StreamReceiver(db, compatibilityChecker, repoId)
+                val streamReceiver = DbV2StreamReceiver(db, repoId, compatibilityChecker)
                val indexProcessor = IndexV2StreamProcessor(streamReceiver, null)
                cIn.use { indexStream ->
                    indexProcessor.process(42, indexStream)
--- a/database/src/androidTest/java/org/fdroid/database/RepositoryDiffTest.kt
+++ b/database/src/androidTest/java/org/fdroid/database/RepositoryDiffTest.kt
@@ -76,47 +76,6 @@ internal class RepositoryDiffTest : DbTest() {
        assertRepoEquals(repo.copy(timestamp = updateTimestamp), repos[0])
    }

-    @Test
-    fun iconDiff() {
-        val repo = getRandomRepo()
-        val updateIcon = getRandomFileV2()
-        val json = """
-            {
-              "icon": ${Json.encodeToString(updateIcon)}
-            }""".trimIndent()
-        testDiff(repo, json) { repos ->
-            assertEquals(updateIcon, repos[0].icon)
-            assertRepoEquals(repo.copy(icon = updateIcon), repos[0])
-        }
-    }
-
-    @Test
-    fun iconPartialDiff() {
-        val repo = getRandomRepo()
-        val updateIcon = repo.icon!!.copy(name = getRandomString())
-        val json = """
-            {
-              "icon": { "name": "${updateIcon.name}" }
-            }""".trimIndent()
-        testDiff(repo, json) { repos ->
-            assertEquals(updateIcon, repos[0].icon)
-            assertRepoEquals(repo.copy(icon = updateIcon), repos[0])
-        }
-    }
-
-    @Test
-    fun iconRemoval() {
-        val repo = getRandomRepo()
-        val json = """
-            {
-              "icon": null
-            }""".trimIndent()
-        testDiff(repo, json) { repos ->
-            assertEquals(null, repos[0].icon)
-            assertRepoEquals(repo.copy(icon = null), repos[0])
-        }
-    }
-
    @Test
    fun mirrorDiff() {
        val repo = getRandomRepo()
--- a/database/src/androidTest/java/org/fdroid/database/UpdateCheckerTest.kt
+++ b/database/src/androidTest/java/org/fdroid/database/UpdateCheckerTest.kt
@@ -1,11 +1,8 @@
 package org.fdroid.database

-import android.content.Context
 import android.util.Log
-import androidx.test.core.app.ApplicationProvider
 import androidx.test.ext.junit.runners.AndroidJUnit4
-import org.apache.commons.io.input.CountingInputStream
-import org.fdroid.index.v1.IndexV1StreamProcessor
+import org.fdroid.index.v2.IndexV2StreamProcessor
 import org.junit.Before
 import org.junit.Test
 import org.junit.runner.RunWith
@@ -15,29 +12,26 @@ import kotlin.time.measureTime
@RunWith(AndroidJUnit4::class)
 internal class UpdateCheckerTest : DbTest() {

-    private lateinit var context: Context
    private lateinit var updateChecker: UpdateChecker

    @Before
    override fun createDb() {
        super.createDb()
-        context = ApplicationProvider.getApplicationContext()
+        // TODO mock packageManager and maybe move to unit tests
        updateChecker = UpdateChecker(db, context.packageManager)
    }

    @Test
    @OptIn(ExperimentalTime::class)
    fun testGetUpdates() {
-        val inputStream = CountingInputStream(context.resources.assets.open("index-v1.json"))
-        val repoId = db.getRepositoryDao().insertEmptyRepo("https://f-droid.org/repo")
-        val indexProcessor = IndexV1StreamProcessor(DbV1StreamReceiver(db, { true }, repoId), null)
-
        db.runInTransaction {
-            inputStream.use { indexStream ->
-                indexProcessor.process(indexStream)
+            val repoId = db.getRepositoryDao().insertEmptyRepo("https://f-droid.org/repo")
+            val streamReceiver = DbV2StreamReceiver(db, repoId) { true }
+            val indexProcessor = IndexV2StreamProcessor(streamReceiver, null)
+            assets.open("resources/index-max-v2.json").use { indexStream ->
+                indexProcessor.process(42, indexStream)
            }
        }
-
        val duration = measureTime {
            updateChecker.getUpdatableApps()
        }
--- a/database/src/androidTest/java/org/fdroid/database/test/TestUtils.kt
+++ b/database/src/androidTest/java/org/fdroid/database/test/TestUtils.kt
@@ -2,12 +2,18 @@ package org.fdroid.database.test

 import androidx.lifecycle.LiveData
 import androidx.lifecycle.Observer
+import org.fdroid.database.App
+import org.fdroid.database.AppVersion
 import org.fdroid.database.Repository
 import org.fdroid.database.toCoreRepository
 import org.fdroid.database.toMirror
 import org.fdroid.database.toRepoAntiFeatures
 import org.fdroid.database.toRepoCategories
 import org.fdroid.database.toRepoReleaseChannel
+import org.fdroid.index.v2.FeatureV2
+import org.fdroid.index.v2.ManifestV2
+import org.fdroid.index.v2.MetadataV2
+import org.fdroid.index.v2.PackageVersionV2
 import org.fdroid.index.v2.RepoV2
 import org.junit.Assert
 import java.util.concurrent.CountDownLatch
@@ -36,6 +42,59 @@ internal object TestUtils {
        assertEquals(coreRepo, repo.repository)
    }

+    internal fun App.toMetadataV2() = MetadataV2(
+        added = metadata.added,
+        lastUpdated = metadata.lastUpdated,
+        name = metadata.name,
+        summary = metadata.summary,
+        description = metadata.description,
+        webSite = metadata.webSite,
+        changelog = metadata.changelog,
+        license = metadata.license,
+        sourceCode = metadata.sourceCode,
+        issueTracker = metadata.issueTracker,
+        translation = metadata.translation,
+        preferredSigner = metadata.preferredSigner,
+        video = metadata.video,
+        authorName = metadata.authorName,
+        authorEmail = metadata.authorEmail,
+        authorWebSite = metadata.authorWebSite,
+        authorPhone = metadata.authorPhone,
+        donate = metadata.donate ?: emptyList(),
+        liberapayID = metadata.liberapayID,
+        liberapay = metadata.liberapay,
+        openCollective = metadata.openCollective,
+        bitcoin = metadata.bitcoin,
+        litecoin = metadata.litecoin,
+        flattrID = metadata.flattrID,
+        categories = metadata.categories ?: emptyList(),
+        icon = icon,
+        featureGraphic = featureGraphic,
+        promoGraphic = promoGraphic,
+        tvBanner = tvBanner,
+        screenshots = screenshots,
+    )
+
+    fun AppVersion.toPackageVersionV2() = PackageVersionV2(
+        added = added,
+        file = file,
+        src = src,
+        manifest = ManifestV2(
+            versionName = manifest.versionName,
+            versionCode = manifest.versionCode,
+            usesSdk = manifest.usesSdk,
+            maxSdkVersion = manifest.maxSdkVersion,
+            signer = manifest.signer,
+            usesPermission = usesPermission?.sortedBy { it.name } ?: emptyList(),
+            usesPermissionSdk23 = usesPermissionSdk23?.sortedBy { it.name } ?: emptyList(),
+            nativecode = manifest.nativecode?.sorted() ?: emptyList(),
+            features = manifest.features?.map { FeatureV2(it) } ?: emptyList(),
+        ),
+        releaseChannels = releaseChannels,
+        antiFeatures = version.antiFeatures ?: emptyMap(),
+        whatsNew = version.whatsNew ?: emptyMap(),
+    )
+
    fun <T> LiveData<T>.getOrAwaitValue(): T? {
        val data = arrayOfNulls<Any>(1)
        val latch = CountDownLatch(1)
--- a/database/src/main/java/org/fdroid/database/App.kt
+++ b/database/src/main/java/org/fdroid/database/App.kt
@@ -327,19 +327,24 @@ internal fun LocalizedFileListV2.toLocalizedFileList(
    packageId: String,
    type: String,
 ): List<LocalizedFileList> = flatMap { (locale, files) ->
-    files.map { file ->
-        LocalizedFileList(
-            repoId = repoId,
-            packageId = packageId,
-            type = type,
-            locale = locale,
-            name = file.name,
-            sha256 = file.sha256,
-            size = file.size,
-        )
-    }
+    files.map { file -> file.toLocalizedFileList(repoId, packageId, type, locale) }
 }

+internal fun FileV2.toLocalizedFileList(
+    repoId: Long,
+    packageId: String,
+    type: String,
+    locale: String,
+) = LocalizedFileList(
+    repoId = repoId,
+    packageId = packageId,
+    type = type,
+    locale = locale,
+    name = name,
+    sha256 = sha256,
+    size = size,
+)
+
 internal fun List<LocalizedFileList>.toLocalizedFileListV2(type: String): LocalizedFileListV2? {
    val map = HashMap<String, List<FileV2>>()
    iterator().forEach { file ->
--- a/database/src/main/java/org/fdroid/database/AppDao.kt
+++ b/database/src/main/java/org/fdroid/database/AppDao.kt
@@ -18,10 +18,20 @@ import androidx.room.OnConflictStrategy
 import androidx.room.Query
 import androidx.room.RoomWarnings.CURSOR_MISMATCH
 import androidx.room.Transaction
+import androidx.room.Update
+import kotlinx.serialization.SerializationException
+import kotlinx.serialization.json.JsonNull
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.decodeFromJsonElement
+import org.fdroid.database.DbDiffUtils.diffAndUpdateListTable
+import org.fdroid.database.DbDiffUtils.diffAndUpdateTable
 import org.fdroid.database.FDroidDatabaseHolder.dispatcher
+import org.fdroid.index.IndexParser.json
+import org.fdroid.index.v2.FileV2
 import org.fdroid.index.v2.LocalizedFileListV2
 import org.fdroid.index.v2.LocalizedFileV2
 import org.fdroid.index.v2.MetadataV2
+import org.fdroid.index.v2.ReflectionDiffer.applyDiff
 import org.fdroid.index.v2.Screenshots

 public interface AppDao {
@@ -64,6 +74,23 @@ public enum class AppListSortOrder {
    LAST_UPDATED, NAME
 }

+/**
+ * A list of unknown fields in [MetadataV2] that we don't allow for [AppMetadata].
+ *
+ * We are applying reflection diffs against internal database classes
+ * and need to prevent the untrusted external JSON input to modify internal fields in those classes.
+ * This list must always hold the names of all those internal FIELDS for [AppMetadata].
+ */
+private val DENY_LIST = listOf("packageId", "repoId")
+
+/**
+ * A list of unknown fields in [LocalizedFileV2] or [LocalizedFileListV2]
+ * that we don't allow for [LocalizedFile] or [LocalizedFileList].
+ *
+ * Similar to [DENY_LIST].
+ */
+private val DENY_FILE_LIST = listOf("packageId", "repoId", "type")
+
@Dao
 internal interface AppDaoInt : AppDao {

@@ -110,6 +137,95 @@ internal interface AppDaoInt : AppDao {
    @Insert(onConflict = OnConflictStrategy.REPLACE)
    fun insertLocalizedFileLists(localizedFiles: List<LocalizedFileList>)

+    @Transaction
+    fun updateApp(
+        repoId: Long,
+        packageId: String,
+        jsonObject: JsonObject?,
+        locales: LocaleListCompat,
+    ) {
+        if (jsonObject == null) {
+            // this app is gone, we need to delete it
+            deleteAppMetadata(repoId, packageId)
+            return
+        }
+        val metadata = getAppMetadata(repoId, packageId)
+        if (metadata == null) { // new app
+            val metadataV2: MetadataV2 = json.decodeFromJsonElement(jsonObject)
+            insert(repoId, packageId, metadataV2)
+        } else { // diff against existing app
+            // ensure that diff does not include internal keys
+            DENY_LIST.forEach { forbiddenKey ->
+                if (jsonObject.containsKey(forbiddenKey)) throw SerializationException(forbiddenKey)
+            }
+            // diff metadata
+            val diffedApp = applyDiff(metadata, jsonObject)
+            val updatedApp =
+                if (jsonObject.containsKey("name") || jsonObject.containsKey("summary")) {
+                    diffedApp.copy(
+                        localizedName = diffedApp.name.getBestLocale(locales),
+                        localizedSummary = diffedApp.summary.getBestLocale(locales),
+                    )
+                } else diffedApp
+            updateAppMetadata(updatedApp)
+            // diff localizedFiles
+            val localizedFiles = getLocalizedFiles(repoId, packageId)
+            localizedFiles.diffAndUpdate(repoId, packageId, "icon", jsonObject)
+            localizedFiles.diffAndUpdate(repoId, packageId, "featureGraphic", jsonObject)
+            localizedFiles.diffAndUpdate(repoId, packageId, "promoGraphic", jsonObject)
+            localizedFiles.diffAndUpdate(repoId, packageId, "tvBanner", jsonObject)
+            // diff localizedFileLists
+            val screenshots = jsonObject["screenshots"]
+            if (screenshots is JsonNull) {
+                deleteLocalizedFileLists(repoId, packageId)
+            } else if (screenshots is JsonObject) {
+                diffAndUpdateLocalizedFileList(repoId, packageId, "phone", screenshots)
+                diffAndUpdateLocalizedFileList(repoId, packageId, "sevenInch", screenshots)
+                diffAndUpdateLocalizedFileList(repoId, packageId, "tenInch", screenshots)
+                diffAndUpdateLocalizedFileList(repoId, packageId, "wear", screenshots)
+                diffAndUpdateLocalizedFileList(repoId, packageId, "tv", screenshots)
+            }
+        }
+    }
+
+    private fun List<LocalizedFile>.diffAndUpdate(
+        repoId: Long,
+        packageId: String,
+        type: String,
+        jsonObject: JsonObject,
+    ) = diffAndUpdateTable(
+        jsonObject = jsonObject,
+        jsonObjectKey = type,
+        itemList = filter { it.type == type },
+        itemFinder = { locale, item -> item.locale == locale },
+        newItem = { locale -> LocalizedFile(repoId, packageId, type, locale, "") },
+        deleteAll = { deleteLocalizedFiles(repoId, packageId, type) },
+        deleteOne = { locale -> deleteLocalizedFile(repoId, packageId, type, locale) },
+        insertReplace = { list -> insert(list) },
+        isNewItemValid = { it.name.isNotEmpty() },
+        keyDenyList = DENY_FILE_LIST,
+    )
+
+    private fun diffAndUpdateLocalizedFileList(
+        repoId: Long,
+        packageId: String,
+        type: String,
+        jsonObject: JsonObject,
+    ) {
+        diffAndUpdateListTable(
+            jsonObject = jsonObject,
+            jsonObjectKey = type,
+            listParser = { locale, jsonArray ->
+                json.decodeFromJsonElement<List<FileV2>>(jsonArray).map {
+                    it.toLocalizedFileList(repoId, packageId, type, locale)
+                }
+            },
+            deleteAll = { deleteLocalizedFileLists(repoId, packageId, type) },
+            deleteList = { locale -> deleteLocalizedFileList(repoId, packageId, type, locale) },
+            insertNewList = { _, fileLists -> insertLocalizedFileLists(fileLists) },
+        )
+    }
+
    /**
     * This is needed to support v1 streaming and shouldn't be used for something else.
     */
@@ -135,6 +251,9 @@ internal interface AppDaoInt : AppDao {
        WHERE repoId = :repoId AND packageId = :packageId""")
    fun updateAppMetadata(repoId: Long, packageId: String, name: String?, summary: String?)

+    @Update
+    fun updateAppMetadata(appMetadata: AppMetadata): Int
+
    override fun getApp(packageId: String): LiveData<App?> {
        return getRepoIdForPackage(packageId).distinctUntilChanged().switchMap { repoId ->
            if (repoId == null) MutableLiveData(null)
@@ -160,7 +279,7 @@ internal interface AppDaoInt : AppDao {

    @Transaction
    override fun getApp(repoId: Long, packageId: String): App? {
-        val metadata = getAppMetadata(repoId, packageId)
+        val metadata = getAppMetadata(repoId, packageId) ?: return null
        val localizedFiles = getLocalizedFiles(repoId, packageId)
        val localizedFileList = getLocalizedFileLists(repoId, packageId)
        return getApp(metadata, localizedFiles, localizedFileList)
@@ -189,7 +308,7 @@ internal interface AppDaoInt : AppDao {
    fun getLiveAppMetadata(repoId: Long, packageId: String): LiveData<AppMetadata>

    @Query("SELECT * FROM AppMetadata WHERE repoId = :repoId AND packageId = :packageId")
-    fun getAppMetadata(repoId: Long, packageId: String): AppMetadata
+    fun getAppMetadata(repoId: Long, packageId: String): AppMetadata?

    @Query("SELECT * FROM AppMetadata")
    fun getAppMetadata(): List<AppMetadata>
@@ -354,10 +473,29 @@ internal interface AppDaoInt : AppDao {
        FROM AppMetadata AS app WHERE repoId = :repoId AND packageId = :packageId""")
    fun getAppOverviewItem(repoId: Long, packageId: String): AppOverviewItem?

-    @VisibleForTesting
    @Query("DELETE FROM AppMetadata WHERE repoId = :repoId AND packageId = :packageId")
    fun deleteAppMetadata(repoId: Long, packageId: String)

+    @Query("""DELETE FROM LocalizedFile
+        WHERE repoId = :repoId AND packageId = :packageId AND type = :type""")
+    fun deleteLocalizedFiles(repoId: Long, packageId: String, type: String)
+
+    @Query("""DELETE FROM LocalizedFile
+        WHERE repoId = :repoId AND packageId = :packageId AND type = :type AND locale = :locale""")
+    fun deleteLocalizedFile(repoId: Long, packageId: String, type: String, locale: String)
+
+    @Query("""DELETE FROM LocalizedFileList
+        WHERE repoId = :repoId AND packageId = :packageId""")
+    fun deleteLocalizedFileLists(repoId: Long, packageId: String)
+
+    @Query("""DELETE FROM LocalizedFileList
+        WHERE repoId = :repoId AND packageId = :packageId AND type = :type""")
+    fun deleteLocalizedFileLists(repoId: Long, packageId: String, type: String)
+
+    @Query("""DELETE FROM LocalizedFileList
+        WHERE repoId = :repoId AND packageId = :packageId AND type = :type AND locale = :locale""")
+    fun deleteLocalizedFileList(repoId: Long, packageId: String, type: String, locale: String)
+
    @VisibleForTesting
    @Query("SELECT COUNT(*) FROM AppMetadata")
    fun countApps(): Int
--- a/database/src/main/java/org/fdroid/database/DbDiffUtils.kt
+++ b/database/src/main/java/org/fdroid/database/DbDiffUtils.kt
@@ -0,0 +1,125 @@
+package org.fdroid.database
+
+import kotlinx.serialization.SerializationException
+import kotlinx.serialization.json.JsonArray
+import kotlinx.serialization.json.JsonNull
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.jsonArray
+import kotlinx.serialization.json.jsonObject
+import org.fdroid.index.v2.ReflectionDiffer
+
+internal object DbDiffUtils {
+
+    /**
+     * Applies the diff from the given [jsonObject] identified by the given [jsonObjectKey]
+     * to [itemList] and updates the DB as needed.
+     *
+     * @param newItem A function to produce a new [T] which typically contains the primary key(s).
+     */
+    @Throws(SerializationException::class)
+    fun <T : Any> diffAndUpdateTable(
+        jsonObject: JsonObject,
+        jsonObjectKey: String,
+        itemList: List<T>,
+        itemFinder: (String, T) -> Boolean,
+        newItem: (String) -> T,
+        deleteAll: () -> Unit,
+        deleteOne: (String) -> Unit,
+        insertReplace: (List<T>) -> Unit,
+        isNewItemValid: (T) -> Boolean = { true },
+        keyDenyList: List<String>? = null,
+    ) {
+        if (!jsonObject.containsKey(jsonObjectKey)) return
+        if (jsonObject[jsonObjectKey] == JsonNull) {
+            deleteAll()
+        } else {
+            val obj = jsonObject[jsonObjectKey]?.jsonObject ?: error("no $jsonObjectKey object")
+            val list = itemList.toMutableList()
+            obj.entries.forEach { (key, value) ->
+                if (value is JsonNull) {
+                    list.removeAll { itemFinder(key, it) }
+                    deleteOne(key)
+                } else {
+                    value.jsonObject.checkDenyList(keyDenyList)
+                    val index = list.indexOfFirst { itemFinder(key, it) }
+                    val item = if (index == -1) null else list[index]
+                    if (item == null) {
+                        val itemToInsert =
+                            ReflectionDiffer.applyDiff(newItem(key), value.jsonObject)
+                        if (!isNewItemValid(itemToInsert)) throw SerializationException("$newItem")
+                        list.add(itemToInsert)
+                    } else {
+                        list[index] = ReflectionDiffer.applyDiff(item, value.jsonObject)
+                    }
+                }
+            }
+            insertReplace(list)
+        }
+    }
+
+    /**
+     * Applies a list diff from a map of lists.
+     * The map is identified by the given [jsonObjectKey] in the given [jsonObject].
+     * The diff is applied for each key
+     * by replacing the existing list using [deleteList] and [insertNewList].
+     *
+     * @param listParser returns a list of [T] from the given [JsonArray].
+     */
+    @Throws(SerializationException::class)
+    fun <T : Any> diffAndUpdateListTable(
+        jsonObject: JsonObject,
+        jsonObjectKey: String,
+        listParser: (String, JsonArray) -> List<T>,
+        deleteAll: () -> Unit,
+        deleteList: (String) -> Unit,
+        insertNewList: (String, List<T>) -> Unit,
+    ) {
+        if (!jsonObject.containsKey(jsonObjectKey)) return
+        if (jsonObject[jsonObjectKey] == JsonNull) {
+            deleteAll()
+        } else {
+            val obj = jsonObject[jsonObjectKey]?.jsonObject ?: error("no $jsonObjectKey object")
+            obj.entries.forEach { (key, list) ->
+                if (list is JsonNull) {
+                    deleteList(key)
+                } else {
+                    val newList = listParser(key, list.jsonArray)
+                    deleteList(key)
+                    insertNewList(key, newList)
+                }
+            }
+        }
+    }
+
+    /**
+     * Applies the list diff from the given [jsonObject] identified by the given [jsonObjectKey]
+     * by replacing an existing list using [deleteList] and [insertNewList].
+     *
+     * @param listParser returns a list of [T] from the given [JsonArray].
+     */
+    @Throws(SerializationException::class)
+    fun <T : Any> diffAndUpdateListTable(
+        jsonObject: JsonObject,
+        jsonObjectKey: String,
+        listParser: (JsonArray) -> List<T>,
+        deleteList: () -> Unit,
+        insertNewList: (List<T>) -> Unit,
+    ) {
+        if (!jsonObject.containsKey(jsonObjectKey)) return
+        if (jsonObject[jsonObjectKey] == JsonNull) {
+            deleteList()
+        } else {
+            val jsonArray = jsonObject[jsonObjectKey]?.jsonArray ?: error("no $jsonObjectKey array")
+            val list = listParser(jsonArray)
+            deleteList()
+            insertNewList(list)
+        }
+    }
+
+    private fun JsonObject.checkDenyList(list: List<String>?) {
+        list?.forEach { forbiddenKey ->
+            if (containsKey(forbiddenKey)) throw SerializationException(forbiddenKey)
+        }
+    }
+
+}
--- a/database/src/main/java/org/fdroid/database/DbV1StreamReceiver.kt
+++ b/database/src/main/java/org/fdroid/database/DbV1StreamReceiver.kt
@@ -19,8 +19,8 @@ import org.fdroid.index.v2.RepoV2
@Deprecated("Use DbV2StreamReceiver instead")
 internal class DbV1StreamReceiver(
    private val db: FDroidDatabaseInt,
-    private val compatibilityChecker: CompatibilityChecker,
    private val repoId: Long,
+    private val compatibilityChecker: CompatibilityChecker,
 ) : IndexV1StreamReceiver {

    private val locales: LocaleListCompat = getLocales(Resources.getSystem().configuration)
--- a/database/src/main/java/org/fdroid/database/DbV2DiffStreamReceiver.kt
+++ b/database/src/main/java/org/fdroid/database/DbV2DiffStreamReceiver.kt
@@ -0,0 +1,40 @@
+package org.fdroid.database
+
+import android.content.res.Resources
+import androidx.core.os.ConfigurationCompat.getLocales
+import androidx.core.os.LocaleListCompat
+import kotlinx.serialization.json.JsonObject
+import org.fdroid.CompatibilityChecker
+import org.fdroid.index.v2.IndexV2DiffStreamReceiver
+
+internal class DbV2DiffStreamReceiver(
+    private val db: FDroidDatabaseInt,
+    private val repoId: Long,
+    private val compatibilityChecker: CompatibilityChecker,
+) : IndexV2DiffStreamReceiver {
+
+    private val locales: LocaleListCompat = getLocales(Resources.getSystem().configuration)
+
+    override fun receiveRepoDiff(repoJsonObject: JsonObject) {
+        db.getRepositoryDao().updateRepository(repoId, repoJsonObject)
+    }
+
+    override fun receivePackageMetadataDiff(packageId: String, packageJsonObject: JsonObject?) {
+        db.getAppDao().updateApp(repoId, packageId, packageJsonObject, locales)
+    }
+
+    override fun receiveVersionsDiff(
+        packageId: String,
+        versionsDiffMap: Map<String, JsonObject?>?,
+    ) {
+        db.getVersionDao().update(repoId, packageId, versionsDiffMap) {
+            compatibilityChecker.isCompatible(it)
+        }
+    }
+
+    @Synchronized
+    override fun onStreamEnded() {
+        db.afterUpdatingRepo(repoId)
+    }
+
+}
--- a/database/src/main/java/org/fdroid/database/DbV2StreamReceiver.kt
+++ b/database/src/main/java/org/fdroid/database/DbV2StreamReceiver.kt
@@ -10,8 +10,8 @@ import org.fdroid.index.v2.RepoV2

 internal class DbV2StreamReceiver(
    private val db: FDroidDatabaseInt,
-    private val compatibilityChecker: CompatibilityChecker,
    private val repoId: Long,
+    private val compatibilityChecker: CompatibilityChecker,
 ) : IndexV2StreamReceiver {

    private val locales: LocaleListCompat = getLocales(Resources.getSystem().configuration)
--- a/database/src/main/java/org/fdroid/database/RepositoryDao.kt
+++ b/database/src/main/java/org/fdroid/database/RepositoryDao.kt
@@ -9,11 +9,10 @@ import androidx.room.Query
 import androidx.room.RewriteQueriesToDropUnusedColumns
 import androidx.room.Transaction
 import androidx.room.Update
-import kotlinx.serialization.json.JsonArray
-import kotlinx.serialization.json.JsonNull
 import kotlinx.serialization.json.JsonObject
 import kotlinx.serialization.json.decodeFromJsonElement
-import kotlinx.serialization.json.jsonObject
+import org.fdroid.database.DbDiffUtils.diffAndUpdateListTable
+import org.fdroid.database.DbDiffUtils.diffAndUpdateTable
 import org.fdroid.index.IndexParser.json
 import org.fdroid.index.v2.MirrorV2
 import org.fdroid.index.v2.ReflectionDiffer.applyDiff
@@ -187,23 +186,25 @@ internal interface RepositoryDaoInt : RepositoryDao {
        val repo = getRepository(repoId) ?: error("Repo $repoId does not exist")
        // update repo with JSON diff
        updateRepository(applyDiff(repo.repository, jsonObject))
-        // replace mirror list, if it is in the diff
-        if (jsonObject.containsKey("mirrors")) {
-            val mirrorArray = jsonObject["mirrors"] as JsonArray
-            val mirrors = json.decodeFromJsonElement<List<MirrorV2>>(mirrorArray).map {
-                it.toMirror(repoId)
-            }
-            // delete and re-insert mirrors, because it is easier than diffing
-            deleteMirrors(repoId)
-            insertMirrors(mirrors)
-        }
+        // replace mirror list (if it is in the diff)
+        diffAndUpdateListTable(
+            jsonObject = jsonObject,
+            jsonObjectKey = "mirrors",
+            listParser = { mirrorArray ->
+                json.decodeFromJsonElement<List<MirrorV2>>(mirrorArray).map {
+                    it.toMirror(repoId)
+                }
+            },
+            deleteList = { deleteMirrors(repoId) },
+            insertNewList = { mirrors -> insertMirrors(mirrors) },
+        )
        // diff and update the antiFeatures
        diffAndUpdateTable(
            jsonObject = jsonObject,
-            key = "antiFeatures",
+            jsonObjectKey = "antiFeatures",
            itemList = repo.antiFeatures,
+            itemFinder = { key, item -> item.id == key },
            newItem = { key -> AntiFeature(repoId, key, null, emptyMap(), emptyMap()) },
-            keyGetter = { item -> item.id },
            deleteAll = { deleteAntiFeatures(repoId) },
            deleteOne = { key -> deleteAntiFeature(repoId, key) },
            insertReplace = { list -> insertAntiFeatures(list) },
@@ -211,10 +212,10 @@ internal interface RepositoryDaoInt : RepositoryDao {
        // diff and update the categories
        diffAndUpdateTable(
            jsonObject = jsonObject,
-            key = "categories",
+            jsonObjectKey = "categories",
            itemList = repo.categories,
+            itemFinder = { key, item -> item.id == key },
            newItem = { key -> Category(repoId, key, null, emptyMap(), emptyMap()) },
-            keyGetter = { item -> item.id },
            deleteAll = { deleteCategories(repoId) },
            deleteOne = { key -> deleteCategory(repoId, key) },
            insertReplace = { list -> insertCategories(list) },
@@ -222,56 +223,16 @@ internal interface RepositoryDaoInt : RepositoryDao {
        // diff and update the releaseChannels
        diffAndUpdateTable(
            jsonObject = jsonObject,
-            key = "releaseChannels",
+            jsonObjectKey = "releaseChannels",
            itemList = repo.releaseChannels,
+            itemFinder = { key, item -> item.id == key },
            newItem = { key -> ReleaseChannel(repoId, key, null, emptyMap(), emptyMap()) },
-            keyGetter = { item -> item.id },
            deleteAll = { deleteReleaseChannels(repoId) },
            deleteOne = { key -> deleteReleaseChannel(repoId, key) },
            insertReplace = { list -> insertReleaseChannels(list) },
        )
    }

-    /**
-     * Applies the diff from [JsonObject] identified by the given [key] of the given [jsonObject]
-     * to the given [itemList] and updates the DB as needed.
-     *
-     * @param newItem A function to produce a new [T] which typically contains the primary key(s).
-     */
-    private fun <T : Any> diffAndUpdateTable(
-        jsonObject: JsonObject,
-        key: String,
-        itemList: List<T>,
-        newItem: (String) -> T,
-        keyGetter: (T) -> String,
-        deleteAll: () -> Unit,
-        deleteOne: (String) -> Unit,
-        insertReplace: (List<T>) -> Unit,
-    ) {
-        if (!jsonObject.containsKey(key)) return
-        if (jsonObject[key] == JsonNull) {
-            deleteAll()
-        } else {
-            val features = jsonObject[key]?.jsonObject ?: error("no $key object")
-            val list = itemList.toMutableList()
-            features.entries.forEach { (key, value) ->
-                if (value is JsonNull) {
-                    list.removeAll { keyGetter(it) == key }
-                    deleteOne(key)
-                } else {
-                    val index = list.indexOfFirst { keyGetter(it) == key }
-                    val item = if (index == -1) null else list[index]
-                    if (item == null) {
-                        list.add(applyDiff(newItem(key), value.jsonObject))
-                    } else {
-                        list[index] = applyDiff(item, value.jsonObject)
-                    }
-                }
-            }
-            insertReplace(list)
-        }
-    }
-
    @Update
    fun updateRepository(repo: CoreRepository): Int

--- a/database/src/main/java/org/fdroid/database/VersionDao.kt
+++ b/database/src/main/java/org/fdroid/database/VersionDao.kt
@@ -1,18 +1,28 @@
 package org.fdroid.database

-import androidx.annotation.VisibleForTesting
 import androidx.lifecycle.LiveData
 import androidx.lifecycle.distinctUntilChanged
 import androidx.lifecycle.liveData
 import androidx.lifecycle.map
 import androidx.room.Dao
 import androidx.room.Insert
-import androidx.room.OnConflictStrategy
+import androidx.room.OnConflictStrategy.REPLACE
 import androidx.room.Query
 import androidx.room.RewriteQueriesToDropUnusedColumns
 import androidx.room.Transaction
+import androidx.room.Update
+import kotlinx.serialization.SerializationException
+import kotlinx.serialization.json.JsonNull
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.decodeFromJsonElement
 import org.fdroid.database.FDroidDatabaseHolder.dispatcher
+import org.fdroid.database.VersionedStringType.PERMISSION
+import org.fdroid.database.VersionedStringType.PERMISSION_SDK_23
+import org.fdroid.index.IndexParser.json
+import org.fdroid.index.v2.ManifestV2
 import org.fdroid.index.v2.PackageVersionV2
+import org.fdroid.index.v2.PermissionV2
+import org.fdroid.index.v2.ReflectionDiffer

 public interface VersionDao {
    public fun insert(
@@ -26,6 +36,15 @@ public interface VersionDao {
    public fun getAppVersions(repoId: Long, packageId: String): List<AppVersion>
 }

+/**
+ * A list of unknown fields in [PackageVersionV2] that we don't allow for [Version].
+ *
+ * We are applying reflection diffs against internal database classes
+ * and need to prevent the untrusted external JSON input to modify internal fields in those classes.
+ * This list must always hold the names of all those internal FIELDS for [Version].
+ */
+private val DENY_LIST = listOf("packageId", "repoId", "versionId")
+
@Dao
 internal interface VersionDaoInt : VersionDao {

@@ -56,12 +75,85 @@ internal interface VersionDaoInt : VersionDao {
        insert(packageVersion.manifest.getVersionedStrings(version))
    }

-    @Insert(onConflict = OnConflictStrategy.REPLACE)
+    @Insert(onConflict = REPLACE)
    fun insert(version: Version)

-    @Insert(onConflict = OnConflictStrategy.REPLACE)
+    @Insert(onConflict = REPLACE)
    fun insert(versionedString: List<VersionedString>)

+    @Update
+    fun update(version: Version)
+
+    fun update(
+        repoId: Long,
+        packageId: String,
+        versionsDiffMap: Map<String, JsonObject?>?,
+        checkIfCompatible: (ManifestV2) -> Boolean,
+    ) {
+        if (versionsDiffMap == null) { // no more versions, delete all
+            deleteAppVersion(repoId, packageId)
+        } else versionsDiffMap.forEach { (versionId, jsonObject) ->
+            if (jsonObject == null) { // delete individual version
+                deleteAppVersion(repoId, packageId, versionId)
+            } else {
+                val version = getVersion(repoId, packageId, versionId)
+                if (version == null) { // new version, parse normally
+                    val packageVersionV2: PackageVersionV2 =
+                        json.decodeFromJsonElement(jsonObject)
+                    val isCompatible = checkIfCompatible(packageVersionV2.manifest)
+                    insert(repoId, packageId, versionId, packageVersionV2, isCompatible)
+                } else { // diff against existing version
+                    diffVersion(version, jsonObject, checkIfCompatible)
+                }
+            }
+        } // end forEach
+    }
+
+    private fun diffVersion(
+        version: Version,
+        jsonObject: JsonObject,
+        checkIfCompatible: (ManifestV2) -> Boolean,
+    ) {
+        // ensure that diff does not include internal keys
+        DENY_LIST.forEach { forbiddenKey ->
+            println("$forbiddenKey ${jsonObject.keys}")
+            if (jsonObject.containsKey(forbiddenKey)) {
+                throw SerializationException(forbiddenKey)
+            }
+        }
+        // diff version
+        val diffedVersion = ReflectionDiffer.applyDiff(version, jsonObject)
+        val isCompatible = checkIfCompatible(diffedVersion.manifest.toManifestV2())
+        update(diffedVersion.copy(isCompatible = isCompatible))
+        // diff versioned strings
+        val manifest = jsonObject["manifest"]
+        if (manifest is JsonNull) { // no more manifest, delete all versionedStrings
+            deleteVersionedStrings(version.repoId, version.packageId, version.versionId)
+        } else if (manifest is JsonObject) {
+            diffVersionedStrings(version, manifest, "usesPermission", PERMISSION)
+            diffVersionedStrings(version, manifest, "usesPermissionSdk23",
+                PERMISSION_SDK_23)
+        }
+    }
+
+    private fun diffVersionedStrings(
+        version: Version,
+        jsonObject: JsonObject,
+        key: String,
+        type: VersionedStringType,
+    ) = DbDiffUtils.diffAndUpdateListTable(
+        jsonObject = jsonObject,
+        jsonObjectKey = key,
+        listParser = { permissionArray ->
+            val list: List<PermissionV2> = json.decodeFromJsonElement(permissionArray)
+            list.toVersionedString(version, type)
+        },
+        deleteList = {
+            deleteVersionedStrings(version.repoId, version.packageId, version.versionId, type)
+        },
+        insertNewList = { versionedStrings -> insert(versionedStrings) },
+    )
+
    override fun getAppVersions(
        packageId: String,
    ): LiveData<List<AppVersion>> = liveData(dispatcher) {
@@ -81,6 +173,10 @@ internal interface VersionDaoInt : VersionDao {
        }
    }

+    @Query("""SELECT * FROM Version
+        WHERE repoId = :repoId AND packageId = :packageId AND versionId = :versionId""")
+    fun getVersion(repoId: Long, packageId: String, versionId: String): Version?
+
    @RewriteQueriesToDropUnusedColumns
    @Query("""SELECT * FROM Version
        JOIN RepositoryPreferences AS pref USING (repoId)
@@ -122,11 +218,26 @@ internal interface VersionDaoInt : VersionDao {
        versionId: String,
    ): List<VersionedString>

-    @VisibleForTesting
+    @Query("""DELETE FROM Version WHERE repoId = :repoId AND packageId = :packageId""")
+    fun deleteAppVersion(repoId: Long, packageId: String)
+
    @Query("""DELETE FROM Version
        WHERE repoId = :repoId AND packageId = :packageId AND versionId = :versionId""")
    fun deleteAppVersion(repoId: Long, packageId: String, versionId: String)

+    @Query("""DELETE FROM VersionedString
+        WHERE repoId = :repoId AND packageId = :packageId AND versionId = :versionId""")
+    fun deleteVersionedStrings(repoId: Long, packageId: String, versionId: String)
+
+    @Query("""DELETE FROM VersionedString WHERE repoId = :repoId
+        AND packageId = :packageId AND versionId = :versionId AND type = :type""")
+    fun deleteVersionedStrings(
+        repoId: Long,
+        packageId: String,
+        versionId: String,
+        type: VersionedStringType,
+    )
+
    @Query("SELECT COUNT(*) FROM Version")
    fun countAppVersions(): Int

--- a/database/src/main/java/org/fdroid/index/v1/IndexV1Updater.kt
+++ b/database/src/main/java/org/fdroid/index/v1/IndexV1Updater.kt
@@ -65,7 +65,7 @@ public class IndexV1Updater(
            db.runInTransaction {
                val cert = verifier.getStreamAndVerify { inputStream ->
                    updateListener?.onStartProcessing() // TODO maybe do more fine-grained reporting
-                    val streamReceiver = DbV1StreamReceiver(db, compatibilityChecker, repoId)
+                    val streamReceiver = DbV1StreamReceiver(db, repoId, compatibilityChecker)
                    val streamProcessor = IndexV1StreamProcessor(streamReceiver, certificate)
                    streamProcessor.process(inputStream)
                }
--- a/database/src/test/java/org/fdroid/database/ConvertersTest.kt
+++ b/database/src/test/java/org/fdroid/database/ConvertersTest.kt
@@ -1,5 +1,6 @@
 package org.fdroid.database

+import org.fdroid.test.TestRepoUtils.getRandomLocalizedFileV2
 import org.fdroid.test.TestUtils.getRandomList
 import org.fdroid.test.TestUtils.getRandomString
 import kotlin.test.Test
@@ -28,4 +29,13 @@ internal class ConvertersTest {
        assertEquals(list, convertedList)
    }

+    @Test
+    fun testFileV2Conversion() {
+        val file = getRandomLocalizedFileV2()
+
+        val str = Converters.localizedFileV2toString(file)
+        val convertedFile = Converters.fromStringToLocalizedFileV2(str)
+        assertEquals(file, convertedFile)
+    }
+
 }