diff --git a/database/src/androidTest/java/org/fdroid/database/DbTest.kt b/database/src/androidTest/java/org/fdroid/database/DbTest.kt index 66e1dc813..1cfe12ec5 100644 --- a/database/src/androidTest/java/org/fdroid/database/DbTest.kt +++ b/database/src/androidTest/java/org/fdroid/database/DbTest.kt @@ -1,6 +1,7 @@ package org.fdroid.database import android.content.Context +import android.content.res.AssetManager import androidx.core.os.LocaleListCompat import androidx.room.Room import androidx.test.core.app.ApplicationProvider @@ -10,11 +11,19 @@ import io.mockk.mockkObject import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.test.setMain +import org.fdroid.database.test.TestUtils.assertRepoEquals +import org.fdroid.database.test.TestUtils.toMetadataV2 +import org.fdroid.database.test.TestUtils.toPackageVersionV2 +import org.fdroid.index.v2.IndexV2 +import org.fdroid.test.TestUtils.sort +import org.fdroid.test.TestUtils.sorted import org.junit.After import org.junit.Before import org.junit.runner.RunWith import java.io.IOException import java.util.Locale +import kotlin.test.assertEquals +import kotlin.test.fail @RunWith(AndroidJUnit4::class) @OptIn(ExperimentalCoroutinesApi::class) @@ -26,11 +35,12 @@ internal abstract class DbTest { internal lateinit var db: FDroidDatabaseInt private val testCoroutineDispatcher = Dispatchers.Unconfined + protected val context: Context = ApplicationProvider.getApplicationContext() + protected val assets: AssetManager = context.resources.assets protected val locales = LocaleListCompat.create(Locale.US) @Before open fun createDb() { - val context = ApplicationProvider.getApplicationContext() db = Room.inMemoryDatabaseBuilder(context, FDroidDatabaseInt::class.java).build() repoDao = db.getRepositoryDao() appDao = db.getAppDao() @@ -48,4 +58,28 @@ internal abstract class DbTest { db.close() } + /** + * Asserts that data associated with the given [repoId] is equal to the given [index]. + */ + protected fun assertDbEquals(repoId: Long, index: IndexV2) { + val repo = repoDao.getRepository(repoId) ?: fail() + val sortedIndex = index.sorted() + assertRepoEquals(sortedIndex.repo, repo) + assertEquals(sortedIndex.packages.size, appDao.countApps(), "number of packages") + sortedIndex.packages.forEach { (packageName, packageV2) -> + assertEquals( + packageV2.metadata, + appDao.getApp(repoId, packageName)?.toMetadataV2()?.sort() + ) + val versions = versionDao.getAppVersions(repoId, packageName).map { + it.toPackageVersionV2() + }.associateBy { it.file.sha256 } + assertEquals(packageV2.versions.size, versions.size, "number of versions") + packageV2.versions.forEach { (versionId, packageVersionV2) -> + val version = versions[versionId] ?: fail() + assertEquals(packageVersionV2, version) + } + } + } + } diff --git a/database/src/androidTest/java/org/fdroid/database/IndexV1InsertTest.kt b/database/src/androidTest/java/org/fdroid/database/IndexV1InsertTest.kt index 48b9b76cd..655c32ff4 100644 --- a/database/src/androidTest/java/org/fdroid/database/IndexV1InsertTest.kt +++ b/database/src/androidTest/java/org/fdroid/database/IndexV1InsertTest.kt @@ -1,149 +1,88 @@ package org.fdroid.database -import android.content.Context -import android.util.Log -import androidx.test.core.app.ApplicationProvider.getApplicationContext import androidx.test.ext.junit.runners.AndroidJUnit4 import kotlinx.serialization.SerializationException import org.apache.commons.io.input.CountingInputStream -import org.fdroid.CompatibilityChecker +import org.fdroid.index.IndexConverter import org.fdroid.index.v1.IndexV1StreamProcessor import org.fdroid.index.v1.IndexV1StreamReceiver import org.fdroid.index.v2.AntiFeatureV2 import org.fdroid.index.v2.CategoryV2 -import org.fdroid.index.v2.IndexV2StreamProcessor import org.fdroid.index.v2.MetadataV2 import org.fdroid.index.v2.PackageVersionV2 import org.fdroid.index.v2.ReleaseChannelV2 import org.fdroid.index.v2.RepoV2 +import org.fdroid.test.TestDataEmptyV1 +import org.fdroid.test.TestDataMaxV1 +import org.fdroid.test.TestDataMidV1 +import org.fdroid.test.TestDataMinV1 import org.junit.Test import org.junit.runner.RunWith -import kotlin.math.roundToInt import kotlin.test.assertEquals import kotlin.test.assertFailsWith import kotlin.test.assertTrue -import kotlin.test.fail @RunWith(AndroidJUnit4::class) internal class IndexV1InsertTest : DbTest() { - @Test - fun testStreamIndexV1IntoDb() { - val c = getApplicationContext() - val fileSize = c.resources.assets.openFd("index-v1.json").use { it.length } - val inputStream = CountingInputStream(c.resources.assets.open("index-v1.json")) - var currentByteCount: Long = 0 - val repoId = db.getRepositoryDao().insertEmptyRepo("https://f-droid.org/repo") - val streamReceiver = TestStreamReceiver(repoId) { - val bytesRead = inputStream.byteCount - val bytesSinceLastCall = bytesRead - currentByteCount - if (bytesSinceLastCall > 0) { - val percent = ((bytesRead.toDouble() / fileSize) * 100).roundToInt() - Log.e("IndexV1InsertTest", - "Stream bytes read: $bytesRead ($percent%) +$bytesSinceLastCall") - } - // the stream gets read in big chunks, but ensure they are not too big, e.g. entire file - assertTrue(bytesSinceLastCall < 600_000, "$bytesSinceLastCall") - currentByteCount = bytesRead - } - val indexProcessor = IndexV1StreamProcessor(streamReceiver, null) + private val indexConverter = IndexConverter() + @Test + fun testStreamEmptyIntoDb() { + val repoId = streamIndex("resources/index-empty-v1.json") + assertEquals(1, repoDao.getRepositories().size) + val index = indexConverter.toIndexV2(TestDataEmptyV1.index) + assertDbEquals(repoId, index) + } + + @Test + fun testStreamMinIntoDb() { + val repoId = streamIndex("resources/index-min-v1.json") + assertTrue(repoDao.getRepositories().size == 1) + val index = indexConverter.toIndexV2(TestDataMinV1.index) + assertDbEquals(repoId, index) + } + + @Test + fun testStreamMidIntoDb() { + val repoId = streamIndex("resources/index-mid-v1.json") + assertTrue(repoDao.getRepositories().size == 1) + val index = indexConverter.toIndexV2(TestDataMidV1.index) + assertDbEquals(repoId, index) + } + + @Test + fun testStreamMaxIntoDb() { + val repoId = streamIndex("resources/index-max-v1.json") + assertTrue(repoDao.getRepositories().size == 1) + val index = indexConverter.toIndexV2(TestDataMaxV1.index) + assertDbEquals(repoId, index) + } + + private fun streamIndex(path: String): Long { + val repoId = db.getRepositoryDao().insertEmptyRepo("https://f-droid.org/repo") + val streamReceiver = TestStreamReceiver(repoId) + val indexProcessor = IndexV1StreamProcessor(streamReceiver, null) db.runInTransaction { - inputStream.use { indexStream -> + assets.open(path).use { indexStream -> indexProcessor.process(indexStream) } } - assertTrue(repoDao.getRepositories().size == 1) - assertTrue(appDao.countApps() > 0) - assertTrue(appDao.countLocalizedFiles() > 0) - assertTrue(appDao.countLocalizedFileLists() > 0) - assertTrue(versionDao.countAppVersions() > 0) - assertTrue(versionDao.countVersionedStrings() > 0) - - println("Apps: " + appDao.countApps()) - println("LocalizedFiles: " + appDao.countLocalizedFiles()) - println("LocalizedFileLists: " + appDao.countLocalizedFileLists()) - println("Versions: " + versionDao.countAppVersions()) - println("Perms/Features: " + versionDao.countVersionedStrings()) - - val version = repoDao.getRepositories()[0].repository.version ?: fail() - insertV2ForComparison(version) - - val repo1 = repoDao.getRepository(1) ?: fail() - val repo2 = repoDao.getRepository(2) ?: fail() - assertEquals(repo1.repository, repo2.repository.copy(repoId = 1)) - assertEquals(repo1.mirrors, repo2.mirrors.map { it.copy(repoId = 1) }) - // TODO enable when better test data -// assertEquals(repo1.antiFeatures, repo2.antiFeatures) -// assertEquals(repo1.categories, repo2.categories) -// assertEquals(repo1.releaseChannels, repo2.releaseChannels) - - val appMetadata = appDao.getAppMetadata() - val appMetadata1 = appMetadata.count { it.repoId == 1L } - val appMetadata2 = appMetadata.count { it.repoId == 2L } - assertEquals(appMetadata1, appMetadata2) - - val localizedFiles = appDao.getLocalizedFiles() - val localizedFiles1 = localizedFiles.count { it.repoId == 1L } - val localizedFiles2 = localizedFiles.count { it.repoId == 2L } - assertEquals(localizedFiles1, localizedFiles2) - - val localizedFileLists = appDao.getLocalizedFileLists() - val localizedFileLists1 = localizedFileLists.count { it.repoId == 1L } - val localizedFileLists2 = localizedFileLists.count { it.repoId == 2L } - assertEquals(localizedFileLists1, localizedFileLists2) - - appMetadata.filter { it.repoId == 2L }.forEach { m -> - val metadata1 = appDao.getAppMetadata(1, m.packageId) - val metadata2 = appDao.getAppMetadata(2, m.packageId) - assertEquals(metadata1, metadata2.copy(repoId = 1, isCompatible = true)) - - val lFiles1 = appDao.getLocalizedFiles(1, m.packageId).toSet() - val lFiles2 = appDao.getLocalizedFiles(2, m.packageId) - assertEquals(lFiles1, lFiles2.map { it.copy(repoId = 1) }.toSet()) - - val lFileLists1 = appDao.getLocalizedFileLists(1, m.packageId).toSet() - val lFileLists2 = appDao.getLocalizedFileLists(2, m.packageId) - assertEquals(lFileLists1, lFileLists2.map { it.copy(repoId = 1) }.toSet()) - - val version1 = versionDao.getVersions(1, m.packageId).toSet() - val version2 = versionDao.getVersions(2, m.packageId) - assertEquals(version1, version2.map { it.copy(repoId = 1) }.toSet()) - - val vStrings1 = versionDao.getVersionedStrings(1, m.packageId).toSet() - val vStrings2 = versionDao.getVersionedStrings(2, m.packageId) - assertEquals(vStrings1, vStrings2.map { it.copy(repoId = 1) }.toSet()) - } - } - - @Suppress("SameParameterValue") - private fun insertV2ForComparison(version: Int) { - val c = getApplicationContext() - val inputStream = CountingInputStream(c.resources.assets.open("index-v2.json")) - val repoId = db.getRepositoryDao().insertEmptyRepo("https://f-droid.org/repo") - val indexProcessor = IndexV2StreamProcessor(DbV2StreamReceiver(db, { true }, repoId), null) - db.runInTransaction { - inputStream.use { indexStream -> - indexProcessor.process(version, indexStream) - } - } + return repoId } @Test fun testExceptionWhileStreamingDoesNotSaveIntoDb() { - val c = getApplicationContext() - val cIn = CountingInputStream(c.resources.assets.open("index-v1.json")) - val compatibilityChecker = CompatibilityChecker { - if (cIn.byteCount > 824096) throw SerializationException() - true - } - val indexProcessor = - IndexV2StreamProcessor(DbV2StreamReceiver(db, compatibilityChecker, 1), null) - + val cIn = CountingInputStream(assets.open("resources/index-max-v1.json")) assertFailsWith { db.runInTransaction { + val repoId = db.getRepositoryDao().insertEmptyRepo("https://f-droid.org/repo") + val streamReceiver = TestStreamReceiver(repoId) { + if (cIn.byteCount > 0) throw SerializationException() + } + val indexProcessor = IndexV1StreamProcessor(streamReceiver, null) cIn.use { indexStream -> - indexProcessor.process(42, indexStream) + indexProcessor.process(indexStream) } } } @@ -155,11 +94,12 @@ internal class IndexV1InsertTest : DbTest() { assertTrue(versionDao.countVersionedStrings() == 0) } + @Suppress("DEPRECATION") inner class TestStreamReceiver( repoId: Long, - private val callback: () -> Unit, + private val callback: () -> Unit = {}, ) : IndexV1StreamReceiver { - private val streamReceiver = DbV1StreamReceiver(db, { true }, repoId) + private val streamReceiver = DbV1StreamReceiver(db, repoId) { true } override fun receive(repo: RepoV2, version: Int, certificate: String?) { streamReceiver.receive(repo, version, certificate) callback() diff --git a/database/src/androidTest/java/org/fdroid/database/IndexV2DiffTest.kt b/database/src/androidTest/java/org/fdroid/database/IndexV2DiffTest.kt new file mode 100644 index 000000000..cebdebe31 --- /dev/null +++ b/database/src/androidTest/java/org/fdroid/database/IndexV2DiffTest.kt @@ -0,0 +1,327 @@ +package org.fdroid.database + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import kotlinx.serialization.SerializationException +import org.fdroid.index.IndexParser +import org.fdroid.index.parseV2 +import org.fdroid.index.v2.IndexV2 +import org.fdroid.index.v2.IndexV2DiffStreamProcessor +import org.fdroid.index.v2.IndexV2StreamProcessor +import org.fdroid.test.TestDataMaxV2 +import org.fdroid.test.TestDataMidV2 +import org.fdroid.test.TestDataMinV2 +import org.junit.Ignore +import org.junit.Test +import org.junit.runner.RunWith +import java.io.ByteArrayInputStream +import java.io.InputStream +import kotlin.test.assertFailsWith + +@RunWith(AndroidJUnit4::class) +internal class IndexV2DiffTest : DbTest() { + + @Test + @Ignore("use for testing specific index on demand") + fun testBrokenIndexDiff() { + val endPath = "resources/tmp/index-end.json" + val endIndex = IndexParser.parseV2(assets.open(endPath)) + testDiff( + startPath = "resources/tmp/index-start.json", + diffPath = "resources/tmp/diff.json", + endIndex = endIndex, + ) + } + + @Test + fun testEmptyToMin() = testDiff( + startPath = "resources/index-empty-v2.json", + diffPath = "resources/diff-empty-min/23.json", + endIndex = TestDataMinV2.index, + ) + + @Test + fun testEmptyToMid() = testDiff( + startPath = "resources/index-empty-v2.json", + diffPath = "resources/diff-empty-mid/23.json", + endIndex = TestDataMidV2.index, + ) + + @Test + fun testEmptyToMax() = testDiff( + startPath = "resources/index-empty-v2.json", + diffPath = "resources/diff-empty-max/23.json", + endIndex = TestDataMaxV2.index, + ) + + @Test + fun testMinToMid() = testDiff( + startPath = "resources/index-min-v2.json", + diffPath = "resources/diff-empty-mid/42.json", + endIndex = TestDataMidV2.index, + ) + + @Test + fun testMinToMax() = testDiff( + startPath = "resources/index-min-v2.json", + diffPath = "resources/diff-empty-max/42.json", + endIndex = TestDataMaxV2.index, + ) + + @Test + fun testMidToMax() = testDiff( + startPath = "resources/index-mid-v2.json", + diffPath = "resources/diff-empty-max/1337.json", + endIndex = TestDataMaxV2.index, + ) + + @Test + fun testMinRemoveApp() { + val diffJson = """{ + "packages": { + "org.fdroid.min1": null + } + }""".trimIndent() + testJsonDiff( + startPath = "resources/index-min-v2.json", + diff = diffJson, + endIndex = TestDataMinV2.index.copy(packages = emptyMap()), + ) + } + + @Test + fun testMinNoMetadataRemoveVersion() { + val diffJson = """{ + "packages": { + "org.fdroid.min1": { + "metadata": { + "added": 0 + }, + "versions": { + "824a109b2352138c3699760e1683385d0ed50ce526fc7982f8d65757743374bf": null + } + } + } + }""".trimIndent() + testJsonDiff( + startPath = "resources/index-min-v2.json", + diff = diffJson, + endIndex = TestDataMinV2.index.copy( + packages = TestDataMinV2.index.packages.mapValues { + it.value.copy(versions = emptyMap()) + } + ), + ) + } + + @Test + fun testMinNoVersionsUnknownKey() { + val diffJson = """{ + "packages": { + "org.fdroid.min1": { + "metadata": { + "added": 42 + }, + "unknownKey": "should get ignored" + } + } + }""".trimIndent() + testJsonDiff( + startPath = "resources/index-min-v2.json", + diff = diffJson, + endIndex = TestDataMinV2.index.copy( + packages = TestDataMinV2.index.packages.mapValues { + it.value.copy(metadata = it.value.metadata.copy(added = 42)) + } + ), + ) + } + + @Test + fun testMinRemoveMetadata() { + val diffJson = """{ + "packages": { + "org.fdroid.min1": { + "metadata": null + } + }, + "unknownKey": "should get ignored" + }""".trimIndent() + testJsonDiff( + startPath = "resources/index-min-v2.json", + diff = diffJson, + endIndex = TestDataMinV2.index.copy( + packages = emptyMap() + ), + ) + } + + @Test + fun testMinRemoveVersions() { + val diffJson = """{ + "packages": { + "org.fdroid.min1": { + "versions": null + } + } + }""".trimIndent() + testJsonDiff( + startPath = "resources/index-min-v2.json", + diff = diffJson, + endIndex = TestDataMinV2.index.copy( + packages = TestDataMinV2.index.packages.mapValues { + it.value.copy(versions = emptyMap()) + } + ), + ) + } + + @Test + fun testMinNoMetadataNoVersion() { + val diffJson = """{ + "packages": { + "org.fdroid.min1": { + } + } + }""".trimIndent() + testJsonDiff( + startPath = "resources/index-min-v2.json", + diff = diffJson, + endIndex = TestDataMinV2.index, + ) + } + + @Test + fun testAppDenyKeyList() { + val diffRepoIdJson = """{ + "packages": { + "org.fdroid.min1": { + "metadata": { + "repoId": 1 + } + } + } + }""".trimIndent() + assertFailsWith { + testJsonDiff( + startPath = "resources/index-min-v2.json", + diff = diffRepoIdJson, + endIndex = TestDataMinV2.index, + ) + } + val diffPackageIdJson = """{ + "packages": { + "org.fdroid.min1": { + "metadata": { + "packageId": "foo" + } + } + } + }""".trimIndent() + assertFailsWith { + testJsonDiff( + startPath = "resources/index-min-v2.json", + diff = diffPackageIdJson, + endIndex = TestDataMinV2.index, + ) + } + } + + @Test + fun testVersionsDenyKeyList() { + assertFailsWith { + testJsonDiff( + startPath = "resources/index-min-v2.json", + diff = getMinVersionJson(""""packageId": "foo""""), + endIndex = TestDataMinV2.index, + ) + } + assertFailsWith { + testJsonDiff( + startPath = "resources/index-min-v2.json", + diff = getMinVersionJson(""""repoId": 1"""), + endIndex = TestDataMinV2.index, + ) + } + assertFailsWith { + testJsonDiff( + startPath = "resources/index-min-v2.json", + diff = getMinVersionJson(""""versionId": "bar""""), + endIndex = TestDataMinV2.index, + ) + } + } + + private fun getMinVersionJson(insert: String) = """{ + "packages": { + "org.fdroid.min1": { + "versions": { + "824a109b2352138c3699760e1683385d0ed50ce526fc7982f8d65757743374bf": { + $insert + } + } + } + }""".trimIndent() + + @Test + fun testMidRemoveScreenshots() { + val diffRepoIdJson = """{ + "packages": { + "org.fdroid.fdroid": { + "metadata": { + "screenshots": null + } + } + } + }""".trimIndent() + val fdroidPackage = TestDataMidV2.packages["org.fdroid.fdroid"]!!.copy( + metadata = TestDataMidV2.packages["org.fdroid.fdroid"]!!.metadata.copy( + screenshots = null, + ) + ) + testJsonDiff( + startPath = "resources/index-mid-v2.json", + diff = diffRepoIdJson, + endIndex = TestDataMidV2.index.copy( + packages = mapOf( + TestDataMidV2.packageName1 to TestDataMidV2.app1, + TestDataMidV2.packageName2 to fdroidPackage, + ) + ), + ) + } + + private fun testJsonDiff(startPath: String, diff: String, endIndex: IndexV2) { + testDiff(startPath, ByteArrayInputStream(diff.toByteArray()), endIndex) + } + + private fun testDiff(startPath: String, diffPath: String, endIndex: IndexV2) { + testDiff(startPath, assets.open(diffPath), endIndex) + } + + private fun testDiff(startPath: String, diffStream: InputStream, endIndex: IndexV2) { + // stream start index into the DB + val repoId = streamIndex(startPath) + + // apply diff stream to the DB + val streamReceiver = DbV2DiffStreamReceiver(db, repoId) { true } + val streamProcessor = IndexV2DiffStreamProcessor(streamReceiver) + db.runInTransaction { + streamProcessor.process(diffStream) + } + // assert that changed DB data is equal to given endIndex + assertDbEquals(repoId, endIndex) + } + + private fun streamIndex(path: String): Long { + val repoId = db.getRepositoryDao().insertEmptyRepo("https://f-droid.org/repo") + val streamReceiver = DbV2StreamReceiver(db, repoId) { true } + val indexProcessor = IndexV2StreamProcessor(streamReceiver, null) + db.runInTransaction { + assets.open(path).use { indexStream -> + indexProcessor.process(42, indexStream) + } + } + return repoId + } + +} diff --git a/database/src/androidTest/java/org/fdroid/database/IndexV2InsertTest.kt b/database/src/androidTest/java/org/fdroid/database/IndexV2InsertTest.kt index 3446dee29..2264ecc28 100644 --- a/database/src/androidTest/java/org/fdroid/database/IndexV2InsertTest.kt +++ b/database/src/androidTest/java/org/fdroid/database/IndexV2InsertTest.kt @@ -1,16 +1,17 @@ package org.fdroid.database -import android.content.Context -import android.util.Log -import androidx.test.core.app.ApplicationProvider.getApplicationContext import androidx.test.ext.junit.runners.AndroidJUnit4 import kotlinx.serialization.SerializationException import org.apache.commons.io.input.CountingInputStream import org.fdroid.CompatibilityChecker import org.fdroid.index.v2.IndexV2StreamProcessor +import org.fdroid.test.TestDataEmptyV2 +import org.fdroid.test.TestDataMaxV2 +import org.fdroid.test.TestDataMidV2 +import org.fdroid.test.TestDataMinV2 import org.junit.Test import org.junit.runner.RunWith -import kotlin.math.roundToInt +import kotlin.test.assertEquals import kotlin.test.assertFailsWith import kotlin.test.assertTrue @@ -18,59 +19,63 @@ import kotlin.test.assertTrue internal class IndexV2InsertTest : DbTest() { @Test - fun testStreamIndexV2IntoDb() { - val c = getApplicationContext() - val fileSize = c.resources.assets.openFd("index-v2.json").use { it.length } - val inputStream = CountingInputStream(c.resources.assets.open("index-v2.json")) - var currentByteCount: Long = 0 - val compatibilityChecker = CompatibilityChecker { - val bytesRead = inputStream.byteCount - val bytesSinceLastCall = bytesRead - currentByteCount - if (bytesSinceLastCall > 0) { - val percent = ((bytesRead.toDouble() / fileSize) * 100).roundToInt() - Log.e("IndexV2InsertTest", - "Stream bytes read: $bytesRead ($percent%) +$bytesSinceLastCall") - } - // the stream gets read in big chunks, but ensure they are not too big, e.g. entire file - assertTrue(bytesSinceLastCall < 400_000, "$bytesSinceLastCall") - currentByteCount = bytesRead - true - } - val repoId = db.getRepositoryDao().insertEmptyRepo("https://f-droid.org/repo") - val streamReceiver = DbV2StreamReceiver(db, compatibilityChecker, repoId) - val indexProcessor = IndexV2StreamProcessor(streamReceiver, null) + fun testStreamEmptyIntoDb() { + val repoId = streamIndex("resources/index-empty-v2.json") + assertEquals(1, repoDao.getRepositories().size) + assertDbEquals(repoId, TestDataEmptyV2.index) + } + @Test + fun testStreamMinIntoDb() { + val repoId = streamIndex("resources/index-min-v2.json") + assertEquals(1, repoDao.getRepositories().size) + assertDbEquals(repoId, TestDataMinV2.index) + } + + @Test + fun testStreamMinReorderedIntoDb() { + val repoId = streamIndex("resources/index-min-reordered-v2.json") + assertEquals(1, repoDao.getRepositories().size) + assertDbEquals(repoId, TestDataMinV2.index) + } + + @Test + fun testStreamMidIntoDb() { + val repoId = streamIndex("resources/index-mid-v2.json") + assertEquals(1, repoDao.getRepositories().size) + assertDbEquals(repoId, TestDataMidV2.index) + } + + @Test + fun testStreamMaxIntoDb() { + val repoId = streamIndex("resources/index-max-v2.json") + assertEquals(1, repoDao.getRepositories().size) + assertDbEquals(repoId, TestDataMaxV2.index) + } + + private fun streamIndex(path: String): Long { + val repoId = db.getRepositoryDao().insertEmptyRepo("https://f-droid.org/repo") + val streamReceiver = DbV2StreamReceiver(db, repoId) { true } + val indexProcessor = IndexV2StreamProcessor(streamReceiver, null) db.runInTransaction { - inputStream.use { indexStream -> + assets.open(path).use { indexStream -> indexProcessor.process(42, indexStream) } } - assertTrue(repoDao.getRepositories().size == 1) - assertTrue(appDao.countApps() > 0) - assertTrue(appDao.countLocalizedFiles() > 0) - assertTrue(appDao.countLocalizedFileLists() > 0) - assertTrue(versionDao.countAppVersions() > 0) - assertTrue(versionDao.countVersionedStrings() > 0) - - println("Apps: " + appDao.countApps()) - println("LocalizedFiles: " + appDao.countLocalizedFiles()) - println("LocalizedFileLists: " + appDao.countLocalizedFileLists()) - println("Versions: " + versionDao.countAppVersions()) - println("Perms/Features: " + versionDao.countVersionedStrings()) + return repoId } @Test fun testExceptionWhileStreamingDoesNotSaveIntoDb() { - val c = getApplicationContext() - val cIn = CountingInputStream(c.resources.assets.open("index-v2.json")) + val cIn = CountingInputStream(assets.open("resources/index-max-v2.json")) val compatibilityChecker = CompatibilityChecker { - if (cIn.byteCount > 824096) throw SerializationException() + if (cIn.byteCount > 0) throw SerializationException() true } assertFailsWith { db.runInTransaction { val repoId = db.getRepositoryDao().insertEmptyRepo("http://example.org") - val streamReceiver = DbV2StreamReceiver(db, compatibilityChecker, repoId) + val streamReceiver = DbV2StreamReceiver(db, repoId, compatibilityChecker) val indexProcessor = IndexV2StreamProcessor(streamReceiver, null) cIn.use { indexStream -> indexProcessor.process(42, indexStream) diff --git a/database/src/androidTest/java/org/fdroid/database/RepositoryDiffTest.kt b/database/src/androidTest/java/org/fdroid/database/RepositoryDiffTest.kt index 962e86b4d..7640ec8ae 100644 --- a/database/src/androidTest/java/org/fdroid/database/RepositoryDiffTest.kt +++ b/database/src/androidTest/java/org/fdroid/database/RepositoryDiffTest.kt @@ -76,47 +76,6 @@ internal class RepositoryDiffTest : DbTest() { assertRepoEquals(repo.copy(timestamp = updateTimestamp), repos[0]) } - @Test - fun iconDiff() { - val repo = getRandomRepo() - val updateIcon = getRandomFileV2() - val json = """ - { - "icon": ${Json.encodeToString(updateIcon)} - }""".trimIndent() - testDiff(repo, json) { repos -> - assertEquals(updateIcon, repos[0].icon) - assertRepoEquals(repo.copy(icon = updateIcon), repos[0]) - } - } - - @Test - fun iconPartialDiff() { - val repo = getRandomRepo() - val updateIcon = repo.icon!!.copy(name = getRandomString()) - val json = """ - { - "icon": { "name": "${updateIcon.name}" } - }""".trimIndent() - testDiff(repo, json) { repos -> - assertEquals(updateIcon, repos[0].icon) - assertRepoEquals(repo.copy(icon = updateIcon), repos[0]) - } - } - - @Test - fun iconRemoval() { - val repo = getRandomRepo() - val json = """ - { - "icon": null - }""".trimIndent() - testDiff(repo, json) { repos -> - assertEquals(null, repos[0].icon) - assertRepoEquals(repo.copy(icon = null), repos[0]) - } - } - @Test fun mirrorDiff() { val repo = getRandomRepo() diff --git a/database/src/androidTest/java/org/fdroid/database/UpdateCheckerTest.kt b/database/src/androidTest/java/org/fdroid/database/UpdateCheckerTest.kt index 9cd38ae1d..a6eec8696 100644 --- a/database/src/androidTest/java/org/fdroid/database/UpdateCheckerTest.kt +++ b/database/src/androidTest/java/org/fdroid/database/UpdateCheckerTest.kt @@ -1,11 +1,8 @@ package org.fdroid.database -import android.content.Context import android.util.Log -import androidx.test.core.app.ApplicationProvider import androidx.test.ext.junit.runners.AndroidJUnit4 -import org.apache.commons.io.input.CountingInputStream -import org.fdroid.index.v1.IndexV1StreamProcessor +import org.fdroid.index.v2.IndexV2StreamProcessor import org.junit.Before import org.junit.Test import org.junit.runner.RunWith @@ -15,29 +12,26 @@ import kotlin.time.measureTime @RunWith(AndroidJUnit4::class) internal class UpdateCheckerTest : DbTest() { - private lateinit var context: Context private lateinit var updateChecker: UpdateChecker @Before override fun createDb() { super.createDb() - context = ApplicationProvider.getApplicationContext() + // TODO mock packageManager and maybe move to unit tests updateChecker = UpdateChecker(db, context.packageManager) } @Test @OptIn(ExperimentalTime::class) fun testGetUpdates() { - val inputStream = CountingInputStream(context.resources.assets.open("index-v1.json")) - val repoId = db.getRepositoryDao().insertEmptyRepo("https://f-droid.org/repo") - val indexProcessor = IndexV1StreamProcessor(DbV1StreamReceiver(db, { true }, repoId), null) - db.runInTransaction { - inputStream.use { indexStream -> - indexProcessor.process(indexStream) + val repoId = db.getRepositoryDao().insertEmptyRepo("https://f-droid.org/repo") + val streamReceiver = DbV2StreamReceiver(db, repoId) { true } + val indexProcessor = IndexV2StreamProcessor(streamReceiver, null) + assets.open("resources/index-max-v2.json").use { indexStream -> + indexProcessor.process(42, indexStream) } } - val duration = measureTime { updateChecker.getUpdatableApps() } diff --git a/database/src/androidTest/java/org/fdroid/database/test/TestUtils.kt b/database/src/androidTest/java/org/fdroid/database/test/TestUtils.kt index 7479ccf47..e2e7724f7 100644 --- a/database/src/androidTest/java/org/fdroid/database/test/TestUtils.kt +++ b/database/src/androidTest/java/org/fdroid/database/test/TestUtils.kt @@ -2,12 +2,18 @@ package org.fdroid.database.test import androidx.lifecycle.LiveData import androidx.lifecycle.Observer +import org.fdroid.database.App +import org.fdroid.database.AppVersion import org.fdroid.database.Repository import org.fdroid.database.toCoreRepository import org.fdroid.database.toMirror import org.fdroid.database.toRepoAntiFeatures import org.fdroid.database.toRepoCategories import org.fdroid.database.toRepoReleaseChannel +import org.fdroid.index.v2.FeatureV2 +import org.fdroid.index.v2.ManifestV2 +import org.fdroid.index.v2.MetadataV2 +import org.fdroid.index.v2.PackageVersionV2 import org.fdroid.index.v2.RepoV2 import org.junit.Assert import java.util.concurrent.CountDownLatch @@ -36,6 +42,59 @@ internal object TestUtils { assertEquals(coreRepo, repo.repository) } + internal fun App.toMetadataV2() = MetadataV2( + added = metadata.added, + lastUpdated = metadata.lastUpdated, + name = metadata.name, + summary = metadata.summary, + description = metadata.description, + webSite = metadata.webSite, + changelog = metadata.changelog, + license = metadata.license, + sourceCode = metadata.sourceCode, + issueTracker = metadata.issueTracker, + translation = metadata.translation, + preferredSigner = metadata.preferredSigner, + video = metadata.video, + authorName = metadata.authorName, + authorEmail = metadata.authorEmail, + authorWebSite = metadata.authorWebSite, + authorPhone = metadata.authorPhone, + donate = metadata.donate ?: emptyList(), + liberapayID = metadata.liberapayID, + liberapay = metadata.liberapay, + openCollective = metadata.openCollective, + bitcoin = metadata.bitcoin, + litecoin = metadata.litecoin, + flattrID = metadata.flattrID, + categories = metadata.categories ?: emptyList(), + icon = icon, + featureGraphic = featureGraphic, + promoGraphic = promoGraphic, + tvBanner = tvBanner, + screenshots = screenshots, + ) + + fun AppVersion.toPackageVersionV2() = PackageVersionV2( + added = added, + file = file, + src = src, + manifest = ManifestV2( + versionName = manifest.versionName, + versionCode = manifest.versionCode, + usesSdk = manifest.usesSdk, + maxSdkVersion = manifest.maxSdkVersion, + signer = manifest.signer, + usesPermission = usesPermission?.sortedBy { it.name } ?: emptyList(), + usesPermissionSdk23 = usesPermissionSdk23?.sortedBy { it.name } ?: emptyList(), + nativecode = manifest.nativecode?.sorted() ?: emptyList(), + features = manifest.features?.map { FeatureV2(it) } ?: emptyList(), + ), + releaseChannels = releaseChannels, + antiFeatures = version.antiFeatures ?: emptyMap(), + whatsNew = version.whatsNew ?: emptyMap(), + ) + fun LiveData.getOrAwaitValue(): T? { val data = arrayOfNulls(1) val latch = CountDownLatch(1) diff --git a/database/src/main/java/org/fdroid/database/App.kt b/database/src/main/java/org/fdroid/database/App.kt index f16d8bc30..b318feb63 100644 --- a/database/src/main/java/org/fdroid/database/App.kt +++ b/database/src/main/java/org/fdroid/database/App.kt @@ -327,19 +327,24 @@ internal fun LocalizedFileListV2.toLocalizedFileList( packageId: String, type: String, ): List = flatMap { (locale, files) -> - files.map { file -> - LocalizedFileList( - repoId = repoId, - packageId = packageId, - type = type, - locale = locale, - name = file.name, - sha256 = file.sha256, - size = file.size, - ) - } + files.map { file -> file.toLocalizedFileList(repoId, packageId, type, locale) } } +internal fun FileV2.toLocalizedFileList( + repoId: Long, + packageId: String, + type: String, + locale: String, +) = LocalizedFileList( + repoId = repoId, + packageId = packageId, + type = type, + locale = locale, + name = name, + sha256 = sha256, + size = size, +) + internal fun List.toLocalizedFileListV2(type: String): LocalizedFileListV2? { val map = HashMap>() iterator().forEach { file -> diff --git a/database/src/main/java/org/fdroid/database/AppDao.kt b/database/src/main/java/org/fdroid/database/AppDao.kt index d2632bef8..9939cf3bb 100644 --- a/database/src/main/java/org/fdroid/database/AppDao.kt +++ b/database/src/main/java/org/fdroid/database/AppDao.kt @@ -18,10 +18,20 @@ import androidx.room.OnConflictStrategy import androidx.room.Query import androidx.room.RoomWarnings.CURSOR_MISMATCH import androidx.room.Transaction +import androidx.room.Update +import kotlinx.serialization.SerializationException +import kotlinx.serialization.json.JsonNull +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.decodeFromJsonElement +import org.fdroid.database.DbDiffUtils.diffAndUpdateListTable +import org.fdroid.database.DbDiffUtils.diffAndUpdateTable import org.fdroid.database.FDroidDatabaseHolder.dispatcher +import org.fdroid.index.IndexParser.json +import org.fdroid.index.v2.FileV2 import org.fdroid.index.v2.LocalizedFileListV2 import org.fdroid.index.v2.LocalizedFileV2 import org.fdroid.index.v2.MetadataV2 +import org.fdroid.index.v2.ReflectionDiffer.applyDiff import org.fdroid.index.v2.Screenshots public interface AppDao { @@ -64,6 +74,23 @@ public enum class AppListSortOrder { LAST_UPDATED, NAME } +/** + * A list of unknown fields in [MetadataV2] that we don't allow for [AppMetadata]. + * + * We are applying reflection diffs against internal database classes + * and need to prevent the untrusted external JSON input to modify internal fields in those classes. + * This list must always hold the names of all those internal FIELDS for [AppMetadata]. + */ +private val DENY_LIST = listOf("packageId", "repoId") + +/** + * A list of unknown fields in [LocalizedFileV2] or [LocalizedFileListV2] + * that we don't allow for [LocalizedFile] or [LocalizedFileList]. + * + * Similar to [DENY_LIST]. + */ +private val DENY_FILE_LIST = listOf("packageId", "repoId", "type") + @Dao internal interface AppDaoInt : AppDao { @@ -110,6 +137,95 @@ internal interface AppDaoInt : AppDao { @Insert(onConflict = OnConflictStrategy.REPLACE) fun insertLocalizedFileLists(localizedFiles: List) + @Transaction + fun updateApp( + repoId: Long, + packageId: String, + jsonObject: JsonObject?, + locales: LocaleListCompat, + ) { + if (jsonObject == null) { + // this app is gone, we need to delete it + deleteAppMetadata(repoId, packageId) + return + } + val metadata = getAppMetadata(repoId, packageId) + if (metadata == null) { // new app + val metadataV2: MetadataV2 = json.decodeFromJsonElement(jsonObject) + insert(repoId, packageId, metadataV2) + } else { // diff against existing app + // ensure that diff does not include internal keys + DENY_LIST.forEach { forbiddenKey -> + if (jsonObject.containsKey(forbiddenKey)) throw SerializationException(forbiddenKey) + } + // diff metadata + val diffedApp = applyDiff(metadata, jsonObject) + val updatedApp = + if (jsonObject.containsKey("name") || jsonObject.containsKey("summary")) { + diffedApp.copy( + localizedName = diffedApp.name.getBestLocale(locales), + localizedSummary = diffedApp.summary.getBestLocale(locales), + ) + } else diffedApp + updateAppMetadata(updatedApp) + // diff localizedFiles + val localizedFiles = getLocalizedFiles(repoId, packageId) + localizedFiles.diffAndUpdate(repoId, packageId, "icon", jsonObject) + localizedFiles.diffAndUpdate(repoId, packageId, "featureGraphic", jsonObject) + localizedFiles.diffAndUpdate(repoId, packageId, "promoGraphic", jsonObject) + localizedFiles.diffAndUpdate(repoId, packageId, "tvBanner", jsonObject) + // diff localizedFileLists + val screenshots = jsonObject["screenshots"] + if (screenshots is JsonNull) { + deleteLocalizedFileLists(repoId, packageId) + } else if (screenshots is JsonObject) { + diffAndUpdateLocalizedFileList(repoId, packageId, "phone", screenshots) + diffAndUpdateLocalizedFileList(repoId, packageId, "sevenInch", screenshots) + diffAndUpdateLocalizedFileList(repoId, packageId, "tenInch", screenshots) + diffAndUpdateLocalizedFileList(repoId, packageId, "wear", screenshots) + diffAndUpdateLocalizedFileList(repoId, packageId, "tv", screenshots) + } + } + } + + private fun List.diffAndUpdate( + repoId: Long, + packageId: String, + type: String, + jsonObject: JsonObject, + ) = diffAndUpdateTable( + jsonObject = jsonObject, + jsonObjectKey = type, + itemList = filter { it.type == type }, + itemFinder = { locale, item -> item.locale == locale }, + newItem = { locale -> LocalizedFile(repoId, packageId, type, locale, "") }, + deleteAll = { deleteLocalizedFiles(repoId, packageId, type) }, + deleteOne = { locale -> deleteLocalizedFile(repoId, packageId, type, locale) }, + insertReplace = { list -> insert(list) }, + isNewItemValid = { it.name.isNotEmpty() }, + keyDenyList = DENY_FILE_LIST, + ) + + private fun diffAndUpdateLocalizedFileList( + repoId: Long, + packageId: String, + type: String, + jsonObject: JsonObject, + ) { + diffAndUpdateListTable( + jsonObject = jsonObject, + jsonObjectKey = type, + listParser = { locale, jsonArray -> + json.decodeFromJsonElement>(jsonArray).map { + it.toLocalizedFileList(repoId, packageId, type, locale) + } + }, + deleteAll = { deleteLocalizedFileLists(repoId, packageId, type) }, + deleteList = { locale -> deleteLocalizedFileList(repoId, packageId, type, locale) }, + insertNewList = { _, fileLists -> insertLocalizedFileLists(fileLists) }, + ) + } + /** * This is needed to support v1 streaming and shouldn't be used for something else. */ @@ -135,6 +251,9 @@ internal interface AppDaoInt : AppDao { WHERE repoId = :repoId AND packageId = :packageId""") fun updateAppMetadata(repoId: Long, packageId: String, name: String?, summary: String?) + @Update + fun updateAppMetadata(appMetadata: AppMetadata): Int + override fun getApp(packageId: String): LiveData { return getRepoIdForPackage(packageId).distinctUntilChanged().switchMap { repoId -> if (repoId == null) MutableLiveData(null) @@ -160,7 +279,7 @@ internal interface AppDaoInt : AppDao { @Transaction override fun getApp(repoId: Long, packageId: String): App? { - val metadata = getAppMetadata(repoId, packageId) + val metadata = getAppMetadata(repoId, packageId) ?: return null val localizedFiles = getLocalizedFiles(repoId, packageId) val localizedFileList = getLocalizedFileLists(repoId, packageId) return getApp(metadata, localizedFiles, localizedFileList) @@ -189,7 +308,7 @@ internal interface AppDaoInt : AppDao { fun getLiveAppMetadata(repoId: Long, packageId: String): LiveData @Query("SELECT * FROM AppMetadata WHERE repoId = :repoId AND packageId = :packageId") - fun getAppMetadata(repoId: Long, packageId: String): AppMetadata + fun getAppMetadata(repoId: Long, packageId: String): AppMetadata? @Query("SELECT * FROM AppMetadata") fun getAppMetadata(): List @@ -354,10 +473,29 @@ internal interface AppDaoInt : AppDao { FROM AppMetadata AS app WHERE repoId = :repoId AND packageId = :packageId""") fun getAppOverviewItem(repoId: Long, packageId: String): AppOverviewItem? - @VisibleForTesting @Query("DELETE FROM AppMetadata WHERE repoId = :repoId AND packageId = :packageId") fun deleteAppMetadata(repoId: Long, packageId: String) + @Query("""DELETE FROM LocalizedFile + WHERE repoId = :repoId AND packageId = :packageId AND type = :type""") + fun deleteLocalizedFiles(repoId: Long, packageId: String, type: String) + + @Query("""DELETE FROM LocalizedFile + WHERE repoId = :repoId AND packageId = :packageId AND type = :type AND locale = :locale""") + fun deleteLocalizedFile(repoId: Long, packageId: String, type: String, locale: String) + + @Query("""DELETE FROM LocalizedFileList + WHERE repoId = :repoId AND packageId = :packageId""") + fun deleteLocalizedFileLists(repoId: Long, packageId: String) + + @Query("""DELETE FROM LocalizedFileList + WHERE repoId = :repoId AND packageId = :packageId AND type = :type""") + fun deleteLocalizedFileLists(repoId: Long, packageId: String, type: String) + + @Query("""DELETE FROM LocalizedFileList + WHERE repoId = :repoId AND packageId = :packageId AND type = :type AND locale = :locale""") + fun deleteLocalizedFileList(repoId: Long, packageId: String, type: String, locale: String) + @VisibleForTesting @Query("SELECT COUNT(*) FROM AppMetadata") fun countApps(): Int diff --git a/database/src/main/java/org/fdroid/database/DbDiffUtils.kt b/database/src/main/java/org/fdroid/database/DbDiffUtils.kt new file mode 100644 index 000000000..cfc1374f5 --- /dev/null +++ b/database/src/main/java/org/fdroid/database/DbDiffUtils.kt @@ -0,0 +1,125 @@ +package org.fdroid.database + +import kotlinx.serialization.SerializationException +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonNull +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonObject +import org.fdroid.index.v2.ReflectionDiffer + +internal object DbDiffUtils { + + /** + * Applies the diff from the given [jsonObject] identified by the given [jsonObjectKey] + * to [itemList] and updates the DB as needed. + * + * @param newItem A function to produce a new [T] which typically contains the primary key(s). + */ + @Throws(SerializationException::class) + fun diffAndUpdateTable( + jsonObject: JsonObject, + jsonObjectKey: String, + itemList: List, + itemFinder: (String, T) -> Boolean, + newItem: (String) -> T, + deleteAll: () -> Unit, + deleteOne: (String) -> Unit, + insertReplace: (List) -> Unit, + isNewItemValid: (T) -> Boolean = { true }, + keyDenyList: List? = null, + ) { + if (!jsonObject.containsKey(jsonObjectKey)) return + if (jsonObject[jsonObjectKey] == JsonNull) { + deleteAll() + } else { + val obj = jsonObject[jsonObjectKey]?.jsonObject ?: error("no $jsonObjectKey object") + val list = itemList.toMutableList() + obj.entries.forEach { (key, value) -> + if (value is JsonNull) { + list.removeAll { itemFinder(key, it) } + deleteOne(key) + } else { + value.jsonObject.checkDenyList(keyDenyList) + val index = list.indexOfFirst { itemFinder(key, it) } + val item = if (index == -1) null else list[index] + if (item == null) { + val itemToInsert = + ReflectionDiffer.applyDiff(newItem(key), value.jsonObject) + if (!isNewItemValid(itemToInsert)) throw SerializationException("$newItem") + list.add(itemToInsert) + } else { + list[index] = ReflectionDiffer.applyDiff(item, value.jsonObject) + } + } + } + insertReplace(list) + } + } + + /** + * Applies a list diff from a map of lists. + * The map is identified by the given [jsonObjectKey] in the given [jsonObject]. + * The diff is applied for each key + * by replacing the existing list using [deleteList] and [insertNewList]. + * + * @param listParser returns a list of [T] from the given [JsonArray]. + */ + @Throws(SerializationException::class) + fun diffAndUpdateListTable( + jsonObject: JsonObject, + jsonObjectKey: String, + listParser: (String, JsonArray) -> List, + deleteAll: () -> Unit, + deleteList: (String) -> Unit, + insertNewList: (String, List) -> Unit, + ) { + if (!jsonObject.containsKey(jsonObjectKey)) return + if (jsonObject[jsonObjectKey] == JsonNull) { + deleteAll() + } else { + val obj = jsonObject[jsonObjectKey]?.jsonObject ?: error("no $jsonObjectKey object") + obj.entries.forEach { (key, list) -> + if (list is JsonNull) { + deleteList(key) + } else { + val newList = listParser(key, list.jsonArray) + deleteList(key) + insertNewList(key, newList) + } + } + } + } + + /** + * Applies the list diff from the given [jsonObject] identified by the given [jsonObjectKey] + * by replacing an existing list using [deleteList] and [insertNewList]. + * + * @param listParser returns a list of [T] from the given [JsonArray]. + */ + @Throws(SerializationException::class) + fun diffAndUpdateListTable( + jsonObject: JsonObject, + jsonObjectKey: String, + listParser: (JsonArray) -> List, + deleteList: () -> Unit, + insertNewList: (List) -> Unit, + ) { + if (!jsonObject.containsKey(jsonObjectKey)) return + if (jsonObject[jsonObjectKey] == JsonNull) { + deleteList() + } else { + val jsonArray = jsonObject[jsonObjectKey]?.jsonArray ?: error("no $jsonObjectKey array") + val list = listParser(jsonArray) + deleteList() + insertNewList(list) + } + } + + private fun JsonObject.checkDenyList(list: List?) { + list?.forEach { forbiddenKey -> + if (containsKey(forbiddenKey)) throw SerializationException(forbiddenKey) + } + } + +} diff --git a/database/src/main/java/org/fdroid/database/DbV1StreamReceiver.kt b/database/src/main/java/org/fdroid/database/DbV1StreamReceiver.kt index 5a0cbe8cd..5bcbb0f41 100644 --- a/database/src/main/java/org/fdroid/database/DbV1StreamReceiver.kt +++ b/database/src/main/java/org/fdroid/database/DbV1StreamReceiver.kt @@ -19,8 +19,8 @@ import org.fdroid.index.v2.RepoV2 @Deprecated("Use DbV2StreamReceiver instead") internal class DbV1StreamReceiver( private val db: FDroidDatabaseInt, - private val compatibilityChecker: CompatibilityChecker, private val repoId: Long, + private val compatibilityChecker: CompatibilityChecker, ) : IndexV1StreamReceiver { private val locales: LocaleListCompat = getLocales(Resources.getSystem().configuration) diff --git a/database/src/main/java/org/fdroid/database/DbV2DiffStreamReceiver.kt b/database/src/main/java/org/fdroid/database/DbV2DiffStreamReceiver.kt new file mode 100644 index 000000000..7ea910b5d --- /dev/null +++ b/database/src/main/java/org/fdroid/database/DbV2DiffStreamReceiver.kt @@ -0,0 +1,40 @@ +package org.fdroid.database + +import android.content.res.Resources +import androidx.core.os.ConfigurationCompat.getLocales +import androidx.core.os.LocaleListCompat +import kotlinx.serialization.json.JsonObject +import org.fdroid.CompatibilityChecker +import org.fdroid.index.v2.IndexV2DiffStreamReceiver + +internal class DbV2DiffStreamReceiver( + private val db: FDroidDatabaseInt, + private val repoId: Long, + private val compatibilityChecker: CompatibilityChecker, +) : IndexV2DiffStreamReceiver { + + private val locales: LocaleListCompat = getLocales(Resources.getSystem().configuration) + + override fun receiveRepoDiff(repoJsonObject: JsonObject) { + db.getRepositoryDao().updateRepository(repoId, repoJsonObject) + } + + override fun receivePackageMetadataDiff(packageId: String, packageJsonObject: JsonObject?) { + db.getAppDao().updateApp(repoId, packageId, packageJsonObject, locales) + } + + override fun receiveVersionsDiff( + packageId: String, + versionsDiffMap: Map?, + ) { + db.getVersionDao().update(repoId, packageId, versionsDiffMap) { + compatibilityChecker.isCompatible(it) + } + } + + @Synchronized + override fun onStreamEnded() { + db.afterUpdatingRepo(repoId) + } + +} diff --git a/database/src/main/java/org/fdroid/database/DbV2StreamReceiver.kt b/database/src/main/java/org/fdroid/database/DbV2StreamReceiver.kt index c06d612a9..c7c77aac6 100644 --- a/database/src/main/java/org/fdroid/database/DbV2StreamReceiver.kt +++ b/database/src/main/java/org/fdroid/database/DbV2StreamReceiver.kt @@ -10,8 +10,8 @@ import org.fdroid.index.v2.RepoV2 internal class DbV2StreamReceiver( private val db: FDroidDatabaseInt, - private val compatibilityChecker: CompatibilityChecker, private val repoId: Long, + private val compatibilityChecker: CompatibilityChecker, ) : IndexV2StreamReceiver { private val locales: LocaleListCompat = getLocales(Resources.getSystem().configuration) diff --git a/database/src/main/java/org/fdroid/database/RepositoryDao.kt b/database/src/main/java/org/fdroid/database/RepositoryDao.kt index 047cf6645..01dc3278b 100644 --- a/database/src/main/java/org/fdroid/database/RepositoryDao.kt +++ b/database/src/main/java/org/fdroid/database/RepositoryDao.kt @@ -9,11 +9,10 @@ import androidx.room.Query import androidx.room.RewriteQueriesToDropUnusedColumns import androidx.room.Transaction import androidx.room.Update -import kotlinx.serialization.json.JsonArray -import kotlinx.serialization.json.JsonNull import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.decodeFromJsonElement -import kotlinx.serialization.json.jsonObject +import org.fdroid.database.DbDiffUtils.diffAndUpdateListTable +import org.fdroid.database.DbDiffUtils.diffAndUpdateTable import org.fdroid.index.IndexParser.json import org.fdroid.index.v2.MirrorV2 import org.fdroid.index.v2.ReflectionDiffer.applyDiff @@ -187,23 +186,25 @@ internal interface RepositoryDaoInt : RepositoryDao { val repo = getRepository(repoId) ?: error("Repo $repoId does not exist") // update repo with JSON diff updateRepository(applyDiff(repo.repository, jsonObject)) - // replace mirror list, if it is in the diff - if (jsonObject.containsKey("mirrors")) { - val mirrorArray = jsonObject["mirrors"] as JsonArray - val mirrors = json.decodeFromJsonElement>(mirrorArray).map { - it.toMirror(repoId) - } - // delete and re-insert mirrors, because it is easier than diffing - deleteMirrors(repoId) - insertMirrors(mirrors) - } + // replace mirror list (if it is in the diff) + diffAndUpdateListTable( + jsonObject = jsonObject, + jsonObjectKey = "mirrors", + listParser = { mirrorArray -> + json.decodeFromJsonElement>(mirrorArray).map { + it.toMirror(repoId) + } + }, + deleteList = { deleteMirrors(repoId) }, + insertNewList = { mirrors -> insertMirrors(mirrors) }, + ) // diff and update the antiFeatures diffAndUpdateTable( jsonObject = jsonObject, - key = "antiFeatures", + jsonObjectKey = "antiFeatures", itemList = repo.antiFeatures, + itemFinder = { key, item -> item.id == key }, newItem = { key -> AntiFeature(repoId, key, null, emptyMap(), emptyMap()) }, - keyGetter = { item -> item.id }, deleteAll = { deleteAntiFeatures(repoId) }, deleteOne = { key -> deleteAntiFeature(repoId, key) }, insertReplace = { list -> insertAntiFeatures(list) }, @@ -211,10 +212,10 @@ internal interface RepositoryDaoInt : RepositoryDao { // diff and update the categories diffAndUpdateTable( jsonObject = jsonObject, - key = "categories", + jsonObjectKey = "categories", itemList = repo.categories, + itemFinder = { key, item -> item.id == key }, newItem = { key -> Category(repoId, key, null, emptyMap(), emptyMap()) }, - keyGetter = { item -> item.id }, deleteAll = { deleteCategories(repoId) }, deleteOne = { key -> deleteCategory(repoId, key) }, insertReplace = { list -> insertCategories(list) }, @@ -222,56 +223,16 @@ internal interface RepositoryDaoInt : RepositoryDao { // diff and update the releaseChannels diffAndUpdateTable( jsonObject = jsonObject, - key = "releaseChannels", + jsonObjectKey = "releaseChannels", itemList = repo.releaseChannels, + itemFinder = { key, item -> item.id == key }, newItem = { key -> ReleaseChannel(repoId, key, null, emptyMap(), emptyMap()) }, - keyGetter = { item -> item.id }, deleteAll = { deleteReleaseChannels(repoId) }, deleteOne = { key -> deleteReleaseChannel(repoId, key) }, insertReplace = { list -> insertReleaseChannels(list) }, ) } - /** - * Applies the diff from [JsonObject] identified by the given [key] of the given [jsonObject] - * to the given [itemList] and updates the DB as needed. - * - * @param newItem A function to produce a new [T] which typically contains the primary key(s). - */ - private fun diffAndUpdateTable( - jsonObject: JsonObject, - key: String, - itemList: List, - newItem: (String) -> T, - keyGetter: (T) -> String, - deleteAll: () -> Unit, - deleteOne: (String) -> Unit, - insertReplace: (List) -> Unit, - ) { - if (!jsonObject.containsKey(key)) return - if (jsonObject[key] == JsonNull) { - deleteAll() - } else { - val features = jsonObject[key]?.jsonObject ?: error("no $key object") - val list = itemList.toMutableList() - features.entries.forEach { (key, value) -> - if (value is JsonNull) { - list.removeAll { keyGetter(it) == key } - deleteOne(key) - } else { - val index = list.indexOfFirst { keyGetter(it) == key } - val item = if (index == -1) null else list[index] - if (item == null) { - list.add(applyDiff(newItem(key), value.jsonObject)) - } else { - list[index] = applyDiff(item, value.jsonObject) - } - } - } - insertReplace(list) - } - } - @Update fun updateRepository(repo: CoreRepository): Int diff --git a/database/src/main/java/org/fdroid/database/VersionDao.kt b/database/src/main/java/org/fdroid/database/VersionDao.kt index acfd7498e..de82509e7 100644 --- a/database/src/main/java/org/fdroid/database/VersionDao.kt +++ b/database/src/main/java/org/fdroid/database/VersionDao.kt @@ -1,18 +1,28 @@ package org.fdroid.database -import androidx.annotation.VisibleForTesting import androidx.lifecycle.LiveData import androidx.lifecycle.distinctUntilChanged import androidx.lifecycle.liveData import androidx.lifecycle.map import androidx.room.Dao import androidx.room.Insert -import androidx.room.OnConflictStrategy +import androidx.room.OnConflictStrategy.REPLACE import androidx.room.Query import androidx.room.RewriteQueriesToDropUnusedColumns import androidx.room.Transaction +import androidx.room.Update +import kotlinx.serialization.SerializationException +import kotlinx.serialization.json.JsonNull +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.decodeFromJsonElement import org.fdroid.database.FDroidDatabaseHolder.dispatcher +import org.fdroid.database.VersionedStringType.PERMISSION +import org.fdroid.database.VersionedStringType.PERMISSION_SDK_23 +import org.fdroid.index.IndexParser.json +import org.fdroid.index.v2.ManifestV2 import org.fdroid.index.v2.PackageVersionV2 +import org.fdroid.index.v2.PermissionV2 +import org.fdroid.index.v2.ReflectionDiffer public interface VersionDao { public fun insert( @@ -26,6 +36,15 @@ public interface VersionDao { public fun getAppVersions(repoId: Long, packageId: String): List } +/** + * A list of unknown fields in [PackageVersionV2] that we don't allow for [Version]. + * + * We are applying reflection diffs against internal database classes + * and need to prevent the untrusted external JSON input to modify internal fields in those classes. + * This list must always hold the names of all those internal FIELDS for [Version]. + */ +private val DENY_LIST = listOf("packageId", "repoId", "versionId") + @Dao internal interface VersionDaoInt : VersionDao { @@ -56,12 +75,85 @@ internal interface VersionDaoInt : VersionDao { insert(packageVersion.manifest.getVersionedStrings(version)) } - @Insert(onConflict = OnConflictStrategy.REPLACE) + @Insert(onConflict = REPLACE) fun insert(version: Version) - @Insert(onConflict = OnConflictStrategy.REPLACE) + @Insert(onConflict = REPLACE) fun insert(versionedString: List) + @Update + fun update(version: Version) + + fun update( + repoId: Long, + packageId: String, + versionsDiffMap: Map?, + checkIfCompatible: (ManifestV2) -> Boolean, + ) { + if (versionsDiffMap == null) { // no more versions, delete all + deleteAppVersion(repoId, packageId) + } else versionsDiffMap.forEach { (versionId, jsonObject) -> + if (jsonObject == null) { // delete individual version + deleteAppVersion(repoId, packageId, versionId) + } else { + val version = getVersion(repoId, packageId, versionId) + if (version == null) { // new version, parse normally + val packageVersionV2: PackageVersionV2 = + json.decodeFromJsonElement(jsonObject) + val isCompatible = checkIfCompatible(packageVersionV2.manifest) + insert(repoId, packageId, versionId, packageVersionV2, isCompatible) + } else { // diff against existing version + diffVersion(version, jsonObject, checkIfCompatible) + } + } + } // end forEach + } + + private fun diffVersion( + version: Version, + jsonObject: JsonObject, + checkIfCompatible: (ManifestV2) -> Boolean, + ) { + // ensure that diff does not include internal keys + DENY_LIST.forEach { forbiddenKey -> + println("$forbiddenKey ${jsonObject.keys}") + if (jsonObject.containsKey(forbiddenKey)) { + throw SerializationException(forbiddenKey) + } + } + // diff version + val diffedVersion = ReflectionDiffer.applyDiff(version, jsonObject) + val isCompatible = checkIfCompatible(diffedVersion.manifest.toManifestV2()) + update(diffedVersion.copy(isCompatible = isCompatible)) + // diff versioned strings + val manifest = jsonObject["manifest"] + if (manifest is JsonNull) { // no more manifest, delete all versionedStrings + deleteVersionedStrings(version.repoId, version.packageId, version.versionId) + } else if (manifest is JsonObject) { + diffVersionedStrings(version, manifest, "usesPermission", PERMISSION) + diffVersionedStrings(version, manifest, "usesPermissionSdk23", + PERMISSION_SDK_23) + } + } + + private fun diffVersionedStrings( + version: Version, + jsonObject: JsonObject, + key: String, + type: VersionedStringType, + ) = DbDiffUtils.diffAndUpdateListTable( + jsonObject = jsonObject, + jsonObjectKey = key, + listParser = { permissionArray -> + val list: List = json.decodeFromJsonElement(permissionArray) + list.toVersionedString(version, type) + }, + deleteList = { + deleteVersionedStrings(version.repoId, version.packageId, version.versionId, type) + }, + insertNewList = { versionedStrings -> insert(versionedStrings) }, + ) + override fun getAppVersions( packageId: String, ): LiveData> = liveData(dispatcher) { @@ -81,6 +173,10 @@ internal interface VersionDaoInt : VersionDao { } } + @Query("""SELECT * FROM Version + WHERE repoId = :repoId AND packageId = :packageId AND versionId = :versionId""") + fun getVersion(repoId: Long, packageId: String, versionId: String): Version? + @RewriteQueriesToDropUnusedColumns @Query("""SELECT * FROM Version JOIN RepositoryPreferences AS pref USING (repoId) @@ -122,11 +218,26 @@ internal interface VersionDaoInt : VersionDao { versionId: String, ): List - @VisibleForTesting + @Query("""DELETE FROM Version WHERE repoId = :repoId AND packageId = :packageId""") + fun deleteAppVersion(repoId: Long, packageId: String) + @Query("""DELETE FROM Version WHERE repoId = :repoId AND packageId = :packageId AND versionId = :versionId""") fun deleteAppVersion(repoId: Long, packageId: String, versionId: String) + @Query("""DELETE FROM VersionedString + WHERE repoId = :repoId AND packageId = :packageId AND versionId = :versionId""") + fun deleteVersionedStrings(repoId: Long, packageId: String, versionId: String) + + @Query("""DELETE FROM VersionedString WHERE repoId = :repoId + AND packageId = :packageId AND versionId = :versionId AND type = :type""") + fun deleteVersionedStrings( + repoId: Long, + packageId: String, + versionId: String, + type: VersionedStringType, + ) + @Query("SELECT COUNT(*) FROM Version") fun countAppVersions(): Int diff --git a/database/src/main/java/org/fdroid/index/v1/IndexV1Updater.kt b/database/src/main/java/org/fdroid/index/v1/IndexV1Updater.kt index e8e947a09..8a3765133 100644 --- a/database/src/main/java/org/fdroid/index/v1/IndexV1Updater.kt +++ b/database/src/main/java/org/fdroid/index/v1/IndexV1Updater.kt @@ -65,7 +65,7 @@ public class IndexV1Updater( db.runInTransaction { val cert = verifier.getStreamAndVerify { inputStream -> updateListener?.onStartProcessing() // TODO maybe do more fine-grained reporting - val streamReceiver = DbV1StreamReceiver(db, compatibilityChecker, repoId) + val streamReceiver = DbV1StreamReceiver(db, repoId, compatibilityChecker) val streamProcessor = IndexV1StreamProcessor(streamReceiver, certificate) streamProcessor.process(inputStream) } diff --git a/database/src/test/java/org/fdroid/database/ConvertersTest.kt b/database/src/test/java/org/fdroid/database/ConvertersTest.kt index 230ac7bd6..37e46bfb1 100644 --- a/database/src/test/java/org/fdroid/database/ConvertersTest.kt +++ b/database/src/test/java/org/fdroid/database/ConvertersTest.kt @@ -1,5 +1,6 @@ package org.fdroid.database +import org.fdroid.test.TestRepoUtils.getRandomLocalizedFileV2 import org.fdroid.test.TestUtils.getRandomList import org.fdroid.test.TestUtils.getRandomString import kotlin.test.Test @@ -28,4 +29,13 @@ internal class ConvertersTest { assertEquals(list, convertedList) } + @Test + fun testFileV2Conversion() { + val file = getRandomLocalizedFileV2() + + val str = Converters.localizedFileV2toString(file) + val convertedFile = Converters.fromStringToLocalizedFileV2(str) + assertEquals(file, convertedFile) + } + }