From a445bee197d6816b9b19e34ae49434677b887ebb Mon Sep 17 00:00:00 2001 From: Torsten Grote Date: Tue, 15 Mar 2022 09:45:29 -0300 Subject: [PATCH] [db] Add support for apps and streaming --- database/build.gradle | 6 + .../java/org/fdroid/database/AppTest.kt | 40 +++++ .../java/org/fdroid/database/DbTest.kt | 27 +-- .../org/fdroid/database/IndexV1InsertTest.kt | 143 +++++++++++++++ .../org/fdroid/database/IndexV2InsertTest.kt | 82 +++++++++ .../org/fdroid/database/RepositoryDiffTest.kt | 17 +- .../org/fdroid/database/RepositoryTest.kt | 31 +++- .../java/org/fdroid/database/VersionTest.kt | 88 +++++++++ .../org/fdroid/database/test/TestAppUtils.kt | 99 ++++++++++ .../org/fdroid/database/test/TestRepoUtils.kt | 80 +++++++++ .../fdroid/database/{ => test}/TestUtils.kt | 48 +---- .../fdroid/database/test/TestVersionUtils.kt | 55 ++++++ .../src/main/java/org/fdroid/database/App.kt | 169 +++++++++++++++++ .../main/java/org/fdroid/database/AppDao.kt | 117 ++++++++++++ .../java/org/fdroid/database/Converters.kt | 23 +++ .../org/fdroid/database/DbStreamReceiver.kt | 20 +++ .../org/fdroid/database/DbV1StreamReceiver.kt | 43 +++++ .../org/fdroid/database/FDroidDatabase.kt | 10 ++ .../java/org/fdroid/database/Repository.kt | 11 +- .../java/org/fdroid/database/RepositoryDao.kt | 30 +++- .../main/java/org/fdroid/database/Version.kt | 170 ++++++++++++++++++ .../java/org/fdroid/database/VersionDao.kt | 79 ++++++++ .../org/fdroid/database/ReflectionTest.kt | 2 +- .../java/org/fdroid/database/TestUtils.kt | 2 +- 24 files changed, 1303 insertions(+), 89 deletions(-) create mode 100644 database/src/androidTest/java/org/fdroid/database/AppTest.kt create mode 100644 database/src/androidTest/java/org/fdroid/database/IndexV1InsertTest.kt create mode 100644 database/src/androidTest/java/org/fdroid/database/IndexV2InsertTest.kt create mode 100644 database/src/androidTest/java/org/fdroid/database/VersionTest.kt create mode 100644 database/src/androidTest/java/org/fdroid/database/test/TestAppUtils.kt create mode 100644 database/src/androidTest/java/org/fdroid/database/test/TestRepoUtils.kt rename database/src/androidTest/java/org/fdroid/database/{ => test}/TestUtils.kt (52%) create mode 100644 database/src/androidTest/java/org/fdroid/database/test/TestVersionUtils.kt create mode 100644 database/src/main/java/org/fdroid/database/App.kt create mode 100644 database/src/main/java/org/fdroid/database/AppDao.kt create mode 100644 database/src/main/java/org/fdroid/database/DbStreamReceiver.kt create mode 100644 database/src/main/java/org/fdroid/database/DbV1StreamReceiver.kt create mode 100644 database/src/main/java/org/fdroid/database/Version.kt create mode 100644 database/src/main/java/org/fdroid/database/VersionDao.kt diff --git a/database/build.gradle b/database/build.gradle index c0cec96d8..354e68e58 100644 --- a/database/build.gradle +++ b/database/build.gradle @@ -34,6 +34,10 @@ android { kotlinOptions { jvmTarget = '1.8' } + aaptOptions { + // needed only for instrumentation tests: assets.openFd() + noCompress "json" + } } dependencies { @@ -54,4 +58,6 @@ dependencies { androidTestImplementation 'org.jetbrains.kotlin:kotlin-test' androidTestImplementation 'androidx.test.ext:junit:1.1.3' androidTestImplementation 'androidx.test.espresso:espresso-core:3.3.0' + + androidTestImplementation 'commons-io:commons-io:2.6' } diff --git a/database/src/androidTest/java/org/fdroid/database/AppTest.kt b/database/src/androidTest/java/org/fdroid/database/AppTest.kt new file mode 100644 index 000000000..16a0ccbc8 --- /dev/null +++ b/database/src/androidTest/java/org/fdroid/database/AppTest.kt @@ -0,0 +1,40 @@ +package org.fdroid.database + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import org.fdroid.database.test.TestAppUtils.assertScreenshotsEqual +import org.fdroid.database.test.TestAppUtils.getRandomMetadataV2 +import org.fdroid.database.test.TestRepoUtils.getRandomRepo +import org.fdroid.database.test.TestUtils.getRandomString +import org.junit.Test +import org.junit.runner.RunWith +import kotlin.test.assertEquals + +@RunWith(AndroidJUnit4::class) +class AppTest : DbTest() { + + private val packageId = getRandomString() + + @Test + fun insertGetDeleteSingleApp() { + val repoId = repoDao.insert(getRandomRepo()) + val metadataV2 = getRandomMetadataV2() + appDao.insert(repoId, packageId, metadataV2) + + val app = appDao.getApp(repoId, packageId) + val metadata = metadataV2.toAppMetadata(repoId, packageId) + assertEquals(metadata.author, app.metadata.author) + assertEquals(metadata.donation, app.metadata.donation) + assertEquals(metadata, app.metadata) + assertEquals(metadataV2.icon, app.icon) + assertEquals(metadataV2.featureGraphic, app.featureGraphic) + assertEquals(metadataV2.promoGraphic, app.promoGraphic) + assertEquals(metadataV2.tvBanner, app.tvBanner) + assertScreenshotsEqual(metadataV2.screenshots, app.screenshots) + + appDao.deleteAppMetadata(repoId, packageId) + assertEquals(0, appDao.getAppMetadata().size) + assertEquals(0, appDao.getLocalizedFiles().size) + assertEquals(0, appDao.getLocalizedFileLists().size) + } + +} diff --git a/database/src/androidTest/java/org/fdroid/database/DbTest.kt b/database/src/androidTest/java/org/fdroid/database/DbTest.kt index fe206fa7c..0bc1bb774 100644 --- a/database/src/androidTest/java/org/fdroid/database/DbTest.kt +++ b/database/src/androidTest/java/org/fdroid/database/DbTest.kt @@ -4,9 +4,7 @@ import android.content.Context import androidx.room.Room import androidx.test.core.app.ApplicationProvider import androidx.test.ext.junit.runners.AndroidJUnit4 -import org.fdroid.index.v2.RepoV2 import org.junit.After -import org.junit.Assert import org.junit.Before import org.junit.runner.RunWith import java.io.IOException @@ -15,13 +13,17 @@ import java.io.IOException abstract class DbTest { internal lateinit var repoDao: RepositoryDaoInt - private lateinit var db: FDroidDatabase + internal lateinit var appDao: AppDaoInt + internal lateinit var versionDao: VersionDaoInt + internal lateinit var db: FDroidDatabase @Before fun createDb() { val context = ApplicationProvider.getApplicationContext() db = Room.inMemoryDatabaseBuilder(context, FDroidDatabase::class.java).build() repoDao = db.getRepositoryDaoInt() + appDao = db.getAppDaoInt() + versionDao = db.getVersionDaoInt() } @After @@ -30,23 +32,4 @@ abstract class DbTest { db.close() } - protected fun assertRepoEquals(repoV2: RepoV2, repo: Repository) { - val repoId = repo.repository.repoId - // mirrors - val expectedMirrors = repoV2.mirrors.map { it.toMirror(repoId) }.toSet() - Assert.assertEquals(expectedMirrors, repo.mirrors.toSet()) - // anti-features - val expectedAntiFeatures = repoV2.antiFeatures.toRepoAntiFeatures(repoId).toSet() - Assert.assertEquals(expectedAntiFeatures, repo.antiFeatures.toSet()) - // categories - val expectedCategories = repoV2.categories.toRepoCategories(repoId).toSet() - Assert.assertEquals(expectedCategories, repo.categories.toSet()) - // release channels - val expectedReleaseChannels = repoV2.releaseChannels.toRepoReleaseChannel(repoId).toSet() - Assert.assertEquals(expectedReleaseChannels, repo.releaseChannels.toSet()) - // core repo - val coreRepo = repoV2.toCoreRepository().copy(repoId = repoId) - Assert.assertEquals(coreRepo, repo.repository) - } - } diff --git a/database/src/androidTest/java/org/fdroid/database/IndexV1InsertTest.kt b/database/src/androidTest/java/org/fdroid/database/IndexV1InsertTest.kt new file mode 100644 index 000000000..4907ec479 --- /dev/null +++ b/database/src/androidTest/java/org/fdroid/database/IndexV1InsertTest.kt @@ -0,0 +1,143 @@ +package org.fdroid.database + +import android.content.Context +import android.util.Log +import androidx.test.core.app.ApplicationProvider.getApplicationContext +import androidx.test.ext.junit.runners.AndroidJUnit4 +import kotlinx.serialization.SerializationException +import org.apache.commons.io.input.CountingInputStream +import org.fdroid.index.v2.IndexStreamProcessor +import org.fdroid.index.IndexV1StreamProcessor +import org.junit.Test +import org.junit.runner.RunWith +import kotlin.math.roundToInt +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +@RunWith(AndroidJUnit4::class) +class IndexV1InsertTest : DbTest() { + + @Test + fun testStreamIndexV1IntoDb() { + val c = getApplicationContext() + val fileSize = c.resources.assets.openFd("index-v1.json").use { it.length } + val inputStream = CountingInputStream(c.resources.assets.open("index-v1.json")) + var currentByteCount: Long = 0 + val indexProcessor = IndexV1StreamProcessor(DbV1StreamReceiver(db)) { + val bytesRead = inputStream.byteCount + val bytesSinceLastCall = bytesRead - currentByteCount + if (bytesSinceLastCall > 0) { + val percent = ((bytesRead.toDouble() / fileSize) * 100).roundToInt() + Log.e("IndexV1InsertTest", + "Stream bytes read: $bytesRead ($percent%) +$bytesSinceLastCall") + } + // the stream gets read in big chunks, but ensure they are not too big, e.g. entire file + assertTrue(bytesSinceLastCall < 600_000, "$bytesSinceLastCall") + currentByteCount = bytesRead + bytesRead + } + + db.runInTransaction { + inputStream.use { indexStream -> + indexProcessor.process(1, indexStream) + } + } + assertTrue(repoDao.getRepositories().size == 1) + assertTrue(appDao.countApps() > 0) + assertTrue(appDao.countLocalizedFiles() > 0) + assertTrue(appDao.countLocalizedFileLists() > 0) + assertTrue(versionDao.countAppVersions() > 0) + assertTrue(versionDao.countVersionedStrings() > 0) + + println("Apps: " + appDao.countApps()) + println("LocalizedFiles: " + appDao.countLocalizedFiles()) + println("LocalizedFileLists: " + appDao.countLocalizedFileLists()) + println("Versions: " + versionDao.countAppVersions()) + println("Perms/Features: " + versionDao.countVersionedStrings()) + + insertV2ForComparison(2) + + val repo1 = repoDao.getRepository(1) + val repo2 = repoDao.getRepository(2) + assertEquals(repo1.repository, repo2.repository.copy(repoId = 1)) + assertEquals(repo1.mirrors, repo2.mirrors.map { it.copy(repoId = 1) }) + assertEquals(repo1.antiFeatures, repo2.antiFeatures) + assertEquals(repo1.categories, repo2.categories) + assertEquals(repo1.releaseChannels, repo2.releaseChannels) + + val appMetadata = appDao.getAppMetadata() + val appMetadata1 = appMetadata.count { it.repoId == 1L } + val appMetadata2 = appMetadata.count { it.repoId == 2L } + assertEquals(appMetadata1, appMetadata2) + + val localizedFiles = appDao.getLocalizedFiles() + val localizedFiles1 = localizedFiles.count { it.repoId == 1L } + val localizedFiles2 = localizedFiles.count { it.repoId == 2L } + assertEquals(localizedFiles1, localizedFiles2) + + val localizedFileLists = appDao.getLocalizedFileLists() + val localizedFileLists1 = localizedFileLists.count { it.repoId == 1L } + val localizedFileLists2 = localizedFileLists.count { it.repoId == 2L } + assertEquals(localizedFileLists1, localizedFileLists2) + + appMetadata.filter { it.repoId ==2L }.forEach { m -> + val metadata1 = appDao.getAppMetadata(1, m.packageId) + val metadata2 = appDao.getAppMetadata(2, m.packageId) + assertEquals(metadata1, metadata2.copy(repoId = 1)) + + val lFiles1 = appDao.getLocalizedFiles(1, m.packageId).toSet() + val lFiles2 = appDao.getLocalizedFiles(2, m.packageId) + assertEquals(lFiles1, lFiles2.map { it.copy(repoId = 1) }.toSet()) + + val lFileLists1 = appDao.getLocalizedFileLists(1, m.packageId).toSet() + val lFileLists2 = appDao.getLocalizedFileLists(2, m.packageId) + assertEquals(lFileLists1, lFileLists2.map { it.copy(repoId = 1) }.toSet()) + + val version1 = versionDao.getVersions(1, m.packageId).toSet() + val version2 = versionDao.getVersions(2, m.packageId) + assertEquals(version1, version2.map { it.copy(repoId = 1) }.toSet()) + + val vStrings1 = versionDao.getVersionedStrings(1, m.packageId).toSet() + val vStrings2 = versionDao.getVersionedStrings(2, m.packageId) + assertEquals(vStrings1, vStrings2.map { it.copy(repoId = 1) }.toSet()) + } + } + + @Suppress("SameParameterValue") + private fun insertV2ForComparison(repoId: Long) { + val c = getApplicationContext() + val inputStream = CountingInputStream(c.resources.assets.open("index-v2.json")) + val indexProcessor = IndexStreamProcessor(DbStreamReceiver(db)) + db.runInTransaction { + inputStream.use { indexStream -> + indexProcessor.process(repoId, indexStream) + } + } + } + + @Test + fun testExceptionWhileStreamingDoesNotSaveIntoDb() { + val c = getApplicationContext() + val cIn = CountingInputStream(c.resources.assets.open("index-v1.json")) + val indexProcessor = IndexStreamProcessor(DbStreamReceiver(db)) { + if (cIn.byteCount > 824096) throw SerializationException() + cIn.byteCount + } + + assertFailsWith { + db.runInTransaction { + cIn.use { indexStream -> + indexProcessor.process(1, indexStream) + } + } + } + assertTrue(repoDao.getRepositories().isEmpty()) + assertTrue(appDao.countApps() == 0) + assertTrue(appDao.countLocalizedFiles() == 0) + assertTrue(appDao.countLocalizedFileLists() == 0) + assertTrue(versionDao.countAppVersions() == 0) + assertTrue(versionDao.countVersionedStrings() == 0) + } + +} diff --git a/database/src/androidTest/java/org/fdroid/database/IndexV2InsertTest.kt b/database/src/androidTest/java/org/fdroid/database/IndexV2InsertTest.kt new file mode 100644 index 000000000..09a2f589f --- /dev/null +++ b/database/src/androidTest/java/org/fdroid/database/IndexV2InsertTest.kt @@ -0,0 +1,82 @@ +package org.fdroid.database + +import android.content.Context +import android.util.Log +import androidx.test.core.app.ApplicationProvider.getApplicationContext +import androidx.test.ext.junit.runners.AndroidJUnit4 +import kotlinx.serialization.SerializationException +import org.apache.commons.io.input.CountingInputStream +import org.fdroid.index.v2.IndexStreamProcessor +import org.junit.Test +import org.junit.runner.RunWith +import kotlin.math.roundToInt +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +@RunWith(AndroidJUnit4::class) +class IndexV2InsertTest : DbTest() { + + @Test + fun testStreamIndexV2IntoDb() { + val c = getApplicationContext() + val fileSize = c.resources.assets.openFd("index-v2.json").use { it.length } + val inputStream = CountingInputStream(c.resources.assets.open("index-v2.json")) + var currentByteCount: Long = 0 + val indexProcessor = IndexStreamProcessor(DbStreamReceiver(db)) { + val bytesRead = inputStream.byteCount + val bytesSinceLastCall = bytesRead - currentByteCount + if (bytesSinceLastCall > 0) { + val percent = ((bytesRead.toDouble() / fileSize) * 100).roundToInt() + Log.e("IndexV2InsertTest", + "Stream bytes read: $bytesRead ($percent%) +$bytesSinceLastCall") + } + // the stream gets read in big chunks, but ensure they are not too big, e.g. entire file + assertTrue(bytesSinceLastCall < 400_000, "$bytesSinceLastCall") + currentByteCount = bytesRead + bytesRead + } + + db.runInTransaction { + inputStream.use { indexStream -> + indexProcessor.process(1, indexStream) + } + } + assertTrue(repoDao.getRepositories().size == 1) + assertTrue(appDao.countApps() > 0) + assertTrue(appDao.countLocalizedFiles() > 0) + assertTrue(appDao.countLocalizedFileLists() > 0) + assertTrue(versionDao.countAppVersions() > 0) + assertTrue(versionDao.countVersionedStrings() > 0) + + println("Apps: " + appDao.countApps()) + println("LocalizedFiles: " + appDao.countLocalizedFiles()) + println("LocalizedFileLists: " + appDao.countLocalizedFileLists()) + println("Versions: " + versionDao.countAppVersions()) + println("Perms/Features: " + versionDao.countVersionedStrings()) + } + + @Test + fun testExceptionWhileStreamingDoesNotSaveIntoDb() { + val c = getApplicationContext() + val cIn = CountingInputStream(c.resources.assets.open("index-v2.json")) + val indexProcessor = IndexStreamProcessor(DbStreamReceiver(db)) { + if (cIn.byteCount > 824096) throw SerializationException() + cIn.byteCount + } + + assertFailsWith { + db.runInTransaction { + cIn.use { indexStream -> + indexProcessor.process(1, indexStream) + } + } + } + assertTrue(repoDao.getRepositories().isEmpty()) + assertTrue(appDao.countApps() == 0) + assertTrue(appDao.countLocalizedFiles() == 0) + assertTrue(appDao.countLocalizedFileLists() == 0) + assertTrue(versionDao.countAppVersions() == 0) + assertTrue(versionDao.countVersionedStrings() == 0) + } + +} diff --git a/database/src/androidTest/java/org/fdroid/database/RepositoryDiffTest.kt b/database/src/androidTest/java/org/fdroid/database/RepositoryDiffTest.kt index 4aa474541..105e4a62b 100644 --- a/database/src/androidTest/java/org/fdroid/database/RepositoryDiffTest.kt +++ b/database/src/androidTest/java/org/fdroid/database/RepositoryDiffTest.kt @@ -4,14 +4,15 @@ import androidx.test.ext.junit.runners.AndroidJUnit4 import kotlinx.serialization.encodeToString import kotlinx.serialization.json.Json import kotlinx.serialization.json.jsonObject -import org.fdroid.database.TestUtils.applyDiff -import org.fdroid.database.TestUtils.getRandomFileV2 -import org.fdroid.database.TestUtils.getRandomLocalizedTextV2 -import org.fdroid.database.TestUtils.getRandomMap -import org.fdroid.database.TestUtils.getRandomMirror -import org.fdroid.database.TestUtils.getRandomRepo -import org.fdroid.database.TestUtils.getRandomString -import org.fdroid.database.TestUtils.randomDiff +import org.fdroid.database.test.TestRepoUtils.assertRepoEquals +import org.fdroid.database.test.TestRepoUtils.getRandomFileV2 +import org.fdroid.database.test.TestRepoUtils.getRandomLocalizedTextV2 +import org.fdroid.database.test.TestRepoUtils.getRandomMirror +import org.fdroid.database.test.TestRepoUtils.getRandomRepo +import org.fdroid.database.test.TestUtils.applyDiff +import org.fdroid.database.test.TestUtils.getRandomMap +import org.fdroid.database.test.TestUtils.getRandomString +import org.fdroid.database.test.TestUtils.randomDiff import org.fdroid.index.v2.AntiFeatureV2 import org.fdroid.index.v2.CategoryV2 import org.fdroid.index.v2.ReleaseChannelV2 diff --git a/database/src/androidTest/java/org/fdroid/database/RepositoryTest.kt b/database/src/androidTest/java/org/fdroid/database/RepositoryTest.kt index f0c47443a..c79d037c5 100644 --- a/database/src/androidTest/java/org/fdroid/database/RepositoryTest.kt +++ b/database/src/androidTest/java/org/fdroid/database/RepositoryTest.kt @@ -1,10 +1,15 @@ package org.fdroid.database import androidx.test.ext.junit.runners.AndroidJUnit4 -import org.fdroid.database.TestUtils.getRandomRepo -import org.junit.Assert.assertEquals +import org.fdroid.database.test.TestAppUtils.getRandomMetadataV2 +import org.fdroid.database.test.TestRepoUtils.assertRepoEquals +import org.fdroid.database.test.TestRepoUtils.getRandomRepo +import org.fdroid.database.test.TestUtils.getRandomString +import org.fdroid.database.test.TestVersionUtils.getRandomPackageVersionV2 import org.junit.Test import org.junit.runner.RunWith +import kotlin.test.assertEquals +import kotlin.test.assertTrue @RunWith(AndroidJUnit4::class) class RepositoryTest : DbTest() { @@ -43,4 +48,26 @@ class RepositoryTest : DbTest() { assertEquals(0, repoDao.getReleaseChannels().size) } + @Test + fun replacingRepoRemovesAllAssociatedData() { + val repoId = repoDao.insert(getRandomRepo()) + val packageId = getRandomString() + val versionId = getRandomString() + appDao.insert(repoId, packageId, getRandomMetadataV2()) + val packageVersion = getRandomPackageVersionV2() + versionDao.insert(repoId, packageId, versionId, packageVersion) + + assertEquals(1, repoDao.getRepositories().size) + assertEquals(1, appDao.getAppMetadata().size) + assertEquals(1, versionDao.getAppVersions(repoId, packageId).size) + assertTrue(versionDao.getVersionedStrings(repoId, packageId).isNotEmpty()) + + repoDao.replace(repoId, getRandomRepo()) + assertEquals(1, repoDao.getRepositories().size) + assertEquals(0, appDao.getAppMetadata().size) + assertEquals(0, appDao.getLocalizedFiles().size) + assertEquals(0, appDao.getLocalizedFileLists().size) + assertEquals(0, versionDao.getAppVersions(repoId, packageId).size) + assertEquals(0, versionDao.getVersionedStrings(repoId, packageId).size) + } } diff --git a/database/src/androidTest/java/org/fdroid/database/VersionTest.kt b/database/src/androidTest/java/org/fdroid/database/VersionTest.kt new file mode 100644 index 000000000..a17eadc4a --- /dev/null +++ b/database/src/androidTest/java/org/fdroid/database/VersionTest.kt @@ -0,0 +1,88 @@ +package org.fdroid.database + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import org.fdroid.database.test.TestAppUtils.getRandomMetadataV2 +import org.fdroid.database.test.TestRepoUtils.getRandomRepo +import org.fdroid.database.test.TestUtils.getRandomString +import org.fdroid.database.test.TestVersionUtils.getRandomPackageVersionV2 +import org.junit.Test +import org.junit.runner.RunWith +import kotlin.test.assertEquals + +@RunWith(AndroidJUnit4::class) +class VersionTest : DbTest() { + + private val packageId = getRandomString() + private val versionId = getRandomString() + + @Test + fun insertGetDeleteSingleVersion() { + val repoId = repoDao.insert(getRandomRepo()) + appDao.insert(repoId, packageId, getRandomMetadataV2()) + val packageVersion = getRandomPackageVersionV2() + versionDao.insert(repoId, packageId, versionId, packageVersion) + + val appVersions = versionDao.getAppVersions(repoId, packageId) + assertEquals(1, appVersions.size) + val appVersion = appVersions[0] + assertEquals(versionId, appVersion.version.versionId) + assertEquals(packageVersion.toVersion(repoId, packageId, versionId), appVersion.version) + val manifest = packageVersion.manifest + assertEquals(manifest.usesPermission.toSet(), appVersion.usesPermission?.toSet()) + assertEquals(manifest.usesPermissionSdk23.toSet(), appVersion.usesPermissionSdk23?.toSet()) + assertEquals(manifest.features.toSet(), appVersion.features?.toSet()) + + val versionedStrings = versionDao.getVersionedStrings(repoId, packageId) + val expectedSize = + manifest.usesPermission.size + manifest.usesPermissionSdk23.size + manifest.features.size + assertEquals(expectedSize, versionedStrings.size) + + versionDao.deleteAppVersion(repoId, packageId, versionId) + assertEquals(0, versionDao.getAppVersions(repoId, packageId).size) + assertEquals(0, versionDao.getVersionedStrings(repoId, packageId).size) + } + + @Test + fun insertGetDeleteTwoVersions() { + // insert two versions along with required objects + val repoId = repoDao.insert(getRandomRepo()) + appDao.insert(repoId, packageId, getRandomMetadataV2()) + val packageVersion1 = getRandomPackageVersionV2() + val version1 = getRandomString() + versionDao.insert(repoId, packageId, version1, packageVersion1) + val packageVersion2 = getRandomPackageVersionV2() + val version2 = getRandomString() + versionDao.insert(repoId, packageId, version2, packageVersion2) + + // get app versions from DB and assign them correctly + val appVersions = versionDao.getAppVersions(repoId, packageId) + assertEquals(2, appVersions.size) + val appVersion = if (version1 == appVersions[0].version.versionId) { + appVersions[0] + } else appVersions[1] + val appVersion2 = if (version2 == appVersions[0].version.versionId) { + appVersions[0] + } else appVersions[1] + + // check first version matches + assertEquals(packageVersion1.toVersion(repoId, packageId, version1), appVersion.version) + val manifest = packageVersion1.manifest + assertEquals(manifest.usesPermission.toSet(), appVersion.usesPermission?.toSet()) + assertEquals(manifest.usesPermissionSdk23.toSet(), appVersion.usesPermissionSdk23?.toSet()) + assertEquals(manifest.features.toSet(), appVersion.features?.toSet()) + + // check second version matches + assertEquals(packageVersion2.toVersion(repoId, packageId, version2), appVersion2.version) + val manifest2 = packageVersion2.manifest + assertEquals(manifest2.usesPermission.toSet(), appVersion2.usesPermission?.toSet()) + assertEquals(manifest2.usesPermissionSdk23.toSet(), + appVersion2.usesPermissionSdk23?.toSet()) + assertEquals(manifest2.features.toSet(), appVersion2.features?.toSet()) + + // delete app and check that all associated data also gets deleted + appDao.deleteAppMetadata(repoId, packageId) + assertEquals(0, versionDao.getAppVersions(repoId, packageId).size) + assertEquals(0, versionDao.getVersionedStrings(repoId, packageId).size) + } + +} diff --git a/database/src/androidTest/java/org/fdroid/database/test/TestAppUtils.kt b/database/src/androidTest/java/org/fdroid/database/test/TestAppUtils.kt new file mode 100644 index 000000000..a2a631289 --- /dev/null +++ b/database/src/androidTest/java/org/fdroid/database/test/TestAppUtils.kt @@ -0,0 +1,99 @@ +package org.fdroid.database.test + +import org.fdroid.database.test.TestRepoUtils.getRandomLocalizedFileV2 +import org.fdroid.database.test.TestRepoUtils.getRandomLocalizedTextV2 +import org.fdroid.database.test.TestUtils.getRandomList +import org.fdroid.database.test.TestUtils.getRandomString +import org.fdroid.database.test.TestUtils.orNull +import org.fdroid.index.v2.Author +import org.fdroid.index.v2.Donation +import org.fdroid.index.v2.LocalizedFileListV2 +import org.fdroid.index.v2.MetadataV2 +import org.fdroid.index.v2.Screenshots +import kotlin.random.Random +import kotlin.test.assertEquals + +internal object TestAppUtils { + + fun getRandomMetadataV2() = MetadataV2( + added = Random.nextLong(), + lastUpdated = Random.nextLong(), + name = getRandomLocalizedTextV2().orNull(), + summary = getRandomLocalizedTextV2().orNull(), + description = getRandomLocalizedTextV2().orNull(), + webSite = getRandomString().orNull(), + changelog = getRandomString().orNull(), + license = getRandomString().orNull(), + sourceCode = getRandomString().orNull(), + issueTracker = getRandomString().orNull(), + translation = getRandomString().orNull(), + preferredSigner = getRandomString().orNull(), + video = getRandomLocalizedTextV2().orNull(), + author = getRandomAuthor().orNull(), + donation = getRandomDonation().orNull(), + icon = getRandomLocalizedFileV2().orNull(), + featureGraphic = getRandomLocalizedFileV2().orNull(), + promoGraphic = getRandomLocalizedFileV2().orNull(), + tvBanner = getRandomLocalizedFileV2().orNull(), + categories = getRandomList { getRandomString() }.orNull() + ?: emptyList(), + screenshots = getRandomScreenshots().orNull(), + ) + + fun getRandomAuthor() = Author( + name = getRandomString().orNull(), + email = getRandomString().orNull(), + website = getRandomString().orNull(), + phone = getRandomString().orNull(), + ) + + fun getRandomDonation() = Donation( + url = getRandomString().orNull(), + liberapay = getRandomString().orNull(), + liberapayID = getRandomString().orNull(), + openCollective = getRandomString().orNull(), + bitcoin = getRandomString().orNull(), + litecoin = getRandomString().orNull(), + flattrID = getRandomString().orNull(), + ) + + fun getRandomScreenshots() = Screenshots( + phone = getRandomLocalizedFileListV2().orNull(), + sevenInch = getRandomLocalizedFileListV2().orNull(), + tenInch = getRandomLocalizedFileListV2().orNull(), + wear = getRandomLocalizedFileListV2().orNull(), + tv = getRandomLocalizedFileListV2().orNull(), + ).takeIf { !it.isNull } + + fun getRandomLocalizedFileListV2() = TestUtils.getRandomMap(Random.nextInt(1, 3)) { + getRandomString() to getRandomList(Random.nextInt(1, + 7)) { TestRepoUtils.getRandomFileV2() } + } + + /** + * [Screenshots] include lists which can be ordered differently, + * so we need to ignore order when comparing them. + */ + fun assertScreenshotsEqual(s1: Screenshots?, s2: Screenshots?) { + if (s1 != null && s2 != null) { + assertLocalizedFileListV2Equal(s1.phone, s2.phone) + assertLocalizedFileListV2Equal(s1.sevenInch, s2.sevenInch) + assertLocalizedFileListV2Equal(s1.tenInch, s2.tenInch) + assertLocalizedFileListV2Equal(s1.wear, s2.wear) + assertLocalizedFileListV2Equal(s1.tv, s2.tv) + } else { + assertEquals(s1, s2) + } + } + + private fun assertLocalizedFileListV2Equal(l1: LocalizedFileListV2?, l2: LocalizedFileListV2?) { + if (l1 != null && l2 != null) { + l1.keys.forEach { key -> + assertEquals(l1[key]?.toSet(), l2[key]?.toSet()) + } + } else { + assertEquals(l1, l2) + } + } + +} diff --git a/database/src/androidTest/java/org/fdroid/database/test/TestRepoUtils.kt b/database/src/androidTest/java/org/fdroid/database/test/TestRepoUtils.kt new file mode 100644 index 000000000..e6268e8f6 --- /dev/null +++ b/database/src/androidTest/java/org/fdroid/database/test/TestRepoUtils.kt @@ -0,0 +1,80 @@ +package org.fdroid.database.test + +import org.fdroid.database.Repository +import org.fdroid.database.test.TestUtils.orNull +import org.fdroid.database.toCoreRepository +import org.fdroid.database.toMirror +import org.fdroid.database.toRepoAntiFeatures +import org.fdroid.database.toRepoCategories +import org.fdroid.database.toRepoReleaseChannel +import org.fdroid.index.v2.AntiFeatureV2 +import org.fdroid.index.v2.CategoryV2 +import org.fdroid.index.v2.FileV2 +import org.fdroid.index.v2.LocalizedTextV2 +import org.fdroid.index.v2.MirrorV2 +import org.fdroid.index.v2.ReleaseChannelV2 +import org.fdroid.index.v2.RepoV2 +import org.junit.Assert +import kotlin.random.Random + +object TestRepoUtils { + + fun getRandomMirror() = MirrorV2( + url = TestUtils.getRandomString(), + location = TestUtils.getRandomString().orNull() + ) + + fun getRandomLocalizedTextV2(size: Int = Random.nextInt(0, 23)): LocalizedTextV2 = buildMap { + repeat(size) { + put(TestUtils.getRandomString(4), TestUtils.getRandomString()) + } + } + + fun getRandomFileV2(sha256Nullable: Boolean = true) = FileV2( + name = TestUtils.getRandomString(), + sha256 = TestUtils.getRandomString(64).also { if (sha256Nullable) orNull() }, + size = Random.nextLong(-1, Long.MAX_VALUE) + ) + + fun getRandomLocalizedFileV2() = TestUtils.getRandomMap(Random.nextInt(1, 8)) { + TestUtils.getRandomString(4) to getRandomFileV2() + } + + fun getRandomRepo() = RepoV2( + name = TestUtils.getRandomString(), + icon = getRandomFileV2(), + address = TestUtils.getRandomString(), + description = getRandomLocalizedTextV2(), + mirrors = TestUtils.getRandomList { getRandomMirror() }, + timestamp = System.currentTimeMillis(), + antiFeatures = TestUtils.getRandomMap { + TestUtils.getRandomString() to AntiFeatureV2(getRandomFileV2(), getRandomLocalizedTextV2()) + }, + categories = TestUtils.getRandomMap { + TestUtils.getRandomString() to CategoryV2(getRandomFileV2(), getRandomLocalizedTextV2()) + }, + releaseChannels = TestUtils.getRandomMap { + TestUtils.getRandomString() to ReleaseChannelV2(getRandomLocalizedTextV2()) + }, + ) + + internal fun assertRepoEquals(repoV2: RepoV2, repo: Repository) { + val repoId = repo.repository.repoId + // mirrors + val expectedMirrors = repoV2.mirrors.map { it.toMirror(repoId) }.toSet() + Assert.assertEquals(expectedMirrors, repo.mirrors.toSet()) + // anti-features + val expectedAntiFeatures = repoV2.antiFeatures.toRepoAntiFeatures(repoId).toSet() + Assert.assertEquals(expectedAntiFeatures, repo.antiFeatures.toSet()) + // categories + val expectedCategories = repoV2.categories.toRepoCategories(repoId).toSet() + Assert.assertEquals(expectedCategories, repo.categories.toSet()) + // release channels + val expectedReleaseChannels = repoV2.releaseChannels.toRepoReleaseChannel(repoId).toSet() + Assert.assertEquals(expectedReleaseChannels, repo.releaseChannels.toSet()) + // core repo + val coreRepo = repoV2.toCoreRepository().copy(repoId = repoId) + Assert.assertEquals(coreRepo, repo.repository) + } + +} diff --git a/database/src/androidTest/java/org/fdroid/database/TestUtils.kt b/database/src/androidTest/java/org/fdroid/database/test/TestUtils.kt similarity index 52% rename from database/src/androidTest/java/org/fdroid/database/TestUtils.kt rename to database/src/androidTest/java/org/fdroid/database/test/TestUtils.kt index bf426ae30..572cdf263 100644 --- a/database/src/androidTest/java/org/fdroid/database/TestUtils.kt +++ b/database/src/androidTest/java/org/fdroid/database/test/TestUtils.kt @@ -1,12 +1,5 @@ -package org.fdroid.database +package org.fdroid.database.test -import org.fdroid.index.v2.AntiFeatureV2 -import org.fdroid.index.v2.CategoryV2 -import org.fdroid.index.v2.FileV2 -import org.fdroid.index.v2.LocalizedTextV2 -import org.fdroid.index.v2.MirrorV2 -import org.fdroid.index.v2.ReleaseChannelV2 -import org.fdroid.index.v2.RepoV2 import kotlin.random.Random object TestUtils { @@ -22,7 +15,7 @@ object TestUtils { size: Int = Random.nextInt(0, 23), factory: () -> T, ): List = if (size == 0) emptyList() else buildList { - repeat(Random.nextInt(0, size)) { + repeat(size) { add(factory()) } } @@ -37,45 +30,10 @@ object TestUtils { } } - private fun T.orNull(): T? { + fun T.orNull(): T? { return if (Random.nextBoolean()) null else this } - fun getRandomMirror() = MirrorV2( - url = getRandomString(), - location = getRandomString().orNull() - ) - - fun getRandomLocalizedTextV2(size: Int = Random.nextInt(0, 23)): LocalizedTextV2 = buildMap { - repeat(size) { - put(getRandomString(4), getRandomString()) - } - } - - fun getRandomFileV2() = FileV2( - name = getRandomString(), - sha256 = getRandomString(64), - size = Random.nextLong(-1, Long.MAX_VALUE) - ) - - fun getRandomRepo() = RepoV2( - name = getRandomString(), - icon = getRandomFileV2(), - address = getRandomString(), - description = getRandomLocalizedTextV2(), - mirrors = getRandomList { getRandomMirror() }, - timestamp = System.currentTimeMillis(), - antiFeatures = getRandomMap { - getRandomString() to AntiFeatureV2(getRandomFileV2(), getRandomLocalizedTextV2()) - }, - categories = getRandomMap { - getRandomString() to CategoryV2(getRandomFileV2(), getRandomLocalizedTextV2()) - }, - releaseChannels = getRandomMap { - getRandomString() to ReleaseChannelV2(getRandomLocalizedTextV2()) - }, - ) - /** * Create a map diff by adding or removing keys. Note that this does not change keys. */ diff --git a/database/src/androidTest/java/org/fdroid/database/test/TestVersionUtils.kt b/database/src/androidTest/java/org/fdroid/database/test/TestVersionUtils.kt new file mode 100644 index 000000000..584226852 --- /dev/null +++ b/database/src/androidTest/java/org/fdroid/database/test/TestVersionUtils.kt @@ -0,0 +1,55 @@ +package org.fdroid.database.test + +import org.fdroid.database.test.TestRepoUtils.getRandomFileV2 +import org.fdroid.database.test.TestRepoUtils.getRandomLocalizedTextV2 +import org.fdroid.database.test.TestUtils.getRandomList +import org.fdroid.database.test.TestUtils.getRandomMap +import org.fdroid.database.test.TestUtils.getRandomString +import org.fdroid.database.test.TestUtils.orNull +import org.fdroid.index.v2.FeatureV2 +import org.fdroid.index.v2.FileV1 +import org.fdroid.index.v2.ManifestV2 +import org.fdroid.index.v2.PackageVersionV2 +import org.fdroid.index.v2.PermissionV2 +import org.fdroid.index.v2.SignatureV2 +import org.fdroid.index.v2.UsesSdkV2 +import kotlin.random.Random + +internal object TestVersionUtils { + + fun getRandomPackageVersionV2() = PackageVersionV2( + added = Random.nextLong(), + file = getRandomFileV2(false).let { + FileV1(it.name, it.sha256!!, it.size) + }, + src = getRandomFileV2().orNull(), + manifest = getRandomManifestV2(), + releaseChannels = getRandomList { getRandomString() }, + antiFeatures = getRandomMap { getRandomString() to getRandomLocalizedTextV2() }, + whatsNew = getRandomLocalizedTextV2(), + ) + + fun getRandomManifestV2() = ManifestV2( + versionName = getRandomString(), + versionCode = Random.nextLong(), + usesSdk = UsesSdkV2( + minSdkVersion = Random.nextInt(), + targetSdkVersion = Random.nextInt(), + ), + maxSdkVersion = Random.nextInt().orNull(), + signer = SignatureV2(getRandomList(Random.nextInt(1, 3)) { + getRandomString(64) + }).orNull(), + usesPermission = getRandomList { + PermissionV2(getRandomString(), Random.nextInt().orNull()) + }, + usesPermissionSdk23 = getRandomList { + PermissionV2(getRandomString(), Random.nextInt().orNull()) + }, + nativeCode = getRandomList(Random.nextInt(0, 4)) { getRandomString() }, + features = getRandomList { + FeatureV2(getRandomString(), Random.nextInt().orNull()) + }, + ) + +} diff --git a/database/src/main/java/org/fdroid/database/App.kt b/database/src/main/java/org/fdroid/database/App.kt new file mode 100644 index 000000000..5021de5f3 --- /dev/null +++ b/database/src/main/java/org/fdroid/database/App.kt @@ -0,0 +1,169 @@ +package org.fdroid.database + +import androidx.room.Embedded +import androidx.room.Entity +import androidx.room.ForeignKey +import org.fdroid.index.v2.Author +import org.fdroid.index.v2.Donation +import org.fdroid.index.v2.FileV2 +import org.fdroid.index.v2.LocalizedFileListV2 +import org.fdroid.index.v2.LocalizedFileV2 +import org.fdroid.index.v2.LocalizedTextV2 +import org.fdroid.index.v2.MetadataV2 +import org.fdroid.index.v2.Screenshots + +@Entity( + primaryKeys = ["repoId", "packageId"], + foreignKeys = [ForeignKey( + entity = CoreRepository::class, + parentColumns = ["repoId"], + childColumns = ["repoId"], + onDelete = ForeignKey.CASCADE, + )], +) +data class AppMetadata( + val repoId: Long, + val packageId: String, + val added: Long, + val lastUpdated: Long, + val name: LocalizedTextV2? = null, + val summary: LocalizedTextV2? = null, + val description: LocalizedTextV2? = null, + val webSite: String? = null, + val changelog: String? = null, + val license: String? = null, + val sourceCode: String? = null, + val issueTracker: String? = null, + val translation: String? = null, + val preferredSigner: String? = null, + val video: LocalizedTextV2? = null, + @Embedded(prefix = "author_") val author: Author? = Author(), + @Embedded(prefix = "donation_") val donation: Donation? = Donation(), + val categories: List? = null, +) + +fun MetadataV2.toAppMetadata(repoId: Long, packageId: String) = AppMetadata( + repoId = repoId, + packageId = packageId, + added = added, + lastUpdated = lastUpdated, + name = name, + summary = summary, + description = description, + webSite = webSite, + changelog = changelog, + license = license, + sourceCode = sourceCode, + issueTracker = issueTracker, + translation = translation, + preferredSigner = preferredSigner, + video = video, + author = if (author?.isNull == true) null else author, + donation = if (donation?.isNull == true) null else donation, + categories = categories, +) + +data class App( + val metadata: AppMetadata, + val icon: LocalizedFileV2? = null, + val featureGraphic: LocalizedFileV2? = null, + val promoGraphic: LocalizedFileV2? = null, + val tvBanner: LocalizedFileV2? = null, + val screenshots: Screenshots? = null, +) + +@Entity( + primaryKeys = ["repoId", "packageId", "type", "locale"], + foreignKeys = [ForeignKey( + entity = AppMetadata::class, + parentColumns = ["repoId", "packageId"], + childColumns = ["repoId", "packageId"], + onDelete = ForeignKey.CASCADE, + )], +) +data class LocalizedFile( + val repoId: Long, + val packageId: String, + val type: String, + val locale: String, + val name: String, + val sha256: String? = null, + val size: Long? = null, +) + +fun LocalizedFileV2.toLocalizedFile( + repoId: Long, + packageId: String, + type: String, +): List = map { (locale, file) -> + LocalizedFile( + repoId = repoId, + packageId = packageId, + type = type, + locale = locale, + name = file.name, + sha256 = file.sha256, + size = file.size, + ) +} + +fun List.toLocalizedFileV2(type: String): LocalizedFileV2? = filter { file -> + file.type == type +}.associate { file -> + file.locale to FileV2( + name = file.name, + sha256 = file.sha256, + size = file.size, + ) +}.ifEmpty { null } + +@Entity( + primaryKeys = ["repoId", "packageId", "type", "locale", "name"], + foreignKeys = [ForeignKey( + entity = AppMetadata::class, + parentColumns = ["repoId", "packageId"], + childColumns = ["repoId", "packageId"], + onDelete = ForeignKey.CASCADE, + )], +) +data class LocalizedFileList( + val repoId: Long, + val packageId: String, + val type: String, + val locale: String, + val name: String, + val sha256: String? = null, + val size: Long? = null, +) + +fun LocalizedFileListV2.toLocalizedFileList( + repoId: Long, + packageId: String, + type: String, +): List = flatMap { (locale, files) -> + files.map { file -> + LocalizedFileList( + repoId = repoId, + packageId = packageId, + type = type, + locale = locale, + name = file.name, + sha256 = file.sha256, + size = file.size, + ) + } +} + +fun List.toLocalizedFileListV2(type: String): LocalizedFileListV2? { + val map = HashMap>() + iterator().forEach { file -> + if (file.type != type) return@forEach + val list = map.getOrPut(file.locale) { ArrayList() } as ArrayList + list.add(FileV2( + name = file.name, + sha256 = file.sha256, + size = file.size, + )) + } + return map.ifEmpty { null } +} diff --git a/database/src/main/java/org/fdroid/database/AppDao.kt b/database/src/main/java/org/fdroid/database/AppDao.kt new file mode 100644 index 000000000..e742535aa --- /dev/null +++ b/database/src/main/java/org/fdroid/database/AppDao.kt @@ -0,0 +1,117 @@ +package org.fdroid.database + +import androidx.annotation.VisibleForTesting +import androidx.room.Dao +import androidx.room.Insert +import androidx.room.OnConflictStrategy +import androidx.room.Query +import androidx.room.Transaction +import org.fdroid.index.v2.LocalizedFileListV2 +import org.fdroid.index.v2.LocalizedFileV2 +import org.fdroid.index.v2.MetadataV2 +import org.fdroid.index.v2.Screenshots + +public interface AppDao { + fun insert(repoId: Long, packageId: String, app: MetadataV2) + fun getApp(repoId: Long, packageId: String): App +} + +@Dao +internal interface AppDaoInt : AppDao { + + @Transaction + override fun insert(repoId: Long, packageId: String, app: MetadataV2) { + insert(app.toAppMetadata(repoId, packageId)) + app.icon.insert(repoId, packageId, "icon") + app.featureGraphic.insert(repoId, packageId, "featureGraphic") + app.promoGraphic.insert(repoId, packageId, "promoGraphic") + app.tvBanner.insert(repoId, packageId, "tvBanner") + app.screenshots?.let { + it.phone.insert(repoId, packageId, "phone") + it.sevenInch.insert(repoId, packageId, "sevenInch") + it.tenInch.insert(repoId, packageId, "tenInch") + it.wear.insert(repoId, packageId, "wear") + it.tv.insert(repoId, packageId, "tv") + } + } + + private fun LocalizedFileV2?.insert(repoId: Long, packageId: String, type: String) { + this?.toLocalizedFile(repoId, packageId, type)?.let { files -> + insert(files) + } + } + + @JvmName("insertLocalizedFileListV2") + private fun LocalizedFileListV2?.insert(repoId: Long, packageId: String, type: String) { + this?.toLocalizedFileList(repoId, packageId, type)?.let { files -> + insertLocalizedFileLists(files) + } + } + + @Insert(onConflict = OnConflictStrategy.REPLACE) + fun insert(appMetadata: AppMetadata) + + @Insert(onConflict = OnConflictStrategy.REPLACE) + fun insert(localizedFiles: List) + + @Insert(onConflict = OnConflictStrategy.REPLACE) + fun insertLocalizedFileLists(localizedFiles: List) + + /** + * This is needed to support v1 streaming and shouldn't be used for something else. + */ + @Query("UPDATE AppMetadata SET preferredSigner = :preferredSigner WHERE repoId = :repoId AND packageId = :packageId") + fun updatePreferredSigner(repoId: Long, packageId: String, preferredSigner: String?) + + @Transaction + override fun getApp(repoId: Long, packageId: String): App { + val localizedFiles = getLocalizedFiles(repoId, packageId) + val localizedFileList = getLocalizedFileLists(repoId, packageId) + return App( + metadata = getAppMetadata(repoId, packageId), + icon = localizedFiles.toLocalizedFileV2("icon"), + featureGraphic = localizedFiles.toLocalizedFileV2("featureGraphic"), + promoGraphic = localizedFiles.toLocalizedFileV2("promoGraphic"), + tvBanner = localizedFiles.toLocalizedFileV2("tvBanner"), + screenshots = if (localizedFileList.isEmpty()) null else Screenshots( + phone = localizedFileList.toLocalizedFileListV2("phone"), + sevenInch = localizedFileList.toLocalizedFileListV2("sevenInch"), + tenInch = localizedFileList.toLocalizedFileListV2("tenInch"), + wear = localizedFileList.toLocalizedFileListV2("wear"), + tv = localizedFileList.toLocalizedFileListV2("tv"), + ) + ) + } + + @Query("SELECT * FROM AppMetadata WHERE repoId = :repoId AND packageId = :packageId") + fun getAppMetadata(repoId: Long, packageId: String): AppMetadata + + @Query("SELECT * FROM AppMetadata") + fun getAppMetadata(): List + + @Query("SELECT * FROM LocalizedFile WHERE repoId = :repoId AND packageId = :packageId") + fun getLocalizedFiles(repoId: Long, packageId: String): List + + @Query("SELECT * FROM LocalizedFileList WHERE repoId = :repoId AND packageId = :packageId") + fun getLocalizedFileLists(repoId: Long, packageId: String): List + + @Query("SELECT * FROM LocalizedFile") + fun getLocalizedFiles(): List + + @Query("SELECT * FROM LocalizedFileList") + fun getLocalizedFileLists(): List + + @VisibleForTesting + @Query("DELETE FROM AppMetadata WHERE repoId = :repoId AND packageId = :packageId") + fun deleteAppMetadata(repoId: Long, packageId: String) + + @Query("SELECT COUNT(*) FROM AppMetadata") + fun countApps(): Int + + @Query("SELECT COUNT(*) FROM LocalizedFile") + fun countLocalizedFiles(): Int + + @Query("SELECT COUNT(*) FROM LocalizedFileList") + fun countLocalizedFileLists(): Int + +} diff --git a/database/src/main/java/org/fdroid/database/Converters.kt b/database/src/main/java/org/fdroid/database/Converters.kt index 46025de69..f5c263fc2 100644 --- a/database/src/main/java/org/fdroid/database/Converters.kt +++ b/database/src/main/java/org/fdroid/database/Converters.kt @@ -9,6 +9,8 @@ import org.fdroid.index.v2.LocalizedTextV2 internal class Converters { private val localizedTextV2Serializer = MapSerializer(String.serializer(), String.serializer()) + private val mapOfLocalizedTextV2Serializer = + MapSerializer(String.serializer(), localizedTextV2Serializer) @TypeConverter fun fromStringToLocalizedTextV2(value: String?): LocalizedTextV2? { @@ -19,4 +21,25 @@ internal class Converters { fun localizedTextV2toString(text: LocalizedTextV2?): String? { return text?.let { json.encodeToString(localizedTextV2Serializer, it) } } + + @TypeConverter + fun fromStringToMapOfLocalizedTextV2(value: String?): Map? { + return value?.let { json.decodeFromString(mapOfLocalizedTextV2Serializer, it) } + } + + @TypeConverter + fun mapOfLocalizedTextV2toString(text: Map?): String? { + return text?.let { json.encodeToString(mapOfLocalizedTextV2Serializer, it) } + } + + @TypeConverter + fun fromStringToListString(value: String?): List { + return value?.split(',') ?: emptyList() + } + + @TypeConverter + fun listStringToString(text: List?): String? { + if (text.isNullOrEmpty()) return null + return text.joinToString(",") { it.replace(',', '_') } + } } diff --git a/database/src/main/java/org/fdroid/database/DbStreamReceiver.kt b/database/src/main/java/org/fdroid/database/DbStreamReceiver.kt new file mode 100644 index 000000000..56692bbe0 --- /dev/null +++ b/database/src/main/java/org/fdroid/database/DbStreamReceiver.kt @@ -0,0 +1,20 @@ +package org.fdroid.database + +import org.fdroid.index.v2.IndexStreamReceiver +import org.fdroid.index.v2.PackageV2 +import org.fdroid.index.v2.RepoV2 + +internal class DbStreamReceiver( + private val db: FDroidDatabase, +) : IndexStreamReceiver { + + override fun receive(repoId: Long, repo: RepoV2) { + db.getRepositoryDaoInt().replace(repoId, repo) + } + + override fun receive(repoId: Long, packageId: String, p: PackageV2) { + db.getAppDaoInt().insert(repoId, packageId, p.metadata) + db.getVersionDaoInt().insert(repoId, packageId, p.versions) + } + +} diff --git a/database/src/main/java/org/fdroid/database/DbV1StreamReceiver.kt b/database/src/main/java/org/fdroid/database/DbV1StreamReceiver.kt new file mode 100644 index 000000000..1a85d854c --- /dev/null +++ b/database/src/main/java/org/fdroid/database/DbV1StreamReceiver.kt @@ -0,0 +1,43 @@ +package org.fdroid.database + +import org.fdroid.index.v1.IndexV1StreamReceiver +import org.fdroid.index.v2.AntiFeatureV2 +import org.fdroid.index.v2.CategoryV2 +import org.fdroid.index.v2.MetadataV2 +import org.fdroid.index.v2.PackageVersionV2 +import org.fdroid.index.v2.ReleaseChannelV2 +import org.fdroid.index.v2.RepoV2 + +internal class DbV1StreamReceiver( + private val db: FDroidDatabase, +) : IndexV1StreamReceiver { + + override fun receive(repoId: Long, repo: RepoV2) { + db.getRepositoryDaoInt().replace(repoId, repo) + } + + override fun receive(repoId: Long, packageId: String, m: MetadataV2) { + db.getAppDaoInt().insert(repoId, packageId, m) + } + + override fun receive(repoId: Long, packageId: String, v: Map) { + db.getVersionDaoInt().insert(repoId, packageId, v) + } + + override fun updateRepo( + repoId: Long, + antiFeatures: Map, + categories: Map, + releaseChannels: Map, + ) { + val repoDao = db.getRepositoryDaoInt() + repoDao.insertAntiFeatures(antiFeatures.toRepoAntiFeatures(repoId)) + repoDao.insertCategories(categories.toRepoCategories(repoId)) + repoDao.insertReleaseChannels(releaseChannels.toRepoReleaseChannel(repoId)) + } + + override fun updateAppMetadata(repoId: Long, packageId: String, preferredSigner: String?) { + db.getAppDaoInt().updatePreferredSigner(repoId, packageId, preferredSigner) + } + +} diff --git a/database/src/main/java/org/fdroid/database/FDroidDatabase.kt b/database/src/main/java/org/fdroid/database/FDroidDatabase.kt index 5f2eed9a5..2c50290ae 100644 --- a/database/src/main/java/org/fdroid/database/FDroidDatabase.kt +++ b/database/src/main/java/org/fdroid/database/FDroidDatabase.kt @@ -7,15 +7,25 @@ import androidx.room.RoomDatabase import androidx.room.TypeConverters @Database(entities = [ + // repo CoreRepository::class, Mirror::class, AntiFeature::class, Category::class, ReleaseChannel::class, + // packages + AppMetadata::class, + LocalizedFile::class, + LocalizedFileList::class, + // versions + Version::class, + VersionedString::class, ], version = 1) @TypeConverters(Converters::class) internal abstract class FDroidDatabase internal constructor() : RoomDatabase() { abstract fun getRepositoryDaoInt(): RepositoryDaoInt + abstract fun getAppDaoInt(): AppDaoInt + abstract fun getVersionDaoInt(): VersionDaoInt companion object { // Singleton prevents multiple instances of database opening at the same time. diff --git a/database/src/main/java/org/fdroid/database/Repository.kt b/database/src/main/java/org/fdroid/database/Repository.kt index dda0b1bad..424ba04dc 100644 --- a/database/src/main/java/org/fdroid/database/Repository.kt +++ b/database/src/main/java/org/fdroid/database/Repository.kt @@ -17,13 +17,14 @@ import org.fdroid.index.v2.RepoV2 data class CoreRepository( @PrimaryKey(autoGenerate = true) val repoId: Long = 0, val name: String, - @Embedded(prefix = "icon") val icon: FileV2?, + @Embedded(prefix = "icon_") val icon: FileV2?, val address: String, val timestamp: Long, val description: LocalizedTextV2 = emptyMap(), ) -fun RepoV2.toCoreRepository() = CoreRepository( +fun RepoV2.toCoreRepository(repoId: Long = 0) = CoreRepository( + repoId = repoId, name = name, icon = icon, address = address, @@ -88,7 +89,7 @@ fun MirrorV2.toMirror(repoId: Long) = Mirror( data class AntiFeature( val repoId: Long, val name: String, - @Embedded(prefix = "icon") val icon: FileV2? = null, + @Embedded(prefix = "icon_") val icon: FileV2? = null, val description: LocalizedTextV2, ) @@ -113,7 +114,7 @@ fun Map.toRepoAntiFeatures(repoId: Long) = map { data class Category( val repoId: Long, val name: String, - @Embedded(prefix = "icon") val icon: FileV2? = null, + @Embedded(prefix = "icon_") val icon: FileV2? = null, val description: LocalizedTextV2, ) @@ -138,7 +139,7 @@ fun Map.toRepoCategories(repoId: Long) = map { data class ReleaseChannel( val repoId: Long, val name: String, - @Embedded(prefix = "icon") val icon: FileV2? = null, + @Embedded(prefix = "icon_") val icon: FileV2? = null, val description: LocalizedTextV2, ) diff --git a/database/src/main/java/org/fdroid/database/RepositoryDao.kt b/database/src/main/java/org/fdroid/database/RepositoryDao.kt index 9090374e7..373636880 100644 --- a/database/src/main/java/org/fdroid/database/RepositoryDao.kt +++ b/database/src/main/java/org/fdroid/database/RepositoryDao.kt @@ -4,7 +4,6 @@ import androidx.annotation.VisibleForTesting import androidx.room.Dao import androidx.room.Delete import androidx.room.Insert -import androidx.room.OnConflictStrategy.ABORT import androidx.room.OnConflictStrategy.REPLACE import androidx.room.Query import androidx.room.Transaction @@ -14,19 +13,28 @@ import kotlinx.serialization.json.JsonNull import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.decodeFromJsonElement import kotlinx.serialization.json.jsonObject -import org.fdroid.index.ReflectionDiffer.applyDiff +import org.fdroid.index.v2.ReflectionDiffer.applyDiff import org.fdroid.index.IndexParser.json import org.fdroid.index.v2.MirrorV2 import org.fdroid.index.v2.RepoV2 public interface RepositoryDao { - fun insert(repository: RepoV2) + /** + * Use when inserting a new repo for the first time. + */ + fun insert(repository: RepoV2): Long + + /** + * Use when replacing an existing repo with a full index. + * This removes all existing index data associated with this repo from the database. + */ + fun replace(repoId: Long, repository: RepoV2) } @Dao internal interface RepositoryDaoInt : RepositoryDao { - @Insert(onConflict = ABORT) + @Insert(onConflict = REPLACE) fun insert(repository: CoreRepository): Long @Insert(onConflict = REPLACE) @@ -42,8 +50,20 @@ internal interface RepositoryDaoInt : RepositoryDao { fun insertReleaseChannels(repoFeature: List) @Transaction - override fun insert(repository: RepoV2) { + override fun insert(repository: RepoV2): Long { val repoId = insert(repository.toCoreRepository()) + insertRepoTables(repoId, repository) + return repoId + } + + @Transaction + override fun replace(repoId: Long, repository: RepoV2) { + val newRepoId = insert(repository.toCoreRepository(repoId)) + require(newRepoId == repoId) { "New repoId $newRepoId did not match old $repoId" } + insertRepoTables(repoId, repository) + } + + private fun insertRepoTables(repoId: Long, repository: RepoV2) { insertMirrors(repository.mirrors.map { it.toMirror(repoId) }) insertAntiFeatures(repository.antiFeatures.toRepoAntiFeatures(repoId)) insertCategories(repository.categories.toRepoCategories(repoId)) diff --git a/database/src/main/java/org/fdroid/database/Version.kt b/database/src/main/java/org/fdroid/database/Version.kt new file mode 100644 index 000000000..0e32dc175 --- /dev/null +++ b/database/src/main/java/org/fdroid/database/Version.kt @@ -0,0 +1,170 @@ +package org.fdroid.database + +import androidx.room.Embedded +import androidx.room.Entity +import androidx.room.ForeignKey +import org.fdroid.database.VersionedStringType.FEATURE +import org.fdroid.database.VersionedStringType.PERMISSION +import org.fdroid.database.VersionedStringType.PERMISSION_SDK_23 +import org.fdroid.index.v2.FeatureV2 +import org.fdroid.index.v2.FileV1 +import org.fdroid.index.v2.FileV2 +import org.fdroid.index.v2.LocalizedTextV2 +import org.fdroid.index.v2.ManifestV2 +import org.fdroid.index.v2.PackageVersionV2 +import org.fdroid.index.v2.PermissionV2 +import org.fdroid.index.v2.SignatureV2 +import org.fdroid.index.v2.UsesSdkV2 + +@Entity( + primaryKeys = ["repoId", "packageId", "versionId"], + foreignKeys = [ForeignKey( + entity = AppMetadata::class, + parentColumns = ["repoId", "packageId"], + childColumns = ["repoId", "packageId"], + onDelete = ForeignKey.CASCADE, + )], +) +data class Version( + val repoId: Long, + val packageId: String, + val versionId: String, + val added: Long, + @Embedded(prefix = "file_") val file: FileV1, + @Embedded(prefix = "src_") val src: FileV2? = null, + @Embedded(prefix = "manifest_") val manifest: AppManifest, + val releaseChannels: List? = emptyList(), + val antiFeatures: Map? = null, + val whatsNew: LocalizedTextV2? = null, +) + +fun PackageVersionV2.toVersion(repoId: Long, packageId: String, versionId: String) = Version( + repoId = repoId, + packageId = packageId, + versionId = versionId, + added = added, + file = file, + src = src, + manifest = manifest.toManifest(), + releaseChannels = releaseChannels, + antiFeatures = antiFeatures, + whatsNew = whatsNew, +) + +data class AppVersion( + val version: Version, + val usesPermission: List? = null, + val usesPermissionSdk23: List? = null, + val features: List? = null, +) + +data class AppManifest( + val versionName: String, + val versionCode: Long, + @Embedded(prefix = "usesSdk_") val usesSdk: UsesSdkV2? = null, + val maxSdkVersion: Int? = null, + @Embedded(prefix = "signer_") val signer: SignatureV2? = null, + val nativecode: List? = emptyList(), +) + +fun ManifestV2.toManifest() = AppManifest( + versionName = versionName, + versionCode = versionCode, + usesSdk = usesSdk, + maxSdkVersion = maxSdkVersion, + signer = signer, + nativecode = nativeCode, +) + +enum class VersionedStringType { + PERMISSION, + PERMISSION_SDK_23, + FEATURE, +} + +@Entity( + primaryKeys = ["repoId", "packageId", "versionId", "type", "name"], + foreignKeys = [ForeignKey( + entity = Version::class, + parentColumns = ["repoId", "packageId", "versionId"], + childColumns = ["repoId", "packageId", "versionId"], + onDelete = ForeignKey.CASCADE, + )], +) +data class VersionedString( + val repoId: Long, + val packageId: String, + val versionId: String, + val type: VersionedStringType, + val name: String, + val version: Int? = null, +) + +fun List.toVersionedString( + version: Version, + type: VersionedStringType, +) = map { permission -> + VersionedString( + repoId = version.repoId, + packageId = version.packageId, + versionId = version.versionId, + type = type, + name = permission.name, + version = permission.maxSdkVersion, + ) +} + +fun List.toVersionedString(version: Version) = map { feature -> + VersionedString( + repoId = version.repoId, + packageId = version.packageId, + versionId = version.versionId, + type = FEATURE, + name = feature.name, + version = feature.version, + ) +} + +fun ManifestV2.getVersionedStrings(version: Version): List { + return usesPermission.toVersionedString(version, PERMISSION) + + usesPermissionSdk23.toVersionedString(version, PERMISSION_SDK_23) + + features.toVersionedString(version) +} + +fun List.getPermissions(version: Version) = mapNotNull { v -> + v.map(version, PERMISSION) { + PermissionV2( + name = v.name, + maxSdkVersion = v.version, + ) + } +} + +fun List.getPermissionsSdk23(version: Version) = mapNotNull { v -> + v.map(version, PERMISSION_SDK_23) { + PermissionV2( + name = v.name, + maxSdkVersion = v.version, + ) + } +} + +fun List.getFeatures(version: Version) = mapNotNull { v -> + v.map(version, FEATURE) { + FeatureV2( + name = v.name, + version = v.version, + ) + } +} + +private fun VersionedString.map( + v: Version, + wantedType: VersionedStringType, + factory: () -> T, +): T? { + return if (repoId != v.repoId || packageId != v.packageId || versionId != v.versionId || + type != wantedType + ) null + else factory() +} diff --git a/database/src/main/java/org/fdroid/database/VersionDao.kt b/database/src/main/java/org/fdroid/database/VersionDao.kt new file mode 100644 index 000000000..06edc6f5a --- /dev/null +++ b/database/src/main/java/org/fdroid/database/VersionDao.kt @@ -0,0 +1,79 @@ +package org.fdroid.database + +import androidx.annotation.VisibleForTesting +import androidx.room.Dao +import androidx.room.Insert +import androidx.room.OnConflictStrategy +import androidx.room.Query +import androidx.room.Transaction +import org.fdroid.index.v2.PackageVersionV2 + +public interface VersionDao { + fun insert(repoId: Long, packageId: String, packageVersions: Map) + fun insert(repoId: Long, packageId: String, versionId: String, packageVersion: PackageVersionV2) + fun getAppVersions(repoId: Long, packageId: String): List +} + +@Dao +internal interface VersionDaoInt : VersionDao { + + @Transaction + override fun insert( + repoId: Long, + packageId: String, + packageVersions: Map, + ) { + // TODO maybe the number of queries here can be reduced + packageVersions.entries.forEach { (versionId, packageVersion) -> + insert(repoId, packageId, versionId, packageVersion) + } + } + + @Transaction + override fun insert( + repoId: Long, + packageId: String, + versionId: String, + packageVersion: PackageVersionV2, + ) { + val version = packageVersion.toVersion(repoId, packageId, versionId) + insert(version) + insert(packageVersion.manifest.getVersionedStrings(version)) + } + + @Insert(onConflict = OnConflictStrategy.REPLACE) + fun insert(version: Version) + + @Insert(onConflict = OnConflictStrategy.REPLACE) + fun insert(versionedString: List) + + @Transaction + override fun getAppVersions(repoId: Long, packageId: String): List { + val versionedStrings = getVersionedStrings(repoId, packageId) + return getVersions(repoId, packageId).map { version -> + AppVersion( + version = version, + usesPermission = versionedStrings.getPermissions(version), + usesPermissionSdk23 = versionedStrings.getPermissionsSdk23(version), + features = versionedStrings.getFeatures(version), + ) + } + } + + @Query("SELECT * FROM Version WHERE repoId = :repoId AND packageId = :packageId") + fun getVersions(repoId: Long, packageId: String): List + + @Query("SELECT * FROM VersionedString WHERE repoId = :repoId AND packageId = :packageId") + fun getVersionedStrings(repoId: Long, packageId: String): List + + @VisibleForTesting + @Query("DELETE FROM Version WHERE repoId = :repoId AND packageId = :packageId AND versionId = :versionId") + fun deleteAppVersion(repoId: Long, packageId: String, versionId: String) + + @Query("SELECT COUNT(*) FROM Version") + fun countAppVersions(): Int + + @Query("SELECT COUNT(*) FROM VersionedString") + fun countVersionedStrings(): Int + +} diff --git a/database/src/test/java/org/fdroid/database/ReflectionTest.kt b/database/src/test/java/org/fdroid/database/ReflectionTest.kt index ebd544414..4f70e11df 100644 --- a/database/src/test/java/org/fdroid/database/ReflectionTest.kt +++ b/database/src/test/java/org/fdroid/database/ReflectionTest.kt @@ -3,7 +3,7 @@ package org.fdroid.database import kotlinx.serialization.encodeToString import kotlinx.serialization.json.Json import kotlinx.serialization.json.jsonObject -import org.fdroid.index.ReflectionDiffer.applyDiff +import org.fdroid.index.v2.ReflectionDiffer.applyDiff import kotlin.random.Random import kotlin.test.Test import kotlin.test.assertEquals diff --git a/database/src/test/java/org/fdroid/database/TestUtils.kt b/database/src/test/java/org/fdroid/database/TestUtils.kt index 5df26cc98..5f43ae973 100644 --- a/database/src/test/java/org/fdroid/database/TestUtils.kt +++ b/database/src/test/java/org/fdroid/database/TestUtils.kt @@ -37,7 +37,7 @@ object TestUtils2 { } } - private fun T.orNull(): T? { + fun T.orNull(): T? { return if (Random.nextBoolean()) null else this }