From 38ce4975258f648d8b97d016af546d352c869a58 Mon Sep 17 00:00:00 2001 From: DerGenaue <9513365-DerGenaue@users.noreply.gitlab.com> Date: Wed, 2 Oct 2024 20:31:26 +0200 Subject: [PATCH 1/3] DB: Force a wal_checkpoint after every repo update to keep the wal file small (fixes #2588) --- .../org/fdroid/fdroid/RepoUpdateManager.kt | 2 ++ .../java/org/fdroid/database/AppPrefsDao.kt | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/app/src/main/java/org/fdroid/fdroid/RepoUpdateManager.kt b/app/src/main/java/org/fdroid/fdroid/RepoUpdateManager.kt index 1ee1a7608..2d7ae90ae 100644 --- a/app/src/main/java/org/fdroid/fdroid/RepoUpdateManager.kt +++ b/app/src/main/java/org/fdroid/fdroid/RepoUpdateManager.kt @@ -104,6 +104,7 @@ class RepoUpdateManager @JvmOverloads constructor( repoErrors.add(Pair(repo, result.e)) } } + db.getAppPrefsDao().walCheckpoint() fdroidPrefs.lastUpdateCheck = System.currentTimeMillis() if (repoErrors.isNotEmpty()) showRepoErrors(repoErrors) if (reposUpdated) { @@ -136,6 +137,7 @@ class RepoUpdateManager @JvmOverloads constructor( } finally { notificationManager.cancelUpdateRepoNotification() _isUpdating.value = false + db.getAppPrefsDao().walCheckpoint() } } diff --git a/libs/database/src/main/java/org/fdroid/database/AppPrefsDao.kt b/libs/database/src/main/java/org/fdroid/database/AppPrefsDao.kt index d6eb2e357..9a7241bd1 100644 --- a/libs/database/src/main/java/org/fdroid/database/AppPrefsDao.kt +++ b/libs/database/src/main/java/org/fdroid/database/AppPrefsDao.kt @@ -8,10 +8,20 @@ import androidx.room.Insert import androidx.room.MapInfo import androidx.room.OnConflictStrategy.Companion.REPLACE import androidx.room.Query +import androidx.room.RawQuery +import androidx.sqlite.db.SimpleSQLiteQuery +import androidx.sqlite.db.SupportSQLiteQuery public interface AppPrefsDao { public fun getAppPrefs(packageName: String): LiveData public fun update(appPrefs: AppPrefs) + + /** + * Force a checkpoint on the SQLite WAL such that the file size gets reduced. + * Blocks until concurrent reads have finished. + * Useful to call after large inserts (repo update) + */ + public fun walCheckpoint() } @Dao @@ -48,4 +58,12 @@ internal interface AppPrefsDaoInt : AppPrefsDao { @Insert(onConflict = REPLACE) override fun update(appPrefs: AppPrefs) + + override fun walCheckpoint() { + rawCheckpoint((SimpleSQLiteQuery("pragma wal_checkpoint(truncate)"))) + } + + @RawQuery + fun rawCheckpoint(supportSQLiteQuery: SupportSQLiteQuery): Int + } From 445f521b1f217206188cd7c2fcee2dc0ce34f230 Mon Sep 17 00:00:00 2001 From: DerGenaue <9513365-DerGenaue@users.noreply.gitlab.com> Date: Thu, 3 Oct 2024 00:52:01 +0200 Subject: [PATCH 2/3] Move walCheckpoint to RepositoryDao --- .../java/org/fdroid/fdroid/RepoUpdateManager.kt | 4 ++-- .../java/org/fdroid/database/AppPrefsDao.kt | 17 ----------------- .../java/org/fdroid/database/RepositoryDao.kt | 16 ++++++++++++++++ 3 files changed, 18 insertions(+), 19 deletions(-) diff --git a/app/src/main/java/org/fdroid/fdroid/RepoUpdateManager.kt b/app/src/main/java/org/fdroid/fdroid/RepoUpdateManager.kt index 2d7ae90ae..0b068617f 100644 --- a/app/src/main/java/org/fdroid/fdroid/RepoUpdateManager.kt +++ b/app/src/main/java/org/fdroid/fdroid/RepoUpdateManager.kt @@ -104,7 +104,7 @@ class RepoUpdateManager @JvmOverloads constructor( repoErrors.add(Pair(repo, result.e)) } } - db.getAppPrefsDao().walCheckpoint() + db.getRepositoryDao().walCheckpoint() fdroidPrefs.lastUpdateCheck = System.currentTimeMillis() if (repoErrors.isNotEmpty()) showRepoErrors(repoErrors) if (reposUpdated) { @@ -137,7 +137,7 @@ class RepoUpdateManager @JvmOverloads constructor( } finally { notificationManager.cancelUpdateRepoNotification() _isUpdating.value = false - db.getAppPrefsDao().walCheckpoint() + db.getRepositoryDao().walCheckpoint() } } diff --git a/libs/database/src/main/java/org/fdroid/database/AppPrefsDao.kt b/libs/database/src/main/java/org/fdroid/database/AppPrefsDao.kt index 9a7241bd1..f878e1466 100644 --- a/libs/database/src/main/java/org/fdroid/database/AppPrefsDao.kt +++ b/libs/database/src/main/java/org/fdroid/database/AppPrefsDao.kt @@ -8,20 +8,10 @@ import androidx.room.Insert import androidx.room.MapInfo import androidx.room.OnConflictStrategy.Companion.REPLACE import androidx.room.Query -import androidx.room.RawQuery -import androidx.sqlite.db.SimpleSQLiteQuery -import androidx.sqlite.db.SupportSQLiteQuery public interface AppPrefsDao { public fun getAppPrefs(packageName: String): LiveData public fun update(appPrefs: AppPrefs) - - /** - * Force a checkpoint on the SQLite WAL such that the file size gets reduced. - * Blocks until concurrent reads have finished. - * Useful to call after large inserts (repo update) - */ - public fun walCheckpoint() } @Dao @@ -59,11 +49,4 @@ internal interface AppPrefsDaoInt : AppPrefsDao { @Insert(onConflict = REPLACE) override fun update(appPrefs: AppPrefs) - override fun walCheckpoint() { - rawCheckpoint((SimpleSQLiteQuery("pragma wal_checkpoint(truncate)"))) - } - - @RawQuery - fun rawCheckpoint(supportSQLiteQuery: SupportSQLiteQuery): Int - } diff --git a/libs/database/src/main/java/org/fdroid/database/RepositoryDao.kt b/libs/database/src/main/java/org/fdroid/database/RepositoryDao.kt index f467d398b..9c56c8fda 100644 --- a/libs/database/src/main/java/org/fdroid/database/RepositoryDao.kt +++ b/libs/database/src/main/java/org/fdroid/database/RepositoryDao.kt @@ -6,9 +6,12 @@ import androidx.room.Dao import androidx.room.Insert import androidx.room.OnConflictStrategy.Companion.REPLACE import androidx.room.Query +import androidx.room.RawQuery import androidx.room.RewriteQueriesToDropUnusedColumns import androidx.room.Transaction import androidx.room.Update +import androidx.sqlite.db.SimpleSQLiteQuery +import androidx.sqlite.db.SupportSQLiteQuery import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.decodeFromJsonElement import org.fdroid.database.DbDiffUtils.diffAndUpdateListTable @@ -87,6 +90,13 @@ public interface RepositoryDao { * Removes all repos and their preferences. */ public fun clearAll() + + /** + * Force a checkpoint on the SQLite WAL such that the file size gets reduced. + * Blocks until concurrent reads have finished. + * Useful to call after large inserts (repo update) + */ + public fun walCheckpoint() } @Dao @@ -527,4 +537,10 @@ internal interface RepositoryDaoInt : RepositoryDao { @Query("SELECT COUNT(*) FROM ${ReleaseChannel.TABLE}") fun countReleaseChannels(): Int + override fun walCheckpoint() { + rawCheckpoint((SimpleSQLiteQuery("pragma wal_checkpoint(truncate)"))) + } + + @RawQuery + fun rawCheckpoint(supportSQLiteQuery: SupportSQLiteQuery): Int } From 61361154b11a9dd8bc2b633527197b1d745dd5b8 Mon Sep 17 00:00:00 2001 From: DerGenaue <9513365-DerGenaue@users.noreply.gitlab.com> Date: Thu, 3 Oct 2024 15:03:22 +0200 Subject: [PATCH 3/3] MR comments --- .../src/main/java/org/fdroid/database/RepositoryDao.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/database/src/main/java/org/fdroid/database/RepositoryDao.kt b/libs/database/src/main/java/org/fdroid/database/RepositoryDao.kt index 9c56c8fda..da14e125c 100644 --- a/libs/database/src/main/java/org/fdroid/database/RepositoryDao.kt +++ b/libs/database/src/main/java/org/fdroid/database/RepositoryDao.kt @@ -95,6 +95,7 @@ public interface RepositoryDao { * Force a checkpoint on the SQLite WAL such that the file size gets reduced. * Blocks until concurrent reads have finished. * Useful to call after large inserts (repo update) + * @see https://www.sqlite.org/wal.html#avoiding_excessively_large_wal_files */ public fun walCheckpoint() } @@ -538,7 +539,7 @@ internal interface RepositoryDaoInt : RepositoryDao { fun countReleaseChannels(): Int override fun walCheckpoint() { - rawCheckpoint((SimpleSQLiteQuery("pragma wal_checkpoint(truncate)"))) + rawCheckpoint(SimpleSQLiteQuery("pragma wal_checkpoint(truncate)")) } @RawQuery