Fix case insensitive search containing diacritics by changing FTS4 tokenizer

The default `simple` tokenzier config used in Room/SQLite3 FTS4 makes only ASCII characters case insensitive. The SQLite FTS3/4 doc [1] says:

> All uppercase characters within the ASCII range (Unicode codepoints less than 128), are transformed to their lowercase equivalents as part of the tokenization process. Thus, full-text queries are case-insensitive when using the simple tokenizer.

> The remove_diacritics option may be set to "0", "1" or "2". The default value is "1". If it is set to "1" or "2", then diacritics are removed from Latin script characters as described above. However, if it is set to "1", then diacritics are not removed in the fairly uncommon case where a single unicode codepoint is used to represent a character with more that one diacritic. [...] This is technically a bug, but cannot be fixed without creating backwards compatibility problems. If this option is set to "2", then diacritics are correctly removed from all Latin characters.

This change makes use of the intended behaviour by using the `unicode61` tokenizer with the `diacritics="0"` option to keep the behaviour similar to the current one. This replaces the previously used `simple` tokenizer. A migration is necessary to recreate the AppMetadataFts table to make use of the different tokenizer.

[1] https://www.sqlite.org/fts3.html#tokenizer

libs/database/src/main/java/org/fdroid/database/App.kt

@@ -125,7 +125,11 @@ internal fun MetadataV2.toAppMetadata(
 )
 
 @Entity(tableName = AppMetadataFts.TABLE)
-@Fts4(contentEntity = AppMetadata::class)
+@Fts4(
+    contentEntity = AppMetadata::class,
+    // make FTS for non-ASCII characters case insensitive, but do not remove diacritics
+    tokenizer = "unicode61 \"remove_diacritics=0\""
+)
 internal data class AppMetadataFts(
     val repoId: Long,
     val packageName: String,

libs/database/src/main/java/org/fdroid/database/FDroidDatabase.kt

@@ -16,7 +16,7 @@ import java.util.concurrent.Callable
     // When bumping this version, please make sure to add one (or more) migration(s) below!
     // Consider also providing tests for that migration.
     // Don't forget to commit the new schema to the git repo as well.
-    version = 5,
+    version = 6,
     entities = [
         // repo
         CoreRepository::class,
@@ -47,6 +47,7 @@ import java.util.concurrent.Callable
         // 2 to 3 is a manual migration
         AutoMigration(3, 4),
         AutoMigration(4, 5),
+        // 5 to 6 is a manual migration
         // add future migrations here (if they are easy enough to be done automatically)
     ],
 )

libs/database/src/main/java/org/fdroid/database/Migrations.kt

@@ -115,3 +115,22 @@ internal val MIGRATION_2_3 = object : Migration(2, 3) {
         db.delete(CoreRepository.TABLE, "certificate IS NULL", null)
     }
 }
+
+/**
+ * The tokenizer of the FTS4 table for the app metadata was modified.
+ * This migration is needed to recreate the FTS table to respect the new tokenizer.
+ */
+internal val MIGRATION_5_6 = object : Migration(5, 6) {
+    override fun migrate(db: SupportSQLiteDatabase) {
+        db.execSQL("DROP TABLE `AppMetadataFts`")
+        // table creation taken from auto-generated code:
+        // build/generated/source/kapt/debug/org/fdroid/database/FDroidDatabaseInt_Impl.java
+        // the corresponding triggers are added automatically
+        db.execSQL("CREATE VIRTUAL TABLE IF NOT EXISTS `AppMetadataFts`" +
+            "USING FTS4(`repoId` INTEGER NOT NULL, `packageName` TEXT NOT NULL, " +
+            "`localizedName` TEXT, `localizedSummary` TEXT, " +
+            "tokenize=unicode61 \"remove_diacritics=0\", content=`AppMetadata`)")
+        // rebuild the FTS table to populate it with the new tokenizer
+        db.execSQL("INSERT INTO AppMetadataFts(AppMetadataFts) VALUES('rebuild')")
+    }
+}