Fix CJK language search in app lists and my apps

app/src/main/kotlin/org/fdroid/ui/apps/MyAppsPresenter.kt

@@ -11,7 +11,8 @@ import org.fdroid.download.NetworkState
 import org.fdroid.install.InstallConfirmationState
 import org.fdroid.install.InstallState
 import org.fdroid.install.InstallStateWithInfo
-import org.fdroid.ui.utils.normalize
+import org.fdroid.ui.search.SearchHelper.fixQuery
+import org.fdroid.ui.search.SearchHelper.normalize
 
 // TODO add tests for this, similar to DetailsPresenter
 @Composable
@@ -29,7 +30,7 @@ fun MyAppsPresenter(
   val appInstallStates = appInstallStatesFlow.collectAsState().value
   val appsWithIssues = appsWithIssuesFlow.collectAsState().value
   val installedApps = installedAppsFlow.collectAsState(null).value
-  val searchQuery = searchQueryFlow.collectAsState().value.normalize()
+  val searchQuery = fixQuery(searchQueryFlow.collectAsState().value)
   val sortOrder = sortOrderFlow.collectAsState().value
   val processedPackageNames = mutableSetOf<String>()
 

app/src/main/kotlin/org/fdroid/ui/lists/AppListPresenter.kt

@@ -10,7 +10,8 @@ import kotlinx.coroutines.flow.StateFlow
 import org.fdroid.database.AppListSortOrder
 import org.fdroid.ui.categories.CategoryItem
 import org.fdroid.ui.repositories.RepositoryItem
-import org.fdroid.ui.utils.normalize
+import org.fdroid.ui.search.SearchHelper.fixQuery
+import org.fdroid.ui.search.SearchHelper.normalize
 
 @Composable
 fun AppListPresenter(
@@ -35,7 +36,7 @@ fun AppListPresenter(
   val filteredAntiFeatureIds = notSelectedAntiFeatureIdsFlow.collectAsState().value
   val repositories = repositoriesFlow.collectAsState(emptyList()).value
   val filteredRepositoryIds = filteredRepositoryIdsFlow.collectAsState().value
-  val searchQuery = searchQueryFlow.collectAsState().value.normalize()
+  val searchQuery = fixQuery(searchQueryFlow.collectAsState().value)
 
   val availableCategoryIds =
     remember(apps) {

app/src/main/kotlin/org/fdroid/ui/search/SearchHelper.kt

@@ -0,0 +1,37 @@
+package org.fdroid.ui.search
+
+import java.text.Normalizer
+import java.text.Normalizer.Form.NFKD
+
+object SearchHelper {
+  private val normalizerRegex = "\\p{M}".toRegex()
+
+  /** Normalizes the string by removing any diacritics that may appear. */
+  fun String.normalize(): String {
+    if (Normalizer.isNormalized(this, NFKD)) return this
+    return Normalizer.normalize(this, NFKD).replace(normalizerRegex, "")
+  }
+
+  /**
+   * Normalize the query by removing diacritics and adding zero-width spaces after ideographic
+   * characters.
+   */
+  fun fixQuery(query: String): String = addZeroWhiteSpaceIfNeeded(query.normalize())
+
+  /**
+   * Inserts a zero-width space after each ideographic character in the query. This is needed,
+   * because for Fts4 search in the DB, we insert zero white space characters between tokens to fake
+   * tokenization. However, when doing more naive [String.contains] searches, we need to add those
+   * also to the query, otherwise nothing will be found.
+   */
+  private fun addZeroWhiteSpaceIfNeeded(query: String): String = buildString {
+    query.forEachIndexed { i, char ->
+      if (Character.isIdeographic(char.code) && i + 1 < query.length) {
+        append(char)
+        append("\u200B")
+      } else {
+        append(char)
+      }
+    }
+  }
+}

app/src/main/kotlin/org/fdroid/ui/search/SearchManager.kt

@@ -26,7 +26,7 @@ import org.fdroid.install.InstalledAppsCache
 import org.fdroid.settings.SettingsManager
 import org.fdroid.ui.categories.CategoryItem
 import org.fdroid.ui.lists.AppListItem
-import org.fdroid.ui.utils.normalize
+import org.fdroid.ui.search.SearchHelper.normalize
 import org.fdroid.utils.IoDispatcher
 
 @Singleton

app/src/main/kotlin/org/fdroid/ui/utils/UiUtils.kt

@@ -17,8 +17,6 @@ import androidx.core.content.ContextCompat
 import androidx.core.graphics.createBitmap
 import com.google.zxing.BarcodeFormat
 import com.google.zxing.qrcode.QRCodeWriter
-import java.text.Normalizer
-import java.text.Normalizer.Form.NFKD
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.withContext
 import org.fdroid.database.Repository
@@ -53,14 +51,6 @@ fun UriHandler.openUriSafe(uri: String) {
   }
 }
 
-private val normalizerRegex = "\\p{M}".toRegex()
-
-/** Normalizes the string by removing any diacritics that may appear. */
-fun String.normalize(): String {
-  if (Normalizer.isNormalized(this, NFKD)) return this
-  return Normalizer.normalize(this, NFKD).replace(normalizerRegex, "")
-}
-
 /**
  * Same as the Java function Utils.generateQrBitmap, but using coroutines instead of Single and
  * Disposable.