Add Android inline keyboard autofill suggestions (#1637)

apps/mobile-app/android/app/build.gradle

@@ -187,6 +187,9 @@ dependencies {
     // Add biometric dependency for credential management
     implementation("androidx.biometric:biometric:1.1.0")
 
+    // Add androidx.autofill for inline suggestion (keyboard) support
+    implementation("androidx.autofill:autofill:1.1.0")
+
     // Add Credential Manager dependencies for passkey support (Android 14+)
     implementation("androidx.credentials:credentials:1.6.0-beta02")
 

apps/mobile-app/android/app/src/main/java/net/aliasvault/app/autofill/AutofillService.kt

@@ -11,6 +11,7 @@ package net.aliasvault.app.autofill
 import android.app.PendingIntent
 import android.content.Intent
 import android.graphics.Typeface
+import android.os.Build
 import android.os.CancellationSignal
 import android.service.autofill.AutofillService
 import android.service.autofill.Dataset
@@ -27,6 +28,7 @@ import android.widget.RemoteViews
 import net.aliasvault.app.R
 import net.aliasvault.app.autofill.utils.AutofillDatasetBuilder
 import net.aliasvault.app.autofill.utils.FieldFinder
+import net.aliasvault.app.autofill.utils.InlinePresentationHelper
 import net.aliasvault.app.autofill.utils.RustItemMatcher
 import net.aliasvault.app.vaultstore.VaultStore
 import net.aliasvault.app.vaultstore.interfaces.ItemOperationCallback
@@ -78,7 +80,8 @@ class AutofillService : AutofillService() {
                 safeCallback()
                 return
             }
-            launchActivityForAutofill(fieldFinder) { response -> safeCallback(response) }
+            val inlinePool = buildInlinePool(request)
+            launchActivityForAutofill(fieldFinder, inlinePool) { response -> safeCallback(response) }
         } catch (e: Exception) {
             Log.e(TAG, "Unexpected error in onFillRequest", e)
             // Provide a simple fallback response to prevent white flash
@@ -96,12 +99,13 @@ class AutofillService : AutofillService() {
                 // Add debug dataset if enabled in settings
                 val sharedPreferences = getSharedPreferences("AliasVaultPrefs", android.content.Context.MODE_PRIVATE)
                 val showSearchText = sharedPreferences.getBoolean("autofill_show_search_text", false)
+                val fallbackPool = buildInlinePool(request)
                 if (showSearchText) {
-                    responseBuilder.addDataset(createSearchDebugDataset(fieldFinder, appInfo ?: "unknown"))
+                    responseBuilder.addDataset(createSearchDebugDataset(fieldFinder, appInfo ?: "unknown", fallbackPool))
                 }
 
                 // Add failed to retrieve dataset
-                responseBuilder.addDataset(createFailedToRetrieveDataset(fieldFinder))
+                responseBuilder.addDataset(createFailedToRetrieveDataset(fieldFinder, fallbackPool))
 
                 safeCallback(responseBuilder.build())
             } catch (fallbackError: Exception) {
@@ -121,7 +125,11 @@ class AutofillService : AutofillService() {
         callback.onSuccess()
     }
 
-    private fun launchActivityForAutofill(fieldFinder: FieldFinder, callback: (FillResponse?) -> Unit) {
+    private fun launchActivityForAutofill(
+        fieldFinder: FieldFinder,
+        inlinePool: InlinePresentationHelper.SpecPool,
+        callback: (FillResponse?) -> Unit,
+    ) {
         // Get the app/website information from assist structure.
         val appInfo = fieldFinder.getAppInfo()
 
@@ -175,7 +183,7 @@ class AutofillService : AutofillService() {
                             val showSearchText = sharedPreferences.getBoolean("autofill_show_search_text", false)
                             val copyTotpOnFill = sharedPreferences.getBoolean("autofill_copy_totp_on_fill", true)
                             if (showSearchText) {
-                                responseBuilder.addDataset(createSearchDebugDataset(fieldFinder, appInfo ?: "unknown"))
+                                responseBuilder.addDataset(createSearchDebugDataset(fieldFinder, appInfo ?: "unknown", inlinePool))
                             }
 
                             // If there are no results, return "no matches" placeholder option.
@@ -189,6 +197,7 @@ class AutofillService : AutofillService() {
                                         this@AutofillService,
                                         fieldFinder.autofillableFields,
                                         appInfo,
+                                        inlinePool.next(),
                                     ),
                                 )
                             } else {
@@ -199,13 +208,14 @@ class AutofillService : AutofillService() {
                                         fields = fieldFinder.autofillableFields,
                                         item = item,
                                         copyTotpOnSelect = copyTotpOnFill && item.hasTotp,
+                                        inlineSpec = inlinePool.next(),
                                     )
                                     responseBuilder.addDataset(dataset)
                                 }
 
                                 // Add "Open app" option at the bottom (when search text is not shown and there are matches)
                                 if (!showSearchText) {
-                                    responseBuilder.addDataset(createOpenAppDataset(fieldFinder))
+                                    responseBuilder.addDataset(createOpenAppDataset(fieldFinder, inlinePool))
                                 }
                             }
 
@@ -217,9 +227,9 @@ class AutofillService : AutofillService() {
                             val sharedPreferences = getSharedPreferences("AliasVaultPrefs", android.content.Context.MODE_PRIVATE)
                             val showSearchText = sharedPreferences.getBoolean("autofill_show_search_text", false)
                             if (showSearchText) {
-                                responseBuilder.addDataset(createSearchDebugDataset(fieldFinder, appInfo ?: "unknown"))
+                                responseBuilder.addDataset(createSearchDebugDataset(fieldFinder, appInfo ?: "unknown", inlinePool))
                             }
-                            responseBuilder.addDataset(createFailedToRetrieveDataset(fieldFinder))
+                            responseBuilder.addDataset(createFailedToRetrieveDataset(fieldFinder, inlinePool))
                             callback(responseBuilder.build())
                         }
                     }
@@ -231,9 +241,9 @@ class AutofillService : AutofillService() {
                         val sharedPreferences = getSharedPreferences("AliasVaultPrefs", android.content.Context.MODE_PRIVATE)
                         val showSearchText = sharedPreferences.getBoolean("autofill_show_search_text", false)
                         if (showSearchText) {
-                            responseBuilder.addDataset(createSearchDebugDataset(fieldFinder, appInfo ?: "unknown"))
+                            responseBuilder.addDataset(createSearchDebugDataset(fieldFinder, appInfo ?: "unknown", inlinePool))
                         }
-                        responseBuilder.addDataset(createFailedToRetrieveDataset(fieldFinder))
+                        responseBuilder.addDataset(createFailedToRetrieveDataset(fieldFinder, inlinePool))
                         callback(responseBuilder.build())
                     }
                 })
@@ -244,13 +254,17 @@ class AutofillService : AutofillService() {
         }
 
         Log.d(TAG, "Vault is locked, requiring authentication before fill")
-        callback(buildLockedFillResponse(fieldFinder, appInfo))
+        callback(buildLockedFillResponse(fieldFinder, appInfo, inlinePool))
     }
 
     /**
      * Build a [FillResponse] whose only entry is a "Vault locked" row.
      */
-    private fun buildLockedFillResponse(fieldFinder: FieldFinder, appInfo: String?): FillResponse {
+    private fun buildLockedFillResponse(
+        fieldFinder: FieldFinder,
+        appInfo: String?,
+        inlinePool: InlinePresentationHelper.SpecPool,
+    ): FillResponse {
         val autofillIds = fieldFinder.autofillableFields.map { it.first }.toTypedArray()
         val fieldTypeOrdinals = IntArray(fieldFinder.autofillableFields.size) { i ->
             fieldFinder.autofillableFields[i].second.ordinal
@@ -268,28 +282,59 @@ class AutofillService : AutofillService() {
             PendingIntent.FLAG_MUTABLE or PendingIntent.FLAG_CANCEL_CURRENT,
         )
 
+        val lockedLabel = getString(R.string.autofill_vault_locked)
         val presentation = RemoteViews(packageName, R.layout.autofill_dataset_item_logo)
-        presentation.setTextViewText(R.id.text, getString(R.string.autofill_vault_locked))
+        presentation.setTextViewText(R.id.text, lockedLabel)
 
-        return FillResponse.Builder()
+        val responseBuilder = FillResponse.Builder()
+        val inlineSpec = inlinePool.next()
+        if (inlineSpec != null && Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
+            val inline = InlinePresentationHelper.buildActionPresentation(this, inlineSpec, lockedLabel)
+            if (inline != null) {
+                val presentations = android.service.autofill.Presentations.Builder()
+                    .setMenuPresentation(presentation)
+                    .setInlinePresentation(inline)
+                    .build()
+                responseBuilder.setAuthentication(autofillIds, pendingIntent.intentSender, presentations)
+                return responseBuilder.build()
+            }
+        }
+        return responseBuilder
             .setAuthentication(autofillIds, pendingIntent.intentSender, presentation)
             .build()
     }
 
+    /**
+     * Creates inline suggestion pool, showing password suggestions in supported keyboards.
+     */
+    private fun buildInlinePool(request: FillRequest): InlinePresentationHelper.SpecPool {
+        val inlineRequest = request.inlineSuggestionsRequest
+            ?: return InlinePresentationHelper.SpecPool(emptyList(), 0)
+        return InlinePresentationHelper.SpecPool(
+            specs = inlineRequest.inlinePresentationSpecs,
+            maxCount = inlineRequest.maxSuggestionCount,
+        )
+    }
+
     /**
      * Create a dataset for the "failed to retrieve" option.
      * @param fieldFinder The field finder
+     * @param inlinePool Keyboard chip slots. If a slot is still free, this row
+     *   also shows as an inline suggestion in the IME; otherwise it appears
+     *   in the dropdown only.
      * @return The dataset
      */
-    private fun createFailedToRetrieveDataset(fieldFinder: FieldFinder): Dataset {
+    private fun createFailedToRetrieveDataset(
+        fieldFinder: FieldFinder,
+        inlinePool: InlinePresentationHelper.SpecPool,
+    ): Dataset {
         // Create presentation for the "failed to retrieve" option
+        val label = getString(R.string.autofill_failed_to_retrieve)
         val presentation = RemoteViews(packageName, R.layout.autofill_dataset_item_logo)
-        presentation.setTextViewText(
-            R.id.text,
-            getString(R.string.autofill_failed_to_retrieve),
-        )
+        presentation.setTextViewText(R.id.text, label)
 
         val dataSetBuilder = Dataset.Builder(presentation)
+        addInlineIfAvailable(dataSetBuilder, inlinePool, label)
 
         // Create deep link URL
         val deepLinkUrl = "aliasvault://reinitialize"
@@ -321,9 +366,16 @@ class AutofillService : AutofillService() {
      * Create a debug dataset showing what string we're searching for, clickable to open the app.
      * @param fieldFinder The field finder
      * @param searchText The text being searched for
+     * @param inlinePool Keyboard chip slots. If a slot is still free, this row
+     *   also shows as an inline suggestion in the IME; otherwise it appears
+     *   in the dropdown only.
      * @return The dataset
      */
-    private fun createSearchDebugDataset(fieldFinder: FieldFinder, searchText: String): Dataset {
+    private fun createSearchDebugDataset(
+        fieldFinder: FieldFinder,
+        searchText: String,
+        inlinePool: InlinePresentationHelper.SpecPool,
+    ): Dataset {
         // Create presentation for the debug option (with search icon)
         val presentation = RemoteViews(packageName, R.layout.autofill_dataset_item_icon)
 
@@ -335,6 +387,7 @@ class AutofillService : AutofillService() {
         presentation.setImageViewResource(R.id.icon, R.drawable.ic_search)
 
         val dataSetBuilder = Dataset.Builder(presentation)
+        addInlineIfAvailable(dataSetBuilder, inlinePool, searchText)
 
         // Get the app/website information to use as item URL
         val appInfo = fieldFinder.getAppInfo()
@@ -369,17 +422,22 @@ class AutofillService : AutofillService() {
     /**
      * Create a dataset for the "open app" option.
      * @param fieldFinder The field finder
+     * @param inlinePool Keyboard chip slots. If a slot is still free, this row
+     *   also shows as an inline suggestion in the IME; otherwise it appears
+     *   in the dropdown only.
      * @return The dataset
      */
-    private fun createOpenAppDataset(fieldFinder: FieldFinder): Dataset {
+    private fun createOpenAppDataset(
+        fieldFinder: FieldFinder,
+        inlinePool: InlinePresentationHelper.SpecPool,
+    ): Dataset {
         // Create presentation for the "open app" option with AliasVault logo
+        val label = getString(R.string.autofill_open_app)
         val presentation = RemoteViews(packageName, R.layout.autofill_dataset_item_logo)
-        presentation.setTextViewText(
-            R.id.text,
-            getString(R.string.autofill_open_app),
-        )
+        presentation.setTextViewText(R.id.text, label)
 
         val dataSetBuilder = Dataset.Builder(presentation)
+        addInlineIfAvailable(dataSetBuilder, inlinePool, label)
 
         // Open the action picker so the user can choose between linking this app
         // to an existing credential or creating a new one.
@@ -409,4 +467,18 @@ class AutofillService : AutofillService() {
 
         return dataSetBuilder.build()
     }
+
+    /**
+     * Also render this row as an inline keyboard chip if the IME has a free
+     * slot.
+     */
+    private fun addInlineIfAvailable(
+        dataSetBuilder: Dataset.Builder,
+        inlinePool: InlinePresentationHelper.SpecPool,
+        label: String,
+    ) {
+        val spec = inlinePool.next() ?: return
+        val inline = InlinePresentationHelper.buildActionPresentation(this, spec, label) ?: return
+        dataSetBuilder.setInlinePresentation(inline)
+    }
 }

apps/mobile-app/android/app/src/main/java/net/aliasvault/app/autofill/utils/AutofillDatasetBuilder.kt

@@ -8,6 +8,7 @@ import android.util.Log
 import android.view.autofill.AutofillId
 import android.view.autofill.AutofillValue
 import android.widget.RemoteViews
+import android.widget.inline.InlinePresentationSpec
 import net.aliasvault.app.R
 import net.aliasvault.app.autofill.AutofillFillActivity
 import net.aliasvault.app.autofill.models.FieldType
@@ -33,6 +34,7 @@ object AutofillDatasetBuilder {
         fields: List<Pair<AutofillId, FieldType>>,
         item: Item,
         copyTotpOnSelect: Boolean,
+        inlineSpec: InlinePresentationSpec? = null,
     ): Dataset {
         val presentation = RemoteViews(context.packageName, R.layout.autofill_dataset_item_icon)
         val builder = Dataset.Builder(presentation)
@@ -64,6 +66,28 @@ object AutofillDatasetBuilder {
             presentation.setImageViewBitmap(R.id.icon, bitmap)
         }
 
+        if (inlineSpec != null) {
+            val itemDeepLink = "aliasvault://items/${item.id.toString().uppercase()}"
+            val attribIntent = InlinePresentationHelper.attributionPendingIntent(
+                context = context,
+                deepLinkUri = itemDeepLink,
+                requestCode = item.id.hashCode(),
+            )
+            val inline = InlinePresentationHelper.buildCredentialPresentation(
+                context = context,
+                spec = inlineSpec,
+                content = InlinePresentationHelper.CredentialContent(
+                    title = item.name.orEmpty(),
+                    subtitle = applyResult.labelSuffix,
+                    icon = bitmap,
+                ),
+                attributionIntent = attribIntent,
+            )
+            if (inline != null) {
+                builder.setInlinePresentation(inline)
+            }
+        }
+
         val autofillIds = fields.map { it.first }.toTypedArray()
         val fieldTypeOrdinals = IntArray(fields.size) { i -> fields[i].second.ordinal }
         val authIntent = Intent(context, AutofillFillActivity::class.java).apply {
@@ -92,15 +116,21 @@ object AutofillDatasetBuilder {
         context: Context,
         fields: List<Pair<AutofillId, FieldType>>,
         appInfo: String?,
+        inlineSpec: InlinePresentationSpec? = null,
     ): Dataset {
+        val label = context.getString(R.string.autofill_no_match_found)
         val presentation = RemoteViews(context.packageName, R.layout.autofill_dataset_item_logo)
-        presentation.setTextViewText(
-            R.id.text,
-            context.getString(R.string.autofill_no_match_found),
-        )
+        presentation.setTextViewText(R.id.text, label)
 
         val dataSetBuilder = Dataset.Builder(presentation)
 
+        if (inlineSpec != null) {
+            val inline = InlinePresentationHelper.buildActionPresentation(context, inlineSpec, label)
+            if (inline != null) {
+                dataSetBuilder.setInlinePresentation(inline)
+            }
+        }
+
         val encodedUrl = appInfo?.let { URLEncoder.encode(it, "UTF-8") } ?: ""
         val deepLinkUrl = "aliasvault://items/autofill-open-app?itemUrl=$encodedUrl"
 

apps/mobile-app/android/app/src/main/java/net/aliasvault/app/autofill/utils/InlinePresentationHelper.kt

@@ -0,0 +1,169 @@
+package net.aliasvault.app.autofill.utils
+
+import android.annotation.SuppressLint
+import android.app.PendingIntent
+import android.content.Context
+import android.content.Intent
+import android.graphics.Bitmap
+import android.graphics.BlendMode
+import android.graphics.drawable.Icon
+import android.service.autofill.InlinePresentation
+import android.util.Log
+import android.widget.inline.InlinePresentationSpec
+import androidx.autofill.inline.UiVersions
+import androidx.autofill.inline.v1.InlineSuggestionUi
+import net.aliasvault.app.MainActivity
+import net.aliasvault.app.R
+
+/**
+ * Builds [InlinePresentation] objects for keyboard inline autofill suggestions.
+ * The IME renders these alongside its own toolbar so users can pick a credential
+ * without ever opening the autofill dropdown.
+ */
+@SuppressLint("RestrictedApi")
+object InlinePresentationHelper {
+    private const val TAG = "AliasVaultAutofill"
+
+    /**
+     * Default attribution target — used when a caller does not supply a more
+     * specific deep link. Long-pressing the inline chip launches MainActivity.
+     */
+    private fun defaultAttributionIntent(context: Context): PendingIntent {
+        val intent = Intent(context, MainActivity::class.java).apply {
+            addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
+        }
+        return PendingIntent.getActivity(
+            context,
+            0,
+            intent,
+            PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE,
+        )
+    }
+
+    /**
+     * Build a long-press attribution PendingIntent that opens the given
+     * deep link (e.g. `aliasvault://items/<id>`). [requestCode] should be
+     * unique per destination so PendingIntent caching does not collapse
+     * separate items onto the same intent.
+     */
+    fun attributionPendingIntent(context: Context, deepLinkUri: String, requestCode: Int): PendingIntent {
+        val intent = Intent(Intent.ACTION_VIEW).apply {
+            data = android.net.Uri.parse(deepLinkUri)
+            addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
+        }
+        return PendingIntent.getActivity(
+            context,
+            requestCode,
+            intent,
+            PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE,
+        )
+    }
+
+    /**
+     * Returns true if the IME-provided [spec] supports the v1 inline UI
+     * surface we know how to build for.
+     */
+    fun supportsV1(spec: InlinePresentationSpec): Boolean {
+        return try {
+            UiVersions.getVersions(spec.style).contains(UiVersions.INLINE_UI_VERSION_1)
+        } catch (e: Exception) {
+            Log.w(TAG, "Failed to read inline spec versions", e)
+            false
+        }
+    }
+
+    /**
+     * Visual payload for a credential inline chip.
+     *
+     * @property title Primary line (typically the credential name).
+     * @property subtitle Secondary line (typically username or email); omitted if null/empty.
+     * @property icon Start-chip icon (typically the site logo); omitted if null.
+     */
+    data class CredentialContent(val title: String, val subtitle: String?, val icon: Bitmap?)
+
+    /**
+     * Build an inline presentation for a credential row. The icon, when
+     * provided, is shown as the start chip.
+     */
+    fun buildCredentialPresentation(
+        context: Context,
+        spec: InlinePresentationSpec,
+        content: CredentialContent,
+        attributionIntent: PendingIntent? = null,
+    ): InlinePresentation? {
+        if (!supportsV1(spec)) {
+            return null
+        }
+        return try {
+            val contentBuilder = InlineSuggestionUi
+                .newContentBuilder(attributionIntent ?: defaultAttributionIntent(context))
+                .setTitle(content.title)
+                .setContentDescription(content.title)
+            if (!content.subtitle.isNullOrEmpty()) {
+                contentBuilder.setSubtitle(content.subtitle)
+            }
+            if (content.icon != null) {
+                val icon = Icon.createWithBitmap(content.icon)
+                icon.setTintBlendMode(BlendMode.DST)
+                contentBuilder.setStartIcon(icon)
+            }
+            InlinePresentation(contentBuilder.build().slice, spec, false)
+        } catch (e: Exception) {
+            Log.w(TAG, "Failed to build inline credential presentation", e)
+            null
+        }
+    }
+
+    /**
+     * Hands out inline-presentation specs in dataset-add order, honouring the
+     * IME's maxSuggestionCount cap. Returns null once the budget is spent so
+     * callers can skip the inline call without changing the dropdown path.
+     */
+    class SpecPool(private val specs: List<InlinePresentationSpec>, maxCount: Int) {
+        private val budget: Int = minOf(maxCount, MAX_INLINE_BUDGET).coerceAtLeast(0)
+        private var consumed: Int = 0
+
+        /**
+         * Returns the next spec for an inline suggestion, or null if the
+         * caller should fall back to a dropdown-only dataset.
+         */
+        fun next(): InlinePresentationSpec? {
+            if (specs.isEmpty() || consumed >= budget) {
+                return null
+            }
+            val spec = specs.getOrElse(consumed) { specs.last() }
+            consumed++
+            return spec
+        }
+
+        companion object {
+            private const val MAX_INLINE_BUDGET = 20
+        }
+    }
+
+    /**
+     * Build an inline presentation for an action chip (open app, no match,
+     * vault locked, etc). Uses the AliasVault launcher icon.
+     */
+    fun buildActionPresentation(
+        context: Context,
+        spec: InlinePresentationSpec,
+        title: String,
+    ): InlinePresentation? {
+        if (!supportsV1(spec)) {
+            return null
+        }
+        return try {
+            val icon = Icon.createWithResource(context, R.drawable.av_logo)
+            val content = InlineSuggestionUi.newContentBuilder(defaultAttributionIntent(context))
+                .setTitle(title)
+                .setContentDescription(title)
+                .setStartIcon(icon)
+                .build()
+            InlinePresentation(content.slice, spec, false)
+        } catch (e: Exception) {
+            Log.w(TAG, "Failed to build inline action presentation", e)
+            null
+        }
+    }
+}