fix(debug): send typed characters to the mirrored device

The mirror's keyboard capture only mapped navigation keys and silently
dropped everything else, so text entry did nothing: printable characters
never left the client even though the firmware already injects kb_char.
Characters now go out as ANYKEY with the code point, Backspace goes as a
character (so text fields delete rather than navigating back, which stays
Esc's job), and the handling moves into a Modifier.remoteKeyInput
extension.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
James RichandClaude Fable 5 committed 2026-09-09 11:19:43 -05:00
1 parent 8e839085f4
commit 67d29570f2
3 files changed
+59 -11

No files matched your search

@@ -55,10 +55,12 @@ import androidx.compose.ui.focus.onFocusChanged
import androidx.compose.ui.geometry.Offset
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.hapticfeedback.HapticFeedbackType
import androidx.compose.ui.input.key.Key
import androidx.compose.ui.input.key.KeyEventType
import androidx.compose.ui.input.key.key
import androidx.compose.ui.input.key.onPreviewKeyEvent
import androidx.compose.ui.input.key.type
import androidx.compose.ui.input.key.utf16CodePoint
import androidx.compose.ui.input.pointer.pointerInput
import androidx.compose.ui.platform.LocalFocusManager
import androidx.compose.ui.platform.LocalHapticFeedback
@@ -160,6 +162,7 @@ fun DisplayMirrorContent(modifier: Modifier = Modifier, viewModel: DisplayMirror
hasTouch = displayInfo?.has_touch == true,
onEvent = viewModel::sendKey,
onTouch = viewModel::sendTouch,
onChar = viewModel::sendChar,
)
}
}
@@ -175,6 +178,7 @@ private fun MirrorWithControls(
hasTouch: Boolean,
onEvent: (Int) -> Unit,
onTouch: (Int, Int, Int) -> Unit,
onChar: (Int) -> Unit,
) {
BoxWithConstraints(modifier = Modifier.fillMaxWidth()) {
if (maxWidth >= SIDE_BY_SIDE_MIN_WIDTH) {
@@ -183,7 +187,15 @@ private fun MirrorWithControls(
verticalAlignment = Alignment.CenterVertically,
modifier = Modifier.fillMaxWidth(),
) {
MirrorSurface(frame, palette, hasTouch, onEvent, onTouch, modifier = Modifier.weight(1f, fill = false))
MirrorSurface(
frame,
palette,
hasTouch,
onEvent,
onTouch,
onChar,
modifier = Modifier.weight(1f, fill = false),
)
DpadCluster(enabled = enabled, onEvent = onEvent)
}
} else {
@@ -192,7 +204,7 @@ private fun MirrorWithControls(
verticalArrangement = Arrangement.spacedBy(12.dp),
modifier = Modifier.fillMaxWidth(),
) {
MirrorSurface(frame, palette, hasTouch, onEvent, onTouch)
MirrorSurface(frame, palette, hasTouch, onEvent, onTouch, onChar)
DpadCluster(enabled = enabled, onEvent = onEvent)
}
}
@@ -212,6 +224,7 @@ private fun MirrorSurface(
hasTouch: Boolean,
onEvent: (Int) -> Unit,
onTouch: (Int, Int, Int) -> Unit,
onChar: (Int) -> Unit,
modifier: Modifier = Modifier,
) {
val focusRequester = remember { FocusRequester() }
@@ -232,12 +245,7 @@ private fun MirrorSurface(
Modifier.focusRequester(focusRequester)
.onFocusChanged { focused = it.isFocused }
.focusable()
.onPreviewKeyEvent { event ->
val mapped = keyToInputEvent(event.key) ?: return@onPreviewKeyEvent false
// Consume KeyUp of mapped keys too, or Space/arrows leak to the scroll container.
if (event.type == KeyEventType.KeyDown) onEvent(mapped)
true
}
.remoteKeyInput(onEvent, onChar)
.pointerInput(hasTouch) {
detectTapGestures(
onTap = { offset ->
@@ -295,6 +303,35 @@ private fun MirrorControlHints(focused: Boolean, hasTouch: Boolean, onToggleKeyb
}
}
/**
* Sends captured key presses to the device: mapped navigation keys as input events, Backspace and printable characters
* as typed characters. Handled keys are consumed on both down and up so nothing leaks into the surrounding scroll
* container.
*/
private fun Modifier.remoteKeyInput(onEvent: (Int) -> Unit, onChar: (Int) -> Unit): Modifier =
onPreviewKeyEvent { event ->
val down = event.type == KeyEventType.KeyDown
val mapped = keyToInputEvent(event.key)
when {
mapped != null -> {
if (down) onEvent(mapped)
true
}
event.key == Key.Backspace -> {
if (down) onChar(CHAR_BACKSPACE)
true
}
event.utf16CodePoint >= FIRST_PRINTABLE_CHAR -> {
if (down) onChar(event.utf16CodePoint)
true
}
else -> false
}
}
/** Converts a completed drag into one direction event along its dominant axis, ignoring short accidental swipes. */
private fun Modifier.swipeToDirection(onEvent: (Int) -> Unit): Modifier = pointerInput(Unit) {
val threshold = SWIPE_THRESHOLD.toPx()
@@ -34,6 +34,16 @@ internal const val INPUT_RIGHT = 20
internal const val INPUT_BACK = 27
internal const val INPUT_USER_PRESS = 28
// ANYKEY carries a character in kb_char rather than a navigation code.
internal const val INPUT_ANYKEY = 0xFF
// LVGL's backspace key value; typed as a character so text fields delete
// rather than navigating back (which is what Esc is for).
internal const val CHAR_BACKSPACE = 8
// Below this, key events are control codes rather than typable text.
internal const val FIRST_PRINTABLE_CHAR = 32
/** Scales a tap position on the scaled-up mirror image back to panel pixel coordinates. */
internal fun Offset.toDeviceX(boxWidthPx: Int, frame: MirrorFrame): Int =
(x / boxWidthPx * frame.width).toInt().coerceIn(0, frame.width - 1)
@@ -55,9 +65,7 @@ internal fun keyToInputEvent(key: Key): Int? = when (key) {
Key.Spacebar,
-> INPUT_SELECT
Key.Escape,
Key.Backspace,
-> INPUT_BACK
Key.Escape -> INPUT_BACK
else -> null
}
@@ -83,6 +83,9 @@ class DisplayMirrorViewModel(
fun sendKey(eventCode: Int) = adminController.sendInputEvent(eventCode)
/** Forwards a typed character; the device UI receives it as a key press rather than a navigation event. */
fun sendChar(codePoint: Int) = adminController.sendInputEvent(INPUT_ANYKEY, kbChar = codePoint)
/** Forwards a tap/long-press on the mirrored image as a device touch event with panel coordinates. */
fun sendTouch(eventCode: Int, x: Int, y: Int) = adminController.sendInputEvent(eventCode, touchX = x, touchY = y)