refactor(debug): harden screen mirror from adversarial review

Manager: validate format (MONO_VLSB only) and geometry (width/height must
describe exactly total_size bytes) so malformed device input can't reach
aspectRatio and crash; unit tests for reassembly, drops, restarts.
AdminController: setDisplayMirror/sendInputEvent become non-suspend
immediate sends (teardown-safe, input never queues behind bulk admin) and
gain requestDisplayFrame. Mirror tab: render each frame once into a 1:1
ImageBitmap scaled with FilterQuality.None (kills the fractional-scale
seams), gate controls on connection state, reset the toggle on disconnect,
stop the stream on tab dispose and ViewModel clear, add a one-shot Refresh,
inject AdminController/ConnectionStateProvider instead of RadioController.
Dispatch: display_frame test coverage incl. revoked-session.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
James RichandClaude Fable 5 committed 2026-09-09 11:19:43 -05:00
1 parent 9d5d07d4c6
commit e34dea7ac1
10 files changed
+448 -73

No files matched your search

@@ -27,9 +27,9 @@ import org.meshtastic.proto.DisplayFrame
/**
* Display frame reassembly state machine.
*
* Chunks arrive in offset order within a frame (the firmware drains one snapshot before capturing the next), so a
* chunk with a new `frame_id` or `offset == 0` starts a new frame and an out-of-sequence chunk drops the partial
* frame. Called sequentially from [FromRadioPacketHandlerImpl] on a single IO coroutine.
* Chunks of one frame arrive contiguously and in offset order (FromRadio is a reliable ordered stream), so a chunk with
* a new `frame_id` or `offset == 0` starts a new frame and an out-of-sequence chunk drops the partial frame. Calls are
* serialized by the single receive-loop collector in MeshServiceOrchestrator (recreated per session).
*/
@Single
class DisplayMirrorManagerImpl : DisplayMirrorManager {
@@ -45,10 +45,7 @@ class DisplayMirrorManagerImpl : DisplayMirrorManager {
val data = chunk.data_.toByteArray()
val total = chunk.total_size
if (total <= 0 || total > MAX_FRAME_BYTES || chunk.offset + data.size > total) {
Logger.w { "DisplayMirror: dropping malformed chunk (offset=${chunk.offset} size=${data.size} total=$total)" }
return
}
if (!isAcceptableChunk(chunk, data.size)) return
if (chunk.offset == 0) {
buffer = ByteArray(total)
@@ -58,7 +55,7 @@ class DisplayMirrorManagerImpl : DisplayMirrorManager {
val buf = buffer
if (buf == null || chunk.frame_id != frameId || chunk.offset != received || buf.size != total) {
Logger.w { "DisplayMirror: out-of-sequence chunk (frame=${chunk.frame_id}/$frameId offset=${chunk.offset}/$received)" }
Logger.w { "DisplayMirror: bad chunk ${chunk.frame_id}@${chunk.offset}, expected $frameId@$received" }
buffer = null
return
}
@@ -72,8 +69,34 @@ class DisplayMirrorManagerImpl : DisplayMirrorManager {
}
}
/**
* Format must be MONO_VLSB and width/height must describe exactly total_size bytes; a zero or lying dimension would
* otherwise reach the renderer (aspectRatio requires > 0).
*/
private fun isAcceptableChunk(chunk: DisplayFrame, dataSize: Int): Boolean {
val total = chunk.total_size
val expectedSize = chunk.width * ((chunk.height + PIXELS_PER_PAGE - 1) / PIXELS_PER_PAGE)
val acceptable =
chunk.format == DisplayFrame.Format.MONO_VLSB &&
chunk.width > 0 &&
chunk.height > 0 &&
expectedSize == total &&
total <= MAX_FRAME_BYTES &&
chunk.offset + dataSize <= total
if (!acceptable) {
Logger.w {
"DisplayMirror: dropping bad chunk (format=${chunk.format} ${chunk.width}x${chunk.height} " +
"offset=${chunk.offset} size=$dataSize total=$total)"
}
}
return acceptable
}
private companion object {
// Generous sanity cap: 320x240 at 1bpp is 9600 bytes.
const val MAX_FRAME_BYTES = 16384
// MONO_VLSB packs 8 vertically adjacent pixels per byte (one "page" row).
const val PIXELS_PER_PAGE = 8
}
}
@@ -46,6 +46,7 @@ import org.meshtastic.core.resources.key_verification_request_title
import org.meshtastic.core.resources.key_verification_title
import org.meshtastic.core.resources.low_entropy_key_title
import org.meshtastic.proto.ClientNotification
import org.meshtastic.proto.DisplayFrame
import org.meshtastic.proto.FromRadio
/** Implementation of [FromRadioPacketHandler] that dispatches [FromRadio] variants to specialized handlers. */
@@ -134,10 +135,7 @@ class FromRadioPacketHandlerImpl(
xmodemPacket != null ->
runIfSessionActive(session, "XModem packet") { xmodemManager.value.handleIncomingXModem(xmodemPacket) }
displayFrame != null ->
runIfSessionActive(session, "display frame") {
displayMirrorManager.value.handleIncomingFrame(displayFrame)
}
displayFrame != null -> handleDisplayFrame(displayFrame, session)
lockdownStatus != null ->
runIfSessionActive(session, "lockdown status") {
@@ -155,6 +153,9 @@ class FromRadioPacketHandlerImpl(
}
}
private fun handleDisplayFrame(frame: DisplayFrame, session: RadioSessionContext) =
runIfSessionActive(session, "display frame") { displayMirrorManager.value.handleIncomingFrame(frame) }
private fun runIfSessionActive(session: RadioSessionContext, operation: String, block: () -> Unit) {
if (!radioInterfaceService.runIfSessionActive(session, block)) {
Logger.d { "Discarding $operation from stale transport session" }
@@ -0,0 +1,134 @@
/*
* Copyright (c) 2026 Meshtastic LLC
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.meshtastic.core.data.manager
import okio.ByteString.Companion.toByteString
import org.meshtastic.proto.DisplayFrame
import kotlin.test.Test
import kotlin.test.assertContentEquals
import kotlin.test.assertEquals
import kotlin.test.assertNull
class DisplayMirrorManagerImplTest {
private val manager = DisplayMirrorManagerImpl()
// 128x64 MONO_VLSB frame = 1024 bytes; the firmware chunks at 384.
private fun chunk(
frameId: Int = 1,
offset: Int = 0,
data: ByteArray,
total: Int = FRAME_BYTES,
width: Int = WIDTH,
height: Int = HEIGHT,
format: DisplayFrame.Format = DisplayFrame.Format.MONO_VLSB,
) = DisplayFrame(
width = width,
height = height,
format = format,
frame_id = frameId,
offset = offset,
total_size = total,
data_ = data.toByteString(),
)
private fun bytes(size: Int, fill: Int) = ByteArray(size) { fill.toByte() }
@Test
fun `reassembles a three-chunk frame in order`() {
manager.handleIncomingFrame(chunk(offset = 0, data = bytes(384, 1)))
manager.handleIncomingFrame(chunk(offset = 384, data = bytes(384, 2)))
assertNull(manager.frame.value)
manager.handleIncomingFrame(chunk(offset = 768, data = bytes(256, 3)))
val frame = manager.frame.value!!
assertEquals(WIDTH, frame.width)
assertEquals(HEIGHT, frame.height)
assertEquals(1, frame.frameId)
assertContentEquals(bytes(384, 1) + bytes(384, 2) + bytes(256, 3), frame.pixels)
}
@Test
fun `single-chunk frame completes immediately`() {
// 64x32 = 256 bytes fits one chunk
manager.handleIncomingFrame(chunk(width = 64, height = 32, total = 256, data = bytes(256, 7)))
assertEquals(256, manager.frame.value?.pixels?.size)
}
@Test
fun `out-of-sequence chunk drops the partial frame`() {
manager.handleIncomingFrame(chunk(offset = 0, data = bytes(384, 1)))
manager.handleIncomingFrame(chunk(offset = 768, data = bytes(256, 3))) // gap: 384 missing
manager.handleIncomingFrame(chunk(offset = 384, data = bytes(384, 2))) // too late
assertNull(manager.frame.value)
}
@Test
fun `offset zero restarts mid-frame on device reboot or torn capture`() {
manager.handleIncomingFrame(chunk(frameId = 5, offset = 0, data = bytes(384, 1)))
// New frame id starting over at offset 0 wins
manager.handleIncomingFrame(chunk(frameId = 6, offset = 0, data = bytes(384, 4)))
manager.handleIncomingFrame(chunk(frameId = 6, offset = 384, data = bytes(384, 5)))
manager.handleIncomingFrame(chunk(frameId = 6, offset = 768, data = bytes(256, 6)))
assertEquals(6, manager.frame.value?.frameId)
}
@Test
fun `frame id change mid-frame without offset zero is dropped`() {
manager.handleIncomingFrame(chunk(frameId = 1, offset = 0, data = bytes(384, 1)))
manager.handleIncomingFrame(chunk(frameId = 2, offset = 384, data = bytes(384, 2)))
assertNull(manager.frame.value)
}
@Test
fun `rejects geometry that does not match total size`() {
manager.handleIncomingFrame(
chunk(width = 64, height = 64, total = 256, data = bytes(256, 1)),
) // 64x64 needs 512
manager.handleIncomingFrame(chunk(width = 0, height = 64, total = 0, data = ByteArray(0)))
manager.handleIncomingFrame(chunk(width = 128, height = 0, total = 0, data = ByteArray(0)))
assertNull(manager.frame.value)
}
@Test
fun `rejects unsupported format`() {
manager.handleIncomingFrame(
chunk(
width = 64,
height = 32,
total = 256,
data = bytes(256, 1),
format = DisplayFrame.Format.FORMAT_UNSPECIFIED,
),
)
assertNull(manager.frame.value)
}
private companion object {
const val WIDTH = 128
const val HEIGHT = 64
const val FRAME_BYTES = 1024
}
}
@@ -0,0 +1,134 @@
/*
* Copyright (c) 2026 Meshtastic LLC
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.meshtastic.core.data.manager
import okio.ByteString.Companion.toByteString
import org.meshtastic.proto.DisplayFrame
import kotlin.test.Test
import kotlin.test.assertContentEquals
import kotlin.test.assertEquals
import kotlin.test.assertNull
class DisplayMirrorManagerImplTest {
private val manager = DisplayMirrorManagerImpl()
// 128x64 MONO_VLSB frame = 1024 bytes; the firmware chunks at 384.
private fun chunk(
frameId: Int = 1,
offset: Int = 0,
data: ByteArray,
total: Int = FRAME_BYTES,
width: Int = WIDTH,
height: Int = HEIGHT,
format: DisplayFrame.Format = DisplayFrame.Format.MONO_VLSB,
) = DisplayFrame(
width = width,
height = height,
format = format,
frame_id = frameId,
offset = offset,
total_size = total,
data_ = data.toByteString(),
)
private fun bytes(size: Int, fill: Int) = ByteArray(size) { fill.toByte() }
@Test
fun `reassembles a three-chunk frame in order`() {
manager.handleIncomingFrame(chunk(offset = 0, data = bytes(384, 1)))
manager.handleIncomingFrame(chunk(offset = 384, data = bytes(384, 2)))
assertNull(manager.frame.value)
manager.handleIncomingFrame(chunk(offset = 768, data = bytes(256, 3)))
val frame = manager.frame.value!!
assertEquals(WIDTH, frame.width)
assertEquals(HEIGHT, frame.height)
assertEquals(1, frame.frameId)
assertContentEquals(bytes(384, 1) + bytes(384, 2) + bytes(256, 3), frame.pixels)
}
@Test
fun `single-chunk frame completes immediately`() {
// 64x32 = 256 bytes fits one chunk
manager.handleIncomingFrame(chunk(width = 64, height = 32, total = 256, data = bytes(256, 7)))
assertEquals(256, manager.frame.value?.pixels?.size)
}
@Test
fun `out-of-sequence chunk drops the partial frame`() {
manager.handleIncomingFrame(chunk(offset = 0, data = bytes(384, 1)))
manager.handleIncomingFrame(chunk(offset = 768, data = bytes(256, 3))) // gap: 384 missing
manager.handleIncomingFrame(chunk(offset = 384, data = bytes(384, 2))) // too late
assertNull(manager.frame.value)
}
@Test
fun `offset zero restarts mid-frame (device reboot or torn capture)`() {
manager.handleIncomingFrame(chunk(frameId = 5, offset = 0, data = bytes(384, 1)))
// New frame id starting over at offset 0 wins
manager.handleIncomingFrame(chunk(frameId = 6, offset = 0, data = bytes(384, 4)))
manager.handleIncomingFrame(chunk(frameId = 6, offset = 384, data = bytes(384, 5)))
manager.handleIncomingFrame(chunk(frameId = 6, offset = 768, data = bytes(256, 6)))
assertEquals(6, manager.frame.value?.frameId)
}
@Test
fun `frame id change mid-frame without offset zero is dropped`() {
manager.handleIncomingFrame(chunk(frameId = 1, offset = 0, data = bytes(384, 1)))
manager.handleIncomingFrame(chunk(frameId = 2, offset = 384, data = bytes(384, 2)))
assertNull(manager.frame.value)
}
@Test
fun `rejects geometry that does not match total size`() {
manager.handleIncomingFrame(
chunk(width = 64, height = 64, total = 256, data = bytes(256, 1)),
) // 64x64 needs 512
manager.handleIncomingFrame(chunk(width = 0, height = 64, total = 0, data = ByteArray(0)))
manager.handleIncomingFrame(chunk(width = 128, height = 0, total = 0, data = ByteArray(0)))
assertNull(manager.frame.value)
}
@Test
fun `rejects unsupported format`() {
manager.handleIncomingFrame(
chunk(
width = 64,
height = 32,
total = 256,
data = bytes(256, 1),
format = DisplayFrame.Format.FORMAT_UNSPECIFIED,
),
)
assertNull(manager.frame.value)
}
private companion object {
const val WIDTH = 128
const val HEIGHT = 64
const val FRAME_BYTES = 1024
}
}
@@ -27,6 +27,7 @@ import dev.mokkery.verify
import dev.mokkery.verify.VerifyMode
import dev.mokkery.verifySuspend
import org.meshtastic.core.model.util.isOtaStatusNotification
import org.meshtastic.core.repository.DisplayMirrorManager
import org.meshtastic.core.repository.FirmwareUpdateStatusRepository
import org.meshtastic.core.repository.MeshConfigFlowManager
import org.meshtastic.core.repository.MeshConfigHandler
@@ -43,6 +44,7 @@ import org.meshtastic.proto.Channel
import org.meshtastic.proto.ClientNotification
import org.meshtastic.proto.Config
import org.meshtastic.proto.DeviceMetadata
import org.meshtastic.proto.DisplayFrame
import org.meshtastic.proto.FromRadio
import org.meshtastic.proto.LoRaRegionPresetMap
import org.meshtastic.proto.LockdownStatus
@@ -68,6 +70,7 @@ class FromRadioPacketHandlerImplTest {
private val configFlowManager: MeshConfigFlowManager = mock(MockMode.autofill)
private val configHandler: MeshConfigHandler = mock(MockMode.autofill)
private val xmodemManager: XModemManager = mock(MockMode.autofill)
private val displayMirrorManager: DisplayMirrorManager = mock(MockMode.autofill)
private val lockdownCoordinator = FakeLockdownCoordinator()
private val firmwareUpdateStatusRepository = FirmwareUpdateStatusRepository()
private val radioInterfaceService: RadioInterfaceService = mock(MockMode.autofill)
@@ -105,6 +108,7 @@ class FromRadioPacketHandlerImplTest {
lazy { configFlowManager },
lazy { configHandler },
lazy { xmodemManager },
lazy { displayMirrorManager },
mqttManager,
packetHandler,
notificationManager,
@@ -220,13 +224,24 @@ class FromRadioPacketHandlerImplTest {
handle(FromRadio(queueStatus = queueStatus))
handle(FromRadio(xmodemPacket = xmodemPacket))
handle(FromRadio(lockdown_status = lockdownStatus))
handle(FromRadio(display_frame = DisplayFrame(width = 128, height = 64)))
verify(mode = VerifyMode.exactly(0)) { mqttManager.handleMqttProxyMessage(any()) }
verify(mode = VerifyMode.exactly(0)) { packetHandler.handleQueueStatus(any()) }
verify(mode = VerifyMode.exactly(0)) { xmodemManager.handleIncomingXModem(any()) }
verify(mode = VerifyMode.exactly(0)) { displayMirrorManager.handleIncomingFrame(any()) }
assertEquals(null, lockdownCoordinator.lastStatus)
}
@Test
fun `handleFromRadio routes DISPLAY_FRAME to displayMirrorManager`() {
val frame = DisplayFrame(width = 128, height = 64, frame_id = 1, total_size = 1024)
handle(FromRadio(display_frame = frame))
verify { displayMirrorManager.handleIncomingFrame(frame) }
}
@Test
fun `handleFromRadio routes LOCKDOWN_STATUS to lockdownCoordinator`() {
val lockdownStatus = LockdownStatus(state = LockdownStatus.State.LOCKED, lock_reason = "token_missing")
@@ -53,15 +53,20 @@ interface AdminController {
/**
* Enables or disables live display mirroring on the locally connected node. While enabled, the device streams its
* framebuffer as `FromRadio.display_frame` chunks — collect them via [DisplayMirrorManager].
* framebuffer as `FromRadio.display_frame` chunks — collect them via [DisplayMirrorManager]. Non-suspend (immediate
* send, bypassing the outbound FIFO) so lifecycle teardown can reliably disable the stream.
*/
suspend fun setDisplayMirror(enabled: Boolean)
fun setDisplayMirror(enabled: Boolean)
/** Requests a single framebuffer frame from the locally connected node, delivered like mirrored frames. */
fun requestDisplayFrame()
/**
* Injects a physical input event (button/key/touch) into the locally connected node's UI via the firmware
* InputBroker. Event codes are the firmware's `input_broker_event` values.
* InputBroker. Event codes are the firmware's `input_broker_event` values. Non-suspend immediate send: input must
* not queue behind bulk admin traffic.
*/
suspend fun sendInputEvent(eventCode: Int, kbChar: Int = 0, touchX: Int = 0, touchY: Int = 0)
fun sendInputEvent(eventCode: Int, kbChar: Int = 0, touchX: Int = 0, touchY: Int = 0)
// ── Remote configuration ────────────────────────────────────────────────
@@ -22,9 +22,9 @@ import org.meshtastic.proto.DisplayFrame
/**
* Reassembles the chunked [DisplayFrame] stream from the connected device into complete framebuffer snapshots.
*
* The device streams its 1bpp framebuffer as `FromRadio.display_frame` chunks while display mirroring is enabled
* (see `AdminMessage.set_display_mirror`). Chunks of one frame share a `frame_id`; a frame is complete when
* `offset + data.size == total_size`.
* The device streams its 1bpp framebuffer as `FromRadio.display_frame` chunks while display mirroring is enabled (see
* `AdminMessage.set_display_mirror`). Chunks of one frame share a `frame_id`; a frame is complete when `offset +
* data.size == total_size`.
*/
interface DisplayMirrorManager {
/** Latest completely reassembled frame, or null before the first one arrives. */
@@ -37,16 +37,15 @@ interface DisplayMirrorManager {
/**
* One complete device framebuffer snapshot.
*
* [pixels] is MONO_VLSB: 1 bit per pixel in vertical LSB-first pages — byte index = `x + (y / 8) * width`,
* bit index = `y % 8`.
* [pixels] is MONO_VLSB: 1 bit per pixel in vertical LSB-first pages — byte index = `x + (y / 8) * width`, bit index =
* `y % 8`.
*/
data class MirrorFrame(val width: Int, val height: Int, val frameId: Int, val pixels: ByteArray) {
override fun equals(other: Any?): Boolean =
other is MirrorFrame &&
other.width == width &&
other.height == height &&
other.frameId == frameId &&
other.pixels.contentEquals(pixels)
override fun equals(other: Any?): Boolean = other is MirrorFrame &&
other.width == width &&
other.height == height &&
other.frameId == frameId &&
other.pixels.contentEquals(pixels)
override fun hashCode(): Int {
var result = width
@@ -125,20 +125,24 @@ internal class AdminControllerImpl(
scope.handledLaunch { radioConfigRepository.setLocalConfig(config) }
}
override suspend fun setDisplayMirror(enabled: Boolean) {
commandSender.sendAdmin(myNodeNum) { AdminMessage(set_display_mirror = enabled) }
override fun setDisplayMirror(enabled: Boolean) {
commandSender.sendAdminImmediate(myNodeNum) { AdminMessage(set_display_mirror = enabled) }
}
override suspend fun sendInputEvent(eventCode: Int, kbChar: Int, touchX: Int, touchY: Int) {
commandSender.sendAdmin(myNodeNum) {
override fun requestDisplayFrame() {
commandSender.sendAdminImmediate(myNodeNum) { AdminMessage(get_display_frame_request = true) }
}
override fun sendInputEvent(eventCode: Int, kbChar: Int, touchX: Int, touchY: Int) {
commandSender.sendAdminImmediate(myNodeNum) {
AdminMessage(
send_input_event =
AdminMessage.InputEvent(
event_code = eventCode,
kb_char = kbChar,
touch_x = touchX,
touch_y = touchY,
),
AdminMessage.InputEvent(
event_code = eventCode,
kb_char = kbChar,
touch_x = touchX,
touch_y = touchY,
),
)
}
}
@@ -252,6 +252,12 @@ class FakeRadioController :
override suspend fun refreshMetadata(destNum: Int) {}
override fun setDisplayMirror(enabled: Boolean) {}
override fun requestDisplayFrame() {}
override fun sendInputEvent(eventCode: Int, kbChar: Int, touchX: Int, touchY: Int) {}
override suspend fun setLocalConfig(config: Config) {
if (throwOnSetLocalConfig) error("Fake local config write failure")
if (rejectLocalConfigWritesRemaining > 0) {
@@ -16,8 +16,7 @@
*/
package org.meshtastic.feature.settings.debugging
import androidx.compose.foundation.Canvas
import androidx.compose.foundation.background
import androidx.compose.foundation.Image
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
@@ -30,32 +29,49 @@ import androidx.compose.foundation.rememberScrollState
import androidx.compose.foundation.verticalScroll
import androidx.compose.material3.FilledTonalButton
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.OutlinedButton
import androidx.compose.material3.Switch
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.DisposableEffect
import androidx.compose.runtime.getValue
import androidx.compose.runtime.remember
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.geometry.Offset
import androidx.compose.ui.geometry.Size
import androidx.compose.ui.graphics.Canvas
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.graphics.FilterQuality
import androidx.compose.ui.graphics.ImageBitmap
import androidx.compose.ui.graphics.drawscope.CanvasDrawScope
import androidx.compose.ui.unit.Density
import androidx.compose.ui.unit.LayoutDirection
import androidx.compose.ui.unit.dp
import androidx.lifecycle.ViewModel
import androidx.lifecycle.compose.collectAsStateWithLifecycle
import androidx.lifecycle.viewModelScope
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.SharingStarted
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.flow.map
import kotlinx.coroutines.flow.stateIn
import kotlinx.coroutines.launch
import org.koin.compose.viewmodel.koinViewModel
import org.koin.core.annotation.KoinViewModel
import org.meshtastic.core.model.ConnectionState
import org.meshtastic.core.repository.AdminController
import org.meshtastic.core.repository.ConnectionStateProvider
import org.meshtastic.core.repository.DisplayMirrorManager
import org.meshtastic.core.repository.MirrorFrame
import org.meshtastic.core.repository.RadioController
// 4x scale for the common 128px-wide OLED; caps the canvas so the D-pad stays above the fold on desktop.
// 4x scale for the common 128px-wide OLED; caps the image so the D-pad stays above the fold on desktop.
private val MAX_CANVAS_WIDTH = 512.dp
// MONO_VLSB packs 8 vertically adjacent pixels per byte (one "page" row).
private const val PIXELS_PER_PAGE = 8
// Firmware input_broker_event codes (src/input/InputBroker.h).
private const val INPUT_SELECT = 10
private const val INPUT_UP = 17
@@ -67,24 +83,41 @@ private const val INPUT_BACK = 27
@KoinViewModel
class DisplayMirrorViewModel(
displayMirrorManager: DisplayMirrorManager,
private val radioController: RadioController,
connectionStateProvider: ConnectionStateProvider,
private val adminController: AdminController,
) : ViewModel() {
val frame: StateFlow<MirrorFrame?> = displayMirrorManager.frame
val connected: StateFlow<Boolean> =
connectionStateProvider.connectionState
.map { it is ConnectionState.Connected }
.stateIn(viewModelScope, SharingStarted.Eagerly, false)
private val _mirroring = MutableStateFlow(false)
val mirroring: StateFlow<Boolean> = _mirroring.asStateFlow()
fun setMirror(enabled: Boolean) {
viewModelScope.launch {
radioController.setDisplayMirror(enabled)
_mirroring.value = enabled
}
init {
// The device forgets the (non-persisted) mirror setting on disconnect/reboot;
// mirror the reset locally so the toggle never claims a dead stream is live.
viewModelScope.launch { connected.collect { if (!it) _mirroring.value = false } }
}
fun sendKey(eventCode: Int) {
viewModelScope.launch { radioController.sendInputEvent(eventCode) }
fun setMirror(enabled: Boolean) {
adminController.setDisplayMirror(enabled)
_mirroring.value = enabled
}
fun requestFrame() = adminController.requestDisplayFrame()
fun sendKey(eventCode: Int) = adminController.sendInputEvent(eventCode)
/** Stops a live stream when the mirror UI goes away; safe to call redundantly. */
fun stopMirroring() {
if (_mirroring.value) setMirror(false)
}
override fun onCleared() = stopMirroring()
}
/** PoC live view of the connected device's screen, with remote D-pad control. Strings are deliberately unlocalized. */
@@ -92,6 +125,10 @@ class DisplayMirrorViewModel(
fun DisplayMirrorContent(modifier: Modifier = Modifier, viewModel: DisplayMirrorViewModel = koinViewModel()) {
val frame by viewModel.frame.collectAsStateWithLifecycle()
val mirroring by viewModel.mirroring.collectAsStateWithLifecycle()
val connected by viewModel.connected.collectAsStateWithLifecycle()
// Don't leave the device streaming to a hidden tab or abandoned screen.
DisposableEffect(Unit) { onDispose { viewModel.stopMirroring() } }
Column(
modifier = modifier.fillMaxSize().verticalScroll(rememberScrollState()).padding(16.dp),
@@ -100,8 +137,9 @@ fun DisplayMirrorContent(modifier: Modifier = Modifier, viewModel: DisplayMirror
) {
val currentFrame = frame
Row(verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(8.dp)) {
Switch(checked = mirroring, onCheckedChange = viewModel::setMirror)
Switch(checked = mirroring, onCheckedChange = viewModel::setMirror, enabled = connected)
Text(text = if (mirroring) "Mirroring" else "Mirror off", style = MaterialTheme.typography.titleMedium)
OutlinedButton(onClick = viewModel::requestFrame, enabled = connected) { Text("Refresh") }
if (currentFrame != null) {
Text(
text = "${currentFrame.width}x${currentFrame.height} frame #${currentFrame.frameId}",
@@ -110,46 +148,62 @@ fun DisplayMirrorContent(modifier: Modifier = Modifier, viewModel: DisplayMirror
}
}
if (currentFrame != null) {
MirrorFrameCanvas(currentFrame)
} else {
Text(text = "No frame received yet — enable mirroring above.")
when {
!connected -> Text(text = "Not connected to a device.")
currentFrame != null -> MirrorFrameImage(currentFrame)
else -> Text(text = "No frame received yet — enable mirroring or tap Refresh.")
}
// Remote D-pad
Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) {
FilledTonalButton(onClick = { viewModel.sendKey(INPUT_UP) }) { Text("Up") }
FilledTonalButton(onClick = { viewModel.sendKey(INPUT_UP) }, enabled = connected) { Text("Up") }
}
Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) {
FilledTonalButton(onClick = { viewModel.sendKey(INPUT_LEFT) }) { Text("Left") }
FilledTonalButton(onClick = { viewModel.sendKey(INPUT_SELECT) }) { Text("OK") }
FilledTonalButton(onClick = { viewModel.sendKey(INPUT_RIGHT) }) { Text("Right") }
FilledTonalButton(onClick = { viewModel.sendKey(INPUT_LEFT) }, enabled = connected) { Text("Left") }
FilledTonalButton(onClick = { viewModel.sendKey(INPUT_SELECT) }, enabled = connected) { Text("OK") }
FilledTonalButton(onClick = { viewModel.sendKey(INPUT_RIGHT) }, enabled = connected) { Text("Right") }
}
Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) {
FilledTonalButton(onClick = { viewModel.sendKey(INPUT_DOWN) }) { Text("Down") }
FilledTonalButton(onClick = { viewModel.sendKey(INPUT_BACK) }) { Text("Back") }
FilledTonalButton(onClick = { viewModel.sendKey(INPUT_DOWN) }, enabled = connected) { Text("Down") }
FilledTonalButton(onClick = { viewModel.sendKey(INPUT_BACK) }, enabled = connected) { Text("Back") }
}
}
}
/** Draws a MONO_VLSB 1bpp framebuffer scaled up, one filled rect per lit pixel. Width-capped so the controls stay in view on desktop. */
/**
* Renders a MONO_VLSB 1bpp framebuffer once per frame into a 1:1 [ImageBitmap] and scales it up with nearest-neighbor
* filtering — crisp device pixels, no fractional-scale seams, one pixel walk per frame instead of per recomposition.
*/
@Composable
private fun MirrorFrameCanvas(frame: MirrorFrame) {
val aspect = frame.width.toFloat() / frame.height.toFloat()
Canvas(
modifier = Modifier.widthIn(max = MAX_CANVAS_WIDTH).fillMaxWidth().aspectRatio(aspect).background(Color.Black),
) {
val scale = size.width / frame.width
val pixel = Size(scale, scale)
private fun MirrorFrameImage(frame: MirrorFrame, modifier: Modifier = Modifier) {
val bitmap = remember(frame) { renderFrame(frame) }
Image(
bitmap = bitmap,
contentDescription = "Device screen",
modifier =
modifier
.widthIn(max = MAX_CANVAS_WIDTH)
.fillMaxWidth()
.aspectRatio(frame.width.toFloat() / frame.height.toFloat()),
filterQuality = FilterQuality.None,
)
}
private fun renderFrame(frame: MirrorFrame): ImageBitmap {
val bitmap = ImageBitmap(frame.width, frame.height)
val size = Size(frame.width.toFloat(), frame.height.toFloat())
CanvasDrawScope().draw(Density(1f), LayoutDirection.Ltr, Canvas(bitmap), size) {
drawRect(color = Color.Black)
val pixel = Size(1f, 1f)
for (y in 0 until frame.height) {
val page = (y / 8) * frame.width
val bit = 1 shl (y % 8)
val page = (y / PIXELS_PER_PAGE) * frame.width
val bit = 1 shl (y % PIXELS_PER_PAGE)
for (x in 0 until frame.width) {
val index = page + x
if (index < frame.pixels.size && frame.pixels[index].toInt() and bit != 0) {
drawRect(color = Color.White, topLeft = Offset(x * scale, y * scale), size = pixel)
if (frame.pixels[page + x].toInt() and bit != 0) {
drawRect(color = Color.White, topLeft = Offset(x.toFloat(), y.toFloat()), size = pixel)
}
}
}
}
return bitmap
}