diff --git a/core/data/src/commonMain/kotlin/org/meshtastic/core/data/manager/DisplayMirrorManagerImpl.kt b/core/data/src/commonMain/kotlin/org/meshtastic/core/data/manager/DisplayMirrorManagerImpl.kt new file mode 100644 index 0000000000..6c273a2caf --- /dev/null +++ b/core/data/src/commonMain/kotlin/org/meshtastic/core/data/manager/DisplayMirrorManagerImpl.kt @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.core.data.manager + +import co.touchlab.kermit.Logger +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.asStateFlow +import org.koin.core.annotation.Single +import org.meshtastic.core.repository.DisplayMirrorManager +import org.meshtastic.core.repository.MirrorFrame +import org.meshtastic.proto.DisplayFrame + +/** + * Display frame reassembly state machine. + * + * Chunks arrive in offset order within a frame (the firmware drains one snapshot before capturing the next), so a + * chunk with a new `frame_id` or `offset == 0` starts a new frame and an out-of-sequence chunk drops the partial + * frame. Called sequentially from [FromRadioPacketHandlerImpl] on a single IO coroutine. + */ +@Single +class DisplayMirrorManagerImpl : DisplayMirrorManager { + + private val _frame = MutableStateFlow(null) + override val frame = _frame.asStateFlow() + + private var buffer: ByteArray? = null + private var frameId = 0 + private var received = 0 + + override fun handleIncomingFrame(chunk: DisplayFrame) { + val data = chunk.data_.toByteArray() + val total = chunk.total_size + + if (total <= 0 || total > MAX_FRAME_BYTES || chunk.offset + data.size > total) { + Logger.w { "DisplayMirror: dropping malformed chunk (offset=${chunk.offset} size=${data.size} total=$total)" } + return + } + + if (chunk.offset == 0) { + buffer = ByteArray(total) + frameId = chunk.frame_id + received = 0 + } + + val buf = buffer + if (buf == null || chunk.frame_id != frameId || chunk.offset != received || buf.size != total) { + Logger.w { "DisplayMirror: out-of-sequence chunk (frame=${chunk.frame_id}/$frameId offset=${chunk.offset}/$received)" } + buffer = null + return + } + + data.copyInto(buf, chunk.offset) + received += data.size + + if (received == total) { + _frame.value = MirrorFrame(width = chunk.width, height = chunk.height, frameId = frameId, pixels = buf) + buffer = null + } + } + + private companion object { + // Generous sanity cap: 320x240 at 1bpp is 9600 bytes. + const val MAX_FRAME_BYTES = 16384 + } +} diff --git a/core/data/src/commonMain/kotlin/org/meshtastic/core/data/manager/FromRadioPacketHandlerImpl.kt b/core/data/src/commonMain/kotlin/org/meshtastic/core/data/manager/FromRadioPacketHandlerImpl.kt index 13cfbfa1c2..72eca2766d 100644 --- a/core/data/src/commonMain/kotlin/org/meshtastic/core/data/manager/FromRadioPacketHandlerImpl.kt +++ b/core/data/src/commonMain/kotlin/org/meshtastic/core/data/manager/FromRadioPacketHandlerImpl.kt @@ -22,6 +22,7 @@ import kotlinx.coroutines.SupervisorJob import org.koin.core.annotation.Single import org.meshtastic.core.common.util.ioDispatcher import org.meshtastic.core.model.util.isOtaStatusNotification +import org.meshtastic.core.repository.DisplayMirrorManager import org.meshtastic.core.repository.FirmwareUpdateStatusRepository import org.meshtastic.core.repository.FromRadioPacketHandler import org.meshtastic.core.repository.LockdownCoordinator @@ -55,6 +56,7 @@ class FromRadioPacketHandlerImpl( private val configFlowManager: Lazy, private val configHandler: Lazy, private val xmodemManager: Lazy, + private val displayMirrorManager: Lazy, private val mqttManager: MqttManager, private val packetHandler: PacketHandler, private val notificationManager: NotificationManager, @@ -84,6 +86,7 @@ class FromRadioPacketHandlerImpl( val regionPresets = proto.region_presets val xmodemPacket = proto.xmodemPacket val lockdownStatus = proto.lockdown_status + val displayFrame = proto.display_frame when { myInfo != null -> configFlowManager.value.handleMyInfo(myInfo, session) @@ -131,6 +134,11 @@ class FromRadioPacketHandlerImpl( xmodemPacket != null -> runIfSessionActive(session, "XModem packet") { xmodemManager.value.handleIncomingXModem(xmodemPacket) } + displayFrame != null -> + runIfSessionActive(session, "display frame") { + displayMirrorManager.value.handleIncomingFrame(displayFrame) + } + lockdownStatus != null -> runIfSessionActive(session, "lockdown status") { lockdownCoordinator.handleLockdownStatus(lockdownStatus) diff --git a/core/repository/src/commonMain/kotlin/org/meshtastic/core/repository/AdminController.kt b/core/repository/src/commonMain/kotlin/org/meshtastic/core/repository/AdminController.kt index 552a56ffcd..0b63234f2b 100644 --- a/core/repository/src/commonMain/kotlin/org/meshtastic/core/repository/AdminController.kt +++ b/core/repository/src/commonMain/kotlin/org/meshtastic/core/repository/AdminController.kt @@ -51,6 +51,18 @@ interface AdminController { /** Updates a local radio channel. Same fire-and-forget contract as [setLocalConfig]. */ suspend fun setLocalChannel(channel: Channel) + /** + * Enables or disables live display mirroring on the locally connected node. While enabled, the device streams its + * framebuffer as `FromRadio.display_frame` chunks — collect them via [DisplayMirrorManager]. + */ + suspend fun setDisplayMirror(enabled: Boolean) + + /** + * Injects a physical input event (button/key/touch) into the locally connected node's UI via the firmware + * InputBroker. Event codes are the firmware's `input_broker_event` values. + */ + suspend fun sendInputEvent(eventCode: Int, kbChar: Int = 0, touchX: Int = 0, touchY: Int = 0) + // ── Remote configuration ──────────────────────────────────────────────── /** Updates the owner (user info) on a remote node. */ diff --git a/core/repository/src/commonMain/kotlin/org/meshtastic/core/repository/DisplayMirrorManager.kt b/core/repository/src/commonMain/kotlin/org/meshtastic/core/repository/DisplayMirrorManager.kt new file mode 100644 index 0000000000..aaabf8a87e --- /dev/null +++ b/core/repository/src/commonMain/kotlin/org/meshtastic/core/repository/DisplayMirrorManager.kt @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.core.repository + +import kotlinx.coroutines.flow.StateFlow +import org.meshtastic.proto.DisplayFrame + +/** + * Reassembles the chunked [DisplayFrame] stream from the connected device into complete framebuffer snapshots. + * + * The device streams its 1bpp framebuffer as `FromRadio.display_frame` chunks while display mirroring is enabled + * (see `AdminMessage.set_display_mirror`). Chunks of one frame share a `frame_id`; a frame is complete when + * `offset + data.size == total_size`. + */ +interface DisplayMirrorManager { + /** Latest completely reassembled frame, or null before the first one arrives. */ + val frame: StateFlow + + /** Routes an incoming display frame chunk from the device to the reassembly state machine. */ + fun handleIncomingFrame(chunk: DisplayFrame) +} + +/** + * One complete device framebuffer snapshot. + * + * [pixels] is MONO_VLSB: 1 bit per pixel in vertical LSB-first pages — byte index = `x + (y / 8) * width`, + * bit index = `y % 8`. + */ +data class MirrorFrame(val width: Int, val height: Int, val frameId: Int, val pixels: ByteArray) { + override fun equals(other: Any?): Boolean = + other is MirrorFrame && + other.width == width && + other.height == height && + other.frameId == frameId && + other.pixels.contentEquals(pixels) + + override fun hashCode(): Int { + var result = width + result = 31 * result + height + result = 31 * result + frameId + result = 31 * result + pixels.contentHashCode() + return result + } +} diff --git a/core/service/src/commonMain/kotlin/org/meshtastic/core/service/AdminControllerImpl.kt b/core/service/src/commonMain/kotlin/org/meshtastic/core/service/AdminControllerImpl.kt index ec4596d4af..9fa9efadad 100644 --- a/core/service/src/commonMain/kotlin/org/meshtastic/core/service/AdminControllerImpl.kt +++ b/core/service/src/commonMain/kotlin/org/meshtastic/core/service/AdminControllerImpl.kt @@ -125,6 +125,24 @@ internal class AdminControllerImpl( scope.handledLaunch { radioConfigRepository.setLocalConfig(config) } } + override suspend fun setDisplayMirror(enabled: Boolean) { + commandSender.sendAdmin(myNodeNum) { AdminMessage(set_display_mirror = enabled) } + } + + override suspend fun sendInputEvent(eventCode: Int, kbChar: Int, touchX: Int, touchY: Int) { + commandSender.sendAdmin(myNodeNum) { + AdminMessage( + send_input_event = + AdminMessage.InputEvent( + event_code = eventCode, + kb_char = kbChar, + touch_x = touchX, + touch_y = touchY, + ), + ) + } + } + override suspend fun setConfig(destNum: Int, config: Config, packetId: Int) { commandSender.sendAdmin(destNum, packetId) { AdminMessage(set_config = config) } if (destNum == nodeManager.myNodeNum.value) { diff --git a/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/debugging/Debug.kt b/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/debugging/Debug.kt index 794a85a23b..6490a5733e 100644 --- a/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/debugging/Debug.kt +++ b/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/debugging/Debug.kt @@ -184,11 +184,21 @@ fun DebugScreen(onNavigateUp: () -> Unit, viewModel: DebugViewModel) { onClick = { selectedTab = 1 }, text = { Text(stringResource(Res.string.debug_tab_app_logs)) }, ) + Tab( + selected = selectedTab == 2, + onClick = { selectedTab = 2 }, + // PoC tab; deliberately unlocalized. + text = { Text("Mirror") }, + ) } if (selectedTab == 1) { LogcatContent(modifier = Modifier.fillMaxSize()) return@Column } + if (selectedTab == 2) { + DisplayMirrorContent(modifier = Modifier.fillMaxSize()) + return@Column + } LazyColumn(modifier = Modifier.fillMaxSize(), state = listState) { stickyHeader { val animatedAlpha by diff --git a/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/debugging/DisplayMirror.kt b/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/debugging/DisplayMirror.kt new file mode 100644 index 0000000000..f3774f0f0a --- /dev/null +++ b/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/debugging/DisplayMirror.kt @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.feature.settings.debugging + +import androidx.compose.foundation.Canvas +import androidx.compose.foundation.background +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.aspectRatio +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.rememberScrollState +import androidx.compose.foundation.verticalScroll +import androidx.compose.material3.FilledTonalButton +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Switch +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.geometry.Offset +import androidx.compose.ui.geometry.Size +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.unit.dp +import androidx.lifecycle.ViewModel +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import androidx.lifecycle.viewModelScope +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.launch +import org.koin.compose.viewmodel.koinViewModel +import org.koin.core.annotation.KoinViewModel +import org.meshtastic.core.repository.DisplayMirrorManager +import org.meshtastic.core.repository.MirrorFrame +import org.meshtastic.core.repository.RadioController + +// Firmware input_broker_event codes (src/input/InputBroker.h). +private const val INPUT_SELECT = 10 +private const val INPUT_UP = 17 +private const val INPUT_DOWN = 18 +private const val INPUT_LEFT = 19 +private const val INPUT_RIGHT = 20 +private const val INPUT_BACK = 27 + +@KoinViewModel +class DisplayMirrorViewModel( + displayMirrorManager: DisplayMirrorManager, + private val radioController: RadioController, +) : ViewModel() { + + val frame: StateFlow = displayMirrorManager.frame + + private val _mirroring = MutableStateFlow(false) + val mirroring: StateFlow = _mirroring.asStateFlow() + + fun setMirror(enabled: Boolean) { + viewModelScope.launch { + radioController.setDisplayMirror(enabled) + _mirroring.value = enabled + } + } + + fun sendKey(eventCode: Int) { + viewModelScope.launch { radioController.sendInputEvent(eventCode) } + } +} + +/** PoC live view of the connected device's screen, with remote D-pad control. Strings are deliberately unlocalized. */ +@Composable +fun DisplayMirrorContent(modifier: Modifier = Modifier, viewModel: DisplayMirrorViewModel = koinViewModel()) { + val frame by viewModel.frame.collectAsStateWithLifecycle() + val mirroring by viewModel.mirroring.collectAsStateWithLifecycle() + + Column( + modifier = modifier.fillMaxSize().verticalScroll(rememberScrollState()).padding(16.dp), + horizontalAlignment = Alignment.CenterHorizontally, + verticalArrangement = Arrangement.spacedBy(12.dp), + ) { + Row(verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(8.dp)) { + Switch(checked = mirroring, onCheckedChange = viewModel::setMirror) + Text(text = if (mirroring) "Mirroring" else "Mirror off", style = MaterialTheme.typography.titleMedium) + } + + val currentFrame = frame + if (currentFrame != null) { + MirrorFrameCanvas(currentFrame) + Text( + text = "${currentFrame.width}x${currentFrame.height} frame #${currentFrame.frameId}", + style = MaterialTheme.typography.labelSmall, + ) + } else { + Text(text = "No frame received yet — enable mirroring above.") + } + + // Remote D-pad + Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) { + FilledTonalButton(onClick = { viewModel.sendKey(INPUT_UP) }) { Text("Up") } + } + Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) { + FilledTonalButton(onClick = { viewModel.sendKey(INPUT_LEFT) }) { Text("Left") } + FilledTonalButton(onClick = { viewModel.sendKey(INPUT_SELECT) }) { Text("OK") } + FilledTonalButton(onClick = { viewModel.sendKey(INPUT_RIGHT) }) { Text("Right") } + } + Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) { + FilledTonalButton(onClick = { viewModel.sendKey(INPUT_DOWN) }) { Text("Down") } + } + Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) { + FilledTonalButton(onClick = { viewModel.sendKey(INPUT_BACK) }) { Text("Back") } + } + } +} + +/** Draws a MONO_VLSB 1bpp framebuffer scaled to the available width, one filled rect per lit pixel. */ +@Composable +private fun MirrorFrameCanvas(frame: MirrorFrame) { + val aspect = frame.width.toFloat() / frame.height.toFloat() + Canvas(modifier = Modifier.fillMaxWidth().aspectRatio(aspect).background(Color.Black)) { + val scale = size.width / frame.width + val pixel = Size(scale, scale) + for (y in 0 until frame.height) { + val page = (y / 8) * frame.width + val bit = 1 shl (y % 8) + for (x in 0 until frame.width) { + val index = page + x + if (index < frame.pixels.size && frame.pixels[index].toInt() and bit != 0) { + drawRect(color = Color.White, topLeft = Offset(x * scale, y * scale), size = pixel) + } + } + } + } +} diff --git a/gradle.properties b/gradle.properties index 887bc1f10c..c54853d66b 100644 --- a/gradle.properties +++ b/gradle.properties @@ -33,3 +33,6 @@ org.gradle.priority=low # stays parallel if Isolated Projects is ever off. org.gradle.tooling.parallel=true org.gradle.welcome=never + +# PoC: resolve org.meshtastic:protobufs 2.99.0-screen-mirror-poc-SNAPSHOT from mavenLocal +useMavenLocal=true diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index e741f537ef..36b883307a 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -104,7 +104,10 @@ mqttastic = "0.8.1" jmdns = "3.6.3" qrcode-kotlin = "4.5.0" takpacket-sdk = "0.9.1" -meshtastic-protobufs = "2.8.0.23-gca2cb1a-SNAPSHOT" +# PoC: DisplayFrame screen mirroring, locally published from protobufs@screen-mirror-poc. +# Revert to main's pin once protobufs#1054 is tagged. main tracks 2.8.0.23-gca2cb1a-SNAPSHOT. +# (requires -PuseMavenLocal or the gradle.properties flag on this branch) +meshtastic-protobufs = "2.99.0-screen-mirror-poc-SNAPSHOT" # Gradle Plugins ccud = "2.8.0"