diff --git a/core/data/src/commonMain/kotlin/org/meshtastic/core/data/manager/DisplayMirrorManagerImpl.kt b/core/data/src/commonMain/kotlin/org/meshtastic/core/data/manager/DisplayMirrorManagerImpl.kt index edb51a8da9..e100865cb2 100644 --- a/core/data/src/commonMain/kotlin/org/meshtastic/core/data/manager/DisplayMirrorManagerImpl.kt +++ b/core/data/src/commonMain/kotlin/org/meshtastic/core/data/manager/DisplayMirrorManagerImpl.kt @@ -22,7 +22,9 @@ import kotlinx.coroutines.flow.asStateFlow import org.koin.core.annotation.Single import org.meshtastic.core.repository.DisplayMirrorManager import org.meshtastic.core.repository.MirrorFrame +import org.meshtastic.core.repository.MirrorPalette import org.meshtastic.proto.DisplayFrame +import org.meshtastic.proto.DisplayPalette /** * Display frame reassembly state machine. @@ -37,10 +39,19 @@ class DisplayMirrorManagerImpl : DisplayMirrorManager { private val _frame = MutableStateFlow(null) override val frame = _frame.asStateFlow() + private val _palette = MutableStateFlow(null) + override val palette = _palette.asStateFlow() + private var buffer: ByteArray? = null private var frameId = 0 private var received = 0 + private var paletteRegions = mutableListOf() + private var paletteSignature = 0 + private var paletteReceived = 0 + private var paletteDefaultOn = 0 + private var paletteDefaultOff = 0 + override fun handleIncomingFrame(chunk: DisplayFrame) { val data = chunk.data_.toByteArray() val total = chunk.total_size @@ -64,11 +75,47 @@ class DisplayMirrorManagerImpl : DisplayMirrorManager { received += data.size if (received == total) { - _frame.value = MirrorFrame(width = chunk.width, height = chunk.height, frameId = frameId, pixels = buf) + _frame.value = + MirrorFrame( + width = chunk.width, + height = chunk.height, + frameId = frameId, + paletteSignature = chunk.palette_signature, + pixels = buf, + ) buffer = null } } + override fun handleIncomingPalette(chunk: DisplayPalette) { + if (chunk.region_offset == 0) { + paletteRegions = mutableListOf() + paletteSignature = chunk.signature + paletteReceived = 0 + // Defaults are authoritative on the first chunk; later chunks may omit them. + paletteDefaultOn = chunk.default_on_color + paletteDefaultOff = chunk.default_off_color + } + if (chunk.signature != paletteSignature || chunk.region_offset != paletteReceived) { + Logger.w { "DisplayMirror: bad palette chunk ${chunk.signature}@${chunk.region_offset}" } + paletteRegions.clear() + paletteReceived = -1 // poison until the next offset-0 chunk restarts + return + } + paletteRegions.addAll(chunk.regions) + paletteReceived += chunk.regions.size + + if (paletteReceived >= chunk.region_total) { + _palette.value = + MirrorPalette( + signature = chunk.signature, + defaultOnColor = paletteDefaultOn, + defaultOffColor = paletteDefaultOff, + regions = paletteRegions.toList(), + ) + } + } + /** * Format must be MONO_VLSB and width/height must describe exactly total_size bytes; a zero or lying dimension would * otherwise reach the renderer (aspectRatio requires > 0). diff --git a/core/data/src/commonMain/kotlin/org/meshtastic/core/data/manager/FromRadioPacketHandlerImpl.kt b/core/data/src/commonMain/kotlin/org/meshtastic/core/data/manager/FromRadioPacketHandlerImpl.kt index 755b20f59f..020613c6b9 100644 --- a/core/data/src/commonMain/kotlin/org/meshtastic/core/data/manager/FromRadioPacketHandlerImpl.kt +++ b/core/data/src/commonMain/kotlin/org/meshtastic/core/data/manager/FromRadioPacketHandlerImpl.kt @@ -47,6 +47,7 @@ import org.meshtastic.core.resources.key_verification_title import org.meshtastic.core.resources.low_entropy_key_title import org.meshtastic.proto.ClientNotification import org.meshtastic.proto.DisplayFrame +import org.meshtastic.proto.DisplayPalette import org.meshtastic.proto.FromRadio /** Implementation of [FromRadioPacketHandler] that dispatches [FromRadio] variants to specialized handlers. */ @@ -88,6 +89,7 @@ class FromRadioPacketHandlerImpl( val xmodemPacket = proto.xmodemPacket val lockdownStatus = proto.lockdown_status val displayFrame = proto.display_frame + val displayPalette = proto.display_palette when { myInfo != null -> configFlowManager.value.handleMyInfo(myInfo, session) @@ -137,6 +139,8 @@ class FromRadioPacketHandlerImpl( displayFrame != null -> handleDisplayFrame(displayFrame, session) + displayPalette != null -> handleDisplayPalette(displayPalette, session) + lockdownStatus != null -> runIfSessionActive(session, "lockdown status") { lockdownCoordinator.handleLockdownStatus(lockdownStatus) @@ -156,6 +160,9 @@ class FromRadioPacketHandlerImpl( private fun handleDisplayFrame(frame: DisplayFrame, session: RadioSessionContext) = runIfSessionActive(session, "display frame") { displayMirrorManager.value.handleIncomingFrame(frame) } + private fun handleDisplayPalette(palette: DisplayPalette, session: RadioSessionContext) = + runIfSessionActive(session, "display palette") { displayMirrorManager.value.handleIncomingPalette(palette) } + private fun runIfSessionActive(session: RadioSessionContext, operation: String, block: () -> Unit) { if (!radioInterfaceService.runIfSessionActive(session, block)) { Logger.d { "Discarding $operation from stale transport session" } diff --git a/core/data/src/commonTest/kotlin/org/meshtastic/core/data/manager/DisplayMirrorManagerImplTest.kt b/core/data/src/commonTest/kotlin/org/meshtastic/core/data/manager/DisplayMirrorManagerImplTest.kt index d50da08a2f..1af13b31cc 100644 --- a/core/data/src/commonTest/kotlin/org/meshtastic/core/data/manager/DisplayMirrorManagerImplTest.kt +++ b/core/data/src/commonTest/kotlin/org/meshtastic/core/data/manager/DisplayMirrorManagerImplTest.kt @@ -18,6 +18,7 @@ package org.meshtastic.core.data.manager import okio.ByteString.Companion.toByteString import org.meshtastic.proto.DisplayFrame +import org.meshtastic.proto.DisplayPalette import kotlin.test.Test import kotlin.test.assertContentEquals import kotlin.test.assertEquals @@ -126,6 +127,50 @@ class DisplayMirrorManagerImplTest { assertNull(manager.frame.value) } + @Test + fun `reassembles a two-chunk palette and tags frames with its signature`() { + val region = + DisplayPalette.ColorRegion(x = 0, y = 0, width = 128, height = 16, on_color = 0xF800, off_color = 0) + manager.handleIncomingPalette( + DisplayPalette( + signature = 42, + default_on_color = 0xFFFF, + default_off_color = 0, + region_offset = 0, + region_total = 3, + regions = listOf(region, region), + ), + ) + assertNull(manager.palette.value) + + manager.handleIncomingPalette( + DisplayPalette(signature = 42, region_offset = 2, region_total = 3, regions = listOf(region)), + ) + + val palette = manager.palette.value!! + assertEquals(42, palette.signature) + assertEquals(0xFFFF, palette.defaultOnColor) + assertEquals(3, palette.regions.size) + } + + @Test + fun `out-of-sequence palette chunk is dropped until a restart`() { + val region = DisplayPalette.ColorRegion(x = 0, y = 0, width = 8, height = 8) + manager.handleIncomingPalette( + DisplayPalette(signature = 7, region_offset = 0, region_total = 2, regions = listOf(region)), + ) + manager.handleIncomingPalette( + DisplayPalette(signature = 7, region_offset = 5, region_total = 2, regions = listOf(region)), + ) + assertNull(manager.palette.value) + + // A fresh offset-0 chunk recovers + manager.handleIncomingPalette( + DisplayPalette(signature = 8, region_offset = 0, region_total = 1, regions = listOf(region)), + ) + assertEquals(8, manager.palette.value?.signature) + } + private companion object { const val WIDTH = 128 const val HEIGHT = 64 diff --git a/core/repository/src/commonMain/kotlin/org/meshtastic/core/repository/DisplayMirrorManager.kt b/core/repository/src/commonMain/kotlin/org/meshtastic/core/repository/DisplayMirrorManager.kt index 3cd303118d..058802d9d5 100644 --- a/core/repository/src/commonMain/kotlin/org/meshtastic/core/repository/DisplayMirrorManager.kt +++ b/core/repository/src/commonMain/kotlin/org/meshtastic/core/repository/DisplayMirrorManager.kt @@ -18,6 +18,7 @@ package org.meshtastic.core.repository import kotlinx.coroutines.flow.StateFlow import org.meshtastic.proto.DisplayFrame +import org.meshtastic.proto.DisplayPalette /** * Reassembles the chunked [DisplayFrame] stream from the connected device into complete framebuffer snapshots. @@ -30,27 +31,52 @@ interface DisplayMirrorManager { /** Latest completely reassembled frame, or null before the first one arrives. */ val frame: StateFlow + /** Latest completely reassembled color palette, or null when the device renders monochrome. */ + val palette: StateFlow + /** Routes an incoming display frame chunk from the device to the reassembly state machine. */ fun handleIncomingFrame(chunk: DisplayFrame) + + /** Routes an incoming palette chunk from the device to the reassembly state machine. */ + fun handleIncomingPalette(chunk: DisplayPalette) } +/** + * A complete color palette for [MirrorFrame]s whose `paletteSignature` matches [signature]: per-region RGB565 on/off + * colors (later regions override earlier ones where they overlap) plus defaults for pixels outside all regions. + */ +data class MirrorPalette( + val signature: Int, + val defaultOnColor: Int, + val defaultOffColor: Int, + val regions: List, +) + /** * One complete device framebuffer snapshot. * * [pixels] is MONO_VLSB: 1 bit per pixel in vertical LSB-first pages — byte index = `x + (y / 8) * width`, bit index = * `y % 8`. */ -data class MirrorFrame(val width: Int, val height: Int, val frameId: Int, val pixels: ByteArray) { +data class MirrorFrame( + val width: Int, + val height: Int, + val frameId: Int, + val paletteSignature: Int, + val pixels: ByteArray, +) { override fun equals(other: Any?): Boolean = other is MirrorFrame && other.width == width && other.height == height && other.frameId == frameId && + other.paletteSignature == paletteSignature && other.pixels.contentEquals(pixels) override fun hashCode(): Int { var result = width result = 31 * result + height result = 31 * result + frameId + result = 31 * result + paletteSignature result = 31 * result + pixels.contentHashCode() return result } diff --git a/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/debugging/DisplayMirror.kt b/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/debugging/DisplayMirror.kt index 2f603685b8..58d4b19d98 100644 --- a/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/debugging/DisplayMirror.kt +++ b/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/debugging/DisplayMirror.kt @@ -17,7 +17,6 @@ package org.meshtastic.feature.settings.debugging import androidx.compose.foundation.ExperimentalFoundationApi -import androidx.compose.foundation.Image import androidx.compose.foundation.background import androidx.compose.foundation.border import androidx.compose.foundation.combinedClickable @@ -28,9 +27,7 @@ import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.Row -import androidx.compose.foundation.layout.aspectRatio import androidx.compose.foundation.layout.fillMaxSize -import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.size import androidx.compose.foundation.layout.widthIn @@ -55,12 +52,7 @@ import androidx.compose.ui.focus.FocusRequester import androidx.compose.ui.focus.focusRequester import androidx.compose.ui.focus.onFocusChanged import androidx.compose.ui.geometry.Offset -import androidx.compose.ui.geometry.Size -import androidx.compose.ui.graphics.Canvas import androidx.compose.ui.graphics.Color -import androidx.compose.ui.graphics.FilterQuality -import androidx.compose.ui.graphics.ImageBitmap -import androidx.compose.ui.graphics.drawscope.CanvasDrawScope import androidx.compose.ui.graphics.vector.ImageVector import androidx.compose.ui.input.key.Key import androidx.compose.ui.input.key.KeyEventType @@ -68,8 +60,6 @@ import androidx.compose.ui.input.key.key import androidx.compose.ui.input.key.onPreviewKeyEvent import androidx.compose.ui.input.key.type import androidx.compose.ui.input.pointer.pointerInput -import androidx.compose.ui.unit.Density -import androidx.compose.ui.unit.LayoutDirection import androidx.compose.ui.unit.dp import androidx.lifecycle.ViewModel import androidx.lifecycle.compose.collectAsStateWithLifecycle @@ -91,6 +81,7 @@ import org.meshtastic.core.repository.AdminController import org.meshtastic.core.repository.ConnectionStateProvider import org.meshtastic.core.repository.DisplayMirrorManager import org.meshtastic.core.repository.MirrorFrame +import org.meshtastic.core.repository.MirrorPalette import org.meshtastic.core.repository.NodeRepository import org.meshtastic.core.ui.icon.KeyboardArrowDown import org.meshtastic.core.ui.icon.KeyboardArrowLeft @@ -99,9 +90,6 @@ import org.meshtastic.core.ui.icon.KeyboardArrowUp import org.meshtastic.core.ui.icon.MeshtasticIcons import org.meshtastic.proto.DisplayInfo -// 4x scale for the common 128px-wide OLED; caps the image so the D-pad stays above the fold on desktop. -private val MAX_CANVAS_WIDTH = 512.dp - // M3 comfortable target for remote-control keys (48dp minimum + breathing room). private val DPAD_KEY_SIZE = 56.dp private val DPAD_GAP = 8.dp @@ -113,9 +101,6 @@ private const val REPEAT_INTERVAL_MS = 100L // Swipes on the mirror shorter than this are ignored as accidental. private val SWIPE_THRESHOLD = 48.dp -// MONO_VLSB packs 8 vertically adjacent pixels per byte (one "page" row). -private const val PIXELS_PER_PAGE = 8 - // Firmware input_broker_event codes (src/input/InputBroker.h). private const val INPUT_SELECT = 10 private const val INPUT_SELECT_LONG = 11 @@ -135,6 +120,8 @@ class DisplayMirrorViewModel( val frame: StateFlow = displayMirrorManager.frame + val palette: StateFlow = displayMirrorManager.palette + val connected: StateFlow = connectionStateProvider.connectionState .map { it is ConnectionState.Connected } @@ -174,6 +161,7 @@ class DisplayMirrorViewModel( @Composable fun DisplayMirrorContent(modifier: Modifier = Modifier, viewModel: DisplayMirrorViewModel = koinViewModel()) { val frame by viewModel.frame.collectAsStateWithLifecycle() + val palette by viewModel.palette.collectAsStateWithLifecycle() val mirroring by viewModel.mirroring.collectAsStateWithLifecycle() val connected by viewModel.connected.collectAsStateWithLifecycle() val displayInfo by viewModel.displayInfo.collectAsStateWithLifecycle() @@ -208,7 +196,7 @@ fun DisplayMirrorContent(modifier: Modifier = Modifier, viewModel: DisplayMirror when { !connected -> Text(text = "Not connected to a device.") - currentFrame != null -> MirrorSurface(currentFrame, onEvent = viewModel::sendKey) + currentFrame != null -> MirrorSurface(currentFrame, palette, onEvent = viewModel::sendKey) else -> Text(text = "No frame received yet — enable mirroring or tap Refresh.") } @@ -221,7 +209,12 @@ fun DisplayMirrorContent(modifier: Modifier = Modifier, viewModel: DisplayMirror * Esc/Backspace = Back), swipe in a cardinal direction for a single direction event. */ @Composable -private fun MirrorSurface(frame: MirrorFrame, onEvent: (Int) -> Unit, modifier: Modifier = Modifier) { +private fun MirrorSurface( + frame: MirrorFrame, + palette: MirrorPalette?, + onEvent: (Int) -> Unit, + modifier: Modifier = Modifier, +) { val focusRequester = remember { FocusRequester() } var focused by remember { mutableStateOf(false) } val focusColor = MaterialTheme.colorScheme.primary @@ -247,7 +240,7 @@ private fun MirrorSurface(frame: MirrorFrame, onEvent: (Int) -> Unit, modifier: .swipeToDirection(onEvent) .border(width = 2.dp, color = if (focused) focusColor else Color.Transparent), ) { - MirrorFrameImage(frame) + MirrorFrameImage(frame, palette) } Text( text = @@ -419,41 +412,3 @@ private fun BackKey(enabled: Boolean, onBack: () -> Unit) { @Composable private fun contentColorFor(enabled: Boolean): Color = if (enabled) MaterialTheme.colorScheme.onSecondaryContainer else MaterialTheme.colorScheme.onSurfaceVariant - -/** - * Renders a MONO_VLSB 1bpp framebuffer once per frame into a 1:1 [ImageBitmap] and scales it up with nearest-neighbor - * filtering — crisp device pixels, no fractional-scale seams, one pixel walk per frame instead of per recomposition. - */ -@Composable -private fun MirrorFrameImage(frame: MirrorFrame, modifier: Modifier = Modifier) { - val bitmap = remember(frame) { renderFrame(frame) } - Image( - bitmap = bitmap, - contentDescription = "Device screen", - modifier = - modifier - .widthIn(max = MAX_CANVAS_WIDTH) - .fillMaxWidth() - .aspectRatio(frame.width.toFloat() / frame.height.toFloat()), - filterQuality = FilterQuality.None, - ) -} - -private fun renderFrame(frame: MirrorFrame): ImageBitmap { - val bitmap = ImageBitmap(frame.width, frame.height) - val size = Size(frame.width.toFloat(), frame.height.toFloat()) - CanvasDrawScope().draw(Density(1f), LayoutDirection.Ltr, Canvas(bitmap), size) { - drawRect(color = Color.Black) - val pixel = Size(1f, 1f) - for (y in 0 until frame.height) { - val page = (y / PIXELS_PER_PAGE) * frame.width - val bit = 1 shl (y % PIXELS_PER_PAGE) - for (x in 0 until frame.width) { - if (frame.pixels[page + x].toInt() and bit != 0) { - drawRect(color = Color.White, topLeft = Offset(x.toFloat(), y.toFloat()), size = pixel) - } - } - } - } - return bitmap -} diff --git a/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/debugging/MirrorFrameRenderer.kt b/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/debugging/MirrorFrameRenderer.kt new file mode 100644 index 0000000000..7bff3ef718 --- /dev/null +++ b/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/debugging/MirrorFrameRenderer.kt @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.feature.settings.debugging + +import androidx.compose.foundation.Image +import androidx.compose.foundation.layout.aspectRatio +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.widthIn +import androidx.compose.runtime.Composable +import androidx.compose.runtime.remember +import androidx.compose.ui.Modifier +import androidx.compose.ui.geometry.Offset +import androidx.compose.ui.geometry.Size +import androidx.compose.ui.graphics.Canvas +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.FilterQuality +import androidx.compose.ui.graphics.ImageBitmap +import androidx.compose.ui.graphics.drawscope.CanvasDrawScope +import androidx.compose.ui.unit.Density +import androidx.compose.ui.unit.LayoutDirection +import androidx.compose.ui.unit.dp +import org.meshtastic.core.repository.MirrorFrame +import org.meshtastic.core.repository.MirrorPalette + +// 4x scale for the common 128px-wide OLED; caps the image so the D-pad stays above the fold on desktop. +private val MAX_CANVAS_WIDTH = 512.dp + +// MONO_VLSB packs 8 vertically adjacent pixels per byte (one "page" row). +private const val PIXELS_PER_PAGE = 8 + +/** + * Renders a MONO_VLSB 1bpp framebuffer once per frame into a 1:1 [ImageBitmap] and scales it up with nearest-neighbor + * filtering — crisp device pixels, no fractional-scale seams, one pixel walk per frame instead of per recomposition. + */ +@Composable +internal fun MirrorFrameImage(frame: MirrorFrame, palette: MirrorPalette?, modifier: Modifier = Modifier) { + // A palette only applies when the frame references it; otherwise render monochrome. + val activePalette = palette?.takeIf { frame.paletteSignature != 0 && it.signature == frame.paletteSignature } + val bitmap = remember(frame, activePalette) { renderFrame(frame, activePalette) } + Image( + bitmap = bitmap, + contentDescription = "Device screen", + modifier = + modifier + .widthIn(max = MAX_CANVAS_WIDTH) + .fillMaxWidth() + .aspectRatio(frame.width.toFloat() / frame.height.toFloat()), + filterQuality = FilterQuality.None, + ) +} + +private class ResolvedRegion( + val left: Int, + val top: Int, + val right: Int, + val bottom: Int, + val on: Color, + val off: Color, +) + +private fun rgb565ToColor(v: Int): Color = Color( + red = ((v ushr 11) and 0x1F) * 255 / 31, + green = ((v ushr 5) and 0x3F) * 255 / 63, + blue = (v and 0x1F) * 255 / 31, +) + +private fun renderFrame(frame: MirrorFrame, palette: MirrorPalette?): ImageBitmap { + val defaultOn = palette?.let { rgb565ToColor(it.defaultOnColor) } ?: Color.White + val defaultOff = palette?.let { rgb565ToColor(it.defaultOffColor) } ?: Color.Black + val regions = + palette?.regions.orEmpty().map { + ResolvedRegion( + left = it.x, + top = it.y, + right = it.x + it.width, + bottom = it.y + it.height, + on = rgb565ToColor(it.on_color), + off = rgb565ToColor(it.off_color), + ) + } + + val bitmap = ImageBitmap(frame.width, frame.height) + val size = Size(frame.width.toFloat(), frame.height.toFloat()) + CanvasDrawScope().draw(Density(1f), LayoutDirection.Ltr, Canvas(bitmap), size) { + drawRect(color = defaultOff) + val pixel = Size(1f, 1f) + for (y in 0 until frame.height) { + val page = (y / PIXELS_PER_PAGE) * frame.width + val bit = 1 shl (y % PIXELS_PER_PAGE) + // Cull to the regions overlapping this row; the last matching region wins, + // matching the firmware's precedence. + val rowRegions = regions.filter { y >= it.top && y < it.bottom } + for (x in 0 until frame.width) { + val set = frame.pixels[page + x].toInt() and bit != 0 + val region = rowRegions.lastOrNull { x >= it.left && x < it.right } + val color = + when { + region != null -> if (set) region.on else region.off + set -> defaultOn + else -> defaultOff + } + if (color != defaultOff) { + drawRect(color = color, topLeft = Offset(x.toFloat(), y.toFloat()), size = pixel) + } + } + } + } + return bitmap +}