From f7548e7c2567cb67d8aae4fba9cdfa590bec1d5e Mon Sep 17 00:00:00 2001 From: Ben Meadors Date: Mon, 11 May 2026 21:42:07 -0500 Subject: [PATCH] Remove gradient sync nonce and simplify replay handling (#10459) * Remove gradient sync nonce and simplify replay handling * Fix ONLY_CONFIG replay gating and stale gradient-sync comments Agent-Logs-Url: https://github.com/meshtastic/firmware/sessions/cfa93978-e2e0-4dc2-ba5f-b82b5b43cef8 Co-authored-by: thebentern <9000580+thebentern@users.noreply.github.com> * Add transport mechanism to replay packets for client filtering * Comments * Update protobuf definitions to include precision_bits in PositionLite * Propagate position precision_bits and remove verbose NodeInfo sync log Agent-Logs-Url: https://github.com/meshtastic/firmware/sessions/41572cbc-408e-499d-b59e-00f330b5789f Co-authored-by: thebentern <9000580+thebentern@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> --- .github/copilot-instructions.md | 24 +- protobufs | 2 +- src/mesh/PhoneAPI.cpp | 344 ++++++++---------- src/mesh/PhoneAPI.h | 37 +- src/mesh/TypeConversions.cpp | 7 + src/mesh/generated/meshtastic/deviceonly.pb.h | 14 +- .../meshtastic/deviceonly_legacy.pb.h | 2 +- 7 files changed, 210 insertions(+), 220 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index bac47853d..d165f2cdb 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -193,22 +193,26 @@ Writers go through `setNodeStatus`, `updatePosition`, `updateTelemetry` (which d Every code path that drops a node from the header table must also evict the satellites. The single chokepoint is `eraseNodeSatellites(NodeNum)`; it's already called from `getOrCreateMeshNode`'s oldest-boring eviction, `removeNodeByNum`, both branches of `resetNodes`, `cleanupMeshDB`, `addFromContact`'s ignored-branch, and `AdminModule`'s `set_ignored_node`. Add new eviction sites here, not by calling `.erase()` directly. -### Gradient sync (opt-in via special nonces) +### Sync flow: thin NodeInfo + post-COMPLETE_ID replay (no opt-in) -`client_capabilities` is **not** a thing in this branch. Phone clients opt into the new sync flow by sending one of two values in the `ToRadio.want_config_id`: +There is no capability flag and no special "gradient" nonce. The **default** sync flow is: -- `SPECIAL_NONCE_GRADIENT_SYNC` (69422) — full config + thin NodeInfo + replay phases. -- `SPECIAL_NONCE_GRADIENT_ONLY_NODES` (69423) — skip config segments, NodeInfo + replay only. +1. Config / module-config / channel / metadata segments (same as before). +2. `STATE_SEND_OWN_NODEINFO` — **our own** NodeInfo, still bundled with our position and device_metrics (because the replay snapshot excludes our own NodeNum). Emitted via `ConvertToNodeInfo(lite)`. +3. `STATE_SEND_OTHER_NODEINFOS` — every other peer's NodeInfo, **always thin** (no `position`, no `device_metrics`). Emitted via `ConvertToNodeInfoThin(lite)`. +4. `STATE_SEND_FILEMANIFEST` → `STATE_SEND_COMPLETE_ID` — the phone sees `config_complete_id` and treats sync as done. +5. `STATE_SEND_PACKETS` — live mesh packets, with a trailing replay drain interleaved. The replay drain walks four cached satellite stores in order (positions → telemetry → environment → status) and emits each cached entry as an ordinary `MeshPacket` on the matching portnum (`POSITION_APP`, `TELEMETRY_APP` device + environment variants, `NODE_STATUS_APP`). These are indistinguishable on the wire from live mesh traffic, so clients need no special handling — any code that already updates UI on `POSITION_APP` etc. works. -`PhoneAPI::clientWantsGradientSync()` is the single switch. When true, `STATE_SEND_OTHER_NODEINFOS` is followed by: +`PhoneAPI::sendConfigComplete()` arms `replayPhase = REPLAY_PHASE_POSITIONS` for default/full sync and `SPECIAL_NONCE_ONLY_NODES`, while `SPECIAL_NONCE_ONLY_CONFIG` skips replay. The drain runs inside `STATE_SEND_PACKETS` via `popReplayPacket()`, lower priority than live traffic. When all four phases drain, `replayPhase` flips back to `REPLAY_PHASE_IDLE` and the snapshot vectors get `shrink_to_fit`ed. -```text -STATE_REPLAY_POSITIONS → STATE_REPLAY_TELEMETRY → STATE_REPLAY_ENVIRONMENT → STATE_REPLAY_STATUS -``` +STM32WL and any other build with all four `MESHTASTIC_EXCLUDE_*DB` flags set produces zero replay packets — `popReplayPacket` advances through each phase in microseconds without emitting anything. -Each replay phase walks the corresponding satellite map and emits synthetic `MeshPacket`s on the matching portnum (`POSITION_APP`, `TELEMETRY_APP` for both device + environment variants, `STATUS_MESSAGE_APP`). Legacy clients (no special nonce) get the bundled-NodeInfo path with position/device_metrics joined back in by `ConvertToNodeInfo(lite, pos*, dm*)` — wire bytes are byte-identical to pre-v25 for them. +Special nonces that still mean something: -`ConvertToNodeInfoThin(lite)` is the gradient-sync emitter (no position/telemetry). +- `SPECIAL_NONCE_ONLY_CONFIG` (69420) — skip node sync entirely, just config. +- `SPECIAL_NONCE_ONLY_NODES` (69421) — skip config segments, jump straight to `STATE_SEND_OWN_NODEINFO`. Still gets the post-COMPLETE_ID replay drain. + +There are no other reserved nonces; everything else is a fresh random `want_config_id` from the client. ### v24 → v25 migration diff --git a/protobufs b/protobufs index f899d3842..ff5b39250 160000 --- a/protobufs +++ b/protobufs @@ -1 +1 @@ -Subproject commit f899d38422ab07e7a973ee1e99fcd5fa1acd8dbd +Subproject commit ff5b392503776bf13073034070543d5c5aa1acf7 diff --git a/src/mesh/PhoneAPI.cpp b/src/mesh/PhoneAPI.cpp index 720743907..ecf6ff809 100644 --- a/src/mesh/PhoneAPI.cpp +++ b/src/mesh/PhoneAPI.cpp @@ -62,7 +62,7 @@ void PhoneAPI::handleStartConfig() onConfigStart(); // even if we were already connected - restart our state machine - if (config_nonce == SPECIAL_NONCE_ONLY_NODES || config_nonce == SPECIAL_NONCE_GRADIENT_ONLY_NODES) { + if (config_nonce == SPECIAL_NONCE_ONLY_NODES) { // If client only wants node info, jump directly to sending nodes state = STATE_SEND_OWN_NODEINFO; LOG_INFO("Client only wants node info, skipping other config"); @@ -138,6 +138,7 @@ void PhoneAPI::close() replayTelemetryIndex = 0; replayEnvironmentIndex = 0; replayStatusIndex = 0; + replayPhase = REPLAY_PHASE_IDLE; } packetForPhone = NULL; filesManifest.clear(); @@ -320,7 +321,7 @@ size_t PhoneAPI::getFromRadio(uint8_t *buf) nodeInfoForPhone.num = 0; } } - if (config_nonce == SPECIAL_NONCE_ONLY_NODES || config_nonce == SPECIAL_NONCE_GRADIENT_ONLY_NODES) { + if (config_nonce == SPECIAL_NONCE_ONLY_NODES) { // If client only wants node info, jump directly to sending nodes state = STATE_SEND_OTHER_NODEINFOS; onNowHasData(0); @@ -535,11 +536,6 @@ size_t PhoneAPI::getFromRadio(uint8_t *buf) // Just in case we stored a different user.id in the past, but should never happen going forward sprintf(infoToSend.user.id, "!%08x", infoToSend.num); - // Logging this really slows down sending nodes on initial connection because the serial console is so slow, so only - // uncomment if you really need to: - // LOG_INFO("nodeinfo: num=0x%x, lastseen=%u, id=%s, name=%s", nodeInfoForPhone.num, nodeInfoForPhone.last_heard, - // nodeInfoForPhone.user.id, nodeInfoForPhone.user.long_name); - fromRadioScratch.which_payload_variant = meshtastic_FromRadio_node_info_tag; fromRadioScratch.node_info = infoToSend; prefetchNodeInfos(); @@ -548,123 +544,8 @@ size_t PhoneAPI::getFromRadio(uint8_t *buf) nodeInfoMutex.lock(); nodeInfoQueue.clear(); nodeInfoMutex.unlock(); - // Replay states no-op for legacy clients / excluded DBs. - state = STATE_REPLAY_POSITIONS; - return getFromRadio(buf); - } - break; - } - - case STATE_REPLAY_POSITIONS: { - if (replayPositionOrder.empty() && replayPositionIndex == 0) - beginReplayPositions(); - prefetchReplayPositions(); - - meshtastic_MeshPacket pkt = {}; - bool havePkt = false; - { - concurrency::LockGuard guard(&nodeInfoMutex); - if (!replayQueue.empty()) { - pkt = replayQueue.front(); - replayQueue.pop_front(); - havePkt = true; - } - } - - if (havePkt) { - fromRadioScratch.which_payload_variant = meshtastic_FromRadio_packet_tag; - fromRadioScratch.packet = pkt; - } else { - LOG_DEBUG("Done replaying positions count=%u millis=%u", (unsigned)replayPositionIndex, millis()); - state = STATE_REPLAY_TELEMETRY; - return getFromRadio(buf); - } - break; - } - - case STATE_REPLAY_TELEMETRY: { - if (replayTelemetryOrder.empty() && replayTelemetryIndex == 0) - beginReplayTelemetry(); - prefetchReplayTelemetry(); - - meshtastic_MeshPacket pkt = {}; - bool havePkt = false; - { - concurrency::LockGuard guard(&nodeInfoMutex); - if (!replayQueue.empty()) { - pkt = replayQueue.front(); - replayQueue.pop_front(); - havePkt = true; - } - } - - if (havePkt) { - fromRadioScratch.which_payload_variant = meshtastic_FromRadio_packet_tag; - fromRadioScratch.packet = pkt; - } else { - LOG_DEBUG("Done replaying telemetry count=%u millis=%u", (unsigned)replayTelemetryIndex, millis()); - state = STATE_REPLAY_ENVIRONMENT; - return getFromRadio(buf); - } - break; - } - - case STATE_REPLAY_ENVIRONMENT: { - if (replayEnvironmentOrder.empty() && replayEnvironmentIndex == 0) - beginReplayEnvironment(); - prefetchReplayEnvironment(); - - meshtastic_MeshPacket pkt = {}; - bool havePkt = false; - { - concurrency::LockGuard guard(&nodeInfoMutex); - if (!replayQueue.empty()) { - pkt = replayQueue.front(); - replayQueue.pop_front(); - havePkt = true; - } - } - - if (havePkt) { - fromRadioScratch.which_payload_variant = meshtastic_FromRadio_packet_tag; - fromRadioScratch.packet = pkt; - } else { - LOG_DEBUG("Done replaying environment count=%u millis=%u", (unsigned)replayEnvironmentIndex, millis()); - state = STATE_REPLAY_STATUS; - return getFromRadio(buf); - } - break; - } - - case STATE_REPLAY_STATUS: { - if (replayStatusOrder.empty() && replayStatusIndex == 0) - beginReplayStatus(); - prefetchReplayStatus(); - - meshtastic_MeshPacket pkt = {}; - bool havePkt = false; - { - concurrency::LockGuard guard(&nodeInfoMutex); - if (!replayQueue.empty()) { - pkt = replayQueue.front(); - replayQueue.pop_front(); - havePkt = true; - } - } - - if (havePkt) { - fromRadioScratch.which_payload_variant = meshtastic_FromRadio_packet_tag; - fromRadioScratch.packet = pkt; - } else { - LOG_DEBUG("Done replaying status count=%u millis=%u", (unsigned)replayStatusIndex, millis()); - replayPositionOrder.clear(); - replayPositionOrder.shrink_to_fit(); - replayTelemetryOrder.clear(); - replayTelemetryOrder.shrink_to_fit(); - replayEnvironmentOrder.clear(); - replayEnvironmentOrder.shrink_to_fit(); - replayStatusOrder.clear(); - replayStatusOrder.shrink_to_fit(); + // Satellite-DB replay (positions/telemetry/environment/status) now happens + // *after* config_complete_id, interleaved with live traffic in STATE_SEND_PACKETS. state = STATE_SEND_FILEMANIFEST; return getFromRadio(buf); } @@ -674,8 +555,7 @@ size_t PhoneAPI::getFromRadio(uint8_t *buf) case STATE_SEND_FILEMANIFEST: { LOG_DEBUG("FromRadio=STATE_SEND_FILEMANIFEST"); // ONLY_NODES variants skip the manifest. - if (config_state == filesManifest.size() || config_nonce == SPECIAL_NONCE_ONLY_NODES || - config_nonce == SPECIAL_NONCE_GRADIENT_ONLY_NODES) { + if (config_state == filesManifest.size() || config_nonce == SPECIAL_NONCE_ONLY_NODES) { config_state = 0; filesManifest.clear(); // Skip to complete packet @@ -720,6 +600,16 @@ size_t PhoneAPI::getFromRadio(uint8_t *buf) fromRadioScratch.which_payload_variant = meshtastic_FromRadio_packet_tag; fromRadioScratch.packet = *packetForPhone; releasePhonePacket(); + } else if (replayPending()) { + // No live packet pending — feed the phone one cached satellite-DB packet. + // popReplayPacket advances through positions->telemetry->environment->status, + // and flips replayPhase back to IDLE when everything has been drained. + meshtastic_MeshPacket replayPkt; + if (popReplayPacket(replayPkt)) { + printPacket("replay packet to phone", &replayPkt); + fromRadioScratch.which_payload_variant = meshtastic_FromRadio_packet_tag; + fromRadioScratch.packet = replayPkt; + } } break; @@ -744,10 +634,19 @@ size_t PhoneAPI::getFromRadio(uint8_t *buf) void PhoneAPI::sendConfigComplete() { LOG_INFO("Config Send Complete millis=%u", millis()); + const bool shouldReplaySatellites = (config_nonce != SPECIAL_NONCE_ONLY_CONFIG); + // The phone sees config_complete_id first (treats sync as done), then the cached + // satellite-DB packets (positions / telemetry / environment / status) trickle in + // afterward as ordinary mesh packets (except SPECIAL_NONCE_ONLY_CONFIG, which + // skips node/satellite sync entirely). Any client that handles live POSITION_APP / + // TELEMETRY_APP / NODE_STATUS_APP packets handles these identically. STM32WL and + // other builds that compile the satellite DBs out produce no replay packets and + // the phase advances to IDLE in microseconds. fromRadioScratch.which_payload_variant = meshtastic_FromRadio_config_complete_id_tag; fromRadioScratch.config_complete_id = config_nonce; config_nonce = 0; state = STATE_SEND_PACKETS; + replayPhase = shouldReplaySatellites ? REPLAY_PHASE_POSITIONS : REPLAY_PHASE_IDLE; if (api_type == TYPE_BLE) { service->api_state = service->STATE_BLE; } else if (api_type == TYPE_WIFI) { @@ -788,7 +687,8 @@ void PhoneAPI::prefetchNodeInfos() { bool added = false; bool wasEmpty = false; - const bool gradient = clientWantsGradientSync(); + // Other-node NodeInfos always go out thin (no bundled position/device_metrics). + // The post-config_complete_id replay drain delivers those as ordinary mesh packets. // Keep the queue topped up so BLE reads stay responsive even if DB fetches take a moment. { concurrency::LockGuard guard(&nodeInfoMutex); @@ -798,8 +698,7 @@ void PhoneAPI::prefetchNodeInfos() if (!nextNode) break; - auto info = - gradient ? TypeConversions::ConvertToNodeInfoThin(nextNode) : TypeConversions::ConvertToNodeInfo(nextNode); + auto info = TypeConversions::ConvertToNodeInfoThin(nextNode); bool isUs = info.num == nodeDB->getNodeNum(); info.hops_away = isUs ? 0 : info.hops_away; info.last_heard = isUs ? getValidTime(RTCQualityFromNet) : info.last_heard; @@ -821,11 +720,20 @@ void PhoneAPI::prefetchNodeInfos() meshtastic_MeshPacket PhoneAPI::makeReplayPositionPacket(NodeNum num, const meshtastic_PositionLite &pos) { + // Shape this exactly like a fresh live broadcast Position from the peer so the + // phone runs it through its normal "live position broadcast" handler path. + // to=ourNum would read as a DM-from-peer and never lands in node detail UI. meshtastic_MeshPacket pkt = meshtastic_MeshPacket_init_default; pkt.from = num; - pkt.to = nodeDB->getNodeNum(); + pkt.to = NODENUM_BROADCAST; pkt.id = generatePacketId(); pkt.rx_time = pos.time; + pkt.channel = 0; + pkt.hop_limit = Default::getConfiguredOrDefaultHopLimit(config.lora.hop_limit); + pkt.hop_start = pkt.hop_limit; + pkt.priority = meshtastic_MeshPacket_Priority_BACKGROUND; + // Mark as if heard over the air, not internally generated + pkt.transport_mechanism = meshtastic_MeshPacket_TransportMechanism_TRANSPORT_LORA; pkt.which_payload_variant = meshtastic_MeshPacket_decoded_tag; pkt.decoded.portnum = meshtastic_PortNum_POSITION_APP; meshtastic_Position fullPos = TypeConversions::ConvertToPosition(pos); @@ -839,11 +747,18 @@ meshtastic_MeshPacket PhoneAPI::makeReplayTelemetryPacket(NodeNum num, const mes { meshtastic_MeshPacket pkt = meshtastic_MeshPacket_init_default; pkt.from = num; - pkt.to = nodeDB->getNodeNum(); + pkt.to = NODENUM_BROADCAST; pkt.id = generatePacketId(); // No native timestamp on telemetry packets here; use last_heard. const meshtastic_NodeInfoLite *header = nodeDB->getMeshNode(num); pkt.rx_time = header ? header->last_heard : 0; + pkt.channel = 0; + pkt.hop_limit = Default::getConfiguredOrDefaultHopLimit(config.lora.hop_limit); + pkt.hop_start = pkt.hop_limit; + pkt.priority = meshtastic_MeshPacket_Priority_BACKGROUND; + // Mark as if heard over the air, not internally generated — iOS client filters + // TRANSPORT_INTERNAL packets out of broadcast peer state updates. + pkt.transport_mechanism = meshtastic_MeshPacket_TransportMechanism_TRANSPORT_LORA; pkt.which_payload_variant = meshtastic_MeshPacket_decoded_tag; pkt.decoded.portnum = meshtastic_PortNum_TELEMETRY_APP; meshtastic_Telemetry fullTel = meshtastic_Telemetry_init_default; @@ -859,16 +774,12 @@ meshtastic_MeshPacket PhoneAPI::makeReplayTelemetryPacket(NodeNum num, const mes void PhoneAPI::beginReplayPositions() { #if MESHTASTIC_EXCLUDE_POSITIONDB - // Build excluded entirely - leave the order list empty so the state arm + // Build excluded entirely - leave the order list empty so the phase // immediately drains and advances. replayPositionOrder.clear(); replayPositionIndex = 0; #else - if (!clientWantsGradientSync()) { - replayPositionOrder.clear(); - replayPositionIndex = 0; - return; - } + // Caller (popReplayPacket) only invokes us when replayPhase is armed. // Snapshot the keyset at phase start so concurrent inserts/erases on the // map don't invalidate iteration. Skip our own node - the phone already // got our position bundled in STATE_SEND_OWN_NODEINFO. @@ -883,8 +794,6 @@ void PhoneAPI::prefetchReplayPositions() #if MESHTASTIC_EXCLUDE_POSITIONDB return; #else - if (!clientWantsGradientSync()) - return; bool added = false; bool wasEmpty = false; { @@ -910,11 +819,6 @@ void PhoneAPI::beginReplayTelemetry() replayTelemetryOrder.clear(); replayTelemetryIndex = 0; #else - if (!clientWantsGradientSync()) { - replayTelemetryOrder.clear(); - replayTelemetryIndex = 0; - return; - } replayTelemetryOrder = nodeDB->snapshotTelemetryNodeNums(nodeDB->getNodeNum()); replayTelemetryIndex = 0; LOG_INFO("Begin telemetry replay: %u entries millis=%u", (unsigned)replayTelemetryOrder.size(), millis()); @@ -926,8 +830,6 @@ void PhoneAPI::prefetchReplayTelemetry() #if MESHTASTIC_EXCLUDE_TELEMETRYDB return; #else - if (!clientWantsGradientSync()) - return; bool added = false; bool wasEmpty = false; { @@ -951,10 +853,17 @@ meshtastic_MeshPacket PhoneAPI::makeReplayEnvironmentPacket(uint32_t num, const { meshtastic_MeshPacket pkt = meshtastic_MeshPacket_init_default; pkt.from = num; - pkt.to = nodeDB->getNodeNum(); + pkt.to = NODENUM_BROADCAST; pkt.id = generatePacketId(); const meshtastic_NodeInfoLite *header = nodeDB->getMeshNode(num); pkt.rx_time = header ? header->last_heard : 0; + pkt.channel = 0; + pkt.hop_limit = Default::getConfiguredOrDefaultHopLimit(config.lora.hop_limit); + pkt.hop_start = pkt.hop_limit; + pkt.priority = meshtastic_MeshPacket_Priority_BACKGROUND; + // Mark as if heard over the air, not internally generated — iOS client filters + // TRANSPORT_INTERNAL packets out of broadcast peer state updates. + pkt.transport_mechanism = meshtastic_MeshPacket_TransportMechanism_TRANSPORT_LORA; pkt.which_payload_variant = meshtastic_MeshPacket_decoded_tag; pkt.decoded.portnum = meshtastic_PortNum_TELEMETRY_APP; meshtastic_Telemetry fullTel = meshtastic_Telemetry_init_default; @@ -973,11 +882,6 @@ void PhoneAPI::beginReplayEnvironment() replayEnvironmentOrder.clear(); replayEnvironmentIndex = 0; #else - if (!clientWantsGradientSync()) { - replayEnvironmentOrder.clear(); - replayEnvironmentIndex = 0; - return; - } replayEnvironmentOrder = nodeDB->snapshotEnvironmentNodeNums(nodeDB->getNodeNum()); replayEnvironmentIndex = 0; LOG_INFO("Begin environment replay: %u entries millis=%u", (unsigned)replayEnvironmentOrder.size(), millis()); @@ -989,8 +893,6 @@ void PhoneAPI::prefetchReplayEnvironment() #if MESHTASTIC_EXCLUDE_ENVIRONMENTDB return; #else - if (!clientWantsGradientSync()) - return; bool added = false; bool wasEmpty = false; { @@ -1014,11 +916,17 @@ meshtastic_MeshPacket PhoneAPI::makeReplayStatusPacket(uint32_t num, const mesht { meshtastic_MeshPacket pkt = meshtastic_MeshPacket_init_default; pkt.from = num; - pkt.to = nodeDB->getNodeNum(); + pkt.to = NODENUM_BROADCAST; pkt.id = generatePacketId(); // StatusMessage has no native timestamp; use last_heard. const meshtastic_NodeInfoLite *header = nodeDB->getMeshNode(num); pkt.rx_time = header ? header->last_heard : 0; + pkt.channel = 0; + pkt.hop_limit = Default::getConfiguredOrDefaultHopLimit(config.lora.hop_limit); + pkt.hop_start = pkt.hop_limit; + pkt.priority = meshtastic_MeshPacket_Priority_BACKGROUND; + // Mark as if heard over the air, not internally generated — client filters + pkt.transport_mechanism = meshtastic_MeshPacket_TransportMechanism_TRANSPORT_LORA; pkt.which_payload_variant = meshtastic_MeshPacket_decoded_tag; pkt.decoded.portnum = meshtastic_PortNum_NODE_STATUS_APP; size_t len = @@ -1033,11 +941,6 @@ void PhoneAPI::beginReplayStatus() replayStatusOrder.clear(); replayStatusIndex = 0; #else - if (!clientWantsGradientSync()) { - replayStatusOrder.clear(); - replayStatusIndex = 0; - return; - } replayStatusOrder = nodeDB->snapshotStatusNodeNums(nodeDB->getNodeNum()); replayStatusIndex = 0; LOG_INFO("Begin status replay: %u entries millis=%u", (unsigned)replayStatusOrder.size(), millis()); @@ -1049,8 +952,6 @@ void PhoneAPI::prefetchReplayStatus() #if MESHTASTIC_EXCLUDE_STATUSDB return; #else - if (!clientWantsGradientSync()) - return; bool added = false; bool wasEmpty = false; { @@ -1070,6 +971,94 @@ void PhoneAPI::prefetchReplayStatus() #endif } +// Pop one cached satellite-DB packet from the active replay phase. +// Phases drain in order: positions -> telemetry -> environment -> status. +// When the current phase's cursor is exhausted (queue empty AND no more entries +// to snapshot), advance to the next phase. When all four phases are done, +// flip replayPhase back to IDLE and release the snapshot vectors. +// +// Returns true if a packet was placed in `out`; false if everything is drained. +bool PhoneAPI::popReplayPacket(meshtastic_MeshPacket &out) +{ + while (replayPhase != REPLAY_PHASE_IDLE) { + // Prime the active phase: seed the snapshot vector on first entry, + // top up replayQueue from the snapshot up to kReplayPrefetchDepth. + switch (replayPhase) { + case REPLAY_PHASE_POSITIONS: + if (replayPositionOrder.empty() && replayPositionIndex == 0) + beginReplayPositions(); + prefetchReplayPositions(); + break; + case REPLAY_PHASE_TELEMETRY: + if (replayTelemetryOrder.empty() && replayTelemetryIndex == 0) + beginReplayTelemetry(); + prefetchReplayTelemetry(); + break; + case REPLAY_PHASE_ENVIRONMENT: + if (replayEnvironmentOrder.empty() && replayEnvironmentIndex == 0) + beginReplayEnvironment(); + prefetchReplayEnvironment(); + break; + case REPLAY_PHASE_STATUS: + if (replayStatusOrder.empty() && replayStatusIndex == 0) + beginReplayStatus(); + prefetchReplayStatus(); + break; + default: + break; + } + + { + concurrency::LockGuard guard(&nodeInfoMutex); + if (!replayQueue.empty()) { + out = replayQueue.front(); + replayQueue.pop_front(); + return true; + } + } + + // Queue empty AND no more entries to feed it — phase is exhausted. + advanceReplayPhase(); + } + return false; +} + +void PhoneAPI::advanceReplayPhase() +{ + switch (replayPhase) { + case REPLAY_PHASE_POSITIONS: + LOG_DEBUG("Replay drain: positions done (count=%u) millis=%u", (unsigned)replayPositionIndex, millis()); + replayPhase = REPLAY_PHASE_TELEMETRY; + break; + case REPLAY_PHASE_TELEMETRY: + LOG_DEBUG("Replay drain: telemetry done (count=%u) millis=%u", (unsigned)replayTelemetryIndex, millis()); + replayPhase = REPLAY_PHASE_ENVIRONMENT; + break; + case REPLAY_PHASE_ENVIRONMENT: + LOG_DEBUG("Replay drain: environment done (count=%u) millis=%u", (unsigned)replayEnvironmentIndex, millis()); + replayPhase = REPLAY_PHASE_STATUS; + break; + case REPLAY_PHASE_STATUS: + LOG_INFO("Replay drain complete (status count=%u) millis=%u", (unsigned)replayStatusIndex, millis()); + replayPositionOrder.clear(); + replayPositionOrder.shrink_to_fit(); + replayTelemetryOrder.clear(); + replayTelemetryOrder.shrink_to_fit(); + replayEnvironmentOrder.clear(); + replayEnvironmentOrder.shrink_to_fit(); + replayStatusOrder.clear(); + replayStatusOrder.shrink_to_fit(); + replayPositionIndex = 0; + replayTelemetryIndex = 0; + replayEnvironmentIndex = 0; + replayStatusIndex = 0; + replayPhase = REPLAY_PHASE_IDLE; + break; + default: + break; + } +} + void PhoneAPI::releaseMqttClientProxyPhonePacket() { if (mqttClientProxyMessageForPhone) { @@ -1116,31 +1105,6 @@ bool PhoneAPI::available() PREFETCH_NODEINFO: prefetchNodeInfos(); return true; - case STATE_REPLAY_POSITIONS: { - // Prime the iterator if we haven't yet, then top up the queue. - if (replayPositionOrder.empty() && replayPositionIndex == 0) - beginReplayPositions(); - prefetchReplayPositions(); - return true; // Always advance state machine; arm itself transitions when drained - } - case STATE_REPLAY_TELEMETRY: { - if (replayTelemetryOrder.empty() && replayTelemetryIndex == 0) - beginReplayTelemetry(); - prefetchReplayTelemetry(); - return true; - } - case STATE_REPLAY_ENVIRONMENT: { - if (replayEnvironmentOrder.empty() && replayEnvironmentIndex == 0) - beginReplayEnvironment(); - prefetchReplayEnvironment(); - return true; - } - case STATE_REPLAY_STATUS: { - if (replayStatusOrder.empty() && replayStatusIndex == 0) - beginReplayStatus(); - prefetchReplayStatus(); - return true; - } case STATE_SEND_PACKETS: { if (!queueStatusPacketForPhone) queueStatusPacketForPhone = service->getQueueStatusForPhone(); @@ -1172,7 +1136,11 @@ bool PhoneAPI::available() if (!packetForPhone) packetForPhone = service->getForPhone(); hasPacket = !!packetForPhone; - return hasPacket; + if (hasPacket) + return true; + // Trailing replay drain — feeds cached satellite-DB packets alongside + // (lower priority than) live traffic. + return replayPending(); } default: LOG_ERROR("PhoneAPI::available unexpected state %d", state); diff --git a/src/mesh/PhoneAPI.h b/src/mesh/PhoneAPI.h index 642fdd7e0..b1bd1fd23 100644 --- a/src/mesh/PhoneAPI.h +++ b/src/mesh/PhoneAPI.h @@ -26,9 +26,6 @@ #define SPECIAL_NONCE_ONLY_CONFIG 69420 #define SPECIAL_NONCE_ONLY_NODES 69421 // ( ͡° ͜ʖ ͡°) -// Gradient sync: phone sends one of these to opt into thin-header + replay. -#define SPECIAL_NONCE_GRADIENT_SYNC 69422 -#define SPECIAL_NONCE_GRADIENT_ONLY_NODES 69423 /** * Provides our protobuf based API which phone/PC clients can use to talk to our device @@ -52,15 +49,22 @@ class PhoneAPI STATE_SEND_CONFIG, // Replacement for the old Radioconfig STATE_SEND_MODULECONFIG, // Send Module specific config STATE_SEND_OTHER_NODEINFOS, // states progress in this order as the device sends to to the client - // Drain satellite DBs as synthetic POSITION_APP / TELEMETRY_APP / - // NODE_STATUS_APP packets when the phone opted into gradient sync. - STATE_REPLAY_POSITIONS, - STATE_REPLAY_TELEMETRY, - STATE_REPLAY_ENVIRONMENT, - STATE_REPLAY_STATUS, - STATE_SEND_FILEMANIFEST, // Send file manifest + STATE_SEND_FILEMANIFEST, // Send file manifest STATE_SEND_COMPLETE_ID, - STATE_SEND_PACKETS // send packets or debug strings + STATE_SEND_PACKETS // live mesh packets + any cached satellite-DB replay that trails sync completion + }; + + // Satellite-DB replay (positions / telemetry / environment / status) used to live + // as four top-level states between STATE_SEND_OTHER_NODEINFOS and STATE_SEND_FILEMANIFEST. + // It now drains *after* config_complete_id has been emitted: the phone considers the + // initial sync done as soon as headers + manifest are delivered, and the cached + // position/telemetry/etc. trickle in alongside live mesh traffic inside STATE_SEND_PACKETS. + enum ReplayPhase : uint8_t { + REPLAY_PHASE_IDLE = 0, // not replaying (legacy clients, no-op DBs, or replay finished) + REPLAY_PHASE_POSITIONS, + REPLAY_PHASE_TELEMETRY, + REPLAY_PHASE_ENVIRONMENT, + REPLAY_PHASE_STATUS, }; State state = STATE_SEND_NOTHING; @@ -114,6 +118,7 @@ class PhoneAPI size_t replayTelemetryIndex = 0; size_t replayEnvironmentIndex = 0; size_t replayStatusIndex = 0; + ReplayPhase replayPhase = REPLAY_PHASE_IDLE; // armed by sendConfigComplete() for full/default sync meshtastic_ToRadio toRadioScratch = { 0}; // this is a static scratch object, any data must be copied elsewhere before returning @@ -164,10 +169,6 @@ class PhoneAPI bool isConnected() { return state != STATE_SEND_NOTHING; } bool isSendingPackets() { return state == STATE_SEND_PACKETS; } - bool clientWantsGradientSync() const - { - return config_nonce == SPECIAL_NONCE_GRADIENT_SYNC || config_nonce == SPECIAL_NONCE_GRADIENT_ONLY_NODES; - } protected: /// Our fromradio packet while it is being assembled @@ -229,6 +230,12 @@ class PhoneAPI meshtastic_MeshPacket makeReplayEnvironmentPacket(uint32_t num, const meshtastic_EnvironmentMetrics &env); meshtastic_MeshPacket makeReplayStatusPacket(uint32_t num, const meshtastic_StatusMessage &status); + // Post-sync replay drain: pop one cached packet from the active phase, advancing + // through positions -> telemetry -> environment -> status until everything is drained. + bool popReplayPacket(meshtastic_MeshPacket &out); + void advanceReplayPhase(); + bool replayPending() const { return replayPhase != REPLAY_PHASE_IDLE; } + void releaseMqttClientProxyPhonePacket(); void releaseClientNotification(); diff --git a/src/mesh/TypeConversions.cpp b/src/mesh/TypeConversions.cpp index 254af6132..cdcf0b328 100644 --- a/src/mesh/TypeConversions.cpp +++ b/src/mesh/TypeConversions.cpp @@ -37,6 +37,7 @@ meshtastic_NodeInfo TypeConversions::ConvertToNodeInfo(const meshtastic_NodeInfo info.position.altitude = position->altitude; info.position.location_source = position->location_source; info.position.time = position->time; + info.position.precision_bits = position->precision_bits; } if (nodeInfoLiteHasUser(lite)) { info.has_user = true; @@ -71,6 +72,7 @@ meshtastic_PositionLite TypeConversions::ConvertToPositionLite(meshtastic_Positi lite.altitude = position.altitude; lite.location_source = position.location_source; lite.time = position.time; + lite.precision_bits = position.precision_bits; return lite; } @@ -89,6 +91,11 @@ meshtastic_Position TypeConversions::ConvertToPosition(meshtastic_PositionLite l position.altitude = lite.altitude; position.location_source = lite.location_source; position.time = lite.time; + // Preserve the peer's broadcast precision; falls back to 0 for entries cached + // before the precision_bits field existed in PositionLite (pre-migration data). + // iOS treats 0 as "unspecified precision" and won't render the pin — so for + // unset values, declare full precision so the stored lat/lon renders as a point. + position.precision_bits = lite.precision_bits == 0 ? 32 : lite.precision_bits; return position; } diff --git a/src/mesh/generated/meshtastic/deviceonly.pb.h b/src/mesh/generated/meshtastic/deviceonly.pb.h index 5e0b844f1..17bec9b3a 100644 --- a/src/mesh/generated/meshtastic/deviceonly.pb.h +++ b/src/mesh/generated/meshtastic/deviceonly.pb.h @@ -33,6 +33,8 @@ typedef struct _meshtastic_PositionLite { uint32_t time; /* TODO: REPLACE */ meshtastic_Position_LocSource location_source; + /* Indicates the bits of precision set by the sending node */ + uint32_t precision_bits; } meshtastic_PositionLite; typedef PB_BYTES_ARRAY_T(32) meshtastic_UserLite_public_key_t; @@ -211,7 +213,7 @@ extern "C" { #endif /* Initializer values for message structs */ -#define meshtastic_PositionLite_init_default {0, 0, 0, 0, _meshtastic_Position_LocSource_MIN} +#define meshtastic_PositionLite_init_default {0, 0, 0, 0, _meshtastic_Position_LocSource_MIN, 0} #define meshtastic_UserLite_init_default {{0}, "", "", _meshtastic_HardwareModel_MIN, 0, _meshtastic_Config_DeviceConfig_Role_MIN, {0, {0}}, false, 0} #define meshtastic_NodeInfoLite_init_default {0, 0, 0, 0, false, 0, 0, 0, "", "", _meshtastic_HardwareModel_MIN, _meshtastic_Config_DeviceConfig_Role_MIN, {0, {0}}} #define meshtastic_DeviceState_init_default {false, meshtastic_MyNodeInfo_init_default, false, meshtastic_User_init_default, 0, {meshtastic_MeshPacket_init_default}, false, meshtastic_MeshPacket_init_default, 0, 0, 0, false, meshtastic_MeshPacket_init_default, 0, {meshtastic_NodeRemoteHardwarePin_init_default, meshtastic_NodeRemoteHardwarePin_init_default, meshtastic_NodeRemoteHardwarePin_init_default, meshtastic_NodeRemoteHardwarePin_init_default, meshtastic_NodeRemoteHardwarePin_init_default, meshtastic_NodeRemoteHardwarePin_init_default, meshtastic_NodeRemoteHardwarePin_init_default, meshtastic_NodeRemoteHardwarePin_init_default, meshtastic_NodeRemoteHardwarePin_init_default, meshtastic_NodeRemoteHardwarePin_init_default, meshtastic_NodeRemoteHardwarePin_init_default, meshtastic_NodeRemoteHardwarePin_init_default}} @@ -222,7 +224,7 @@ extern "C" { #define meshtastic_NodeDatabase_init_default {0, {0}, {0}, {0}, {0}, {0}} #define meshtastic_ChannelFile_init_default {0, {meshtastic_Channel_init_default, meshtastic_Channel_init_default, meshtastic_Channel_init_default, meshtastic_Channel_init_default, meshtastic_Channel_init_default, meshtastic_Channel_init_default, meshtastic_Channel_init_default, meshtastic_Channel_init_default}, 0} #define meshtastic_BackupPreferences_init_default {0, 0, false, meshtastic_LocalConfig_init_default, false, meshtastic_LocalModuleConfig_init_default, false, meshtastic_ChannelFile_init_default, false, meshtastic_User_init_default} -#define meshtastic_PositionLite_init_zero {0, 0, 0, 0, _meshtastic_Position_LocSource_MIN} +#define meshtastic_PositionLite_init_zero {0, 0, 0, 0, _meshtastic_Position_LocSource_MIN, 0} #define meshtastic_UserLite_init_zero {{0}, "", "", _meshtastic_HardwareModel_MIN, 0, _meshtastic_Config_DeviceConfig_Role_MIN, {0, {0}}, false, 0} #define meshtastic_NodeInfoLite_init_zero {0, 0, 0, 0, false, 0, 0, 0, "", "", _meshtastic_HardwareModel_MIN, _meshtastic_Config_DeviceConfig_Role_MIN, {0, {0}}} #define meshtastic_DeviceState_init_zero {false, meshtastic_MyNodeInfo_init_zero, false, meshtastic_User_init_zero, 0, {meshtastic_MeshPacket_init_zero}, false, meshtastic_MeshPacket_init_zero, 0, 0, 0, false, meshtastic_MeshPacket_init_zero, 0, {meshtastic_NodeRemoteHardwarePin_init_zero, meshtastic_NodeRemoteHardwarePin_init_zero, meshtastic_NodeRemoteHardwarePin_init_zero, meshtastic_NodeRemoteHardwarePin_init_zero, meshtastic_NodeRemoteHardwarePin_init_zero, meshtastic_NodeRemoteHardwarePin_init_zero, meshtastic_NodeRemoteHardwarePin_init_zero, meshtastic_NodeRemoteHardwarePin_init_zero, meshtastic_NodeRemoteHardwarePin_init_zero, meshtastic_NodeRemoteHardwarePin_init_zero, meshtastic_NodeRemoteHardwarePin_init_zero, meshtastic_NodeRemoteHardwarePin_init_zero}} @@ -240,6 +242,7 @@ extern "C" { #define meshtastic_PositionLite_altitude_tag 3 #define meshtastic_PositionLite_time_tag 4 #define meshtastic_PositionLite_location_source_tag 5 +#define meshtastic_PositionLite_precision_bits_tag 6 #define meshtastic_UserLite_macaddr_tag 1 #define meshtastic_UserLite_long_name_tag 2 #define meshtastic_UserLite_short_name_tag 3 @@ -298,7 +301,8 @@ X(a, STATIC, SINGULAR, SFIXED32, latitude_i, 1) \ X(a, STATIC, SINGULAR, SFIXED32, longitude_i, 2) \ X(a, STATIC, SINGULAR, INT32, altitude, 3) \ X(a, STATIC, SINGULAR, FIXED32, time, 4) \ -X(a, STATIC, SINGULAR, UENUM, location_source, 5) +X(a, STATIC, SINGULAR, UENUM, location_source, 5) \ +X(a, STATIC, SINGULAR, UINT32, precision_bits, 6) #define meshtastic_PositionLite_CALLBACK NULL #define meshtastic_PositionLite_DEFAULT NULL @@ -447,10 +451,10 @@ extern const pb_msgdesc_t meshtastic_BackupPreferences_msg; #define meshtastic_DeviceState_size 1737 #define meshtastic_NodeEnvironmentEntry_size 170 #define meshtastic_NodeInfoLite_size 105 -#define meshtastic_NodePositionEntry_size 36 +#define meshtastic_NodePositionEntry_size 42 #define meshtastic_NodeStatusEntry_size 89 #define meshtastic_NodeTelemetryEntry_size 35 -#define meshtastic_PositionLite_size 28 +#define meshtastic_PositionLite_size 34 #define meshtastic_UserLite_size 98 #ifdef __cplusplus diff --git a/src/mesh/generated/meshtastic/deviceonly_legacy.pb.h b/src/mesh/generated/meshtastic/deviceonly_legacy.pb.h index 916951419..6beee0809 100644 --- a/src/mesh/generated/meshtastic/deviceonly_legacy.pb.h +++ b/src/mesh/generated/meshtastic/deviceonly_legacy.pb.h @@ -115,7 +115,7 @@ extern const pb_msgdesc_t meshtastic_NodeDatabase_Legacy_msg; /* Maximum encoded size of messages (where known) */ /* meshtastic_NodeDatabase_Legacy_size depends on runtime parameters */ #define MESHTASTIC_MESHTASTIC_DEVICEONLY_LEGACY_PB_H_MAX_SIZE meshtastic_NodeInfoLite_Legacy_size -#define meshtastic_NodeInfoLite_Legacy_size 196 +#define meshtastic_NodeInfoLite_Legacy_size 202 #ifdef __cplusplus } /* extern "C" */