From c55fc438f10c7142935b39db8a8524a4d4d910ad Mon Sep 17 00:00:00 2001 From: Jonathan Bennett Date: Tue, 3 Nov 2026 19:06:51 +0000 Subject: [PATCH 1/2] NodeDB: encode satellite maps in place when saving (#12293) * NodeDB: encode satellite maps in place when saving saveNodeDatabaseToDisk() copied each satellite map into nodeDatabase's vectors just to encode them, so every save needed a second, contiguous copy: up to ~19 KB at the nRF52 cap of 40, 40 x 312 B for environment alone. On busy meshes this ran nRF52 sensor nodes out of heap (new(10920) failed in saveNodeDatabaseToDisk()). Mirror the decode path: arm per-map encode sources for the duration of the save, and have the NodeDatabase encode callback walk the map, one stack entry at a time. Same entries in the same order, so nodes.proto is byte-identical. Co-Authored-By: Claude Opus 5.5 * NodeDB: clear satellite vectors before encoding the save With a satellite DB compiled out, nodes.proto's entries for it still decode into nodeDatabase's vector. The projection used to clear those vectors before encoding; without that, the first save after boot wrote the stale entries back. Clear all four before arming the encode sources; an armed source ignores its vector anyway. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- src/mesh/NodeDB.cpp | 163 ++++++++++++++++++++++++++++++-------------- src/mesh/NodeDB.h | 2 + 2 files changed, 112 insertions(+), 53 deletions(-) diff --git a/src/mesh/NodeDB.cpp b/src/mesh/NodeDB.cpp index e63e90abf3..06354ba443 100644 --- a/src/mesh/NodeDB.cpp +++ b/src/mesh/NodeDB.cpp @@ -259,11 +259,40 @@ std::map *s_decodeEnvironmentTarget = nu std::map *s_decodeStatusTarget = nullptr; #endif +// When armed by saveNodeDatabaseToDisk, the encode callback reads satellite entries straight from these maps, so a +// save never builds a second, contiguous copy of them. Nullptr = encode from the vectors. +#if !MESHTASTIC_EXCLUDE_POSITIONDB +const std::map *s_encodePositionsSource = nullptr; +#endif +#if !MESHTASTIC_EXCLUDE_TELEMETRYDB +const std::map *s_encodeTelemetrySource = nullptr; +#endif +#if !MESHTASTIC_EXCLUDE_ENVIRONMENTDB +const std::map *s_encodeEnvironmentSource = nullptr; +#endif +#if !MESHTASTIC_EXCLUDE_STATUSDB +const std::map *s_encodeStatusSource = nullptr; +#endif + // Keys that can never name a real node. [[maybe_unused]] inline bool isUsableSatelliteKey(NodeNum n) { return n != 0 && !isBroadcast(n); } + +// Encode a satellite map as repeated entries, building one entry at a time on the stack. +template +[[maybe_unused]] bool encodeSatelliteMap(pb_ostream_t *ostream, const pb_field_iter_t *iter, const pb_msgdesc_t *fields, + const std::map &source, Fill fill) +{ + for (const auto &kv : source) { + Entry entry = {}; + fill(entry, kv.first, kv.second); + if (!pb_encode_tag_for_field(ostream, iter) || !pb_encode_submessage(ostream, fields, &entry)) + return false; + } + return true; +} } // namespace bool meshtastic_NodeDatabase_callback(pb_istream_t *istream, pb_ostream_t *ostream, const pb_field_t *field) @@ -304,6 +333,16 @@ bool meshtastic_NodeDatabase_callback(pb_istream_t *istream, pb_ostream_t *ostre } case meshtastic_NodeDatabase_positions_tag: { if (ostream) { +#if !MESHTASTIC_EXCLUDE_POSITIONDB + if (s_encodePositionsSource) + return encodeSatelliteMap( + ostream, iter, meshtastic_NodePositionEntry_fields, *s_encodePositionsSource, + [](meshtastic_NodePositionEntry &e, NodeNum num, const meshtastic_PositionLite &v) { + e.num = num; + e.has_position = true; + e.position = v; + }); +#endif const auto *vec = static_cast *>(iter->pData); for (const auto &item : *vec) { if (!pb_encode_tag_for_field(ostream, iter)) @@ -330,6 +369,16 @@ bool meshtastic_NodeDatabase_callback(pb_istream_t *istream, pb_ostream_t *ostre } case meshtastic_NodeDatabase_telemetry_tag: { if (ostream) { +#if !MESHTASTIC_EXCLUDE_TELEMETRYDB + if (s_encodeTelemetrySource) + return encodeSatelliteMap( + ostream, iter, meshtastic_NodeTelemetryEntry_fields, *s_encodeTelemetrySource, + [](meshtastic_NodeTelemetryEntry &e, NodeNum num, const meshtastic_DeviceMetrics &v) { + e.num = num; + e.has_device_metrics = true; + e.device_metrics = v; + }); +#endif const auto *vec = static_cast *>(iter->pData); for (const auto &item : *vec) { if (!pb_encode_tag_for_field(ostream, iter)) @@ -356,6 +405,16 @@ bool meshtastic_NodeDatabase_callback(pb_istream_t *istream, pb_ostream_t *ostre } case meshtastic_NodeDatabase_status_tag: { if (ostream) { +#if !MESHTASTIC_EXCLUDE_STATUSDB + if (s_encodeStatusSource) + return encodeSatelliteMap( + ostream, iter, meshtastic_NodeStatusEntry_fields, *s_encodeStatusSource, + [](meshtastic_NodeStatusEntry &e, NodeNum num, const meshtastic_StatusMessage &v) { + e.num = num; + e.has_status = true; + e.status = v; + }); +#endif const auto *vec = static_cast *>(iter->pData); for (const auto &item : *vec) { if (!pb_encode_tag_for_field(ostream, iter)) @@ -382,6 +441,16 @@ bool meshtastic_NodeDatabase_callback(pb_istream_t *istream, pb_ostream_t *ostre } case meshtastic_NodeDatabase_environment_tag: { if (ostream) { +#if !MESHTASTIC_EXCLUDE_ENVIRONMENTDB + if (s_encodeEnvironmentSource) + return encodeSatelliteMap( + ostream, iter, meshtastic_NodeEnvironmentEntry_fields, *s_encodeEnvironmentSource, + [](meshtastic_NodeEnvironmentEntry &e, NodeNum num, const meshtastic_EnvironmentMetrics &v) { + e.num = num; + e.has_environment_metrics = true; + e.environment_metrics = v; + }); +#endif const auto *vec = static_cast *>(iter->pData); for (const auto &item : *vec) { if (!pb_encode_tag_for_field(ostream, iter)) @@ -431,6 +500,38 @@ void NodeDB::armNodeDatabaseDecodeTargets() #endif } +void NodeDB::armNodeDatabaseEncodeSources() +{ +#if !MESHTASTIC_EXCLUDE_POSITIONDB + s_encodePositionsSource = &nodePositions; +#endif +#if !MESHTASTIC_EXCLUDE_TELEMETRYDB + s_encodeTelemetrySource = &nodeTelemetry; +#endif +#if !MESHTASTIC_EXCLUDE_ENVIRONMENTDB + s_encodeEnvironmentSource = &nodeEnvironment; +#endif +#if !MESHTASTIC_EXCLUDE_STATUSDB + s_encodeStatusSource = &nodeStatus; +#endif +} + +void NodeDB::disarmNodeDatabaseEncodeSources() +{ +#if !MESHTASTIC_EXCLUDE_POSITIONDB + s_encodePositionsSource = nullptr; +#endif +#if !MESHTASTIC_EXCLUDE_TELEMETRYDB + s_encodeTelemetrySource = nullptr; +#endif +#if !MESHTASTIC_EXCLUDE_ENVIRONMENTDB + s_encodeEnvironmentSource = nullptr; +#endif +#if !MESHTASTIC_EXCLUDE_STATUSDB + s_encodeStatusSource = nullptr; +#endif +} + void NodeDB::disarmNodeDatabaseDecodeTargets() { #if !MESHTASTIC_EXCLUDE_POSITIONDB @@ -3297,64 +3398,20 @@ bool NodeDB::saveNodeDatabaseToDisk() spiLock->unlock(); #endif - // Project the maps into the on-disk vectors just before encoding; cleared - // again on the way out so we don't carry duplicate state. + // Encode the satellite maps in place. Projecting them into the vectors first needed a second, contiguous copy on + // every save (40 x 312 B for environment alone), which ran nRF52 out of heap. The lock keeps the maps unchanged + // between the sizing pass and the write. concurrency::LockGuard guard(&satelliteMutex); -#if !MESHTASTIC_EXCLUDE_POSITIONDB + // An armed source encodes from its map; a compiled-out DB's decoded leftovers in the vectors must not be re-saved nodeDatabase.positions.clear(); - nodeDatabase.positions.reserve(nodePositions.size()); - for (const auto &kv : nodePositions) { - meshtastic_NodePositionEntry entry = meshtastic_NodePositionEntry_init_default; - entry.num = kv.first; - entry.has_position = true; - entry.position = kv.second; - nodeDatabase.positions.push_back(entry); - } -#else - nodeDatabase.positions.clear(); -#endif - -#if !MESHTASTIC_EXCLUDE_TELEMETRYDB nodeDatabase.telemetry.clear(); - nodeDatabase.telemetry.reserve(nodeTelemetry.size()); - for (const auto &kv : nodeTelemetry) { - meshtastic_NodeTelemetryEntry entry = meshtastic_NodeTelemetryEntry_init_default; - entry.num = kv.first; - entry.has_device_metrics = true; - entry.device_metrics = kv.second; - nodeDatabase.telemetry.push_back(entry); - } -#else - nodeDatabase.telemetry.clear(); -#endif - -#if !MESHTASTIC_EXCLUDE_ENVIRONMENTDB nodeDatabase.environment.clear(); - nodeDatabase.environment.reserve(nodeEnvironment.size()); - for (const auto &kv : nodeEnvironment) { - meshtastic_NodeEnvironmentEntry entry = meshtastic_NodeEnvironmentEntry_init_default; - entry.num = kv.first; - entry.has_environment_metrics = true; - entry.environment_metrics = kv.second; - nodeDatabase.environment.push_back(entry); - } -#else - nodeDatabase.environment.clear(); -#endif - -#if !MESHTASTIC_EXCLUDE_STATUSDB nodeDatabase.status.clear(); - nodeDatabase.status.reserve(nodeStatus.size()); - for (const auto &kv : nodeStatus) { - meshtastic_NodeStatusEntry entry = meshtastic_NodeStatusEntry_init_default; - entry.num = kv.first; - entry.has_status = true; - entry.status = kv.second; - nodeDatabase.status.push_back(entry); - } -#else - nodeDatabase.status.clear(); -#endif + armNodeDatabaseEncodeSources(); + struct Disarm { + NodeDB &self; + ~Disarm() { self.disarmNodeDatabaseEncodeSources(); } + } disarm{*this}; size_t nodeDatabaseSize; pb_get_encoded_size(&nodeDatabaseSize, meshtastic_NodeDatabase_fields, &nodeDatabase); diff --git a/src/mesh/NodeDB.h b/src/mesh/NodeDB.h index d270b7c437..73af9cf17d 100644 --- a/src/mesh/NodeDB.h +++ b/src/mesh/NodeDB.h @@ -823,6 +823,8 @@ class NodeDB // temp vectors. Must be paired - disarm before any other NodeDatabase decode. void armNodeDatabaseDecodeTargets(); void disarmNodeDatabaseDecodeTargets(); + void armNodeDatabaseEncodeSources(); + void disarmNodeDatabaseEncodeSources(); }; extern NodeDB *nodeDB; From e441cb1d745f1aba0b6bfcdac8750241991e94f0 Mon Sep 17 00:00:00 2001 From: Jonathan Bennett Date: Tue, 3 Nov 2026 19:32:16 +0000 Subject: [PATCH 2/2] CI: gate release uploads on manual runs, not the default branch (#12296) The 'Add sources' and 'Add bins and debug elfs' steps only ran when the dispatch targeted the default branch, so a release run from a maintenance branch created an empty draft release. Gate them on workflow_dispatch so any manually dispatched release run uploads its assets. Claude-Session: https://claude.ai/code/session_011C7D9eV9iMt7VkhQcLo961 Co-authored-by: Claude --- .github/workflows/main_matrix.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main_matrix.yml b/.github/workflows/main_matrix.yml index bcfde2bb56..7cbeefc955 100644 --- a/.github/workflows/main_matrix.yml +++ b/.github/workflows/main_matrix.yml @@ -576,8 +576,8 @@ jobs: path: firmware-${{ needs.version.outputs.long }}.json - name: Add sources to GitHub Release - # Only run when targeting the default branch with workflow_dispatch - if: github.ref_name == github.event.repository.default_branch + # Only upload on manual (workflow_dispatch) release runs, from any branch + if: github.event_name == 'workflow_dispatch' run: | gh release upload v${{ needs.version.outputs.long }} ./firmware-${{ needs.version.outputs.long }}.json gh release upload v${{ needs.version.outputs.long }} ./output/meshtasticd-${{ needs.version.outputs.deb }}-src.zip @@ -643,8 +643,8 @@ jobs: run: ls -lR - name: Add bins and debug elfs to GitHub Release - # Only run when targeting the default branch with workflow_dispatch - if: github.ref_name == github.event.repository.default_branch + # Only upload on manual (workflow_dispatch) release runs, from any branch + if: github.event_name == 'workflow_dispatch' run: | gh release upload v${{ needs.version.outputs.long }} ./firmware-${{matrix.arch}}-${{ needs.version.outputs.long }}.zip gh release upload v${{ needs.version.outputs.long }} ./debug-elfs-${{matrix.arch}}-${{ needs.version.outputs.long }}.zip