portduino Sensors: run commands on a worker thread

CommandSensor ran its commands from getMetrics(), on the main loop that
also drives the radio. Each expired sensor could stall the loop for up to
its 2 s timeout, and on Windows, where popen() has no timeout, a command
that never exits froze the node.

- A worker thread re-runs each command every 30 s and stores the result.
  getMetrics() only reads the latest results under a mutex, so it never
  waits on a command.
- The worker never calls LOG_* (portduino's log buffer is not
  thread-safe). It records why a read failed, and getMetrics() logs each
  new result on the main thread, as NativeDiscovery does.
- execCommand() returns an ExecResult instead of logging. exec() logs
  timeouts and start failures for its main-thread callers (HostMetrics).
- The sensor copies its config at init and joins the worker on
  destruction.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011C7D9dE4aUYBrGdpPyHs25

src/modules/Telemetry/Sensor/CommandSensor.cpp

@@ -4,69 +4,130 @@
 
 #include "CommandSensor.h"
 #include "CommandSensorFields.h"
-#include "PortduinoGlue.h"
 #include "UptimeClock.h"
 #include "mesh/Throttle.h"
+#include <chrono>
 #include <cmath>
 
-// getMetrics() also runs for screen refreshes and phone requests; this keeps those from re-running every command.
-static constexpr uint32_t COMMAND_SENSOR_CACHE_MS = 30 * 1000;
+// How often each command is re-run.
+static constexpr uint32_t COMMAND_SENSOR_REFRESH_MS = 30 * 1000;
+// How often the worker wakes to check for due commands and for shutdown.
+static constexpr uint32_t COMMAND_SENSOR_POLL_MS = 100;
+static constexpr size_t COMMAND_SENSOR_MAX_QUOTED_OUTPUT = 64;
 
 CommandSensor::CommandSensor() : TelemetrySensor(meshtastic_TelemetrySensorType_SENSOR_UNSET, "Command") {}
 
+CommandSensor::~CommandSensor()
+{
+    stopping = true;
+    if (worker.joinable())
+        worker.join();
+}
+
 bool CommandSensor::initDevice(TwoWire *bus, ScanI2C::FoundDevice *dev)
 {
     if (portduino_config.command_sensors.empty())
         return false;
 
-    readings.assign(portduino_config.command_sensors.size(), Reading());
-    LOG_INFO("Init sensor: %s, %u from config.yaml Sensors", sensorName,
-             static_cast<unsigned>(portduino_config.command_sensors.size()));
+    configs = portduino_config.command_sensors;
+    readings.assign(configs.size(), Reading());
+    loggedGenerations.assign(configs.size(), 0);
+    LOG_INFO("Init sensor: %s, %u from config.yaml Sensors", sensorName, static_cast<unsigned>(configs.size()));
     status = 1;
     initialized = true;
+    worker = std::thread(&CommandSensor::run, this);
     return true;
 }
 
-void CommandSensor::refresh(size_t index)
+// Worker thread. Must not call LOG_*: portduino leaves RedirectablePrint's buffer unlocked, so results
+// and problems are handed to getMetrics() to log on the main thread.
+void CommandSensor::run()
 {
-    const commandSensorConfig &sensor = portduino_config.command_sensors[index];
-    Reading &reading = readings[index];
+    while (!stopping) {
+        for (size_t i = 0; i < configs.size() && !stopping; i++) {
+            uint32_t readAtMs;
+            {
+                std::lock_guard<std::mutex> lock(mu);
+                readAtMs = readings[i].readAtMs;
+            }
+            if (readAtMs != 0 && !Throttle::hasElapsed(readAtMs, COMMAND_SENSOR_REFRESH_MS))
+                continue;
+
+            Reading fresh = read(configs[i]);
+            std::lock_guard<std::mutex> lock(mu);
+            fresh.generation = readings[i].generation + 1;
+            readings[i] = fresh;
+        }
+        std::this_thread::sleep_for(std::chrono::milliseconds(COMMAND_SENSOR_POLL_MS));
+    }
+}
+
+CommandSensor::Reading CommandSensor::read(const commandSensorConfig &sensor)
+{
+    Reading reading;
     reading.readAtMs = Time::stampMillis();
-    reading.valid = false;
 
     std::string output;
-    if (!execCommand(sensor.command.c_str(), COMMAND_SENSOR_TIMEOUT_MS, output)) {
-        LOG_WARN("Sensor %s: command failed or timed out: %s", sensor.type.c_str(), sensor.command.c_str());
-        return;
+    switch (execCommand(sensor.command.c_str(), COMMAND_SENSOR_TIMEOUT_MS, output)) {
+    case ExecResult::Ok:
+        break;
+    case ExecResult::TimedOut:
+        reading.problem = "timed out after " + std::to_string(COMMAND_SENSOR_TIMEOUT_MS) + "ms and was killed";
+        return reading;
+    case ExecResult::NotStarted:
+        reading.problem = "could not be started";
+        return reading;
+    default:
+        reading.problem = "exited with an error";
+        return reading;
     }
+
     float value;
     if (!CommandSensorFields::parseValue(output, value)) {
         while (!output.empty() && (output.back() == '\n' || output.back() == '\r'))
             output.pop_back();
-        LOG_WARN("Sensor %s: command output is not a number: '%s'", sensor.type.c_str(), output.c_str());
-        return;
+        if (output.size() > COMMAND_SENSOR_MAX_QUOTED_OUTPUT)
+            output = output.substr(0, COMMAND_SENSOR_MAX_QUOTED_OUTPUT) + "...";
+        reading.problem = "printed '" + output + "', which is not a number";
+        return reading;
     }
     const float scaled = value * sensor.scale;
     if (!std::isfinite(scaled)) {
-        LOG_WARN("Sensor %s: %f scaled by %f is not a finite number", sensor.type.c_str(), value, sensor.scale);
-        return;
+        reading.problem = "printed a value that is not a finite number once scaled";
+        return reading;
     }
     reading.value = scaled;
     reading.valid = true;
-    LOG_DEBUG("Sensor %s: %f", sensor.type.c_str(), reading.value);
+    return reading;
 }
 
 bool CommandSensor::getMetrics(meshtastic_Telemetry *measurement)
 {
     bool any = false;
+    std::lock_guard<std::mutex> lock(mu);
     for (size_t i = 0; i < readings.size(); i++) {
-        if (readings[i].readAtMs == 0 || Throttle::hasElapsed(readings[i].readAtMs, COMMAND_SENSOR_CACHE_MS))
-            refresh(i);
-        if (readings[i].valid && CommandSensorFields::apply(measurement->variant.environment_metrics,
-                                                            portduino_config.command_sensors[i].type, readings[i].value))
+        const Reading &reading = readings[i];
+        const commandSensorConfig &sensor = configs[i];
+        if (reading.generation != loggedGenerations[i]) {
+            loggedGenerations[i] = reading.generation;
+            if (reading.valid)
+                LOG_DEBUG("Sensor %s: %f", sensor.type.c_str(), reading.value);
+            else
+                LOG_WARN("Sensor %s: command %s: %s", sensor.type.c_str(), reading.problem.c_str(), sensor.command.c_str());
+        }
+        if (reading.valid && CommandSensorFields::apply(measurement->variant.environment_metrics, sensor.type, reading.value))
             any = true;
     }
     return any;
 }
 
+uint32_t CommandSensor::finishedReads()
+{
+    std::lock_guard<std::mutex> lock(mu);
+    uint32_t total = 0;
+    for (const auto &reading : readings)
+        total += reading.generation;
+    return total;
+}
+
 #endif