From 047a56d5540f4c8d9a176e49d1b5f7956d18df70 Mon Sep 17 00:00:00 2001
From: Kevin Hester <kevinh@geeksville.com>
Date: Wed, 26 Jun 2024 12:59:28 -0700
Subject: [PATCH] speed up file writing

---
 meshtastic/slog/arrow.py |  4 +++-
 meshtastic/slog/slog.py  | 10 +++++++---
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/meshtastic/slog/arrow.py b/meshtastic/slog/arrow.py
index eae77c2..a2d32e6 100644
--- a/meshtastic/slog/arrow.py
+++ b/meshtastic/slog/arrow.py
@@ -33,7 +33,8 @@ class ArrowWriter:
         """Write the new rows to the file."""
         if len(self.new_rows) > 0:
             if self.schema is None:
-                self.schema = pa.Table.from_pylist(self.new_rows).schema
+                # only need to look at the first row to learn the schema
+                self.schema = pa.Table.from_pylist([self.new_rows[0]]).schema
                 self.writer = pa.ipc.new_stream(self.sink, self.schema)
 
             self.writer.write_batch(pa.RecordBatch.from_pylist(self.new_rows))
@@ -75,3 +76,4 @@ class FeatherWriter(ArrowWriter):
 
             # See https://stackoverflow.com/a/72406099 for more info and performance testing measurements
             feather.write_feather(array, dest_name, compression="zstd")
+            os.remove(src_name)
diff --git a/meshtastic/slog/slog.py b/meshtastic/slog/slog.py
index 8279130..7f99d38 100644
--- a/meshtastic/slog/slog.py
+++ b/meshtastic/slog/slog.py
@@ -5,6 +5,7 @@ import logging
 import os
 import re
 import threading
+import io
 import time
 from dataclasses import dataclass
 from datetime import datetime
@@ -99,7 +100,7 @@ class StructuredLogger:
         """
         self.client = client
         self.writer = FeatherWriter(f"{dir_path}/slog")
-        self.raw_file = open(  # pylint: disable=consider-using-with
+        self.raw_file: Optional[io.TextIOWrapper] = open(  # pylint: disable=consider-using-with
             f"{dir_path}/raw.txt", "w", encoding="utf8"
         )
         self.listener = pub.subscribe(self._onLogMessage, TOPIC_MESHTASTIC_LOG_LINE)
@@ -108,7 +109,9 @@ class StructuredLogger:
         """Stop logging."""
         pub.unsubscribe(self.listener, TOPIC_MESHTASTIC_LOG_LINE)
         self.writer.close()
-        self.raw_file.close()  # Close the raw.txt file
+        f = self.raw_file
+        self.raw_file = None  # mark that we are shutting down
+        f.close()  # Close the raw.txt file
 
     def _onLogMessage(
         self, line: str, interface: MeshInterface  # pylint: disable=unused-argument
@@ -134,7 +137,8 @@ class StructuredLogger:
                     logging.warning(f"Failed to parse slog {line} with {d.format}")
             else:
                 logging.warning(f"Unknown Structured Log: {line}")
-        self.raw_file.write(line + "\n")  # Write the raw log
+        if self.raw_file:
+            self.raw_file.write(line + "\n")  # Write the raw log
 
 
 class LogSet: