add more testing

README.md

@@ -220,6 +220,10 @@ Be sure to hit the reset button on the device after the test is completed.
 
 ```
 pytest -msmoke1 meshtastic/test/test_smoke1.py::test_smoke1_info
+# or to run a specific smoke2 test
+pytest -m smoke2 meshtastic/test/test_smoke2.py::test_smoke2_info
+# or to run a specific smoke_wifi test
+pytest -m smokewifi meshtastic/test/test_smoke_wifi.py::test_smokewifi_info
 ```
 
 * To add another classification of tests such as "unit" or "smoke1", see [pytest.ini](pytest.ini).

meshtastic/__main__.py

@@ -599,7 +599,9 @@ def common():
             parser.print_help(sys.stderr)
             sys.exit(1)
         elif args.test:
-            test.testAll()
+            result = test.testAll()
+            if not result:
+                our_exit("Warning: Test was not successful.")
         else:
             if args.seriallog == "stdout":
                 logfile = sys.stdout

meshtastic/test.py

@@ -1,4 +1,5 @@
-""" Testing
+""" With two radios connected serially, send and receive test
+    messages and report back if successful.
 """
 import logging
 import time
@@ -10,6 +11,7 @@ from . import util
 from .__init__ import BROADCAST_NUM
 from .serial_interface import SerialInterface
 from .tcp_interface import TCPInterface
+from .util import our_exit
 
 """The interfaces we are using for our tests"""
 interfaces = None
@@ -107,25 +109,23 @@ def runTests(numTests=50, wantAck=False, maxFailures=0):
             logging.info(
                 f"Test {testNumber} succeeded {numSuccess} successes {numFail} failures so far")
 
-        # if numFail >= 3:
-        #    for i in interfaces:
-        #        i.close()
-        #    return
-
         time.sleep(1)
 
     if numFail > maxFailures:
         logging.error("Too many failures! Test failed!")
-
-    return numFail
+        return False
+    return True
 
 
-def testThread(numTests=50):
+def testThread(numTests=5):
     """Test thread"""
     logging.info("Found devices, starting tests...")
-    runTests(numTests, wantAck=True)
-    # Allow a few dropped packets
-    runTests(numTests, wantAck=False, maxFailures=5)
+    result = runTests(numTests, wantAck=True)
+    if result:
+        # Run another test
+        # Allow a few dropped packets
+        result = runTests(numTests, wantAck=False, maxFailures=1)
+    return result
 
 
 def onConnection(topic=pub.AUTO_TOPIC):
@@ -143,13 +143,12 @@ def openDebugLog(portName):
 def testAll():
     """
     Run a series of tests using devices we can find.
+    This is called from the cli with the "--test" option.
 
-    Raises:
-        Exception: If not enough devices are found
     """
     ports = util.findPorts()
     if len(ports) < 2:
-        raise Exception("Must have at least two devices connected to USB")
+        our_exit("Warning: Must have at least two devices connected to USB.")
 
     pub.subscribe(onConnection, "meshtastic.connection")
     pub.subscribe(onReceive, "meshtastic.receive")
@@ -158,11 +157,13 @@ def testAll():
         port, debugOut=openDebugLog(port), connectNow=True), ports))
 
     logging.info("Ports opened, starting test")
-    testThread()
+    result = testThread()
 
     for i in interfaces:
         i.close()
 
+    return result
+
 
 def testSimulator():
     """

meshtastic/test/test_smoke1.py

@@ -1,4 +1,4 @@
-"""Meshtastic smoke tests with a single device"""
+"""Meshtastic smoke tests with a single device via USB"""
 import re
 import subprocess
 import time
@@ -39,6 +39,25 @@ def test_smoke1_info():
     assert return_value == 0
 
 
+@pytest.mark.smoke1
+def test_smoke1_sendping():
+    """Test --sendping"""
+    return_value, out = subprocess.getstatusoutput('meshtastic --sendping')
+    assert re.match(r'Connected to radio', out)
+    assert re.search(r'^Sending ping message', out, re.MULTILINE)
+    assert return_value == 0
+
+
+@pytest.mark.smoke1
+def test_smoke1_test():
+    """Test --test
+       Note: Since only one device is connected, it will not do much.
+    """
+    return_value, out = subprocess.getstatusoutput('meshtastic --test')
+    assert re.search(r'^Warning: Must have at least two devices', out, re.MULTILINE)
+    assert return_value == 1
+
+
 @pytest.mark.smoke1
 def test_smoke1_debug():
     """Test --debug"""

meshtastic/test/test_smoke2.py

@@ -0,0 +1,23 @@
+"""Meshtastic smoke tests with 2 devices connected via USB"""
+import re
+import subprocess
+
+import pytest
+
+
+@pytest.mark.smoke2
+def test_smoke2_info():
+    """Test --info with 2 devices connected serially"""
+    return_value, out = subprocess.getstatusoutput('meshtastic --info')
+    assert re.search(r'Warning: Multiple', out, re.MULTILINE)
+    assert return_value == 1
+
+
+@pytest.mark.smoke2
+def test_smoke2_test():
+    """Test --test"""
+    return_value, out = subprocess.getstatusoutput('meshtastic --test')
+    assert re.search(r'Writing serial debugging', out, re.MULTILINE)
+    assert re.search(r'Ports opened', out, re.MULTILINE)
+    assert re.search(r'Running 50 tests', out, re.MULTILINE)
+    assert return_value == 0

meshtastic/test/test_smoke_wifi.py

@@ -0,0 +1,23 @@
+"""Meshtastic smoke tests a device setup with wifi.
+
+   Need to have run the following on an esp32 device:
+      meshtastic --set wifi_ssid 'foo' --set wifi_password 'sekret'
+"""
+import re
+import subprocess
+
+import pytest
+
+
+@pytest.mark.smokewifi
+def test_smokewifi_info():
+    """Test --info"""
+    return_value, out = subprocess.getstatusoutput('meshtastic --info --host meshtastic.local')
+    assert re.search(r'^Owner', out, re.MULTILINE)
+    assert re.search(r'^My info', out, re.MULTILINE)
+    assert re.search(r'^Nodes in mesh', out, re.MULTILINE)
+    assert re.search(r'^Preferences', out, re.MULTILINE)
+    assert re.search(r'^Channels', out, re.MULTILINE)
+    assert re.search(r'^  PRIMARY', out, re.MULTILINE)
+    assert re.search(r'^Primary channel URL', out, re.MULTILINE)
+    assert return_value == 0

pytest.ini

@@ -1,8 +1,10 @@
 [pytest]
 
-addopts = -m "not smoke1"
+addopts = -m "not smoke1 and not smoke2 and not smokewifi"
 
 markers =
     unit: marks tests as unit tests
     int: marks tests as integration tests
-    smoke1: runs smoke test on one device
+    smoke1: runs smoke tests on a single device connected via USB
+    smoke2: runs smoke tests on a two devices connected via USB
+    smokewifi: runs smoke test on an esp32 device setup with wifi