From 88b73611f0f085e9413b7ed982a00b1f98e239a9 Mon Sep 17 00:00:00 2001 From: Kevin Hester Date: Wed, 26 Jun 2024 09:43:56 -0700 Subject: [PATCH] Move protobufs to meshtastic.protobuf python namespace --- .github/workflows/update_protobufs.yml | 2 +- bin/regen-protobufs.sh | 48 ++++++++++++++++++----- meshtastic/__init__.py | 9 +++-- meshtastic/__main__.py | 3 +- meshtastic/mesh_interface.py | 5 ++- meshtastic/node.py | 2 +- meshtastic/remote_hardware.py | 2 +- meshtastic/tests/test_main.py | 2 +- meshtastic/tests/test_mesh_interface.py | 3 +- meshtastic/tests/test_node.py | 4 +- meshtastic/tests/test_serial_interface.py | 2 +- meshtastic/tests/test_tcp_interface.py | 2 +- meshtastic/tests/test_util.py | 4 +- meshtastic/tunnel.py | 3 +- 14 files changed, 65 insertions(+), 26 deletions(-) diff --git a/.github/workflows/update_protobufs.yml b/.github/workflows/update_protobufs.yml index 5cec898..c7b9375 100644 --- a/.github/workflows/update_protobufs.yml +++ b/.github/workflows/update_protobufs.yml @@ -37,5 +37,5 @@ jobs: git config --global user.email 'bot@noreply.github.com' git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }} git add protobufs - git add meshtastic + git add meshtastic/protobuf git commit -m "Update protobuf submodule" && git push || echo "No changes to commit" diff --git a/bin/regen-protobufs.sh b/bin/regen-protobufs.sh index 6b993c8..a41f7ba 100755 --- a/bin/regen-protobufs.sh +++ b/bin/regen-protobufs.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e + #Uncomment to run hack #gsed -i 's/import "\//import ".\//g' ./protobufs/meshtastic/* #gsed -i 's/package meshtastic;//g' ./protobufs/meshtastic/* @@ -7,17 +9,45 @@ # protoc looks for mypy plugin in the python path source $(poetry env info --path)/bin/activate -./nanopb-0.4.8/generator-bin/protoc -I=protobufs --python_out ./ --mypy_out ./ ./protobufs/meshtastic/*.proto -./nanopb-0.4.8/generator-bin/protoc -I=protobufs --python_out ./meshtastic/ --mypy_out ./meshtastic/ ./protobufs/nanopb.proto +# Put our temp files in the poetry build directory +TMPDIR=./build/meshtastic/protofixup +echo "Fixing up protobuf paths in ${TMPDIR} temp directory" -# workaround for import bug in protoc https://github.com/protocolbuffers/protobuf/issues/1491#issuecomment-690618628 +# Ensure a clean build +rm -r "${TMPDIR}" + +INDIR=${TMPDIR}/in/meshtastic/protobuf +OUTDIR=${TMPDIR}/out +PYIDIR=${TMPDIR}/out +mkdir -p "${OUTDIR}" "${INDIR}" "${PYIDIR}" +cp ./protobufs/meshtastic/*.proto "${INDIR}" + +# OS-X sed is apparently a little different and expects an arg for -i if [[ $OSTYPE == 'darwin'* ]]; then - sed -i '' -E 's/^(import.*_pb2)/from . \1/' meshtastic/*.py - # automate the current workaround (may be related to Meshtastic-protobufs issue #27 https://github.com/meshtastic/protobufs/issues/27) - sed -i '' -E "s/^None = 0/globals()['None'] = 0/" meshtastic/mesh_pb2.py + SEDCMD="sed -i '' -E" else - sed -i -e 's/^import.*_pb2/from . \0/' meshtastic/*.py - # automate the current workaround (may be related to Meshtastic-protobufs issue #27 https://github.com/meshtastic/protobufs/issues/27) - sed -i -e "s/^None = 0/globals()['None'] = 0/" meshtastic/mesh_pb2.py + SEDCMD="sed -i -E" fi + + +# change the package names to meshtastic.protobuf +$SEDCMD 's/^package meshtastic;/package meshtastic.protobuf;/' "${INDIR}/"*.proto +# fix the imports to match +$SEDCMD 's/^import "meshtastic\//import "meshtastic\/protobuf\//' "${INDIR}/"*.proto + +# Generate the python files +./nanopb-0.4.8/generator-bin/protoc -I=$TMPDIR/in --python_out "${OUTDIR}" "--mypy_out=${PYIDIR}" $INDIR/*.proto + +# Change "from meshtastic.protobuf import" to "from . import" +$SEDCMD 's/^from meshtastic.protobuf import/from . import/' "${OUTDIR}"/meshtastic/protobuf/*pb2*.py[i] + +# Create a __init__.py in the out directory +touch "${OUTDIR}/meshtastic/protobuf/__init__.py" + +# Copy to the source controlled tree +mkdir -p meshtastic/protobuf +rm -rf meshtastic/protobuf/*pb2*.py +cp "${OUTDIR}/meshtastic/protobuf"/* meshtastic/protobuf + +exit 0 diff --git a/meshtastic/__init__.py b/meshtastic/__init__.py index 313cb42..e445aa7 100644 --- a/meshtastic/__init__.py +++ b/meshtastic/__init__.py @@ -81,7 +81,10 @@ from google.protobuf.json_format import MessageToJson from pubsub import pub # type: ignore[import-untyped] from tabulate import tabulate -from meshtastic import ( +from meshtastic.node import Node +from meshtastic.util import DeferredExecution, Timeout, catchAndIgnore, fixme, stripnl + +from .protobuf import ( admin_pb2, apponly_pb2, channel_pb2, @@ -93,10 +96,10 @@ from meshtastic import ( remote_hardware_pb2, storeforward_pb2, telemetry_pb2, +) +from . import ( util, ) -from meshtastic.node import Node -from meshtastic.util import DeferredExecution, Timeout, catchAndIgnore, fixme, stripnl # Note: To follow PEP224, comments should be after the module variable. diff --git a/meshtastic/__main__.py b/meshtastic/__main__.py index fa71adc..781f6bf 100644 --- a/meshtastic/__main__.py +++ b/meshtastic/__main__.py @@ -17,7 +17,8 @@ from pubsub import pub # type: ignore[import-untyped] import meshtastic.test import meshtastic.util from meshtastic import mt_config -from meshtastic import channel_pb2, config_pb2, portnums_pb2, remote_hardware, BROADCAST_ADDR +from meshtastic.protobuf import channel_pb2, config_pb2, portnums_pb2 +from meshtastic import remote_hardware, BROADCAST_ADDR from meshtastic.version import get_active_version from meshtastic.ble_interface import BLEInterface from meshtastic.mesh_interface import MeshInterface diff --git a/meshtastic/mesh_interface.py b/meshtastic/mesh_interface.py index 66ce7f2..49e87fd 100644 --- a/meshtastic/mesh_interface.py +++ b/meshtastic/mesh_interface.py @@ -18,10 +18,13 @@ from pubsub import pub # type: ignore[import-untyped] from tabulate import tabulate import meshtastic.node -from meshtastic import ( + +from meshtastic.protobuf import ( mesh_pb2, portnums_pb2, telemetry_pb2, +) +from meshtastic import ( BROADCAST_ADDR, BROADCAST_NUM, LOCAL_ADDR, diff --git a/meshtastic/node.py b/meshtastic/node.py index 419b712..31a5b03 100644 --- a/meshtastic/node.py +++ b/meshtastic/node.py @@ -7,7 +7,7 @@ import time from typing import Union -from meshtastic import admin_pb2, apponly_pb2, channel_pb2, localonly_pb2, mesh_pb2, portnums_pb2 +from meshtastic.protobuf import admin_pb2, apponly_pb2, channel_pb2, localonly_pb2, mesh_pb2, portnums_pb2 from meshtastic.util import ( Timeout, camel_to_snake, diff --git a/meshtastic/remote_hardware.py b/meshtastic/remote_hardware.py index 55c8c18..73836f3 100644 --- a/meshtastic/remote_hardware.py +++ b/meshtastic/remote_hardware.py @@ -4,7 +4,7 @@ import logging from pubsub import pub # type: ignore[import-untyped] -from meshtastic import portnums_pb2, remote_hardware_pb2 +from meshtastic.protobuf import portnums_pb2, remote_hardware_pb2 from meshtastic.util import our_exit diff --git a/meshtastic/tests/test_main.py b/meshtastic/tests/test_main.py index 18fa497..6f2dd26 100644 --- a/meshtastic/tests/test_main.py +++ b/meshtastic/tests/test_main.py @@ -21,7 +21,7 @@ from meshtastic.__main__ import ( ) from meshtastic import mt_config -from ..channel_pb2 import Channel # pylint: disable=E0611 +from ..protobuf.channel_pb2 import Channel # pylint: disable=E0611 # from ..ble_interface import BLEInterface from ..node import Node diff --git a/meshtastic/tests/test_mesh_interface.py b/meshtastic/tests/test_mesh_interface.py index 5e8441c..2f8dc83 100644 --- a/meshtastic/tests/test_mesh_interface.py +++ b/meshtastic/tests/test_mesh_interface.py @@ -7,7 +7,8 @@ from unittest.mock import MagicMock, patch import pytest from hypothesis import given, strategies as st -from .. import mesh_pb2, config_pb2, BROADCAST_ADDR, LOCAL_ADDR +from ..protobuf import mesh_pb2, config_pb2 +from .. import BROADCAST_ADDR, LOCAL_ADDR from ..mesh_interface import MeshInterface, _timeago from ..node import Node diff --git a/meshtastic/tests/test_node.py b/meshtastic/tests/test_node.py index 5bacd9d..df3dffb 100644 --- a/meshtastic/tests/test_node.py +++ b/meshtastic/tests/test_node.py @@ -6,8 +6,8 @@ from unittest.mock import MagicMock, patch import pytest -from .. import localonly_pb2, config_pb2 -from ..channel_pb2 import Channel # pylint: disable=E0611 +from ..protobuf import localonly_pb2, config_pb2 +from ..protobuf.channel_pb2 import Channel # pylint: disable=E0611 from ..node import Node from ..serial_interface import SerialInterface from ..mesh_interface import MeshInterface diff --git a/meshtastic/tests/test_serial_interface.py b/meshtastic/tests/test_serial_interface.py index 739380c..797c6d7 100644 --- a/meshtastic/tests/test_serial_interface.py +++ b/meshtastic/tests/test_serial_interface.py @@ -6,7 +6,7 @@ from unittest.mock import mock_open, patch import pytest from ..serial_interface import SerialInterface -from .. import config_pb2 +from ..protobuf import config_pb2 @pytest.mark.unit diff --git a/meshtastic/tests/test_tcp_interface.py b/meshtastic/tests/test_tcp_interface.py index 03d317c..44e79de 100644 --- a/meshtastic/tests/test_tcp_interface.py +++ b/meshtastic/tests/test_tcp_interface.py @@ -5,7 +5,7 @@ from unittest.mock import patch import pytest -from .. import config_pb2 +from ..protobuf import config_pb2 from ..tcp_interface import TCPInterface diff --git a/meshtastic/tests/test_util.py b/meshtastic/tests/test_util.py index 74f66d9..1ea80da 100644 --- a/meshtastic/tests/test_util.py +++ b/meshtastic/tests/test_util.py @@ -9,7 +9,7 @@ import pytest from hypothesis import given, strategies as st from meshtastic.supported_device import SupportedDevice -from meshtastic.mesh_pb2 import MyNodeInfo +from meshtastic.protobuf import mesh_pb2 from meshtastic.util import ( Timeout, active_ports_on_supported_devices, @@ -555,7 +555,7 @@ def test_active_ports_on_supported_devices_mac_duplicates_check(mock_platform, m @pytest.mark.unit def test_message_to_json_shows_all(): """Test that message_to_json prints fields that aren't included in data passed in""" - actual = json.loads(message_to_json(MyNodeInfo())) + actual = json.loads(message_to_json(mesh_pb2.MyNodeInfo())) expected = { "myNodeNum": 0, "rebootCount": 0, "minAppVersion": 0 } assert actual == expected diff --git a/meshtastic/tunnel.py b/meshtastic/tunnel.py index 40a1c2e..409e3d4 100644 --- a/meshtastic/tunnel.py +++ b/meshtastic/tunnel.py @@ -22,7 +22,8 @@ import threading from pubsub import pub # type: ignore[import-untyped] from pytap2 import TapDevice -from meshtastic import portnums_pb2, mt_config +from meshtastic.protobuf import portnums_pb2 +from meshtastic import mt_config from meshtastic.util import ipstr, readnet_u16