Merge pull request #606 from geeksville/pr-poetry

Change build to use poetry

.github/workflows/ci.yml

@@ -18,30 +18,29 @@ jobs:
           - "3.10"
           - "3.11"
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Install Python 3
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
       - name: Uninstall meshtastic
         run: |
-          pip3 uninstall meshtastic
+          pip3 uninstall -y meshtastic
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip3 install -r requirements.txt
+          pip3 install poetry
       - name: Install meshtastic from local
         run: |
-          pip3 install .
-          which meshtastic
-          meshtastic --version
+          poetry install
+          poetry run meshtastic --version
       - name: Run pylint
-        run: pylint meshtastic examples/ --ignore-patterns ".*_pb2.pyi?$"
+        run: poetry run pylint meshtastic examples/ --ignore-patterns ".*_pb2.pyi?$"
       - name: Check types with mypy
-        run: mypy meshtastic/
+        run: poetry run mypy meshtastic/
       - name: Run tests with pytest
-        run: pytest --cov=meshtastic
+        run: poetry run pytest --cov=meshtastic
       - name: Generate coverage report
         run: |
-          pytest --cov=meshtastic --cov-report=xml
+          poetry run pytest --cov=meshtastic --cov-report=xml
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
         with:
@@ -62,11 +61,12 @@ jobs:
           - "3.10"
           - "3.11"
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Install Python 3
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
       - name: Install meshtastic from local
         run: |
-          pip3 install .
-          which meshtastic
-          meshtastic --version
+          python -m pip install --upgrade pip
+          pip3 install poetry
+          poetry install
+          poetry run meshtastic --version

.github/workflows/release.yml

@@ -12,26 +12,36 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.9"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip3 install poetry
 
       - name: Bump version
         run: >-
-          bin/bump_version.py
+          poetry version patch
 
-      - name: Commit updated version.py
+      - name: Commit updated version.
         id: commit_updated
         run: |
           git config --global user.name 'github-actions'
           git config --global user.email 'bot@noreply.github.com'
           git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}
-          git add setup.py
+          git add pyproject.toml
           git commit -m "bump version" && git push || echo "No changes to commit"
           git log -n 1 --pretty=format:"%H" | tail -n 1 | awk '{print "::set-output name=sha::"$0}'
 
       - name: Get version
         id: get_version
         run: >-
-          bin/show_version.py
+          poetry version
 
       - name: Create GitHub release
         uses: actions/create-release@v1
@@ -47,26 +57,9 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-
-      - name: Install pypa/build
-        run: >-
-          python -m
-          pip install
-          build
-          --user
-
       - name: Build a binary wheel and a source tarball
         run: >-
-          python -m
-          build
-          --sdist
-          --wheel
-          --outdir dist/
-          .
+          poetry build
 
       - name: Publish to PyPI
         uses: pypa/gh-action-pypi-publish@master
@@ -79,14 +72,14 @@ jobs:
   #   needs: release_create
   #   steps:
   #     - name: Checkout
-  #       uses: actions/checkout@v3
+  #       uses: actions/checkout@v4
   #       with:
   #         ref: ${{ needs.release_create.outputs.new_sha }}
 
   #     - name: Set up Python 3.9
-  #       uses: actions/setup-python@v2
+  #       uses: actions/setup-python@v5
   #       with:
-  #         python-version: 3.9
+  #         python-version: "3.9"
 
   #     - name: Setup code signing
   #       env:
@@ -125,21 +118,19 @@ jobs:
     needs: release_create
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ needs.release_create.outputs.new_sha }}
 
       - name: Set up Python 3.9
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: "3.9"
 
       - name: Build
         run: |
-          pip install pyinstaller
-          pip install -r requirements.txt
-          pip install .
-          pyinstaller -F -n meshtastic --collect-all meshtastic meshtastic/__main__.py
+          pip install poetry
+          bin/build-bin.sh
 
       - name: Add ubuntu to release
         uses: actions/upload-release-asset@v1
@@ -166,21 +157,19 @@ jobs:
     needs: release_create
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ needs.release_create.outputs.new_sha }}
 
       - name: Set up Python 3.9
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: "3.9"
 
       - name: Build
         run: |
-          pip install pyinstaller
-          pip install -r requirements.txt
-          pip install .
-          pyinstaller -F -n meshtastic --collect-all meshtastic meshtastic/__main__.py
+          pip install poetry
+          bin/build-bin.sh
 
       - name: Add windows to release
         uses: actions/upload-release-asset@v1

.github/workflows/update_protobufs.yml

@@ -7,7 +7,7 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           submodules: true
 
@@ -18,9 +18,14 @@ jobs:
 
       - name: Download nanopb
         run: |
-          wget https://jpa.kapsi.fi/nanopb/download/nanopb-0.4.6-linux-x86.tar.gz
-          tar xvzf nanopb-0.4.6-linux-x86.tar.gz
-          mv nanopb-0.4.6-linux-x86 nanopb-0.4.6
+          wget https://jpa.kapsi.fi/nanopb/download/nanopb-0.4.8-linux-x86.tar.gz
+          tar xvzf nanopb-0.4.8-linux-x86.tar.gz
+          mv nanopb-0.4.8-linux-x86 nanopb-0.4.8
+
+      - name: Install poetry (needed by regen-protobufs.sh)
+        run: |
+          python -m pip install --upgrade pip
+          pip3 install poetry
 
       - name: Re-generate protocol buffers
         run: |

.gitignore

@@ -16,3 +16,4 @@ __pycache__
 examples/__pycache__
 meshtastic.spec
 .hypothesis/
+coverage.xml

bin/build-bin.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -e
+
+echo Building ubuntu binary
+poetry install
+source $(poetry env info --path)/bin/activate
+pyinstaller -F -n meshtastic --collect-all meshtastic meshtastic/__main__.py
+

bin/bump_version.py

@@ -1,30 +0,0 @@
-#!/usr/bin/env python
-"""Bump the version number"""
-import re
-
-version_filename = "setup.py"
-
-lines = None
-
-with open(version_filename, "r", encoding="utf-8") as f:
-    lines = f.readlines()
-
-with open(version_filename, "w", encoding="utf-8") as f:
-    for line in lines:
-        if line.lstrip().startswith("version="):
-            # get rid of quotes around the version
-            line = line.replace('"', "")
-            # get rid of trailing comma
-            line = line.replace(",", "")
-            # split on '='
-            words = line.split("=")
-            # split the version into parts (by period)
-            v = words[1].split(".")
-            build_num = re.findall(r"\d+", v[2])[0]
-            new_build_num = str(int(build_num) + 1)
-            ver = f"{v[0]}.{v[1]}.{v[2].replace(build_num, new_build_num)}".replace(
-                "\n", ""
-            )
-            f.write(f'    version="{ver}",\n')
-        else:
-            f.write(line)

bin/prerelease-tests.sh

@@ -3,20 +3,24 @@ set -e
 # You may consider running: "pytest -m smoke1" instead of this test.
 
 echo "Running (crude) prerelease tests to verify sanity"
+
+# Use the python environment created by poetry
+source $(poetry env info --path)/bin/activate
+
 echo running hello
 python3 tests/hello_world.py
-# bin/run.sh --help
+# meshtastic --help
 echo toggling router
-bin/run.sh --set is_router true
-bin/run.sh --set is_router false
+meshtastic --set is_router true
+meshtastic --set is_router false
 # TODO: This does not seem to work.
 echo setting channel
-bin/run.sh --seturl "https://www.meshtastic.org/c/#GAMiENTxuzogKQdZ8Lz_q89Oab8qB0RlZmF1bHQ="
+meshtastic --seturl "https://www.meshtastic.org/c/#GAMiENTxuzogKQdZ8Lz_q89Oab8qB0RlZmF1bHQ="
 echo setting owner
-bin/run.sh --set-owner "Test Build"
+meshtastic --set-owner "Test Build"
 echo setting position
-bin/run.sh --setlat 32.7767 --setlon -96.7970 --setalt 1337
+meshtastic --setlat 32.7767 --setlon -96.7970 --setalt 1337
 echo dumping info
-bin/run.sh --info
+meshtastic run meshtastic --info
 echo sending closing message
-bin/run.sh --sendtext "Sanity complete"
+meshtastic --sendtext "Sanity complete"

bin/regen-protobufs.sh

@@ -4,8 +4,11 @@
 #gsed -i 's/import "\//import ".\//g' ./protobufs/meshtastic/*
 #gsed -i 's/package meshtastic;//g' ./protobufs/meshtastic/*
 
-./nanopb-0.4.7/generator-bin/protoc -I=protobufs --python_out ./ --mypy_out ./ ./protobufs/meshtastic/*.proto
-./nanopb-0.4.7/generator-bin/protoc -I=protobufs --python_out ./meshtastic/ --mypy_out ./meshtastic/ ./protobufs/nanopb.proto
+# protoc looks for mypy plugin in the python path
+source $(poetry env info --path)/bin/activate
+
+./nanopb-0.4.8/generator-bin/protoc -I=protobufs --python_out ./ --mypy_out ./ ./protobufs/meshtastic/*.proto
+./nanopb-0.4.8/generator-bin/protoc -I=protobufs --python_out ./meshtastic/ --mypy_out ./meshtastic/ ./protobufs/nanopb.proto
 
 # workaround for import bug in protoc https://github.com/protocolbuffers/protobuf/issues/1491#issuecomment-690618628
 

bin/run-ci-local.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# This script lets you run github ci actions locally
+# You need to have act installed.  You can get it at https://nektosact.com/
+
+# by default it simulates a push event
+# other useful options
+# -j build-and-publish-ubuntu
+
+# also: we only run one of the 4 matrix tests, because otherwise it absolutely hammers the CPU (so many containers and threads)
+act -P ubuntu-latest=-self-hosted --matrix "python-version:3.8" "$@"

bin/run.sh

@@ -1,2 +0,0 @@
-rm log_*
-python3 -m meshtastic "$@"

bin/show_version.py

@@ -1,20 +0,0 @@
-#!/usr/bin/env python
-"""Show the version number"""
-
-version_filename = "setup.py"
-
-lines = None
-
-with open(version_filename, "r", encoding="utf-8") as f:
-    lines = f.readlines()
-
-for line in lines:
-    if line.lstrip().startswith("version="):
-        # get rid of quotes around the version
-        line2 = line.replace('"', "")
-        # get rid of the trailing comma
-        line2 = line2.replace(",", "")
-        # split on =
-        words = line2.split("=")
-        # Note: This format is for github actions
-        print(f"::set-output name=version::{words[1].strip()}")

bin/test-release.sh

@@ -3,8 +3,6 @@ set -e
 
 bin/regen-docs.sh
 pandoc --from=markdown --to=rst --output=README README.md
-python3 setup.py sdist bdist_wheel
-python3 -m twine check dist/*
-# test the upload
-python3 -m twine upload --repository-url https://test.pypi.org/legacy/ dist/*
+
+poetry publish -r test-pypi --build
 echo "view the upload at https://test.pypi.org/ it it looks good upload for real"

bin/upload-release.sh

@@ -1,5 +1,8 @@
 rm dist/*
 set -e
 
-python3 setup.py sdist bdist_wheel
-python3 -m twine upload dist/*
+poetry build
+poetry run pytest
+poetry publish 
+#python3 setup.py sdist bdist_wheel
+#python3 -m twine upload dist/*

pyproject.toml

@@ -0,0 +1,51 @@
+[tool.poetry]
+name = "meshtastic"
+version = "2.3.11"
+description = "Python API & client shell for talking to Meshtastic devices"
+authors = ["Meshtastic Developers <contact@meshtastic.org>"]
+license = "GPL-3.0-only"
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.8,<3.13" # was 3.7 for production but, 3.8 is needed for pytap2, 3.9 is needed for pandas, bleak requires a max of 3.13 for some reason
+pyserial = "^3.5"
+protobuf = ">=5.26.0"
+dotmap = "^1.3.30"
+pexpect = "^4.9.0"
+pyqrcode = "^1.2.1"
+tabulate = "^0.9.0"
+webencodings = "^0.5.1"
+requests = "^2.31.0"
+pyparsing = "^3.1.2"
+pyyaml = "^6.0.1"
+pypubsub = "^4.0.3"
+bleak = "^0.21.1"
+packaging = "^24.0"
+
+[tool.poetry.group.dev.dependencies]
+hypothesis = "^6.103.2"
+pytest = "^8.2.2"
+pytest-cov = "^5.0.0"
+pdoc3 = "^0.10.0"
+autopep8 = "^2.1.0"
+pylint = "^3.2.3"
+pytap2 = "^2.3.0"
+pyinstaller = "^6.8.0"
+mypy = "^1.10.0"
+mypy-protobuf = "^3.6.0"
+types-protobuf = "^5.26.0.20240422"
+types-tabulate = "^0.9.0.20240106"
+types-requests = "^2.31.0.20240406"
+types-setuptools = "^69.5.0.20240423"
+types-pyyaml = "^6.0.12.20240311"
+
+[tool.poetry.extras]
+tunnel = ["pytap2"]
+
+[tool.poetry.scripts]
+meshtastic = "meshtastic.__main__:main"
+mesh-tunnel = "meshtastic.__main__:tunnelMain [tunnel]"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

requirements.txt

@@ -1,30 +0,0 @@
-markdown
-pyserial
-protobuf>=5.26.0
-dotmap
-pexpect
-pyqrcode
-tabulate
-timeago
-webencodings
-requests
-pyparsing
-twine
-autopep8
-pylint
-pytest
-pytest-cov
-hypothesis
-pyyaml
-pytap2
-pdoc3
-pypubsub
-bleak
-packaging
-mypy
-mypy-protobuf
-types-protobuf>=5.26.0
-types-tabulate
-types-requests
-types-setuptools
-types-PyYAML

setup.py

@@ -1,55 +0,0 @@
-# Note: you shouldn't need to run this script manually.  It is run implicitly by the pip3 install command.
-
-import pathlib
-
-from setuptools import setup
-
-# The directory containing this file
-HERE = pathlib.Path(__file__).parent
-
-with open("README.md", "r") as fh:
-    long_description = fh.read()
-
-# This call to setup() does all the work
-setup(
-    name="meshtastic",
-    version="2.3.11",
-    description="Python API & client shell for talking to Meshtastic devices",
-    long_description=long_description,
-    long_description_content_type="text/markdown",
-    url="https://github.com/meshtastic/python",
-    author="Meshtastic Developers",
-    author_email="contact@meshtastic.org",
-    license="GPL-3.0-only",
-    classifiers=[
-        "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
-        "Development Status :: 4 - Beta",
-        "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
-        "Programming Language :: Python :: 3.10",
-        "Programming Language :: Python :: 3.11",
-    ],
-    packages=["meshtastic"],
-    include_package_data=True,
-    install_requires=[
-        "pyserial>=3.4",
-        "protobuf>=5.26.0",
-        "requests>=2.25.0",
-        "pypubsub>=4.0.3",
-        "dotmap>=1.3.14",
-        "pyqrcode>=1.2.1",
-        "tabulate>=0.8.9",
-        "pyyaml",
-        "bleak>=0.21.1",
-        "packaging",
-    ],
-    extras_require={"tunnel": ["pytap2>=2.0.0"]},
-    python_requires=">=3.7",
-    entry_points={
-        "console_scripts": [
-            "meshtastic=meshtastic.__main__:main",
-            "mesh-tunnel=meshtastic.__main__:tunnelMain [tunnel]",
-        ]
-    },
-)