From a981d824467b36ae2563a1fa516907e68761edcc Mon Sep 17 00:00:00 2001 From: Tom Keffer Date: Wed, 24 Jan 2024 15:45:59 -0800 Subject: [PATCH 1/6] Remove no longer needed (or never used) class TimeDerivative --- src/weeutil/tests/test_timediff.py | 62 ------------------------------ src/weeutil/timediff.py | 52 ------------------------- 2 files changed, 114 deletions(-) delete mode 100644 src/weeutil/tests/test_timediff.py delete mode 100644 src/weeutil/timediff.py diff --git a/src/weeutil/tests/test_timediff.py b/src/weeutil/tests/test_timediff.py deleted file mode 100644 index 35a7c010..00000000 --- a/src/weeutil/tests/test_timediff.py +++ /dev/null @@ -1,62 +0,0 @@ -# -# Copyright (c) 2022 Tom Keffer -# -# See the file LICENSE.txt for your full rights. -# -"""Test routines for weeutil.timediff.""" - -import time -import unittest - -import weewx -from weeutil.timediff import * - -start = time.mktime((2013, 3, 10, 0, 0, 0, 0, 0, -1)) - -record1 = { - 'dateTime': start, - 'outTemp': 20.0, -} - -record2 = { - 'dateTime': start + 100, - 'outTemp': 21.0, -} - -record3 = { - 'dateTime': start + 400, - 'outTemp': 21.0, -} - - -class TimeDiffTest(unittest.TestCase): - - def test_simple(self): - time_diff = TimeDerivative('outTemp', 300) - self.assertIsNone(time_diff.add_record(record1)) - self.assertEqual(time_diff.add_record(record2), .01) - - def test_backwards(self): - time_diff = TimeDerivative('outTemp', 300) - self.assertIsNone(time_diff.add_record(record2)) - with self.assertRaises(weewx.ViolatedPrecondition): - time_diff.add_record(record1) - - def test_no_fwd(self): - time_diff = TimeDerivative('outTemp', 300) - self.assertIsNone(time_diff.add_record(record1)) - self.assertIsNone(time_diff.add_record(record1)) - - def test_too_old(self): - time_diff = TimeDerivative('outTemp', 300) - self.assertIsNone(time_diff.add_record(record1)) - self.assertIsNone(time_diff.add_record(record3)) - - def test_not_there(self): - time_diff = TimeDerivative('outTemp', 300) - with self.assertRaises(weewx.CannotCalculate): - time_diff.add_record({'dateTime': start}) - - -if __name__ == '__main__': - unittest.main() diff --git a/src/weeutil/timediff.py b/src/weeutil/timediff.py deleted file mode 100644 index bc88125c..00000000 --- a/src/weeutil/timediff.py +++ /dev/null @@ -1,52 +0,0 @@ -# -# Copyright (c) 2019-2021 Tom Keffer -# -# See the file LICENSE.txt for your full rights. -# -"""Class for calculating time derivatives""" -import weewx - - -class TimeDerivative(object): - """Calculate time derivative for a specific observation type.""" - - def __init__(self, obs_type, stale_age): - """Initialize. - - obs_type: the observation type for which the derivative will be calculated. - - stale_age: Derivatives are calculated as a difference over time. This is how old the old value - can be and still be considered useful. - """ - self.obs_type = obs_type - self.stale_age = stale_age - self.old_timestamp = None - self.old_value = None - - def add_record(self, record): - """Add a new record, then return the difference in value divided by the difference in time.""" - - # If the type does not appear in the incoming record, then we can't calculate the derivative - if self.obs_type not in record: - raise weewx.CannotCalculate(self.obs_type) - - derivative = None - if record[self.obs_type] is not None: - # We can't calculate anything if we don't have an old record - if self.old_timestamp: - # Check to make sure the incoming record is later than the retained record - if record['dateTime'] < self.old_timestamp: - raise weewx.ViolatedPrecondition("Records presented out of order (%s vs %s)" - % (record['dateTime'], self.old_timestamp)) - # Calculate the time derivative only if there is a delta in time, - # and the old record is not too old. - if record['dateTime'] != self.old_timestamp \ - and (record['dateTime'] - self.old_timestamp) <= self.stale_age: - # All OK. - derivative = (record[self.obs_type] - self.old_value) \ - / (record['dateTime'] - self.old_timestamp) - # Save the current values - self.old_timestamp = record['dateTime'] - self.old_value = record[self.obs_type] - - return derivative From b32ca4069e519a50dce636d6c8730958b0c1cb00 Mon Sep 17 00:00:00 2001 From: Matthew Wall Date: Wed, 24 Jan 2024 19:40:08 -0500 Subject: [PATCH 2/6] fix redhat/suse udev packaging rule --- pkg/weewx.spec.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/weewx.spec.in b/pkg/weewx.spec.in index 8aa85931..e6167ac5 100644 --- a/pkg/weewx.spec.in +++ b/pkg/weewx.spec.in @@ -204,7 +204,7 @@ fi # install the udev rules if [ -d %{udev_dir} ]; then sed \ - -e "s/GROUP=\"weewx\"/GROUP=\"{$WEEWX_GROUP}\"/" \ + -e "s/GROUP=\"weewx\"/GROUP=\"${WEEWX_GROUP}\"/" \ %{dst_cfg_dir}/udev/weewx.rules > %{udev_dir}/60-weewx.rules fi From 4005e1e9f62bfb75c2d353c8accde04436bdce40 Mon Sep 17 00:00:00 2001 From: Matthew Wall Date: Wed, 24 Jan 2024 21:06:21 -0500 Subject: [PATCH 3/6] use /var/lib/weewx as HOME for weewx on redhat/suse --- docs_src/changes.md | 5 +++++ pkg/debian/postinst | 16 ++++++++-------- pkg/weewx.spec.in | 11 ++++++----- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/docs_src/changes.md b/docs_src/changes.md index f9488eee..888a8571 100644 --- a/docs_src/changes.md +++ b/docs_src/changes.md @@ -18,6 +18,11 @@ PR #919. Chinese language code changed to `zh`. Fixes issue #912. +Fixed bug in redhat/suse scriptlet that incorrectly substituted "{weewx}" +instead of "weewx" in the udev rules file. + +In the redhat/suse installers, use /var/lib/weewx as the HOME for user weewx. + ### 5.0.0 01/14/2024 diff --git a/pkg/debian/postinst b/pkg/debian/postinst index 86f3d659..43a1746e 100755 --- a/pkg/debian/postinst +++ b/pkg/debian/postinst @@ -20,10 +20,10 @@ set -e cfgfile=/etc/weewx/weewx.conf cfgapp=/usr/bin/weectl -WEEWX_USER=weewx -WEEWX_GROUP=weewx -WEEWX_HOME=/var/lib/weewx -WEEWX_HTML=/var/www/html/weewx +WEEWX_USER="${WEEWX_USER:-weewx}" +WEEWX_GROUP="${WEEWX_GROUP:-weewx}" +WEEWX_HOME="${WEEWX_HOME:-/var/lib/weewx}" +WEEWX_HTMLDIR="${WEEWX_HTMLDIR:-/var/www/html/weewx}" WEEWX_USERDIR=/etc/weewx/bin/user # insert the driver and stanza into the configuration file @@ -398,10 +398,10 @@ setup_database_dir() { # create the reporting directory setup_reporting_dir() { - echo "Configuring reporting directory $WEEWX_HTML" - mkdir -p $WEEWX_HTML - chmod 2775 $WEEWX_HTML - chown -R $WEEWX_USER:$WEEWX_GROUP $WEEWX_HTML + echo "Configuring reporting directory $WEEWX_HTMLDIR" + mkdir -p $WEEWX_HTMLDIR + chmod 2775 $WEEWX_HTMLDIR + chown -R $WEEWX_USER:$WEEWX_GROUP $WEEWX_HTMLDIR } # set the permissions on the configuration, skins, and extensions diff --git a/pkg/weewx.spec.in b/pkg/weewx.spec.in index e6167ac5..406a29da 100644 --- a/pkg/weewx.spec.in +++ b/pkg/weewx.spec.in @@ -127,14 +127,15 @@ done # if there is already a database directory, then use ownership of that to # determine what user/group we should use for permissions and running. # otherwise, use 'weewx' for user and group. -WEEWX_USER=weewx -WEEWX_GROUP=weewx +WEEWX_USER="${WEEWX_USER:-weewx}" +WEEWX_GROUP="${WEEWX_GROUP:-weewx}" +WEEWX_HOME="${WEEWX_HOME:-/var/lib/weewx}" if [ -d /var/lib/weewx ]; then WEEWX_USER=$(stat -c "%%U" /var/lib/weewx) WEEWX_GROUP=$(stat -c "%%G" /var/lib/weewx) fi /usr/bin/getent group $WEEWX_GROUP || /usr/sbin/groupadd -r $WEEWX_GROUP -/usr/bin/getent passwd $WEEWX_USER || /usr/sbin/useradd -r -g $WEEWX_GROUP -M -s /sbin/nologin $WEEWX_USER +/usr/bin/getent passwd $WEEWX_USER || /usr/sbin/useradd -r -g $WEEWX_GROUP -M -d $WEEWX_HOME -s /sbin/nologin $WEEWX_USER if [ $1 -gt 1 ]; then # this is an upgrade @@ -166,8 +167,8 @@ get_conf_version() { } # figure out which user should own everything -WEEWX_USER=weewx -WEEWX_GROUP=weewx +WEEWX_USER="${WEEWX_USER:-weewx}" +WEEWX_GROUP="${WEEWX_GROUP:-weewx}" if [ -d /var/lib/weewx ]; then WEEWX_USER=$(stat -c "%%U" /var/lib/weewx) WEEWX_GROUP=$(stat -c "%%G" /var/lib/weewx) From a73b5aba18f840c76118056a6e12bb672ff8ca56 Mon Sep 17 00:00:00 2001 From: matthew wall Date: Thu, 25 Jan 2024 21:42:17 -0500 Subject: [PATCH 4/6] clarify what happens when signing in debian world --- DEV_NOTES.txt | 17 +++++++++-------- makefile | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/DEV_NOTES.txt b/DEV_NOTES.txt index ce9b1d5d..0947f37a 100644 --- a/DEV_NOTES.txt +++ b/DEV_NOTES.txt @@ -348,10 +348,11 @@ installations. signing packages -------------------------------------------------------------- -gpg is used to sign the deb and rpm packages. SHA1 is no longer acceptable for -signing, so use SHA256 instead. this should be the default when building on -redhat9 and later. if it is not the default, then it can be forced with a -change to the signing macro in .rpmmacros - add the line --digest-algo sha256 +gpg is used to sign the deb repository and rpm packages. SHA1 is no longer +acceptable for signing, so be sure that your gpg keys and the signing command +use SHA256 instead. this should be the default when building on redhat9 and +later. if it is not the default, then it can be forced with a change to the +signing macro in .rpmmacros - add the line --digest-algo sha256 %__gpg_sign_cmd %{__gpg} \ gpg --no-verbose --no-armor \ @@ -363,10 +364,10 @@ change to the signing macro in .rpmmacros - add the line --digest-algo sha256 In the debian world, you sign the repository (specifically the file 'Release'), not the individual .deb files. So if you need to re-sign, you re-build and -re-sign the repository; there is no need touch the individual .deb files. This -is a bit odd, since the dpkg-build has options -us and -uc that tell it to *not* -sign the .deb files, and if you do not specify -us and -uc, the .deb files are -not signed, but dpkg-build will complain. +re-sign the repository; there is no need touch the individual .deb files. +Signing is controlled by the -us and -uc options to dpkg-build. If you do not +specify those options, then dpkg-build will try to sign the .dsc, .buildinfo, +and .changes files. The .deb itself is not signed. unit tests -------------------------------------------------------------------- diff --git a/makefile b/makefile index 06fdd490..e210da2f 100644 --- a/makefile +++ b/makefile @@ -229,7 +229,7 @@ debian-changelog: fi # use dpkg-buildpackage to create the debian package -# -us -uc - skip gpg signature on .dsc and .changes +# -us -uc - skip gpg signature on .dsc, .buildinfo, and .changes # the latest version in the debian changelog must match the packaging version DEBARCH=all DEBBLDDIR=$(BLDDIR)/weewx-$(VERSION) From eacab1ba3c266a96093c702092b768f8a80309b9 Mon Sep 17 00:00:00 2001 From: matthew wall Date: Thu, 25 Jan 2024 22:02:57 -0500 Subject: [PATCH 5/6] note some niggling bits with deb/rpm packages --- TODO.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/TODO.md b/TODO.md index f4e1f848..47c4b62e 100644 --- a/TODO.md +++ b/TODO.md @@ -1,3 +1,19 @@ +## 5.0.1 + +- mw using 'sudo weectl' on a deb/rpm install breaks permissions. new files + are owned by root:root instead of weewx:weewx. update the docs to use + "sudo -u weewx weectl ..." for deb/rpm installs, but only if running as + weewx. probably biggest problem is when manipulating database, since + confs and skins will still work when owned root:root. another example + of trying to do things the 'right' way for security ends up causing more + hassle. + +- mw doing 'apt purge weewx' does the right thing if your first install was + v5. but if your first install was v4, then doing a purge will destroy + the skins. this is probably because the skins were declared as conffiles + in v4. is there a way to remove that in v5 so that v5 'fixes' the confness + of skins? + ## Testing - mw convert to pytest From d1ac83bc599e47637cb2b1c5a50ccf1d0db19cc9 Mon Sep 17 00:00:00 2001 From: Tom Keffer Date: Fri, 26 Jan 2024 15:47:21 -0800 Subject: [PATCH 6/6] Fix problem with rebuilding daily summaries with dropped columns If a column was dropped from a database using weectl database drop-columns, then its daily summaries rebuilt, user would get a "table already exists error." --- docs_src/changes.md | 15 ++++++----- src/weectllib/database_actions.py | 43 +++++++++++++++++++++---------- src/weewx/manager.py | 2 +- 3 files changed, 40 insertions(+), 20 deletions(-) diff --git a/docs_src/changes.md b/docs_src/changes.md index 888a8571..e06361d2 100644 --- a/docs_src/changes.md +++ b/docs_src/changes.md @@ -5,23 +5,26 @@ WeeWX change history Include backwards compatible reference to `weewx.UnknownType`. -Fixed problem with installing extensions into installations that used V4 config +Fix problem with installing extensions into installations that used V4 config files that were installed by a package installer. Fix problem with `weectl device` when using drivers that were installed using the extension installer. Fixes issue #918. -Allow the use of the tilde (~) prefix with `--config` options. +Fix problem that prevented daily summaries from being rebuilt if they had been +modified by using `weectl database drop-columns`. + +Allow the use of the tilde (`~`) prefix with `--config` options. Minor corrections to the Norwegian translations. Thanks to user Aslak! PR #919. -Chinese language code changed to `zh`. Fixes issue #912. +Change Chinese language code to `zh`. Fixes issue #912. -Fixed bug in redhat/suse scriptlet that incorrectly substituted "{weewx}" -instead of "weewx" in the udev rules file. +Fix bug in redhat/suse scriptlet that incorrectly substituted `{weewx}` +instead of `weewx` in the udev rules file. -In the redhat/suse installers, use /var/lib/weewx as the HOME for user weewx. +In the redhat/suse installers, use `/var/lib/weewx` as `HOME` for user `weewx`. ### 5.0.0 01/14/2024 diff --git a/src/weectllib/database_actions.py b/src/weectllib/database_actions.py index 3a6374b1..79f5eba7 100644 --- a/src/weectllib/database_actions.py +++ b/src/weectllib/database_actions.py @@ -111,22 +111,37 @@ def rebuild_daily(config_dict, t1 = time.time() - log.info("Rebuilding daily summaries in database '%s' ..." % database_name) - print("Rebuilding daily summaries in database '%s' ..." % database_name) - if dry_run: - return + msg = f"Rebuilding daily summaries in database '{database_name}' ..." + log.info(msg) + print(msg) - # Open up the database. This will create the tables necessary for the daily - # summaries if they don't already exist: - with weewx.manager.open_manager_with_config(config_dict, - db_binding, initialize=True) as dbm: - # Do the actual rebuild - nrecs, ndays = dbm.backfill_day_summary(start_d=from_d, - stop_d=to_d, - trans_days=20) + # Open the database using the Manager object, which does not use daily summaries. This allows + # us to retrieve the SQL keys without triggering an exception because of the missing daily + # summaries. + with weewx.manager.Manager.open(manager_dict['database_dict'], + manager_dict['table_name']) as db: + sqlkeys = db.sqlkeys + # From the keys, build a schema for the daily summaries that reflects what is in the database + day_summaries_schemas = [(e, 'scalar') for e in sqlkeys + if e not in ('dateTime', 'usUnits', 'interval')] + if 'windSpeed' in sqlkeys: + # For backwards compatibility, include 'wind' + day_summaries_schemas += [('wind', 'vector')] + # Replace the static schema with the one we just built: + manager_dict['schema'] = {'day_summaries': day_summaries_schemas} + + # Open up the database. Use initialize=True, so the daily summary tables will be created: + with weewx.manager.open_manager(manager_dict, initialize=True) as dbm: + if dry_run: + nrecs = ndays = 0 + else: + # Do the actual rebuild + nrecs, ndays = dbm.backfill_day_summary(start_d=from_d, + stop_d=to_d, + trans_days=20) tdiff = time.time() - t1 # advise the user/log what we did - log.info("Rebuild of daily summaries in database '%s' complete." % database_name) + log.info(f"Rebuild of daily summaries in database '{database_name}' complete.") if nrecs: sys.stdout.flush() # fix a bit of formatting inconsistency if less than 1000 records @@ -140,6 +155,8 @@ def rebuild_daily(config_dict, f"{tdiff:.2f} seconds." print(msg) print(f"Rebuild of daily summaries in database '{database_name}' complete.") + elif dry_run: + print("Dry run: no records processed.") else: print(f"Daily summaries up to date in '{database_name}'.") diff --git a/src/weewx/manager.py b/src/weewx/manager.py index 7cf469dc..7eddd6e1 100644 --- a/src/weewx/manager.py +++ b/src/weewx/manager.py @@ -164,7 +164,7 @@ class Manager(object): 'host': 'localhost', 'user': 'weewx', 'password': 'weewx-password', - 'database_name' : 'weeewx', + 'database_name' : 'weewx', 'driver' : 'weedb.mysql' }