From c566d7abe5d5e4143ce15f915560d7eb31c08612 Mon Sep 17 00:00:00 2001 From: gjr80 Date: Sun, 8 Sep 2019 09:22:40 +1000 Subject: [PATCH 01/20] First cut of python logging in wee_import (#441) * First cut of python logging in wee_import Utilities Guide updates to be done separately * Refactored logging. Removed class WeeImportLog. All logging now done directly to module level logger. Console output now done directly from the module concerned using print function. * Replaced log method calls with info/debug/error method calls --- bin/wee_import | 267 +++++++++++++++++++-------------- bin/weeimport/csvimport.py | 71 ++++++--- bin/weeimport/cumulusimport.py | 45 ++++-- bin/weeimport/wdimport.py | 60 +++++--- bin/weeimport/weeimport.py | 231 ++++++++++++---------------- bin/weeimport/wuimport.py | 61 +++++--- 6 files changed, 414 insertions(+), 321 deletions(-) diff --git a/bin/wee_import b/bin/wee_import index fe975667..375bae74 100755 --- a/bin/wee_import +++ b/bin/wee_import @@ -9,47 +9,51 @@ Compatibility: - wee_import can import from a Comma Separated Values (CSV) format file, - directly from the historical records of a Weather Underground Personal - Weather Station or from one or more Cumulus monthly log files. CSV format - files must have a comma separated list of field names on the first line. + wee_import can import from: + - a Comma Separated Values (CSV) format file + - the historical records of a Weather Underground Personal + Weather Station + - one or more Cumulus monthly log files + - one or more Weather Display monthly log files Design - wee_import utilises a config file (the import config file) and a number of - command line options to control the import. The config file defines the type - of input to be performed and the import data source as well as more advanced - options such as field maps etc. Details of the supported command line - parameters/options can be viewed by entering wee_import --help at the command - line. Details of the wee_import config file settings can be found in example - import config files distributed in the weewx/util/import directory. + wee_import utilises an import config file and a number of command line + options to control the import. The import config file defines the type of + input to be performed and the import data source as well as more advanced + options such as field maps etc. Details of the supported command line + parameters/options can be viewed by entering wee_import --help at the + command line. Details of the wee_import import config file settings can be + found in the example import config files distributed in the + weewx/util/import directory. - wee_import utilises an abstract base class Source that defines the majority - of the wee_import functionality. The abstract base class and other supporting - structures are in bin/weeimport/weeimport.py. Child classes are created from - the base class for each different import type supported by wee_import. The - child classes set a number of import type specific properties as well as - defining a getData() method that reads the raw data to be imported and a - period_generator() method that generates a sequence of objects to be imported - (eg monthly log files). This way wee_import can be extended to support other - sources by defining a new child class, its specific properties as well as - getData() and period_generator() methods. The child class for a given import - type are defined in the bin/weeimport/xxximport.py files. + wee_import utilises an abstract base class Source that defines the majority + of the wee_import functionality. The abstract base class and other + supporting structures are in bin/weeimport/weeimport.py. Child classes are + created from the base class for each different import type supported by + wee_import. The child classes set a number of import type specific + properties as well as defining getData() and period_generator() methods that + read the raw data to be imported and generates a sequence of objects to be + imported (eg monthly log files) respectively. This way wee_import can be + extended to support other sources by defining a new child class, its + specific properties as well as getData() and period_generator() methods. The + child class for a given import type are defined in the + bin/weeimport/xxximport.py files. - As with other WeeWX utilities, wee_import advises the user of basic - configuration, action taken and results via stdout. However, since - wee_import can make substantial changes to the WeeWX archive, wee_import also - logs to file by default. This functionality is controlled via a command - line option. + As with other WeeWX utilities, wee_import advises the user of basic + configuration, action taken and results via the console. However, since + wee_import can make substantial changes to the WeeWX archive, wee_import + also logs to WeeWX log system file. Console and log output can be controlled + via a number of command line options. Prerequisites - wee_import uses a number of WeeWX API calls and therefore must have a - functional WeeWX installation. wee_import requires WeeWX 3.6.0 or later. + wee_import uses a number of WeeWX API calls and therefore must have a + functional WeeWX installation. wee_import requires WeeWX 4.0.0 or later. Configuration - A number of parameters can be defined in the import config file as follows: + A number of parameters can be defined in the import config file as follows: # EXAMPLE WEE_IMPORT CONFIGURATION FILE # @@ -63,7 +67,7 @@ Configuration # WU - import obs from a Weather Underground PWS history # Cumulus - import obs from a one or more Cumulus monthly log files # Format is: -# source = (CSV | WU | Cumulus) +# source = (CSV | WU | Cumulus | WD) source = CSV ############################################################################## @@ -511,10 +515,10 @@ source = CSV # inTemp # outHumidity # outTemp - # radiation (if Cumulus data available) - # rain (requires Cumulus 1.9.4 or later) + # radiation (if WD radiation data available) + # rain # rainRate - # UV (if Cumulus data available) + # UV (if WD UV data available) # windDir # windGust # windSpeed @@ -557,8 +561,8 @@ source = CSV # setting is best used if the records to be imported are # equally based in time but there are some missing records. # This setting is recommended for WU imports. - # To import Cumulus records it is recommended that the interval setting - # be set to the value used in Cumulus as the 'data log interval'. + # To import WD records it is recommended that the interval setting be set to + # the value used in WD as the 'data log interval'. # Format is: # interval = (derive | conf | x) interval = x @@ -580,17 +584,17 @@ source = CSV # calc_missing = (True | False) calc_missing = True - # Specify the character used as the field delimiter as Cumulus monthly log - # files may not always use a comma to delimit fields in the monthly log - # files. The character must be enclosed in quotes. Must not be the same - # as the decimal setting below. Format is: + # Specify the character used as the field delimiter as WD monthly log files + # may not always use a comma to delimit fields in the monthly log files. The + # character must be enclosed in quotes. Must not be the same as the decimal + # setting below. Format is: # delimiter = ',' delimiter = ',' - # Specify the character used as the decimal point. Cumulus monthly log - # files may not always use a fullstop character as the decimal point. The - # character must be enclosed in quotes. Must not be the same as the - # delimiter setting. Format is: + # Specify the character used as the decimal point. WD monthly log file may + # not always use a full stop character as the decimal point. The character + # must be enclosed in quotes. Must not be the same as the delimiter setting. + # Format is: # decimal = '.' decimal = '.' @@ -705,20 +709,27 @@ Adding a New Import Source # Python imports from __future__ import absolute_import from __future__ import print_function + +import logging import optparse -import syslog from distutils.version import StrictVersion # WeeWX imports +import weecfg import weewx import weeimport import weeimport.weeimport +import weeutil.logger +import weeutil.weeutil + + +log = logging.getLogger(__name__) # wee_import version number -WEE_IMPORT_VERSION = '0.3' +WEE_IMPORT_VERSION = '0.4' # minimum WeeWX version required for this version of wee_import -REQUIRED_WEEWX = "3.6.0" +REQUIRED_WEEWX = "4.0.0a7" description = """Import observation data into a WeeWX archive.""" @@ -730,7 +741,6 @@ usage = """wee_import --help [--dry-run] [--verbose] [--suppress-warnings] - [--log=-] """ epilog = """wee_import will import data from an external source into a WeeWX @@ -742,6 +752,9 @@ epilog = """wee_import will import data from an external source into a WeeWX def main(): """The main routine that kicks everything off.""" + # Set defaults for logging: + weeutil.logger.setup('wee_import', {}) + # Create a command line parser: parser = optparse.OptionParser(description=description, usage=usage, @@ -749,8 +762,8 @@ def main(): # Add the various options: parser.add_option("--config", dest="config_path", type=str, - metavar="CONFIG_FILE", default="weewx.conf", - help="Use WeeWX configuration file CONFIG_FILE.") + metavar="CONFIG_FILE", + help="Use configuration file CONFIG_FILE.") parser.add_option("--import-config", dest="import_config_path", type=str, metavar="IMPORT_CONFIG_FILE", help="Use import configuration file IMPORT_CONFIG_FILE.") @@ -764,15 +777,8 @@ def main(): parser.add_option("--to", dest="date_to", type=str, metavar="YYYY-mm-dd[THH:MM]", help="Import data up until this date or date-time. Format " "is YYYY-mm-dd[THH:MM].") - parser.add_option("--log", dest="logging", type=str, metavar="-", - help="Control wee_import log output. By default log output " - "is sent to the WeeWX log file. Log output may be " - "disabled by using '--log=-'. Some WeeWX API log " - "output cannot be controlled by wee_import and will " - "be sent to the default log file irrespective of the " - "'--log' option.") parser.add_option("--verbose", action="store_true", dest="verbose", - help="Print useful extra output.") + help="Print and log useful extra output.") parser.add_option("--suppress-warnings", action="store_true", dest="suppress", help="Suppress warnings to stdout. Warnings are still logged.") parser.add_option("--version", dest="version", action="store_true", @@ -787,74 +793,103 @@ def main(): weewx.__version__)) exit(1) + # get config_dict to use + config_path, config_dict = weecfg.read_config(options.config_path, args) + print("Using WeeWX configuration file %s" % config_path) + + # Set weewx.debug as necessary: + weewx.debug = weeutil.weeutil.to_int(config_dict.get('debug', 0)) + + # Now we can set up the user customized logging: + weeutil.logger.setup('wee_import', config_dict) + # display wee_import version info if options.version: print("wee_import version: %s" % WEE_IMPORT_VERSION) exit(0) - # Set up logging - wlog = weeimport.weeimport.WeeImportLog(options.logging, - options.verbose, - options.suppress, - options.dry_run) + # to do anything more we need an import config file, check if one was + # provided + if options.import_config_path: + # we have something so try to start - # advise the user we are starting up - wlog.printlog(syslog.LOG_INFO, "Starting wee_import...") + # advise the user we are starting up + print("Starting wee_import...") + log.info("Starting wee_import...") - # If we got this far we must want to import something so get a Source - # object from our factory and try to import. Be prepared to catch any - # errors though. - try: - source_obj = weeimport.weeimport.Source.sourceFactory(options, - args, - wlog) - source_obj.run() - except weeimport.weeimport.WeeImportOptionError as e: - wlog.printlog(syslog.LOG_INFO, "**** Command line option error.") - wlog.printlog(syslog.LOG_INFO, "**** %s" % e) - print("**** Nothing done, exiting.") - wlog.logonly(syslog.LOG_INFO, "**** Nothing done.") - exit(1) - except weeimport.weeimport.WeeImportIOError as e: - wlog.printlog(syslog.LOG_INFO, "**** Unable to load source file.") - wlog.printlog(syslog.LOG_INFO, "**** %s" % e) - print("**** Nothing done, exiting.") - wlog.logonly(syslog.LOG_INFO, "**** Nothing done.") - exit(1) - except weeimport.weeimport.WeeImportFieldError as e: - wlog.printlog(syslog.LOG_INFO, "**** Unable to map source data.") - wlog.printlog(syslog.LOG_INFO, "**** %s" % e) - print("**** Nothing done, exiting.") - wlog.logonly(syslog.LOG_INFO, "**** Nothing done.") - exit(1) - except weeimport.weeimport.WeeImportMapError as e: - wlog.printlog(syslog.LOG_INFO, - "**** Unable to parse source-to-WeeWX field map.") - wlog.printlog(syslog.LOG_INFO, "**** %s" % e) - print("**** Nothing done, exiting.") - wlog.logonly(syslog.LOG_INFO, "**** Nothing done.") - exit(1) - except (weewx.ViolatedPrecondition, weewx.UnsupportedFeature) as e: - wlog.printlog(syslog.LOG_INFO, "**** %s" % e) - print("**** Nothing done, exiting.") - wlog.logonly(syslog.LOG_INFO, "**** Nothing done.") + # If we got this far we must want to import something so get a Source + # object from our factory and try to import. Be prepared to catch any + # errors though. + try: + source_obj = weeimport.weeimport.Source.sourceFactory(options, + args) + source_obj.run() + except weeimport.weeimport.WeeImportOptionError as e: + print("**** Command line option error.") + log.info("**** Command line option error.") + print("**** %s" % e) + log.info("**** %s" % e) + print("**** Nothing done, exiting.") + log.info("**** Nothing done.") + exit(1) + except weeimport.weeimport.WeeImportIOError as e: + print("**** Unable to load source file.") + log.info("**** Unable to load source file.") + print("**** %s" % e) + log.info("**** %s" % e) + print("**** Nothing done, exiting.") + log.info("**** Nothing done.") + exit(1) + except weeimport.weeimport.WeeImportFieldError as e: + print("**** Unable to map source data.") + log.info("**** Unable to map source data.") + print("**** %s" % e) + log.info("**** %s" % e) + print("**** Nothing done, exiting.") + log.info("**** Nothing done.") + exit(1) + except weeimport.weeimport.WeeImportMapError as e: + print("**** Unable to parse source-to-WeeWX field map.") + log.info("**** Unable to parse source-to-WeeWX field map.") + print("**** %s" % e) + log.info("**** %s" % e) + print("**** Nothing done, exiting.") + log.info("**** Nothing done.") + exit(1) + except (weewx.ViolatedPrecondition, weewx.UnsupportedFeature) as e: + print("**** %s" % e) + log.info("**** %s" % e) + print("**** Nothing done, exiting.") + log.info("**** Nothing done.") + print() + parser.print_help() + exit(1) + except SystemExit as e: + print(e) + exit(0) + except (ValueError, weewx.UnitError) as e: + print("**** %s" % e) + log.info("**** %s" % e) + print("**** Nothing done, exiting.") + log.info("**** Nothing done.") + exit(1) + except IOError as e: + print("**** Unable to load config file.") + log.info("**** Unable to load config file.") + print("**** %s" % e) + log.info("**** %s" % e) + print("**** Nothing done, exiting.") + log.info("**** Nothing done.") + exit(1) + else: + # we have no import config file so display a suitable message followed + # by the help text then exit + print("**** No import config file specified.") + print("**** Nothing done.") print() parser.print_help() exit(1) - except SystemExit as e: - print(e) - exit(0) - except (ValueError, weewx.UnitError) as e: - wlog.printlog(syslog.LOG_INFO, "**** %s" % e) - print("**** Nothing done, exiting.") - wlog.logonly(syslog.LOG_INFO, "**** Nothing done.") - exit(1) - except IOError as e: - wlog.printlog(syslog.LOG_INFO, "**** Unable to load config file.") - wlog.printlog(syslog.LOG_INFO, "**** %s" % e) - print("**** Nothing done, exiting.") - wlog.logonly(syslog.LOG_INFO, "**** Nothing done.") - exit(1) + # execute our main code if __name__ == "__main__": diff --git a/bin/weeimport/csvimport.py b/bin/weeimport/csvimport.py index ef3bc9f6..8f999708 100644 --- a/bin/weeimport/csvimport.py +++ b/bin/weeimport/csvimport.py @@ -15,8 +15,8 @@ from __future__ import print_function # Python imports import csv +import logging import os -import syslog # WeeWX imports from . import weeimport @@ -25,6 +25,8 @@ import weewx from weeutil.weeutil import timestamp_to_string, option_as_list from weewx.units import unit_nicknames +log = logging.getLogger(__name__) + # ============================================================================ # class CSVSource @@ -42,13 +44,12 @@ class CSVSource(weeimport.Source): # these details are specified by the user in the wee_import config file. _header_map = None - def __init__(self, config_dict, config_path, csv_config_dict, import_config_path, options, log): + def __init__(self, config_dict, config_path, csv_config_dict, import_config_path, options): # call our parents __init__ super(CSVSource, self).__init__(config_dict, csv_config_dict, - options, - log) + options) # save our import config path self.import_config_path = import_config_path @@ -86,12 +87,17 @@ class CSVSource(weeimport.Source): # tell the user/log what we intend to do _msg = "A CSV import from source file '%s' has been requested." % self.source - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) _msg = "The following options will be used:" - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " config=%s, import-config=%s" % (config_path, self.import_config_path) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) if options.date: _msg = " source=%s, date=%s" % (self.source, options.date) else: @@ -99,37 +105,62 @@ class CSVSource(weeimport.Source): _msg = " source=%s, from=%s, to=%s" % (self.source, options.date_from, options.date_to) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " dry-run=%s, calc_missing=%s, ignore_invalid_data=%s" % (self.dry_run, self.calc_missing, self.ignore_invalid_data) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " tranche=%s, interval=%s, date/time_string_format=%s" % (self.tranche, self.interval, self.raw_datetime_format) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " rain=%s, wind_direction=%s" % (self.rain, self.wind_dir) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " UV=%s, radiation=%s" % (self.UV_sensor, self.solar_sensor) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = "Using database binding '%s', which is bound to database '%s'" % (self.db_binding_wx, self.dbm.database_name) - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) _msg = "Destination table '%s' unit system is '%#04x' (%s)." % (self.dbm.table_name, self.archive_unit_sys, unit_nicknames[self.archive_unit_sys]) - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) if self.calc_missing: - print("Missing derived observations will be calculated.") + _msg = "Missing derived observations will be calculated." + print(_msg) + log.info(_msg) + if not self.UV_sensor: - print("All WeeWX UV fields will be set to None.") + _msg = "All WeeWX UV fields will be set to None." + print(_msg) + log.info(_msg) if not self.solar_sensor: - print("All WeeWX radiation fields will be set to None.") + _msg = "All WeeWX radiation fields will be set to None." + print(_msg) + log.info(_msg) if options.date or options.date_from: - print("Observations timestamped after %s and up to and" % timestamp_to_string(self.first_ts)) - print("including %s will be imported." % timestamp_to_string(self.last_ts)) + _msg = "Observations timestamped after %s and up to and" % timestamp_to_string(self.first_ts) + print(_msg) + log.info(_msg) + _msg = "including %s will be imported." % timestamp_to_string(self.last_ts) + print(_msg) + log.info(_msg) if self.dry_run: - print("This is a dry run, imported data will not be saved to archive.") + _msg = "This is a dry run, imported data will not be saved to archive." + print(_msg) + log.info(_msg) def getRawData(self, period): """Obtain an iterable containing the raw data to be imported. diff --git a/bin/weeimport/cumulusimport.py b/bin/weeimport/cumulusimport.py index e5f93518..4bce8969 100644 --- a/bin/weeimport/cumulusimport.py +++ b/bin/weeimport/cumulusimport.py @@ -15,8 +15,8 @@ from __future__ import print_function # Python imports import csv import glob +import logging import os -import syslog import time # WeeWX imports @@ -26,6 +26,8 @@ import weewx from weeutil.weeutil import timestamp_to_string from weewx.units import unit_nicknames +log = logging.getLogger(__name__) + # Dict to lookup rainRate units given rain units rain_units_dict = {'inch': 'inch_per_hour', 'mm': 'mm_per_hour'} @@ -89,13 +91,12 @@ class CumulusSource(weeimport.Source): 'cur_app_temp': {'map_to': 'appTemp'} } - def __init__(self, config_dict, config_path, cumulus_config_dict, import_config_path, options, log): + def __init__(self, config_dict, config_path, cumulus_config_dict, import_config_path, options): # call our parents __init__ super(CumulusSource, self).__init__(config_dict, cumulus_config_dict, - options, - log) + options) # save our import config path self.import_config_path = import_config_path @@ -207,7 +208,8 @@ class CumulusSource(weeimport.Source): try: self.source = cumulus_config_dict['directory'] except KeyError: - raise weewx.ViolatedPrecondition("Cumulus monthly logs directory not specified in '%s'." % import_config_path) + _msg = "Cumulus monthly logs directory not specified in '%s'." % import_config_path + raise weewx.ViolatedPrecondition(_msg) # property holding the current log file name being processed self.file_name = None @@ -223,34 +225,49 @@ class CumulusSource(weeimport.Source): # tell the user/log what we intend to do _msg = "Cumulus monthly log files in the '%s' directory will be imported" % self.source - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) _msg = "The following options will be used:" - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " config=%s, import-config=%s" % (config_path, self.import_config_path) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) if options.date: _msg = " date=%s" % options.date else: # we must have --from and --to _msg = " from=%s, to=%s" % (options.date_from, options.date_to) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " dry-run=%s, calc_missing=%s, ignore_invalid_data=%s" % (self.dry_run, self.calc_missing, self.ignore_invalid_data) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " tranche=%s, interval=%s" % (self.tranche, self.interval) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " UV=%s, radiation=%s" % (self.UV_sensor, self.solar_sensor) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = "Using database binding '%s', which is bound to database '%s'" % (self.db_binding_wx, self.dbm.database_name) - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) _msg = "Destination table '%s' unit system is '%#04x' (%s)." % (self.dbm.table_name, self.archive_unit_sys, unit_nicknames[self.archive_unit_sys]) - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) if self.calc_missing: print("Missing derived observations will be calculated.") if not self.UV_sensor: diff --git a/bin/weeimport/wdimport.py b/bin/weeimport/wdimport.py index bee7cff1..98f3e0ca 100644 --- a/bin/weeimport/wdimport.py +++ b/bin/weeimport/wdimport.py @@ -17,9 +17,9 @@ import collections import csv import datetime import glob +import logging import operator import os -import syslog import time # WeeWX imports @@ -30,6 +30,7 @@ import weewx from weeutil.weeutil import timestamp_to_string from weewx.units import unit_nicknames +log = logging.getLogger(__name__) # ============================================================================ # class WDSource @@ -183,11 +184,10 @@ class WDSource(weeimport.Source): 'hum7': {'units': 'percent', 'map_to': 'extraHumid7'} } - def __init__(self, config_dict, config_path, wd_config_dict, import_config_path, options, log): + def __init__(self, config_dict, config_path, wd_config_dict, import_config_path, options): # call our parents __init__ - super(WDSource, self).__init__(config_dict, wd_config_dict, - options, log) + super(WDSource, self).__init__(config_dict, wd_config_dict, options) # save the import config path self.import_config_path = import_config_path @@ -397,48 +397,71 @@ class WDSource(weeimport.Source): # tell the user/log what we intend to do _msg = "Weather Display monthly log files in the '%s' directory will be imported" % self.source - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) _msg = "The following options will be used:" - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " config=%s, import-config=%s" % (config_path, self.import_config_path) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) if options.date: _msg = " date=%s" % options.date else: # we must have --from and --to _msg = " from=%s, to=%s" % (options.date_from, options.date_to) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " dry-run=%s, calc_missing=%s, ignore_invalid_data=%s" % (self.dry_run, self.calc_missing, self.ignore_invalid_data) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) if log_unit_sys is not None and log_unit_sys.upper() in ['METRIC', 'US']: # valid unit system specified _msg = " monthly logs are in %s units" % log_unit_sys.upper() - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) else: # group units specified _msg = " monthly logs use the following units:" - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " temperature=%s pressure=%s" % (temp_u, press_u) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " rain=%s speed=%s" % (rain_u, speed_u) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " tranche=%s, interval=%s" % (self.tranche, self.interval) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " UV=%s, radiation=%s ignore extreme temperature and humidity=%s" % (self.UV_sensor, self.solar_sensor, self.ignore_extreme_temp_hum) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = "Using database binding '%s', which is bound to database '%s'" % (self.db_binding_wx, self.dbm.database_name) - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) _msg = "Destination table '%s' unit system is '%#04x' (%s)." % (self.dbm.table_name, self.archive_unit_sys, unit_nicknames[self.archive_unit_sys]) - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) if self.calc_missing: print("Missing derived observations will be calculated.") if not self.UV_sensor: @@ -512,7 +535,8 @@ class WDSource(weeimport.Source): _msg = "Unexpected number of columns found in '%s': %s v %s" % (_fn, len(_row.split(_del)), len(self.logs[lg]['fields'])) - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) # make sure we have full stops as decimal points _clean_data.append(_row.replace(self.decimal, '.')) diff --git a/bin/weeimport/weeimport.py b/bin/weeimport/weeimport.py index 19dc5c7f..fcf01c0f 100644 --- a/bin/weeimport/weeimport.py +++ b/bin/weeimport/weeimport.py @@ -15,9 +15,9 @@ from __future__ import absolute_import # Python imports import datetime +import logging import re import sys -import syslog import time from datetime import datetime as dt @@ -34,8 +34,10 @@ from weewx.manager import open_manager_with_config from weewx.units import unit_constants, unit_nicknames, convertStd, to_std_system, ValueTuple from weeutil.weeutil import timestamp_to_string, option_as_list, to_int, tobool, _get_object +log = logging.getLogger(__name__) + # List of sources we support -SUPPORTED_SOURCES = ['CSV', 'WU', 'Cumulus'] +SUPPORTED_SOURCES = ['CSV', 'WU', 'Cumulus', 'WD'] # Minimum requirements in any explicit or implicit WeeWX field-to-import field # map @@ -114,7 +116,7 @@ class Source(object): # reg expression to match any HTML tag of the form <...> _tags = re.compile(r'\<.*\>') - def __init__(self, config_dict, import_config_dict, options, log): + def __init__(self, config_dict, import_config_dict, options): """A generic initialisation. Set some realistic default values for options read from the import @@ -123,8 +125,8 @@ class Source(object): know what records to import. """ - # give our source object some logging abilities - self.wlog = log +# # give our source object some logging abilities +# self.wlog = log # save our WeeWX config dict self.config_dict = config_dict @@ -198,6 +200,7 @@ class Source(object): # Process our command line options self.dry_run = options.dry_run self.verbose = options.verbose + self.suppress = options.suppress # By processing any --date, --from and --to options we need to derive # self.first_ts and self.last_ts; the earliest and latest (inclusive) @@ -293,7 +296,7 @@ class Source(object): self.period_duplicates = set() @staticmethod - def sourceFactory(options, args, log): + def sourceFactory(options, args): """Factory to produce a Source object. Returns an appropriate object depending on the source type. Raises a @@ -302,9 +305,7 @@ class Source(object): # get some key WeeWX parameters # first the config dict to use - config_path, config_dict = weecfg.read_config(None, - args, - file_name=options.config_path) + config_path, config_dict = weecfg.read_config(options.config_path, args) # get wee_import config dict if it exists import_config_path, import_config_dict = weecfg.read_config(None, args, @@ -333,8 +334,7 @@ class Source(object): config_path, import_config_dict.get(source, {}), import_config_path, - options, - log) + options) def run(self): """Main entry point for importing from an external source. @@ -374,25 +374,40 @@ class Source(object): # get the raw data _msg = 'Obtaining raw import data for period %d...' % self.period_no - self.wlog.verboselog(syslog.LOG_INFO, _msg) + if self.verbose: + print(_msg) + log.info(_msg) _raw_data = self.getRawData(period) _msg = 'Raw import data read successfully for period %d.' % self.period_no - self.wlog.verboselog(syslog.LOG_INFO, _msg) + if self.verbose: + print(_msg) + log.info(_msg) # map the raw data to a WeeWX archive compatible dictionary _msg = 'Mapping raw import data for period %d...' % self.period_no - self.wlog.verboselog(syslog.LOG_INFO, _msg) + if self.verbose: + print(_msg) + log.info(_msg) _mapped_data = self.mapRawData(_raw_data, self.archive_unit_sys) _msg = 'Raw import data mapped successfully for period %d.' % self.period_no - self.wlog.verboselog(syslog.LOG_INFO, _msg) + if self.verbose: + print(_msg) + log.info(_msg) # save the mapped data to archive - _msg = 'Saving mapped data to archive for period %d...' % self.period_no - self.wlog.verboselog(syslog.LOG_INFO, _msg) + # first advise the user and log, but only if its not a dry run + if not self.dry_run: + _msg = 'Saving mapped data to archive for period %d...' % self.period_no + if self.verbose: + print(_msg) + log.info(_msg) self.saveToArchive(archive, _mapped_data) - _msg = 'Mapped data saved to archive successfully for period %d.' % self.period_no - self.wlog.verboselog(syslog.LOG_INFO, _msg) - + # advise the user and log, but only if its not a dry run + if not self.dry_run: + _msg = 'Mapped data saved to archive successfully for period %d.' % self.period_no + if self.verbose: + print(_msg) + log.info(_msg) # increment our period counter self.period_no += 1 # Provide some summary info now that we have finished the import. @@ -401,37 +416,48 @@ class Source(object): if self.total_rec_proc == 0: # nothing imported so say so _msg = 'No records were identified for import. Exiting. Nothing done.' - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) else: # we imported something total_rec = self.total_rec_proc + self.total_duplicate_rec if self.dry_run: # but it was a dry run - self.wlog.printlog(syslog.LOG_INFO, "Finished dry run import") + _msg = "Finished dry run import" + print(_msg) + log.info(_msg) _msg = "%d records were processed and %d unique records would "\ "have been imported." % (total_rec, self.total_rec_proc) - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) if self.total_duplicate_rec > 1: _msg = "%d duplicate records were ignored." % self.total_duplicate_rec - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) elif self.total_duplicate_rec == 1: - self.wlog.printlog(syslog.LOG_INFO, - "1 duplicate record was ignored.") + _msg = "1 duplicate record was ignored." + print(_msg) + log.info(_msg) else: # something should have been saved to database - self.wlog.printlog(syslog.LOG_INFO, "Finished import") + _msg = "Finished import" + print(_msg) + log.info(_msg) _msg = "%d records were processed and %d unique records " \ "imported in %.2f seconds." % (total_rec, self.total_rec_proc, self.tdiff) - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) if self.total_duplicate_rec > 1: _msg = "%d duplicate records were ignored." % self.total_duplicate_rec - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) elif self.total_duplicate_rec == 1: - self.wlog.printlog(syslog.LOG_INFO, - "1 duplicate record was ignored.") + _msg = "1 duplicate record was ignored." + print(_msg) + log.info(_msg) print("Those records with a timestamp already in the archive will not have been") print("imported. Confirm successful import in the WeeWX log file.") @@ -573,9 +599,8 @@ class Source(object): # will use _msg = "The following imported field-to-WeeWX field map will be used:" if self.verbose: - self.wlog.verboselog(syslog.LOG_INFO, _msg) - else: - self.wlog.logonly(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) for _key, _val in six.iteritems(_map): if 'field_name' in _val: _units_msg = "" @@ -585,9 +610,8 @@ class Source(object): _units_msg, _key) if self.verbose: - self.wlog.verboselog(syslog.LOG_INFO, _msg) - else: - self.wlog.logonly(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) else: # no [[FieldMap]] stanza and no _header_map so raise an error as we # don't know what to map @@ -813,19 +837,19 @@ class Source(object): if self.map[_field]['field_name'] not in _warned: _msg = "Warning: Import field '%s' is mapped to WeeWX " \ "field '%s' but the" % (self.map[_field]['field_name'], - _field) - self.wlog.printlog(syslog.LOG_INFO, - _msg, - can_suppress=True) + _field) + if not self.suppress: + print(_msg) + log.info(_msg) _msg = " import field '%s' could not be found " \ "in one or more records." % self.map[_field]['field_name'] - self.wlog.printlog(syslog.LOG_INFO, - _msg, - can_suppress=True) + if not self.suppress: + print(_msg) + log.info(_msg) _msg = " WeeWX field '%s' will be set to 'None' in these records." % _field - self.wlog.printlog(syslog.LOG_INFO, - _msg, - can_suppress=True) + if not self.suppress: + print(_msg) + log.info(_msg) # make sure we do this warning once only _warned.append(self.map[_field]['field_name']) # if we have a mapped field for a unit system with a valid value, @@ -865,7 +889,8 @@ class Source(object): # we had more than one unique value for interval, warn the user _msg = "Warning: Records to be imported contain multiple " \ "different 'interval' values." - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) print(" This may mean the imported data is missing some records and it may lead") print(" to data integrity issues. If the raw data has a known, fixed interval") print(" value setting the relevant 'interval' setting in wee_import config to") @@ -887,15 +912,20 @@ class Source(object): print("Import aborted by user. No records saved to archive.") _msg = "User chose to abort import. %d records were processed. " \ "Exiting." % self.total_rec_proc - self.wlog.logonly(syslog.LOG_INFO, _msg) + log.info(_msg) raise SystemExit('Exiting. Nothing done.') - self.wlog.verboselog(syslog.LOG_INFO, - "Mapped %d records." % len(_records)) + _msg = "Mapped %d records." % len(_records) + if self.verbose: + print(_msg) + log.info(_msg) # the user wants to continue or we have only one unique value for # interval so return the records return _records else: - self.wlog.verboselog(syslog.LOG_INFO, "Mapped 0 records.") + _msg = "Mapped 0 records." + if self.verbose: + print(_msg) + log.info(_msg) # we have no records to return so return None return None @@ -945,16 +975,16 @@ class Source(object): if _interval < 0: # so raise an error _msg = "Cannot derive 'interval' for record timestamp: %s." % timestamp_to_string(current_ts) - self.wlog.printlog(syslog.LOG_INFO, _msg) - raise ValueError( - "Raw data is not in ascending date time order.") + print(_msg) + log.info(_msg) + raise ValueError("Raw data is not in ascending date time order.") except TypeError: _interval = None return _interval else: # we don't know what to do so raise an error - raise ValueError( - "Cannot derive 'interval'. Unknown 'interval' setting in %s." % self.import_config_path) + _msg = "Cannot derive 'interval'. Unknown 'interval' setting in %s." % self.import_config_path + raise ValueError(_msg) @staticmethod def getRain(last_rain, current_rain): @@ -1101,7 +1131,7 @@ class Source(object): # tell the user what we have done _msg = "Unique records processed: %d; Last timestamp: %s\r" % (nrecs, timestamp_to_string(_final_rec['dateTime'])) - print(_msg, end=' ', file=sys.stdout) + print(_msg, end='', file=sys.stdout) sys.stdout.flush() _tranche = [] # we have processed all records but do we have any records left @@ -1118,7 +1148,7 @@ class Source(object): # tell the user what we have done _msg = "Unique records processed: %d; Last timestamp: %s\r" % (nrecs, timestamp_to_string(_final_rec['dateTime'])) - print(_msg, end=' ', file=sys.stdout) + print(_msg, end='', file=sys.stdout) print() sys.stdout.flush() # update our counts @@ -1133,11 +1163,14 @@ class Source(object): else: _msg = " %d duplicate records were identified in period %d:" % (num_duplicates, self.period_no) - self.wlog.printlog(syslog.LOG_INFO, _msg, can_suppress=True) + if not self.suppress: + print(_msg) + log.info(_msg) for ts in sorted(self.period_duplicates): _msg = " %s" % timestamp_to_string(ts) - self.wlog.printlog(syslog.LOG_INFO, _msg, - can_suppress=True) + if not self.suppress: + print(_msg) + log.info(_msg) # add the period duplicates to the overall duplicates self.duplicates |= self.period_duplicates # reset the period duplicates @@ -1145,8 +1178,9 @@ class Source(object): elif self.ans == 'n': # user does not want to import so display a message and then # ask to exit - self.wlog.printlog(syslog.LOG_INFO, - 'User chose not to import records. Exiting. Nothing done.') + _msg = "User chose not to import records. Exiting. Nothing done." + print(_msg) + log.info(_msg) raise SystemExit('Exiting. Nothing done.') else: # we have no records to import, advise the user but what we say @@ -1163,74 +1197,6 @@ class Source(object): self.tdiff = time.time() - self.t1 -# ============================================================================ -# class WeeImportLog -# ============================================================================ - - -class WeeImportLog(object): - """Class to handle wee_import logging. - - This class provides a wrapper around the python syslog module to handle - wee_import logging requirements. The --log=- command line option disables - log output otherwise log output is sent to the same log used by WeeWX. - """ - - def __init__(self, opt_logging, opt_verbose, opt_suppress, opt_dry_run): - """Initialise our log environment.""" - - # first check if we are turning off log to file or not - if opt_logging: - log_bool = opt_logging.strip() == '-' - else: - log_bool = False - # Flag to indicate whether we are logging to file or not. Log to file - # every time except when logging is explicitly turned off on the - # command line or its a dry run. - self.log = not (opt_dry_run or log_bool) - # if we are logging then setup our syslog environment - # if --verbose we log up to syslog.LOG_DEBUG - # otherwise just log up to syslog.LOG_INFO - if self.log: - syslog.openlog('wee_import', - logoption=syslog.LOG_PID | syslog.LOG_CONS) - if opt_verbose: - syslog.setlogmask(syslog.LOG_UPTO(syslog.LOG_DEBUG)) - else: - syslog.setlogmask(syslog.LOG_UPTO(syslog.LOG_INFO)) - # logging by other modules (eg WxCalculate) does not use WeeImportLog - # but we can disable most logging by raising the log priority if its a - # dry run - if opt_dry_run: - syslog.setlogmask(syslog.LOG_UPTO(syslog.LOG_CRIT)) - # keep opt_verbose for later - self.verbose = opt_verbose - self.suppress = opt_suppress - - def logonly(self, level, message): - """Log to file only.""" - - # are we logging ? - if self.log: - # add a little preamble to say this is wee_import - _message = 'wee_import: ' + message - syslog.syslog(level, _message) - - def printlog(self, level, message, can_suppress=False): - """Print to screen and log to file.""" - - if not(can_suppress and self.suppress): - print(message) - self.logonly(level, message) - - def verboselog(self, level, message): - """Print to screen if --verbose and log to file always.""" - - if self.verbose: - print(message) - self.logonly(level, message) - - # ============================================================================ # Utility functions # ============================================================================ @@ -1247,4 +1213,3 @@ def get_binding(config_dict): else: db_binding_wx = None return db_binding_wx - diff --git a/bin/weeimport/wuimport.py b/bin/weeimport/wuimport.py index 8014d332..6a448a22 100644 --- a/bin/weeimport/wuimport.py +++ b/bin/weeimport/wuimport.py @@ -15,8 +15,8 @@ from __future__ import absolute_import from __future__ import print_function import csv import datetime +import logging import socket -import syslog from datetime import datetime as dt from six.moves import urllib @@ -28,6 +28,7 @@ import weewx from weeutil.weeutil import timestamp_to_string, option_as_list, startOfDay from weewx.units import unit_nicknames +log = logging.getLogger(__name__) # ============================================================================ # class WUSource @@ -75,13 +76,12 @@ class WUSource(weeimport.Source): 'map_to': 'radiation'} } - def __init__(self, config_dict, config_path, wu_config_dict, import_config_path, options, log): + def __init__(self, config_dict, config_path, wu_config_dict, import_config_path, options): # call our parents __init__ super(WUSource, self).__init__(config_dict, wu_config_dict, - options, - log) + options) # save our import config path self.import_config_path = import_config_path @@ -92,7 +92,8 @@ class WUSource(weeimport.Source): try: self.station_id = wu_config_dict['station_id'] except KeyError: - raise weewx.ViolatedPrecondition("Weather Underground station ID not specified in '%s'." % import_config_path) + _msg = "Weather Underground station ID not specified in '%s'." % import_config_path + raise weewx.ViolatedPrecondition(_msg) # wind dir bounds _wind_direction = option_as_list(wu_config_dict.get('wind_direction', @@ -130,12 +131,17 @@ class WUSource(weeimport.Source): # tell the user/log what we intend to do _msg = "Observation history for Weather Underground station '%s' will be imported." % self.station_id - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) _msg = "The following options will be used:" - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " config=%s, import-config=%s" % (config_path, self.import_config_path) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) if options.date: _msg = " station=%s, date=%s" % (self.station_id, options.date) else: @@ -143,22 +149,30 @@ class WUSource(weeimport.Source): _msg = " station=%s, from=%s, to=%s" % (self.station_id, options.date_from, options.date_to) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " dry-run=%s, calc_missing=%s, ignore_invalid_data=%s" % (self.dry_run, self.calc_missing, self.ignore_invalid_data) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " tranche=%s, interval=%s, wind_direction=%s" % (self.tranche, self.interval, self.wind_dir) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = "Using database binding '%s', which is bound to database '%s'" % (self.db_binding_wx, self.dbm.database_name) - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) _msg = "Destination table '%s' unit system is '%#04x' (%s)." % (self.dbm.table_name, self.archive_unit_sys, unit_nicknames[self.archive_unit_sys]) - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) if self.calc_missing: print("Missing derived observations will be calculated.") if options.date or options.date_from: @@ -188,7 +202,8 @@ class WUSource(weeimport.Source): which raw obs data will be read. """ - # the date for which we want the WU data is held in a datetime object, we need to convert it to a timetuple + # the date for which we want the WU data is held in a datetime object, + # we need to convert it to a timetuple date_tt = period.timetuple() # construct our URL using station ID and day, month, year _url = "http://www.wunderground.com/weatherstation/WXDailyHistory.asp?ID=%s&" \ @@ -200,14 +215,20 @@ class WUSource(weeimport.Source): try: _wudata = urllib.request.urlopen(_url) except urllib.error.URLError as e: - self.wlog.printlog(syslog.LOG_ERR, - "Unable to open Weather Underground station %s" % self.station_id) - self.wlog.printlog(syslog.LOG_ERR, " **** %s" % e) + _msg = "Unable to open Weather Underground station %s" % self.station_id + print(_msg) + log.error(_msg) + _msg = " **** %s" % e + print(_msg) + log.error(_msg) raise except socket.timeout as e: - self.wlog.printlog(syslog.LOG_ERR, - "Socket timeout for Weather Underground station %s" % self.station_id) - self.wlog.printlog(syslog.LOG_ERR, " **** %s" % e) + _msg = "Socket timeout for Weather Underground station %s" % self.station_id + print(_msg) + log.error(_msg) + _msg = " **** %s" % e + print(_msg) + log.error(_msg) raise # because the data comes back with lots of HTML tags and whitespace we From 3eb97346b92b588beb451449d67aab30b30ad6f3 Mon Sep 17 00:00:00 2001 From: sshambar Date: Sun, 8 Sep 2019 18:39:28 +0100 Subject: [PATCH 02/20] Fix engine to exit correctly on Terminate signal (#442) Re-send SIGTERM to self after exiting main loop so process exits with expected exit code. --- bin/weewx/engine.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/weewx/engine.py b/bin/weewx/engine.py index 6b750bcb..81b8d5c7 100644 --- a/bin/weewx/engine.py +++ b/bin/weewx/engine.py @@ -953,8 +953,9 @@ def main(options, args, engine_class=StdEngine): except Terminate: syslog.syslog(syslog.LOG_INFO, "engine: Terminating weewx version %s" % weewx.__version__) weeutil.weeutil.log_traceback(" **** ", syslog.LOG_DEBUG) - # Reraise the exception (this should cause the program to exit) - raise + # Reraise the signal (this should cause the program to exit) + signal.signal(signal.SIGTERM, signal.SIG_DFL) + os.kill(0, signal.SIGTERM) # Catch any keyboard interrupts and log them except KeyboardInterrupt: From a843b59d7eab785ec0438aea42efd9072e213c15 Mon Sep 17 00:00:00 2001 From: gjr80 Date: Mon, 9 Sep 2019 13:54:17 +1000 Subject: [PATCH 03/20] Updated wee_import section to reflect logging system changes --- docs/utilities.htm | 75 ++++++++++++++++------------------------------ 1 file changed, 25 insertions(+), 50 deletions(-) diff --git a/docs/utilities.htm b/docs/utilities.htm index 8c871a64..e7d25f02 100644 --- a/docs/utilities.htm +++ b/docs/utilities.htm @@ -1013,14 +1013,13 @@ No extensions installed [--dry-run] [--verbose] [--suppress-warnings] - [--log=-] Import observation data into a WeeWX archive. Options: -h, --help show this help message and exit - --config=CONFIG_FILE Use WeeWX configuration file CONFIG_FILE. + --config=CONFIG_FILE Use configuration file CONFIG_FILE. --import-config=IMPORT_CONFIG_FILE Use import configuration file IMPORT_CONFIG_FILE. --dry-run Print what would happen but do not do it. @@ -1031,13 +1030,7 @@ Options: --to=YYYY-mm-dd[THH:MM] Import data up until this date or date-time. Format is YYYY-mm-dd[THH:MM]. - --log=- Control wee_import log output. By default log output - is sent to the WeeWX log file. Log output may be - disabled by using '--log=-'. Some WeeWX API log output - cannot be controlled by wee_import and will be sent to - the default log file irrespective of the '--log' - option. - --verbose Print useful extra output. + --verbose Print and log useful extra output. --suppress-warnings Suppress warnings to stdout. Warnings are still logged. --version Display wee_import version number. @@ -1191,19 +1184,6 @@ summaries using the wee_database utility. importing from Weather Underground or to import all available records when importing from any other source.

-

Option --log

- -

The --log option controls the wee_import log output. - Omitting the option will result in wee_import log output being sent to the WeeWX - log file (nominally the system log, refer to Monitoring - WeeWX and Where to find things to - find it). wee_import log output can be disabled by using --log=-. - The --log option is used as follows: -

- - 
wee_import --import-config=/directory/import.conf --log=-
-
-

Option --verbose

Inclusion of the --verbose option will cause additional information to be printed @@ -1221,7 +1201,7 @@ summaries using the wee_database utility. a given timestamp or there being no data found for a mapped import field. These warnings do not necessarily require action, but they can consist of extensive output and thus make it difficult to follow the import progress. Irrespective of whether --suppress-warnings is used all warnings are sent - to log unless the --log=- option is used. + to log. 

wee_import --import-config=/directory/import.conf --suppress-warnings
@@ -2295,7 +2275,8 @@ summaries using the wee_database utility.

The output should be something like this:

- 
Starting wee_import...
+                
Using WeeWX configuration file /home/weewx/weewx.conf
+Starting wee_import...
 A CSV import from source file '/var/tmp/data.csv' has been requested.
 Using database binding 'wx_binding', which is bound to database 'weewx.sdb'
 Destination table 'archive' unit system is '0x01' (US).
@@ -2312,14 +2293,6 @@ Finished dry run import
                     data will be processed. The import will then be performed but no data will be written to the WeeWX
                     database. Upon completion a brief summary of the records processed is provided.
                 

-
-                

-                    Note
As the WeeWX database is not altered when the --dry-run option is used, wee_import log output is
-                    suspended during a dry run import. In effect, the use of --dry-run is
-                    equivalent to --dry-run --log=-. During a dry run import the only wee_import output is that displayed on stdout.
-                

             
 
             
  • Once the dry run results are satisfactory the data can be imported using the following command:
@@ -2330,7 +2303,8 @@ Finished dry run import
                     there will be a prompt:
                 
    
 
-                
    Starting wee_import...
+                
    Using WeeWX configuration file /home/weewx/weewx.conf
+Starting wee_import...
 A CSV import from source file '/var/tmp/data.csv' has been requested.
 Using database binding 'wx_binding', which is bound to database 'weewx.sdb'
 Destination table 'archive' unit system is '0x01' (US).
@@ -2556,7 +2530,8 @@ Aug 22 14:38:28 stretch12 weewx[863]: manager: unable to add record 2018-09-04 0
                 
    
                 
    The output should be similar to:
    
 
-                
    Starting wee_import...
+                
    Using WeeWX configuration file /home/weewx/weewx.conf
+Starting wee_import...
 Observation history for Weather Underground station 'ISTATION123' will be imported.
 Using database binding 'wx_binding', which is bound to database 'weewx.sdb'
 Destination table 'archive' unit system is '0x01' (US).
@@ -2577,14 +2552,6 @@ Unique records processed: 71; Last timestamp: 2016-01-23 06:00:00 AEST (14534928
 Finished dry run import
 657 records were processed and 657 unique records would have been imported.
 
    
-
-                
    
-                    Note
    As the WeeWX database is not altered when the --dry-run option is used, wee_import log output is
-                    suspended during a dry run import. In effect, the use of --dry-run is
-                    equivalent to --dry-run --log=-. During a dry run import the only wee_import output is that displayed on stdout(console).
-                
    
             
    • 
 
             
  • Once the dry run results are satisfactory the source data can be imported using the following command:
@@ -2596,7 +2563,8 @@ Finished dry run import
                     will be a prompt:
                 
    
 
-                
    Starting wee_import...
+                
    Using WeeWX configuration file /home/weewx/weewx.conf
+Starting wee_import...
 Observation history for Weather Underground station 'ISTATION123' will be imported.
 Using database binding 'wx_binding', which is bound to database 'weewx.sdb'
 Destination table 'archive' unit system is '0x01' (US).
@@ -2831,7 +2799,8 @@ Aug 22 14:38:28 stretch12 weewx[863]: manager: unable to add record 2018-09-04 0
                 
    
                 
    The output should be similar to:
    
 
-                
    Starting wee_import...
+                
    Using WeeWX configuration file /home/weewx/weewx.conf
+Starting wee_import...
 Cumulus monthly log files in the '/var/tmp/cumulus' directory will be imported
 Using database binding 'wx_binding', which is bound to database 'weewx.sdb'
 Destination table 'archive' unit system is '0x01' (US).
@@ -2876,7 +2845,8 @@ Finished dry run import
                     a prompt:
                 
    
 
-                
    Starting wee_import...
+                
    Using WeeWX configuration file /home/weewx/weewx.conf
+Starting wee_import...
 Cumulus monthly log files in the '/var/tmp/cumulus' directory will be imported
 Using database binding 'wx_binding', which is bound to database 'weewx.sdb'
 Destination table 'archive' unit system is '0x01' (US).
@@ -2895,7 +2865,8 @@ Are you sure you want to proceed (y/n)?
                     preamble may look similar to:
                 
    
 
-                
    Starting wee_import...
+                
    Using WeeWX configuration file /home/weewx/weewx.conf
+Starting wee_import...
 Cumulus monthly log files in the '/var/tmp/cumulus' directory will be imported
 Using database binding 'wx_binding', which is bound to database 'weewx.sdb'
 Destination table 'archive' unit system is '0x01' (US).
@@ -3143,7 +3114,8 @@ Aug 22 14:38:28 stretch12 weewx[863]: manager: unable to add record 2018-09-04 0
                 
    
                 
    The output should be similar to:
    
 
-                
    Starting wee_import...
+                
    Using WeeWX configuration file /home/weewx/weewx.conf
+Starting wee_import...
 Weather Display monthly log files in the '/var/tmp/WD' directory will be imported
 Using database binding 'wx_binding', which is bound to database 'weewx.sdb'
 Destination table 'archive' unit system is '0x01' (US).
@@ -3185,7 +3157,8 @@ Finished dry run import
 
                 
    The output should be similar to:
    
 
-                
    Starting wee_import...
+                
    Using WeeWX configuration file /home/weewx/weewx.conf
+Starting wee_import...
 Weather Display monthly log files in the '/var/tmp/WD' directory will be imported
 Using database binding 'wx_binding', which is bound to database 'weewx.sdb'
 Destination table 'archive' unit system is '0x01' (US).
@@ -3288,7 +3261,8 @@ Finished dry run import
                     a prompt:
                 
    
 
-                
    Starting wee_import...
+                
    Using WeeWX configuration file /home/weewx/weewx.conf
+Starting wee_import...
 Weather Display monthly log files in the '/var/tmp/WD' directory will be imported
 Using database binding 'wx_binding', which is bound to database 'weewx.sdb'
 Destination table 'archive' unit system is '0x01' (US).
@@ -3307,7 +3281,8 @@ Are you sure you want to proceed (y/n)?
                     is ignored. In such cases the preamble may look similar to:
                 
    
 
-                
    SStarting wee_import...
+                
    Using WeeWX configuration file /home/weewx/weewx.conf
+Starting wee_import...
 Weather Display monthly log files in the '/var/tmp/WD' directory will be imported
 Using database binding 'wx_binding', which is bound to database 'weewx.sdb'
 Destination table 'archive' unit system is '0x01' (US).

From a21d51b963f4671e30ddc70b248399ba169a0ded Mon Sep 17 00:00:00 2001
From: gjr80 
Date: Mon, 9 Sep 2019 14:04:48 +1000
Subject: [PATCH 04/20] Added wee_import WD import capability

---
 docs/changes.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changes.txt b/docs/changes.txt
index 1ef91db6..3bfac586 100644
--- a/docs/changes.txt
+++ b/docs/changes.txt
@@ -48,6 +48,8 @@ API key, which you can get from the WU. Put it in weewx.conf.
 Unfortunately, WU is very unreliable whether reposts "stick".
 Fixes issue #414
 
+Wee_import can now import Weather Display monthly log files.
+
 Fixed problem where sub-sections DegreeDays and Trend were located under the
 wrong weewx.conf section. Fixes issue #432. Thanks to user mph for spotting
 this!

From 4a0ae2eeb718cd0c4f31d70812016ee5acdefda2 Mon Sep 17 00:00:00 2001
From: gjr80 
Date: Thu, 12 Sep 2019 09:48:30 +1000
Subject: [PATCH 05/20] Various updates: - updated console output to reflect
 changed input options - revised --log option instructions - updated example
 console output to reflect changed output format - added note re what fields
 are displayed

---
 docs/utilities.htm | 43 +++++++++++++++++++++----------------------
 1 file changed, 21 insertions(+), 22 deletions(-)

diff --git a/docs/utilities.htm b/docs/utilities.htm
index e7d25f02..540b36ed 100644
--- a/docs/utilities.htm
+++ b/docs/utilities.htm
@@ -3520,10 +3520,7 @@ Options:
                         Socket timeout in seconds. Default is 10.
   -v, --verbose         Print useful extra output.
   -l LOG_FACILITY, --log=LOG_FACILITY
-                        Log selected output to syslog. If omitted no syslog
-                        logging occurs. If LOG_FACILITY is 'weewx' then logs
-                        are written to the same log used by weewx. Any other
-                        parameter will log to syslog.
+                        OBSOLETE. Logging will always occur.
   -t, --test            Test what would happen, but don't do anything.
   -q, --query           For each record, query the user before making a
                         change.
@@ -3621,14 +3618,11 @@ command line, or in the configuration file.
    
 
         
    Option --log
    
 
-        
    Control the wunderfixer log output. The default is no logging. If --log=weewx is used then wunderfixer logs to the same log file
-            as used by WeeWX. Any other setting will result in wunderfixer logs being written
-            to syslog. The --log option is used as follows:
+        
    The --log option is obsolete and no longer performs any function. The
+            --log option has been retained for backwards compatibility with users existing
+            scripts. All wunderfixer log output is always logged via the WeeWX log facility.
         
    
 
-        
    wunderfixer --log=weewx
    
-
         
    Option --test
    
 
         
    The --test option will cause wunderfixer to do everything
@@ -3668,6 +3662,11 @@ command line, or in the configuration file.
    
             options.
         
    
 
+        
    
+            Note
     The data fields shown in the example outputs below are indicative only and the
+            actual output may or may not include other fields.
+        
    
+
         
    To run wunderfixer directly:
    
 
         
      
@@ -3678,10 +3677,10 @@ command line, or in the configuration file.
    
 
                 
    Using configuration file /home/weewx/weewx.conf.
 Using database binding 'wx_binding', which is bound to database 'archive_sqlite'
-2016-09-22 06:30:00 AEST (1474489800); 29.920";  58.9F;  79%; 1.0 mph; 248 deg; 6.0 mph gust;  52.4F; 0.00" rain; 0.01" daily rain  ... skipped.
-2016-09-22 07:35:00 AEST (1474493700); 29.931";  64.9F;  65%; 2.0 mph; 180 deg; 7.0 mph gust;  52.8F; 0.00" rain; 0.01" daily rain  ... skipped.
-2016-09-22 07:55:00 AEST (1474494900); 29.934";  65.8F;  63%; 2.0 mph; 180 deg;10.0 mph gust;  52.8F; 0.00" rain; 0.01" daily rain  ... skipped.
-2016-09-22 08:20:00 AEST (1474496400); 29.938";  66.5F;  59%; 5.0 mph; 180 deg;12.0 mph gust;  51.7F; 0.00" rain; 0.01" daily rain  ... skipped.
+2016-09-22 06:30:00 AEST (1474489800); 29.920";  58.9F;  79%; 1.0 mph; 248 deg; 6.0 mph gust;  52.4F; 0.00" rain; 0.01" ... skipped.
+2016-09-22 07:35:00 AEST (1474493700); 29.931";  64.9F;  65%; 2.0 mph; 180 deg; 7.0 mph gust;  52.8F; 0.00" rain; 0.01" ... skipped.
+2016-09-22 07:55:00 AEST (1474494900); 29.934";  65.8F;  63%; 2.0 mph; 180 deg;10.0 mph gust;  52.8F; 0.00" rain; 0.01" ... skipped.
+2016-09-22 08:20:00 AEST (1474496400); 29.938";  66.5F;  59%; 5.0 mph; 180 deg;12.0 mph gust;  51.7F; 0.00" rain; 0.01" ... skipped.
 
    
                 
    This output indicates that four records were found to be missing. The word 'skipped' at the end of each
                     line indicates that whilst wunderfixer detected the record as missing, the
@@ -3703,10 +3702,10 @@ Using database binding 'wx_binding', which is bound to database 'archive_sqlite'
 
             
    Using configuration file /home/weewx/weewx.conf.
 Using database binding 'wx_binding', which is bound to database 'archive_sqlite'
-2016-09-22 06:30:00 AEST (1474489800); 29.920";  58.9F;  79%; 1.0 mph; 248 deg; 6.0 mph gust;  52.4F; 0.00" rain; 0.01" daily rain  ... published.
-2016-09-22 07:35:00 AEST (1474493700); 29.931";  64.9F;  65%; 2.0 mph; 180 deg; 7.0 mph gust;  52.8F; 0.00" rain; 0.01" daily rain  ... published.
-2016-09-22 07:55:00 AEST (1474494900); 29.934";  65.8F;  63%; 2.0 mph; 180 deg;10.0 mph gust;  52.8F; 0.00" rain; 0.01" daily rain  ... published.
-2016-09-22 08:20:00 AEST (1474496400); 29.938";  66.5F;  59%; 5.0 mph; 180 deg;12.0 mph gust;  51.7F; 0.00" rain; 0.01" daily rain  ... published.
+2016-09-22 06:30:00 AEST (1474489800); 29.920";  58.9F;  79%; 1.0 mph; 248 deg; 6.0 mph gust;  52.4F; 0.00" rain; 0.01" ... published.
+2016-09-22 07:35:00 AEST (1474493700); 29.931";  64.9F;  65%; 2.0 mph; 180 deg; 7.0 mph gust;  52.8F; 0.00" rain; 0.01" ... published.
+2016-09-22 07:55:00 AEST (1474494900); 29.934";  65.8F;  63%; 2.0 mph; 180 deg;10.0 mph gust;  52.8F; 0.00" rain; 0.01" ... published.
+2016-09-22 08:20:00 AEST (1474496400); 29.938";  66.5F;  59%; 5.0 mph; 180 deg;12.0 mph gust;  51.7F; 0.00" rain; 0.01" ... published.
 
    
 
             
    This output indicates that four records were found to be missing. This time word 'skipped' at the end of
@@ -3727,13 +3726,13 @@ Number of WU records:          284
 Number of missing records:     4
 
 Missing records:
-2016-09-22 06:30:00 AEST (1474489800); 29.920";  58.9F;  79%; 1.0 mph; 248 deg; 6.0 mph gust;  52.4F; 0.00" rain; 0.01" daily rain  ...fix? (y/n/a/q):y
+2016-09-22 06:30:00 AEST (1474489800); 29.920";  58.9F;  79%; 1.0 mph; 248 deg; 6.0 mph gust;  52.4F; 0.00" rain; 0.01" ...fix? (y/n/a/q):y
  ...published.
-2016-09-22 07:35:00 AEST (1474493700); 29.931";  64.9F;  65%; 2.0 mph; 180 deg; 7.0 mph gust;  52.8F; 0.00" rain; 0.01" daily rain  ...fix? (y/n/a/q):y
+2016-09-22 07:35:00 AEST (1474493700); 29.931";  64.9F;  65%; 2.0 mph; 180 deg; 7.0 mph gust;  52.8F; 0.00" rain; 0.01" ...fix? (y/n/a/q):y
  ...published.
-2016-09-22 07:55:00 AEST (1474494900); 29.934";  65.8F;  63%; 2.0 mph; 180 deg;10.0 mph gust;  52.8F; 0.00" rain; 0.01" daily rain  ...fix? (y/n/a/q):y
+2016-09-22 07:55:00 AEST (1474494900); 29.934";  65.8F;  63%; 2.0 mph; 180 deg;10.0 mph gust;  52.8F; 0.00" rain; 0.01" ...fix? (y/n/a/q):y
  ...published.
-2016-09-22 08:20:00 AEST (1474496400); 29.938";  66.5F;  59%; 5.0 mph; 180 deg;12.0 mph gust;  51.7F; 0.00" rain; 0.01" daily rain  ...fix? (y/n/a/q):
+2016-09-22 08:20:00 AEST (1474496400); 29.938";  66.5F;  59%; 5.0 mph; 180 deg;12.0 mph gust;  51.7F; 0.00" rain; 0.01" ...fix? (y/n/a/q):
 
    
 
             
    • 

From 33918ce09e3ba4154d75bfe67914ab3acf85a5f2 Mon Sep 17 00:00:00 2001
From: gjr80 
Date: Sun, 15 Sep 2019 12:12:07 +1000
Subject: [PATCH 06/20] First cut implementation of WU API as the source for WU
 imports. Works but there is still an issue with the apparent caching of WU
 API results that can affect the availability of records to import for the
 current day.

---
 bin/weeimport/weeimport.py |  27 ++++-
 bin/weeimport/wuimport.py  | 242 ++++++++++++++++++++++++-------------
 2 files changed, 178 insertions(+), 91 deletions(-)

diff --git a/bin/weeimport/weeimport.py b/bin/weeimport/weeimport.py
index fcf01c0f..ac3d0d80 100644
--- a/bin/weeimport/weeimport.py
+++ b/bin/weeimport/weeimport.py
@@ -1,5 +1,5 @@
 #
-#    Copyright (c) 2009-2016 Tom Keffer  and
+#    Copyright (c) 2009-2019 Tom Keffer  and
 #                            Gary Roderick
 #
 #    See the file LICENSE.txt for your full rights.
@@ -16,6 +16,7 @@ from __future__ import absolute_import
 # Python imports
 import datetime
 import logging
+import numbers
 import re
 import sys
 import time
@@ -219,7 +220,7 @@ class Source(object):
                 _msg = "Invalid --date option specified."
                 raise WeeImportOptionError(_msg)
             else:
-                # we have a valid date so do soem date arithmetic
+                # we have a valid date so do some date arithmetic
                 _last_dt = _first_dt + datetime.timedelta(days=1)
                 self.first_ts = time.mktime(_first_dt.timetuple())
                 self.last_ts = time.mktime(_last_dt.timetuple())
@@ -659,7 +660,7 @@ class Source(object):
                     _msg = "Field '%s' not found in source data." % self.map['dateTime']['field_name']
                     raise WeeImportFieldError(_msg)
                 # now process the raw date time data
-                if _raw_dateTime.isdigit():
+                if isinstance(_raw_dateTime, numbers.Number) or _raw_dateTime.isdigit():
                     # Our dateTime is a number, is it a timestamp already?
                     # Try to use it and catch the error if there is one and
                     # raise it higher.
@@ -671,8 +672,8 @@ class Source(object):
                                                _raw_dateTime)
                         raise ValueError(_msg)
                 else:
-                    # it's a string so try to parse it and catch the error if
-                    # there is one and raise it higher
+                    # it's a non-numeric string so try to parse it and catch
+                    # the error if there is one and raise it higher
                     try:
                         _datetm = time.strptime(_raw_dateTime,
                                                 self.raw_datetime_format)
@@ -685,7 +686,7 @@ class Source(object):
                 # if we have a timeframe of concern does our record fall within
                 # it
                 if (self.first_ts is None and self.last_ts is None) or \
-                        self.first_ts <= _rec_dateTime <= self.last_ts:
+                        self.first_ts < _rec_dateTime <= self.last_ts:
                     # we have no timeframe or if we do it falls within it so
                     # save the dateTime
                     _rec['dateTime'] = _rec_dateTime
@@ -759,6 +760,20 @@ class Source(object):
                         # can't catch the error
                         try:
                             _temp = float(_row[self.map[_field]['field_name']].strip())
+                        except AttributeError:
+                            # the data has not strip() attribute so chances are
+                            # it's a number already
+                            if isinstance(_row[self.map[_field]['field_name']], numbers.Number):
+                                _temp = _row[self.map[_field]['field_name']]
+                            elif _row[self.map[_field]['field_name']] is None:
+                                _temp = None
+                            else:
+                                # we raise the error
+                                _msg = "%s: cannot convert '%s' to float at " \
+                                       "timestamp '%s'." % (_field,
+                                                            _row[self.map[_field]['field_name']],
+                                                            timestamp_to_string(_rec['dateTime']))
+                                raise ValueError(_msg)
                         except TypeError:
                             # perhaps we have a None, so return None for our field
                             _temp = None
diff --git a/bin/weeimport/wuimport.py b/bin/weeimport/wuimport.py
index 6a448a22..886718be 100644
--- a/bin/weeimport/wuimport.py
+++ b/bin/weeimport/wuimport.py
@@ -1,24 +1,31 @@
 #
-#    Copyright (c) 2009-2016 Tom Keffer  and
+#    Copyright (c) 2009-2019 Tom Keffer  and
 #                            Gary Roderick
 #
 #    See the file LICENSE.txt for your full rights.
 #
 
-"""Module to interact with Weather Underground PWS history and import raw
-observational data for use with weeimport.
+"""Module to interact with the Weather Underground API and obtain raw
+observational data for use with wee_import.
 """
 
 # Python imports
 from __future__ import with_statement
 from __future__ import absolute_import
 from __future__ import print_function
-import csv
+
 import datetime
+import gzip
+import json
 import logging
+import numbers
 import socket
+import sys
+
 from datetime import datetime as dt
 
+# python3 compatibility shims
+import six
 from six.moves import urllib
 
 # WeeWX imports
@@ -36,45 +43,36 @@ log = logging.getLogger(__name__)
 
 
 class WUSource(weeimport.Source):
-    """Class to interact with the Weather Underground.
+    """Class to interact with the Weather Underground API.
 
-    Uses WXDailyHistory.asp call via http to obtain historical daily weather
-    observations for a given PWS. WU uses geolocation of the requester to
-    determine the units to use when providing historical PWS records. Fields
-    that can be provided with multiple possible units have the units in use
-    appended to the returned field name. This means that a request for a user
-    in a given location for historical data from a given station may well
-    return different results to the same request being made from another
-    location. This requires a mechanism to both determine the units in use from
-    returned data as well as mapping a number of different possible field names
-    to a given WeeWX archive field name.
+    Uses PWS history call via http to obtain historical daily weather
+    observations for a given PWS. Unlike the previous WU import module the use
+    of the API requires an API key.
     """
 
     # Dict to map all possible WU field names to WeeWX archive field names and
     # units
-    _header_map = {'Time': {'units': 'unix_epoch', 'map_to': 'dateTime'},
-                   'TemperatureC': {'units': 'degree_C', 'map_to': 'outTemp'},
-                   'TemperatureF': {'units': 'degree_F', 'map_to': 'outTemp'},
-                   'DewpointC': {'units': 'degree_C', 'map_to': 'dewpoint'},
-                   'DewpointF': {'units': 'degree_F', 'map_to': 'dewpoint'},
-                   'PressurehPa': {'units': 'hPa', 'map_to': 'barometer'},
-                   'PressureIn': {'units': 'inHg', 'map_to': 'barometer'},
-                   'WindDirectionDegrees': {'units': 'degree_compass',
-                                            'map_to': 'windDir'},
-                   'WindSpeedKMH': {'units': 'km_per_hour',
+    _header_map = {'epoch': {'units': 'unix_epoch', 'map_to': 'dateTime'},
+                   'tempAvg': {'units': 'degree_F', 'map_to': 'outTemp'},
+                   'dewptAvg': {'units': 'degree_F', 'map_to': 'dewpoint'},
+                   'heatindexAvg': {'units': 'degree_F', 'map_to': 'heatindex'},
+                   'windchillAvg': {'units': 'degree_F', 'map_to': 'windchill'},
+                   'pressureAvg': {'units': 'inHg', 'map_to': 'barometer'},
+                   'winddirAvg': {'units': 'degree_compass',
+                                  'map_to': 'windDir'},
+                   'windspeedAvg': {'units': 'mile_per_hour',
                                     'map_to': 'windSpeed'},
-                   'WindSpeedMPH': {'units': 'mile_per_hour',
-                                    'map_to': 'windSpeed'},
-                   'WindSpeedGustKMH': {'units': 'km_per_hour',
-                                        'map_to': 'windGust'},
-                   'WindSpeedGustMPH': {'units': 'mile_per_hour',
-                                        'map_to': 'windGust'},
-                   'Humidity': {'units': 'percent', 'map_to': 'outHumidity'},
-                   'dailyrainMM': {'units': 'mm', 'map_to': 'rain'},
-                   'dailyrainin': {'units': 'inch', 'map_to': 'rain'},
-                   'SolarRadiationWatts/m^2': {'units': 'watt_per_meter_squared',
-                                               'map_to': 'radiation'}
+                   'windgustHigh': {'units': 'mile_per_hour',
+                                    'map_to': 'windGust'},
+                   'humidityAvg': {'units': 'percent', 'map_to': 'outHumidity'},
+                   'precipTotal': {'units': 'inch', 'map_to': 'rain'},
+                   'precipRate': {'units': 'inch_per_hour',
+                                  'map_to': 'rainRate'},
+                   'solarRadiationHigh': {'units': 'watt_per_meter_squared',
+                                          'map_to': 'radiation'},
+                   'uvHigh': {'units': 'uv_index', 'map_to': 'UV'}
                    }
+    _extras = ['pressureMin', 'pressureMax']
 
     def __init__(self, config_dict, config_path, wu_config_dict, import_config_path, options):
 
@@ -95,6 +93,13 @@ class WUSource(weeimport.Source):
             _msg = "Weather Underground station ID not specified in '%s'." % import_config_path
             raise weewx.ViolatedPrecondition(_msg)
 
+        # get our WU API key
+        try:
+            self.api_key = wu_config_dict['api_key']
+        except KeyError:
+            _msg = "Weather Underground API key not specified in '%s'." % import_config_path
+            raise weewx.ViolatedPrecondition(_msg)
+
         # wind dir bounds
         _wind_direction = option_as_list(wu_config_dict.get('wind_direction',
                                                             '0,360'))
@@ -185,16 +190,9 @@ class WUSource(weeimport.Source):
         """Get raw observation data and construct a map from WU to WeeWX
             archive fields.
 
-        Obtain raw observational data from WU using a http WXDailyHistory
-        request. This raw data needs to be cleaned of unnecessary
-        characters/codes and an iterable returned.
-
-        Since WU geolocates any http request we do not know what units our WU
-        data will use until we actually receive the data. A further
-        complication is that WU appends the unit abbreviation to the end of the
-        returned field name for fields that can have different units. So once
-        we have the data have received the response we need to determine the
-        units and create a dict to map the WU fields to WeeWX archive fields.
+        Obtain raw observational data from WU via the WU API. This raw data
+        needs some basic processing to place it in a format suitable for
+        wee_import to ingest.
 
         Input parameters:
 
@@ -204,53 +202,127 @@ class WUSource(weeimport.Source):
 
         # the date for which we want the WU data is held in a datetime object,
         # we need to convert it to a timetuple
-        date_tt = period.timetuple()
-        # construct our URL using station ID and day, month, year
-        _url = "http://www.wunderground.com/weatherstation/WXDailyHistory.asp?ID=%s&" \
-               "month=%d&day=%d&year=%d&format=1" % (self.station_id,
-                                                     date_tt[1],
-                                                     date_tt[2],
-                                                     date_tt[0])
-        # hit the WU site, wrap in a try..except so we can catch any errors
+        day_tt = period.timetuple()
+        # and then format the date suitable for use in the WU API URL
+        day = "%4d%02d%02d" % (day_tt.tm_year,
+                               day_tt.tm_mon,
+                               day_tt.tm_mday)
+
+        # construct the URL to be used
+        url = "https://api.weather.com/v2/pws/history/all?" \
+              "stationId=%s&format=json&units=e&numericPrecision=decimal&date=%s&apiKey=%s" \
+              % (self.station_id, day, self.api_key)
+        # create a Request object using the constructed URL
+        request_obj = urllib.request.Request(url)
+        # add necessary headers
+        request_obj.add_header('Cache-Control', 'no-cache')
+        request_obj.add_header('Accept-Encoding', 'gzip')
+        # hit the API wrapping in a try..except to catch any errors
         try:
-            _wudata = urllib.request.urlopen(_url)
+            response = urllib.request.urlopen(request_obj)
         except urllib.error.URLError as e:
-            _msg = "Unable to open Weather Underground station %s" % self.station_id
-            print(_msg)
-            log.error(_msg)
-            _msg = "   **** %s" % e
-            print(_msg)
-            log.error(_msg)
+            print("Unable to open Weather Underground station " + self.station_id, " or ", e, file=sys.stderr)
+            log.error("Unable to open Weather Underground station %s or %s" % (self.station_id, e))
             raise
         except socket.timeout as e:
-            _msg = "Socket timeout for Weather Underground station %s" % self.station_id
-            print(_msg)
-            log.error(_msg)
-            _msg = "   **** %s" % e
-            print(_msg)
-            log.error(_msg)
+            print("Socket timeout for Weather Underground station " + self.station_id, file=sys.stderr)
+            log.error("Socket timeout for Weather Underground station %s" % self.station_id)
+            print("   **** %s" % e, file=sys.stderr)
+            log.error("   **** %s" % e)
             raise
+        # check the response code and raise an exception if there was an error
+        if hasattr(response, 'code') and response.code != 200:
+            if response.code == 204:
+                raise IOError("Probably a bad station ID or invalid date")
+            else:
+                raise IOError("Bad response code returned: %d" % response.code)
 
-        # because the data comes back with lots of HTML tags and whitespace we
-        # need a bit of logic to clean it up.
-        _cleanWUdata = []
-        for _row in _wudata:
-            # Convert from byte-string to string
-            _urow = _row.decode('ascii')
-            # get rid of any HTML tags
-            _line = ''.join(WUSource._tags.split(_urow))
-            # get rid of any blank lines
-            if _line != "\n":
-                # save what's left
-                _cleanWUdata.append(_line)
+        # The WU API says that compression is required, but let's be prepared
+        # if compression is not used
+        if response.info().get('Content-Encoding') == 'gzip':
+            buf = six.StringIO(response.read())
+            f = gzip.GzipFile(fileobj=buf)
+            _raw_data = f.read()
+            # decode the json data
+            _raw_decoded_data = json.loads(_raw_data)
+        else:
+            _raw_data = response
+            # decode the json data
+            _raw_decoded_data = json.load(_raw_data)
 
-        # now create a dictionary CSV reader, the first line is used as keys to
-        # the dictionary
-        _reader = csv.DictReader(_cleanWUdata)
+        # The raw WU response is not suitable to return as is, we need to
+        # return an iterable that provides a dict of observational data for each
+        # available timestamp. In this case a list of dicts is appropriate.
+
+        # initialise a list of dicts
+        wu_data = []
+        # first check we have some observational data
+        if 'observations' in _raw_decoded_data:
+            # iterate over each record in the WU data
+            for record in _raw_decoded_data['observations']:
+                # initialise a dict to hold the resulting data for this record
+                _flat_record = {}
+                # iterate over each WU API response field that we can use
+                _fields = self._header_map.keys() + self._extras
+                for obs in _fields:
+                    # The field may appear as a top level field in the WU data
+                    # or it may be embedded in the dict in the WU data that
+                    # contains variable unit data. Look in the top level record
+                    # first. If its there uses it, otherwise look in the
+                    # variable units dict. If it can't be fond then skip it.
+                    if obs in record:
+                        # it's in the top level record
+                        _flat_record[obs] = record[obs]
+                    else:
+                        # it's not in the top level so look in the variable
+                        # units dict
+                        try:
+                            _flat_record[obs] = record['imperial'][obs]
+                        except KeyError:
+                            # it's not there so skip it
+                            pass
+                    if obs == 'epoch':
+                        try:
+                            _date = datetime.date.fromtimestamp(_flat_record['epoch'])
+                        except ValueError:
+                            _flat_record['epoch'] = _flat_record['epoch'] // 1000
+                # WU in its wisdom provides min and max pressure but no average
+                # pressure (unlike other obs) so we need to calculate it. If
+                # both min and max are numeric use a simple average of the two
+                # (they will likely be the same anyway for non-RF stations).
+                # Otherwise use max if numeric, then use min if numeric
+                # otherwise skip.
+                self.calc_pressure(_flat_record)
+                # append the data dict for the current record to the list of
+                # dicts for this period
+                wu_data.append(_flat_record)
         # finally, get our database-source mapping
-        self.map = self.parseMap('WU', _reader, self.wu_config_dict)
-        # return our dict reader
-        return _reader
+        self.map = self.parseMap('WU', wu_data, self.wu_config_dict)
+        # return our dict
+        return wu_data
+
+    @staticmethod
+    def calc_pressure(record):
+        """Calculate pressureAvg field.
+
+        The WU API provides min and max pressure but no average pressure.
+        Calculate an average pressure to be used in the import using one of the
+        following (in order):
+
+        1. simple average of min and max pressure
+        2. max pressure
+        3. min pressure
+        4. None
+        """
+
+        if 'pressureMin' in record and 'pressureMax' in record and isinstance(record['pressureMin'], numbers.Number) and isinstance(record['pressureMax'], numbers.Number):
+            record['pressureAvg'] = (record['pressureMin'] + record['pressureMax'])/2.0
+        elif 'pressureMax' in record and isinstance(record['pressureMax'], numbers.Number):
+            record['pressureAvg'] = record['pressureMax']
+        elif 'pressureMin' in record and isinstance(record['pressureMin'], numbers.Number):
+            record['pressureAvg'] = record['pressureMin']
+        elif 'pressureMin' in record or 'pressureMax' in record:
+            record['pressureAvg'] = None
 
     def period_generator(self):
         """Generator function yielding a sequence of datetime objects.

From 5fe2b768899272191093915dc9f16f38c0c6bf8a Mon Sep 17 00:00:00 2001
From: gjr80 
Date: Wed, 18 Sep 2019 09:40:37 +1000
Subject: [PATCH 07/20] Added obfuscated API key entry to verbose WU import
 output

---
 bin/weeimport/wuimport.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/bin/weeimport/wuimport.py b/bin/weeimport/wuimport.py
index 886718be..c2189c5b 100644
--- a/bin/weeimport/wuimport.py
+++ b/bin/weeimport/wuimport.py
@@ -157,6 +157,11 @@ class WUSource(weeimport.Source):
         if self.verbose:
             print(_msg)
         log.debug(_msg)
+        _obf_api_key_msg = '='.join(['     apiKey',
+                                     '*'*(len(self.api_key) - 4) + self.api_key[-4:]])
+        if self.verbose:
+            print(_obf_api_key_msg)
+        log.debug(_obf_api_key_msg)
         _msg = "     dry-run=%s, calc_missing=%s, ignore_invalid_data=%s" % (self.dry_run,
                                                                              self.calc_missing,
                                                                              self.ignore_invalid_data)

From ff3065de454d96e94f6671972c8dd4cf95d173bc Mon Sep 17 00:00:00 2001
From: gjr80 
Date: Wed, 18 Sep 2019 09:51:35 +1000
Subject: [PATCH 08/20] Updated WU import config comments/example to reflect
 usage of WU API Minor comment updates

---
 bin/wee_import                   | 12 ++++++++----
 util/import/csv-example.conf     |  2 +-
 util/import/cumulus-example.conf |  2 +-
 util/import/wu-example.conf      | 12 ++++++++----
 4 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/bin/wee_import b/bin/wee_import
index 375bae74..09a50185 100755
--- a/bin/wee_import
+++ b/bin/wee_import
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 #
-#    Copyright (c) 2009-2016 Tom Keffer  and
+#    Copyright (c) 2009-2019 Tom Keffer  and
 #                            Gary Roderick
 #
 #    See the file LICENSE.txt for your rights.
@@ -235,6 +235,9 @@ source = CSV
     # WU PWS Station ID to be used for import.
     station_id = XXXXXXXX123
 
+    # WU API key to be used for import.
+    api_key = XXXXXXXXXXXXXXXXXXXXXX1234567890
+
     #
     # When importing WU data the following WeeWX database fields will be
     # populated directly by the imported data (provided the corresponding data
@@ -242,13 +245,17 @@ source = CSV
     #   barometer
     #   dateTime
     #   dewpoint
+    #   heatindex
     #   outHumidity
     #   outTemp
     #   radiation
     #   rain
+    #   rainRate
+    #   windchill
     #   windDir
     #   windGust
     #   windSpeed
+    #   UV
     #
     # The following WeeWX database fields will be populated from other
     # settings/config files:
@@ -260,10 +267,7 @@ source = CSV
     # used during import:
     #   altimeter
     #   ET
-    #   heatindex
     #   pressure
-    #   rainRate
-    #   windchill
     #
     # The following WeeWX fields will be populated with derived values from the
     # imported data provided the --calc-missing command line option is used
diff --git a/util/import/csv-example.conf b/util/import/csv-example.conf
index 6d20b6cd..21a727d3 100644
--- a/util/import/csv-example.conf
+++ b/util/import/csv-example.conf
@@ -1,6 +1,6 @@
 # EXAMPLE CONFIGURATION FILE FOR IMPORTING FROM CSV FILES
 #
-# Copyright (c) 2009-2016 Tom Keffer  and Gary Roderick.
+# Copyright (c) 2009-2019 Tom Keffer  and Gary Roderick.
 # See the file LICENSE.txt for your rights.
 
 ##############################################################################
diff --git a/util/import/cumulus-example.conf b/util/import/cumulus-example.conf
index 14074b9c..ecae3a4f 100644
--- a/util/import/cumulus-example.conf
+++ b/util/import/cumulus-example.conf
@@ -1,6 +1,6 @@
 # EXAMPLE CONFIGURATION FILE FOR IMPORTING FROM CUMULUS
 #
-# Copyright (c) 2009-2016 Tom Keffer  and Gary Roderick.
+# Copyright (c) 2009-2019 Tom Keffer  and Gary Roderick.
 # See the file LICENSE.txt for your rights.
 
 ##############################################################################
diff --git a/util/import/wu-example.conf b/util/import/wu-example.conf
index 623b2125..2a6d2aa6 100644
--- a/util/import/wu-example.conf
+++ b/util/import/wu-example.conf
@@ -1,6 +1,6 @@
 # EXAMPLE CONFIGURATION FILE FOR IMPORTING FROM THE WEATHER UNDERGROUND
 #
-# Copyright (c) 2009-2016 Tom Keffer  and Gary Roderick.
+# Copyright (c) 2009-2019 Tom Keffer  and Gary Roderick.
 # See the file LICENSE.txt for your rights.
 
 ##############################################################################
@@ -22,6 +22,9 @@ source = WU
     # WU PWS Station ID to be used for import.
     station_id = XXXXXXXX123
 
+    # WU API key to be used for import.
+    api_key = XXXXXXXXXXXXXXXXXXXXXX1234567890
+
     #
     # When importing WU data the following WeeWX database fields will be
     # populated directly by the imported data (provided the corresponding data
@@ -29,13 +32,17 @@ source = WU
     #   barometer
     #   dateTime
     #   dewpoint
+    #   heatindex
     #   outHumidity
     #   outTemp
     #   radiation
     #   rain
+    #   rainRate
+    #   windchill
     #   windDir
     #   windGust
     #   windSpeed
+    #   UV
     #
     # The following WeeWX database fields will be populated from other
     # settings/config files:
@@ -47,10 +54,7 @@ source = WU
     # used during import:
     #   altimeter
     #   ET
-    #   heatindex
     #   pressure
-    #   rainRate
-    #   windchill
     #
     # The following WeeWX fields will be populated with derived values from the
     # imported data provided the --calc-missing command line option is used

From 4fc258c19aa6a568c9d913be16cca48a2d978f7b Mon Sep 17 00:00:00 2001
From: gjr80 
Date: Wed, 18 Sep 2019 09:59:07 +1000
Subject: [PATCH 09/20] Updated wee_import section to include WU API usage

---
 docs/utilities.htm | 97 ++++++++++++++++++++++++++++------------------
 1 file changed, 60 insertions(+), 37 deletions(-)

diff --git a/docs/utilities.htm b/docs/utilities.htm
index 540b36ed..4f832450 100644
--- a/docs/utilities.htm
+++ b/docs/utilities.htm
@@ -1235,7 +1235,7 @@ summaries using the wee_database utility.
    
         
      
             
    • CSV to import from a single CSV format file.
             
      • 
-            
    • WU to import from a Weather Underground PWS daily history.
+            
    • WU to import from a Weather Underground PWS history.
             
      • 
             
    • Cumulus to import from one or more Cumulus monthly log files.
             
      • 
@@ -1561,22 +1561,33 @@ summaries using the wee_database utility.

    [WU]

    The [WU] section contains the options relating to the import of - observational data from a Weather Underground PWS daily history. + observational data from a Weather Underground PWS history.

    station_id

    -

    The Weather Underground weather station ID of the PWS from which the daily history will be imported. There is +

    The Weather Underground weather station ID of the PWS from which the historical data will be imported. There is no default.

    +

    api_key

    + +

    The Weather Underground API key to be used to obtain the PWS history data. There is no default.

    + +

    + Note
    The API key is a seemingly random string of 32 characters used to access the new + (2019) Weather Underground API. PWS contributors can obtain an API key by logging onto the Weather + Underground internet site and accessing Member Settings. 16 character API keys used with the previous Weather + Underground API are not supported. +

    +

    interval

    Determines how the time interval (WeeWX database field interval) between successive observations is determined. This option is identical in operation to the CSV interval - option but applies to Weather Underground imports only. As Weather Underground often skips observation - records when responding to a daily history query, the use of interval = derive may - give incorrect or inconsistent interval values. Better results may be obtained by using interval = conf + option but applies to Weather Underground imports only. As a Weather Underground PWS history sometimes has + missing records, the use of interval = derive may give incorrect or inconsistent + interval values. Better results may be obtained by using interval = conf if the current WeeWX installation has the same archive_interval as the Weather Underground data, or by using interval = x where x is the time interval in minutes used to upload the Weather Underground data. The most appropriate setting will @@ -2359,12 +2370,12 @@ Aug 22 14:38:28 stretch12 weewx[863]: manager: unable to add record 2018-09-04 0

    Importing from Weather Underground

    -

    wee_import can import data from the daily history of a Weather Undeground PWS. A - Weather Underground daily history provides weather station observations received by Weather Underground for - the PWS concerned on a day by day basis. As such, the data is analogous to the WeeWX archive table. When - wee_import imports data from a Weather Underground daily history each day is - considered a 'period'. wee_import processes one period at a time in chronological - order (oldest to newest) and provides import summary data on a per period basis. +

    wee_import can import historical observation data for a Weather Underground PWS via + the Weather Underground API. The Weather Underground API provides historical weather station observations + received by Weather Underground for the PWS concerned on a day by day basis. As such, the data is analogous + to the WeeWX archive table. When wee_import imports data from the Weather + Underground API each day is considered a 'period'. wee_import processes one period + at a time in chronological order (oldest to newest) and provides import summary data on a per period basis.

    Mapping data to archive fields

    @@ -2372,28 +2383,34 @@ Aug 22 14:38:28 stretch12 weewx[863]: manager: unable to add record 2018-09-04 0

    A Weather Underground import will populate WeeWX archive fields as follows:

      -
    • Provided data exists for each field in the Weather Underground PWS daily history, the following WeeWX - archive fields will be directly populated by imported data: +
    • Provided data exists for each field returned by the Weather Underground API, the following WeeWX archive + fields will be directly populated by imported data:
      • dateTime
      • barometer
      • dewpoint
        • +
      • heatindex
      • outHumidity
      • outTemp
      • radiation
      • rain
        • +
      • rainRate
        • +
      • UV
        • +
      • windchill
      • windDir
      • windGust
      • windSpeed

      - Note
      If an appropriate field does not exist in the Weather Underground daily - history then the corresponding WeeWX archive field will be set to None/null. For example, if there is no solar radiation sensor then radiation will be null, or if outHumidity was never - uploaded to Weather Undeground then outHumidity will be null. + Note
      If an appropriate field is not returned by the Weather Underground API then + the corresponding WeeWX archive field will contain no data. If the API returns an appropriate field but + with no data, the corresponding WeeWX archive field will be set to None/null. + For example, if the API response has no solar radiation field the WeeWX + radiation archive field will have no data stored. However, if the API + response has a solar radiation field but contains no data, the WeeWX + radiation archive field will be None/null.

      • @@ -2411,10 +2428,7 @@ Aug 22 14:38:28 stretch12 weewx[863]: manager: unable to add record 2018-09-04 0
      • altimeter
      • ET
        • -
      • heatindex
      • pressure
        • -
      • rainRate
        • -
      • windchill

      @@ -2428,7 +2442,7 @@ Aug 22 14:38:28 stretch12 weewx[863]: manager: unable to add record 2018-09-04 0

      Step-by-step instructions

      -

      To import observations from the daily history of a Weather Underground PWS:

      +

      To import observations from a Weather Underground PWS history:

      1. Obtain the weather station ID of the Weather Underground PWS from which data is to be imported. The @@ -2437,6 +2451,11 @@ Aug 22 14:38:28 stretch12 weewx[863]: manager: unable to add record 2018-09-04 0 of ISTATION123 will be used.
        2. +
      2. Obtain the API key to be used to access the Weather Underground API. This will be a seemingly random + alphanumeric sequence of 32 characters. API keys are available to Weather Underground PWS contributors + by logging on to their Weather Underground account and accessing Member Settings. +
        3. +
      3. Make a backup of the WeeWX database in case the import should go awry.
        4. @@ -2451,7 +2470,6 @@ Aug 22 14:38:28 stretch12 weewx[863]: manager: unable to add record 2018-09-04 0 
        source = WU
        -
      4. Confirm that the following options in the [WU] section are correctly set:
          @@ -2459,6 +2477,9 @@ Aug 22 14:38:28 stretch12 weewx[863]: manager: unable to add record 2018-09-04 0 character weather station ID of the Weather Underground PWS that will be the source of the imported data. +
        • api_key. The 32 character + API key to be used to access the Weather Underground API. +
        • interval. Determines how the WeeWX interval field is derived.
          • @@ -2579,9 +2600,9 @@ Are you sure you want to proceed (y/n)?

          - Note
          wee_import obtains Weather Underground daily - history data one day at a time via a HTTP request and as such the import of large time spans of data - may take some time. Such imports may be best handled as a series of imports of smaller time spans. + Note
          wee_import obtains Weather Underground data + one day at a time via a HTTP request and as such the import of large time spans of data may take some + time. Such imports may be best handled as a series of imports of smaller time spans.

          @@ -2611,15 +2632,17 @@ imported. Confirm successful import in the WeeWX log file.

          - Note
          It is not unusual to see a Weather Underground import return a different - number of records for the same import performed at different times. If importing the current day - this could be because an additional record may have been added between wee_import runs. For periods before today, this behaviour appears to be a vagary - of Weather Underground. The only solution appears to be to repeat the import with the same --date option setting and observe whether the missing records are imported. - Repeating the import will not adversely affect any existing data as records with timestamps that are - already in the WeeWX archive will be ignored. It may; however, generated many UNIQUE constraint failed: archive.dateTime - messages in the WeeWX log. + Note
          The new (2019) Weather Underground API appears to have an issue when + obtaining historical data for the current day. The first time the API is queried the API returns all + historical data up to and including the most recent record. However, subsequent later API queries + during the same day return the same set of records rather than all records up to and including the + time of the latest API query. Users importing Weather Underground data that includes data from the + current day are advised to carefully check the WeeWX log to ensure that all expected records were + imported. If some records are missing from the current day try running an import for the current day + again using the --date option setting. If this fails then wait until the + following day and perform another import for the day concerned again using the + --date option setting. In all cases confirm what data has been imported by + referring to the WeeWX log.

          @@ -3849,7 +3872,7 @@ Missing records: will depend on the --log option used and your version of Linux. -
        • Check the Weather Undeground daily history for the station concerned to ensure that any missing +
        • Check the Weather Underground Weather History for the station concerned to ensure that any missing data was accepted by Weather Underground.
        From 0be384a13740aa791a3910df3cc9bd88b0bdfc7d Mon Sep 17 00:00:00 2001 From: Tom Keffer Date: Wed, 18 Sep 2019 05:42:17 -0700 Subject: [PATCH 10/20] Renamed weeutil.weeutil._get_object() to get_object(). --- bin/weeimport/weeimport.py | 12 ++++++------ bin/weeutil/weeutil.py | 6 +++++- bin/weewx/cheetahgenerator.py | 2 +- bin/weewx/engine.py | 2 +- bin/weewx/manager.py | 4 ++-- bin/weewx/reportengine.py | 2 +- 6 files changed, 16 insertions(+), 12 deletions(-) diff --git a/bin/weeimport/weeimport.py b/bin/weeimport/weeimport.py index ac3d0d80..53b47f88 100644 --- a/bin/weeimport/weeimport.py +++ b/bin/weeimport/weeimport.py @@ -33,7 +33,7 @@ import weewx.wxservices from weewx.manager import open_manager_with_config from weewx.units import unit_constants, unit_nicknames, convertStd, to_std_system, ValueTuple -from weeutil.weeutil import timestamp_to_string, option_as_list, to_int, tobool, _get_object +from weeutil.weeutil import timestamp_to_string, option_as_list, to_int, tobool, get_object log = logging.getLogger(__name__) @@ -331,11 +331,11 @@ class Source(object): module_class = '.'.join(['weeimport', source.lower() + 'import', source + 'Source']) - return _get_object(module_class)(config_dict, - config_path, - import_config_dict.get(source, {}), - import_config_path, - options) + return get_object(module_class)(config_dict, + config_path, + import_config_dict.get(source, {}), + import_config_path, + options) def run(self): """Main entry point for importing from an external source. diff --git a/bin/weeutil/weeutil.py b/bin/weeutil/weeutil.py index b93050ef..efd4ec47 100644 --- a/bin/weeutil/weeutil.py +++ b/bin/weeutil/weeutil.py @@ -1095,7 +1095,7 @@ def latlon_string(ll, hemi, which, format_list=None): hemi[0] if ll >= 0 else hemi[1]) -def _get_object(module_class): +def get_object(module_class): """Given a string with a module class name, it imports and returns the class.""" # Split the path into its parts parts = module_class.split('.') @@ -1115,6 +1115,10 @@ def _get_object(module_class): return mod +# For backwards compatibility: +_get_object = get_object + + class GenWithPeek(object): """Generator object which allows a peek at the next object to be returned. diff --git a/bin/weewx/cheetahgenerator.py b/bin/weewx/cheetahgenerator.py index ce39cabe..81f3e8e9 100644 --- a/bin/weewx/cheetahgenerator.py +++ b/bin/weewx/cheetahgenerator.py @@ -186,7 +186,7 @@ class CheetahGenerator(weewx.reportengine.ReportGenerator): x = c.strip() if x: # Get the class - class_ = weeutil.weeutil._get_object(x) + class_ = weeutil.weeutil.get_object(x) # Then instantiate the class, passing self as the sole argument self.search_list_objs.append(class_(self)) diff --git a/bin/weewx/engine.py b/bin/weewx/engine.py index 5f17ed79..b1d31c92 100644 --- a/bin/weewx/engine.py +++ b/bin/weewx/engine.py @@ -142,7 +142,7 @@ class StdEngine(object): # passing self and the configuration dictionary as the # arguments: log.debug("Loading service %s", svc) - self.service_obj.append(weeutil.weeutil._get_object(svc)(self, config_dict)) + self.service_obj.append(weeutil.weeutil.get_object(svc)(self, config_dict)) log.debug("Finished loading service %s", svc) except Exception: # An exception occurred. Shut down any running services, then diff --git a/bin/weewx/manager.py b/bin/weewx/manager.py index ee6f2b04..b65c540c 100644 --- a/bin/weewx/manager.py +++ b/bin/weewx/manager.py @@ -1015,7 +1015,7 @@ def get_manager_dict_from_config(config_dict, data_binding, manager_dict['schema'] = [(col_name, manager_dict['schema'][col_name]) for col_name in manager_dict['schema']] else: # Schema is a string, with the name of the schema object - manager_dict['schema'] = weeutil.weeutil._get_object(schema_name) + manager_dict['schema'] = weeutil.weeutil.get_object(schema_name) return manager_dict @@ -1031,7 +1031,7 @@ def get_manager_dict(bindings_dict, databases_dict, data_binding, def open_manager(manager_dict, initialize=False): - manager_cls = weeutil.weeutil._get_object(manager_dict['manager']) + manager_cls = weeutil.weeutil.get_object(manager_dict['manager']) if initialize: return manager_cls.open_with_create(manager_dict['database_dict'], manager_dict['table_name'], diff --git a/bin/weewx/reportengine.py b/bin/weewx/reportengine.py index bb1d1381..5f7cff7d 100644 --- a/bin/weewx/reportengine.py +++ b/bin/weewx/reportengine.py @@ -182,7 +182,7 @@ class StdReportEngine(threading.Thread): try: # Instantiate an instance of the class. - obj = weeutil.weeutil._get_object(generator)( + obj = weeutil.weeutil.get_object(generator)( self.config_dict, skin_dict, self.gen_ts, From d1c3abee6f7357119276907b0de8d949a71587dc Mon Sep 17 00:00:00 2001 From: Tom Keffer Date: Wed, 18 Sep 2019 07:53:42 -0700 Subject: [PATCH 11/20] Fix WU bug for locations ahead of the GMT date. Fixes issue #445. --- bin/wunderfixer | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/bin/wunderfixer b/bin/wunderfixer index 8e436042..cede1838 100755 --- a/bin/wunderfixer +++ b/bin/wunderfixer @@ -372,13 +372,21 @@ class WunderStation(weewx.restx.AmbientThread): return: a set containing the timestamps in epoch time """ - day_tt = day_requested.timetuple() - day = "%4d%02d%02d" % (day_tt.tm_year, - day_tt.tm_mon, - day_tt.tm_mday) - url = "https://api.weather.com/v2/pws/history/all?stationId=%s&format=json&units=m&date=%s&apiKey=%s" \ - % (self.station, day, self.api_key) + # To get around a WU bug for locations ahead of the GMT date, we have to use an alternative API. + # See Issue #445 (https://github.com/weewx/weewx/issues/445) + today_date = datetime.date.today() + if day_requested >= today_date: + url = "https://api.weather.com/v2/pws/observations/all/1day?stationId=%s&format=json&units=m&apiKey=%s" \ + % (self.station, self.api_key) + else: + day_tt = day_requested.timetuple() + day = "%4d%02d%02d" % (day_tt.tm_year, + day_tt.tm_mon, + day_tt.tm_mday) + + url = "https://api.weather.com/v2/pws/history/all?stationId=%s&format=json&units=m&date=%s&apiKey=%s" \ + % (self.station, day, self.api_key) request = urllib.request.Request(url) request.add_header('Accept-Encoding', 'gzip') From 56f3feccc92bd04788891d0162367b77d088a972 Mon Sep 17 00:00:00 2001 From: Tom Keffer Date: Wed, 18 Sep 2019 07:57:09 -0700 Subject: [PATCH 12/20] Added WU api_key to weewx.conf --- docs/changes.txt | 7 +++---- weewx.conf | 4 ++++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/changes.txt b/docs/changes.txt index 3bfac586..7f84462e 100644 --- a/docs/changes.txt +++ b/docs/changes.txt @@ -43,10 +43,9 @@ Ported OS uptime to OpenBSD. Fixes issue #428. Thanks to user Jeff Ross! Catch SSL certificate errors in uploaders. Retry after an hour. Fixes issue #413. -Wunderfixer has been ported to the new WU API. This API requires an -API key, which you can get from the WU. Put it in weewx.conf. -Unfortunately, WU is very unreliable whether reposts "stick". -Fixes issue #414 +Wunderfixer has been ported to the new WU API. This API requires an API key, +which you can get from the WU. Put it in weewx.conf. Unfortunately, WU is +very unreliable whether reposts "stick". Fixes issues #414 and #445. Wee_import can now import Weather Display monthly log files. diff --git a/weewx.conf b/weewx.conf index 22b8c3dc..97977e23 100644 --- a/weewx.conf +++ b/weewx.conf @@ -118,6 +118,10 @@ version = 4.0.0a8 station = replace_me password = "replace_me" + # If you plan on using wunderfixer, set the following + # to your API key: + api_key = replace_me + # Set the following to True to have weewx use the WU "Rapidfire" # protocol. Not all hardware can support it. See the User's Guide. rapidfire = False From 04d1985f7ea83f3db33dba1b012e54ed29c6c937 Mon Sep 17 00:00:00 2001 From: Tom Keffer Date: Wed, 18 Sep 2019 15:03:27 -0700 Subject: [PATCH 13/20] Revert "Fix WU bug for locations ahead of the GMT date. Fixes issue #445." This reverts commit d1c3abee --- bin/wunderfixer | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/bin/wunderfixer b/bin/wunderfixer index cede1838..8e436042 100755 --- a/bin/wunderfixer +++ b/bin/wunderfixer @@ -372,21 +372,13 @@ class WunderStation(weewx.restx.AmbientThread): return: a set containing the timestamps in epoch time """ + day_tt = day_requested.timetuple() + day = "%4d%02d%02d" % (day_tt.tm_year, + day_tt.tm_mon, + day_tt.tm_mday) - # To get around a WU bug for locations ahead of the GMT date, we have to use an alternative API. - # See Issue #445 (https://github.com/weewx/weewx/issues/445) - today_date = datetime.date.today() - if day_requested >= today_date: - url = "https://api.weather.com/v2/pws/observations/all/1day?stationId=%s&format=json&units=m&apiKey=%s" \ - % (self.station, self.api_key) - else: - day_tt = day_requested.timetuple() - day = "%4d%02d%02d" % (day_tt.tm_year, - day_tt.tm_mon, - day_tt.tm_mday) - - url = "https://api.weather.com/v2/pws/history/all?stationId=%s&format=json&units=m&date=%s&apiKey=%s" \ - % (self.station, day, self.api_key) + url = "https://api.weather.com/v2/pws/history/all?stationId=%s&format=json&units=m&date=%s&apiKey=%s" \ + % (self.station, day, self.api_key) request = urllib.request.Request(url) request.add_header('Accept-Encoding', 'gzip') From 9e54078f0f7cd5af29a1844fe298b820ca04a041 Mon Sep 17 00:00:00 2001 From: Tom Keffer Date: Fri, 20 Sep 2019 06:01:44 -0700 Subject: [PATCH 14/20] Deprecated the use of month_delta and year_delta in spans. See issue #436. --- bin/weeutil/weeutil.py | 4 +++- docs/customizing.htm | 12 ------------ 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/bin/weeutil/weeutil.py b/bin/weeutil/weeutil.py index efd4ec47..4301f40c 100644 --- a/bin/weeutil/weeutil.py +++ b/bin/weeutil/weeutil.py @@ -389,7 +389,9 @@ def archiveHoursAgoSpan(time_ts, hours_ago=0, grace=1): def archiveSpanSpan(time_ts, time_delta=0, hour_delta=0, day_delta=0, week_delta=0, month_delta=0, year_delta=0): """ Returns a TimeSpan for the last xxx seconds where xxx equals time_delta sec + hour_delta hours + day_delta days + week_delta weeks + month_delta months + year_delta years - Note: For month_delta, 1 month = 30 days, For year_delta, 1 year = 365 days + + NOTE: Use of month_delta and year_delta is deprecated. + See issue #436 (https://github.com/weewx/weewx/issues/436) Example: >>> os.environ['TZ'] = 'Australia/Brisbane' diff --git a/docs/customizing.htm b/docs/customizing.htm index 7356c580..36e73ed8 100644 --- a/docs/customizing.htm +++ b/docs/customizing.htm @@ -2436,18 +2436,6 @@ or in foobar units: $day.barometer.min.foobar The maximum barometric pressure over the last immediate 2 weeks. - - $month_delta=months - $span($month_delta=3).outTemp.min - The minimum temperture over the last immediate 3 months (90 days). - - - - $year_delta=years - $span($year_delta=1).windchill.min - The minimum wind chill over the last immediate 1 year (365 days). - - From e98c52dee9a11c744bd78f08645fc8ea35d15d6a Mon Sep 17 00:00:00 2001 From: Tom Keffer Date: Fri, 20 Sep 2019 10:28:01 -0700 Subject: [PATCH 15/20] Document other parameters that can get overridden in $almanac --- docs/customizing.htm | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/customizing.htm b/docs/customizing.htm index 36e73ed8..7ae05413 100644 --- a/docs/customizing.htm +++ b/docs/customizing.htm @@ -2826,11 +2826,16 @@ Sunrise, sunset: $almanac.sunrise $almanac.sunset 

        $almanac(pressure=0, horizon=-6).sun(use_center=1).rise

        The general syntax is:

        - 
        $almanac(pressure=pressure, horizon=horizon,
-         temperature=temperature_C).heavenly_body(use_center=[01]).attribute
+        
        $almanac(almanac_time=time,            ## Unix epoch time
+         lat=latitude, lon=longitude,  ## degrees
+         altitude=altitude,            ## meters
+         pressure=pressure,            ## mbars
+         horizon=horizon,              ## degrees
+         temperature=temperature_C     ## degrees C
+       ).heavenly_body(use_center=[01]).attribute
       
        
-        
        As you can see, in addition to the horizon angle, you can also override atmospheric pressure and temperature
-            (degrees Celsius).
+        
        
+            As you can see, many other properties can be overridden besides pressure and the horizon angle.
         
        
 
         
        

From ed28486c59b080d9871ff690b2e3cf06a0e5267f Mon Sep 17 00:00:00 2001
From: Tom Keffer 
Date: Fri, 20 Sep 2019 12:11:51 -0700
Subject: [PATCH 16/20] Log "No database specified" only if debug>=2. Prevents
 a lot of chatter for uploaders that do not use the database.

---
 bin/weewx/restx.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/weewx/restx.py b/bin/weewx/restx.py
index 1926f319..4175da8d 100644
--- a/bin/weewx/restx.py
+++ b/bin/weewx/restx.py
@@ -270,7 +270,7 @@ class RESTThread(threading.Thread):
 
         if dbmanager is None:
             # If we don't have a database, we can't do anything
-            if self.log_failure:
+            if self.log_failure and weewx.debug >= 2:
                 log.debug("No database specified. Augmentation from database skipped.")
             return record
 

From 92dcb7337e91df73fc621e1d98142a8b9f6fa436 Mon Sep 17 00:00:00 2001
From: Tom Keffer 
Date: Fri, 20 Sep 2019 12:12:09 -0700
Subject: [PATCH 17/20] v4.0.0a9

---
 bin/weecfg/tests/expected/weewx40_user_expected.conf | 2 +-
 bin/weewx/__init__.py                                | 2 +-
 weewx.conf                                           | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/bin/weecfg/tests/expected/weewx40_user_expected.conf b/bin/weecfg/tests/expected/weewx40_user_expected.conf
index 5ffc73e3..05ef28a1 100644
--- a/bin/weecfg/tests/expected/weewx40_user_expected.conf
+++ b/bin/weecfg/tests/expected/weewx40_user_expected.conf
@@ -26,7 +26,7 @@ log_failure = True
 socket_timeout = 20
 
 # Do not modify this - it is used by setup.py when installing and updating.
-version = 4.0.0a8
+version = 4.0.0a9
 
 ##############################################################################
 
diff --git a/bin/weewx/__init__.py b/bin/weewx/__init__.py
index 8c302308..90cacc7d 100644
--- a/bin/weewx/__init__.py
+++ b/bin/weewx/__init__.py
@@ -7,7 +7,7 @@
 from __future__ import absolute_import
 import time
 
-__version__="4.0.0a8"
+__version__="4.0.0a9"
 
 # Holds the program launch time in unix epoch seconds:
 # Useful for calculating 'uptime.'
diff --git a/weewx.conf b/weewx.conf
index 97977e23..aef33669 100644
--- a/weewx.conf
+++ b/weewx.conf
@@ -23,7 +23,7 @@ log_failure = True
 socket_timeout = 20
 
 # Do not modify this. It is used when installing and updating weewx.
-version = 4.0.0a8
+version = 4.0.0a9
 
 ##############################################################################
 

From 6b06e7e10550b780f459d0e565912e87a820a4ab Mon Sep 17 00:00:00 2001
From: Tom Keffer 
Date: Fri, 20 Sep 2019 16:27:37 -0700
Subject: [PATCH 18/20] Handle errors from WOW more generally.

---
 bin/weewx/restx.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/weewx/restx.py b/bin/weewx/restx.py
index 4175da8d..9d378d9b 100644
--- a/bin/weewx/restx.py
+++ b/bin/weewx/restx.py
@@ -1041,8 +1041,8 @@ class WOWThread(AmbientThread):
             # WOW signals a bad login with a HTML Error 403 code:
             if e.code == 403:
                 raise BadLogin(e)
-            elif e.code == 429:
-                raise FailedPost("Too many requests; data already seen; or too out of date.")
+            elif e.code >= 400:
+                raise FailedPost(e)
             else:
                 raise
         else:

From 417aab3a8ff11428439974f52bcaafba46ecafc1 Mon Sep 17 00:00:00 2001
From: Tom Keffer 
Date: Fri, 20 Sep 2019 17:01:39 -0700
Subject: [PATCH 19/20] Updated benchmarks for creating reports.

---
 docs/usersguide.htm | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/docs/usersguide.htm b/docs/usersguide.htm
index a5125297..d7063fcd 100644
--- a/docs/usersguide.htm
+++ b/docs/usersguide.htm
@@ -107,9 +107,11 @@ Version: 4.0
             consumes about 5% of the CPU, 100 MB of virtual memory, and 20 MB of real memory.
         
        
 
-        
        WeeWX also runs great on a Raspberry Pi, although report generation will take longer. For example, the 12 "To
-            Date" templates take about 5.1 seconds on the RPi, compared to 3.0 seconds on my Fit-PC, and a mere 0.9
-            seconds on my vintage Dell Optiplex 745.
+        
        
+            WeeWX also runs great on a Raspberry Pi, although report generation will take longer. For example,
+            the 12 "To Date" templates of the "Standard" report take about 5.1 seconds on a RPi B+,
+            compared to 3.0 seconds on my Fit-PC, 0.9 seconds on my vintage Dell Optiplex 745, and 0.3 seconds on
+            a NUC with a 4th gen i5 processor.
         
        
 
         
        Time
        

From f888305329fc5159c733e60581527213df5ee640 Mon Sep 17 00:00:00 2001
From: gjr80 
Date: Sun, 22 Sep 2019 13:52:00 +1000
Subject: [PATCH 20/20] Remove space that appears at the start of each line

---
 bin/weewx/manager.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/weewx/manager.py b/bin/weewx/manager.py
index ee6f2b04..d154c2a9 100644
--- a/bin/weewx/manager.py
+++ b/bin/weewx/manager.py
@@ -1093,7 +1093,7 @@ def drop_database_with_config(config_dict, data_binding,
 def show_progress(nrec, last_time):
     """Utility function to show our progress while backfilling"""
     print("Records processed: %d; Last date: %s\r" %
-          (nrec, weeutil.weeutil.timestamp_to_string(last_time)), end=' ', file=sys.stdout)
+          (nrec, weeutil.weeutil.timestamp_to_string(last_time)), end='', file=sys.stdout)
     sys.stdout.flush()