diff --git a/bin/wee_import b/bin/wee_import index fe975667..09a50185 100755 --- a/bin/wee_import +++ b/bin/wee_import @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Copyright (c) 2009-2016 Tom Keffer and +# Copyright (c) 2009-2019 Tom Keffer and # Gary Roderick # # See the file LICENSE.txt for your rights. @@ -9,47 +9,51 @@ Compatibility: - wee_import can import from a Comma Separated Values (CSV) format file, - directly from the historical records of a Weather Underground Personal - Weather Station or from one or more Cumulus monthly log files. CSV format - files must have a comma separated list of field names on the first line. + wee_import can import from: + - a Comma Separated Values (CSV) format file + - the historical records of a Weather Underground Personal + Weather Station + - one or more Cumulus monthly log files + - one or more Weather Display monthly log files Design - wee_import utilises a config file (the import config file) and a number of - command line options to control the import. The config file defines the type - of input to be performed and the import data source as well as more advanced - options such as field maps etc. Details of the supported command line - parameters/options can be viewed by entering wee_import --help at the command - line. Details of the wee_import config file settings can be found in example - import config files distributed in the weewx/util/import directory. + wee_import utilises an import config file and a number of command line + options to control the import. The import config file defines the type of + input to be performed and the import data source as well as more advanced + options such as field maps etc. Details of the supported command line + parameters/options can be viewed by entering wee_import --help at the + command line. Details of the wee_import import config file settings can be + found in the example import config files distributed in the + weewx/util/import directory. - wee_import utilises an abstract base class Source that defines the majority - of the wee_import functionality. The abstract base class and other supporting - structures are in bin/weeimport/weeimport.py. Child classes are created from - the base class for each different import type supported by wee_import. The - child classes set a number of import type specific properties as well as - defining a getData() method that reads the raw data to be imported and a - period_generator() method that generates a sequence of objects to be imported - (eg monthly log files). This way wee_import can be extended to support other - sources by defining a new child class, its specific properties as well as - getData() and period_generator() methods. The child class for a given import - type are defined in the bin/weeimport/xxximport.py files. + wee_import utilises an abstract base class Source that defines the majority + of the wee_import functionality. The abstract base class and other + supporting structures are in bin/weeimport/weeimport.py. Child classes are + created from the base class for each different import type supported by + wee_import. The child classes set a number of import type specific + properties as well as defining getData() and period_generator() methods that + read the raw data to be imported and generates a sequence of objects to be + imported (eg monthly log files) respectively. This way wee_import can be + extended to support other sources by defining a new child class, its + specific properties as well as getData() and period_generator() methods. The + child class for a given import type are defined in the + bin/weeimport/xxximport.py files. - As with other WeeWX utilities, wee_import advises the user of basic - configuration, action taken and results via stdout. However, since - wee_import can make substantial changes to the WeeWX archive, wee_import also - logs to file by default. This functionality is controlled via a command - line option. + As with other WeeWX utilities, wee_import advises the user of basic + configuration, action taken and results via the console. However, since + wee_import can make substantial changes to the WeeWX archive, wee_import + also logs to WeeWX log system file. Console and log output can be controlled + via a number of command line options. Prerequisites - wee_import uses a number of WeeWX API calls and therefore must have a - functional WeeWX installation. wee_import requires WeeWX 3.6.0 or later. + wee_import uses a number of WeeWX API calls and therefore must have a + functional WeeWX installation. wee_import requires WeeWX 4.0.0 or later. Configuration - A number of parameters can be defined in the import config file as follows: + A number of parameters can be defined in the import config file as follows: # EXAMPLE WEE_IMPORT CONFIGURATION FILE # @@ -63,7 +67,7 @@ Configuration # WU - import obs from a Weather Underground PWS history # Cumulus - import obs from a one or more Cumulus monthly log files # Format is: -# source = (CSV | WU | Cumulus) +# source = (CSV | WU | Cumulus | WD) source = CSV ############################################################################## @@ -231,6 +235,9 @@ source = CSV # WU PWS Station ID to be used for import. station_id = XXXXXXXX123 + # WU API key to be used for import. + api_key = XXXXXXXXXXXXXXXXXXXXXX1234567890 + # # When importing WU data the following WeeWX database fields will be # populated directly by the imported data (provided the corresponding data @@ -238,13 +245,17 @@ source = CSV # barometer # dateTime # dewpoint + # heatindex # outHumidity # outTemp # radiation # rain + # rainRate + # windchill # windDir # windGust # windSpeed + # UV # # The following WeeWX database fields will be populated from other # settings/config files: @@ -256,10 +267,7 @@ source = CSV # used during import: # altimeter # ET - # heatindex # pressure - # rainRate - # windchill # # The following WeeWX fields will be populated with derived values from the # imported data provided the --calc-missing command line option is used @@ -511,10 +519,10 @@ source = CSV # inTemp # outHumidity # outTemp - # radiation (if Cumulus data available) - # rain (requires Cumulus 1.9.4 or later) + # radiation (if WD radiation data available) + # rain # rainRate - # UV (if Cumulus data available) + # UV (if WD UV data available) # windDir # windGust # windSpeed @@ -557,8 +565,8 @@ source = CSV # setting is best used if the records to be imported are # equally based in time but there are some missing records. # This setting is recommended for WU imports. - # To import Cumulus records it is recommended that the interval setting - # be set to the value used in Cumulus as the 'data log interval'. + # To import WD records it is recommended that the interval setting be set to + # the value used in WD as the 'data log interval'. # Format is: # interval = (derive | conf | x) interval = x @@ -580,17 +588,17 @@ source = CSV # calc_missing = (True | False) calc_missing = True - # Specify the character used as the field delimiter as Cumulus monthly log - # files may not always use a comma to delimit fields in the monthly log - # files. The character must be enclosed in quotes. Must not be the same - # as the decimal setting below. Format is: + # Specify the character used as the field delimiter as WD monthly log files + # may not always use a comma to delimit fields in the monthly log files. The + # character must be enclosed in quotes. Must not be the same as the decimal + # setting below. Format is: # delimiter = ',' delimiter = ',' - # Specify the character used as the decimal point. Cumulus monthly log - # files may not always use a fullstop character as the decimal point. The - # character must be enclosed in quotes. Must not be the same as the - # delimiter setting. Format is: + # Specify the character used as the decimal point. WD monthly log file may + # not always use a full stop character as the decimal point. The character + # must be enclosed in quotes. Must not be the same as the delimiter setting. + # Format is: # decimal = '.' decimal = '.' @@ -705,20 +713,27 @@ Adding a New Import Source # Python imports from __future__ import absolute_import from __future__ import print_function + +import logging import optparse -import syslog from distutils.version import StrictVersion # WeeWX imports +import weecfg import weewx import weeimport import weeimport.weeimport +import weeutil.logger +import weeutil.weeutil + + +log = logging.getLogger(__name__) # wee_import version number -WEE_IMPORT_VERSION = '0.3' +WEE_IMPORT_VERSION = '0.4' # minimum WeeWX version required for this version of wee_import -REQUIRED_WEEWX = "3.6.0" +REQUIRED_WEEWX = "4.0.0a7" description = """Import observation data into a WeeWX archive.""" @@ -730,7 +745,6 @@ usage = """wee_import --help [--dry-run] [--verbose] [--suppress-warnings] - [--log=-] """ epilog = """wee_import will import data from an external source into a WeeWX @@ -742,6 +756,9 @@ epilog = """wee_import will import data from an external source into a WeeWX def main(): """The main routine that kicks everything off.""" + # Set defaults for logging: + weeutil.logger.setup('wee_import', {}) + # Create a command line parser: parser = optparse.OptionParser(description=description, usage=usage, @@ -749,8 +766,8 @@ def main(): # Add the various options: parser.add_option("--config", dest="config_path", type=str, - metavar="CONFIG_FILE", default="weewx.conf", - help="Use WeeWX configuration file CONFIG_FILE.") + metavar="CONFIG_FILE", + help="Use configuration file CONFIG_FILE.") parser.add_option("--import-config", dest="import_config_path", type=str, metavar="IMPORT_CONFIG_FILE", help="Use import configuration file IMPORT_CONFIG_FILE.") @@ -764,15 +781,8 @@ def main(): parser.add_option("--to", dest="date_to", type=str, metavar="YYYY-mm-dd[THH:MM]", help="Import data up until this date or date-time. Format " "is YYYY-mm-dd[THH:MM].") - parser.add_option("--log", dest="logging", type=str, metavar="-", - help="Control wee_import log output. By default log output " - "is sent to the WeeWX log file. Log output may be " - "disabled by using '--log=-'. Some WeeWX API log " - "output cannot be controlled by wee_import and will " - "be sent to the default log file irrespective of the " - "'--log' option.") parser.add_option("--verbose", action="store_true", dest="verbose", - help="Print useful extra output.") + help="Print and log useful extra output.") parser.add_option("--suppress-warnings", action="store_true", dest="suppress", help="Suppress warnings to stdout. Warnings are still logged.") parser.add_option("--version", dest="version", action="store_true", @@ -787,74 +797,103 @@ def main(): weewx.__version__)) exit(1) + # get config_dict to use + config_path, config_dict = weecfg.read_config(options.config_path, args) + print("Using WeeWX configuration file %s" % config_path) + + # Set weewx.debug as necessary: + weewx.debug = weeutil.weeutil.to_int(config_dict.get('debug', 0)) + + # Now we can set up the user customized logging: + weeutil.logger.setup('wee_import', config_dict) + # display wee_import version info if options.version: print("wee_import version: %s" % WEE_IMPORT_VERSION) exit(0) - # Set up logging - wlog = weeimport.weeimport.WeeImportLog(options.logging, - options.verbose, - options.suppress, - options.dry_run) + # to do anything more we need an import config file, check if one was + # provided + if options.import_config_path: + # we have something so try to start - # advise the user we are starting up - wlog.printlog(syslog.LOG_INFO, "Starting wee_import...") + # advise the user we are starting up + print("Starting wee_import...") + log.info("Starting wee_import...") - # If we got this far we must want to import something so get a Source - # object from our factory and try to import. Be prepared to catch any - # errors though. - try: - source_obj = weeimport.weeimport.Source.sourceFactory(options, - args, - wlog) - source_obj.run() - except weeimport.weeimport.WeeImportOptionError as e: - wlog.printlog(syslog.LOG_INFO, "**** Command line option error.") - wlog.printlog(syslog.LOG_INFO, "**** %s" % e) - print("**** Nothing done, exiting.") - wlog.logonly(syslog.LOG_INFO, "**** Nothing done.") - exit(1) - except weeimport.weeimport.WeeImportIOError as e: - wlog.printlog(syslog.LOG_INFO, "**** Unable to load source file.") - wlog.printlog(syslog.LOG_INFO, "**** %s" % e) - print("**** Nothing done, exiting.") - wlog.logonly(syslog.LOG_INFO, "**** Nothing done.") - exit(1) - except weeimport.weeimport.WeeImportFieldError as e: - wlog.printlog(syslog.LOG_INFO, "**** Unable to map source data.") - wlog.printlog(syslog.LOG_INFO, "**** %s" % e) - print("**** Nothing done, exiting.") - wlog.logonly(syslog.LOG_INFO, "**** Nothing done.") - exit(1) - except weeimport.weeimport.WeeImportMapError as e: - wlog.printlog(syslog.LOG_INFO, - "**** Unable to parse source-to-WeeWX field map.") - wlog.printlog(syslog.LOG_INFO, "**** %s" % e) - print("**** Nothing done, exiting.") - wlog.logonly(syslog.LOG_INFO, "**** Nothing done.") - exit(1) - except (weewx.ViolatedPrecondition, weewx.UnsupportedFeature) as e: - wlog.printlog(syslog.LOG_INFO, "**** %s" % e) - print("**** Nothing done, exiting.") - wlog.logonly(syslog.LOG_INFO, "**** Nothing done.") + # If we got this far we must want to import something so get a Source + # object from our factory and try to import. Be prepared to catch any + # errors though. + try: + source_obj = weeimport.weeimport.Source.sourceFactory(options, + args) + source_obj.run() + except weeimport.weeimport.WeeImportOptionError as e: + print("**** Command line option error.") + log.info("**** Command line option error.") + print("**** %s" % e) + log.info("**** %s" % e) + print("**** Nothing done, exiting.") + log.info("**** Nothing done.") + exit(1) + except weeimport.weeimport.WeeImportIOError as e: + print("**** Unable to load source file.") + log.info("**** Unable to load source file.") + print("**** %s" % e) + log.info("**** %s" % e) + print("**** Nothing done, exiting.") + log.info("**** Nothing done.") + exit(1) + except weeimport.weeimport.WeeImportFieldError as e: + print("**** Unable to map source data.") + log.info("**** Unable to map source data.") + print("**** %s" % e) + log.info("**** %s" % e) + print("**** Nothing done, exiting.") + log.info("**** Nothing done.") + exit(1) + except weeimport.weeimport.WeeImportMapError as e: + print("**** Unable to parse source-to-WeeWX field map.") + log.info("**** Unable to parse source-to-WeeWX field map.") + print("**** %s" % e) + log.info("**** %s" % e) + print("**** Nothing done, exiting.") + log.info("**** Nothing done.") + exit(1) + except (weewx.ViolatedPrecondition, weewx.UnsupportedFeature) as e: + print("**** %s" % e) + log.info("**** %s" % e) + print("**** Nothing done, exiting.") + log.info("**** Nothing done.") + print() + parser.print_help() + exit(1) + except SystemExit as e: + print(e) + exit(0) + except (ValueError, weewx.UnitError) as e: + print("**** %s" % e) + log.info("**** %s" % e) + print("**** Nothing done, exiting.") + log.info("**** Nothing done.") + exit(1) + except IOError as e: + print("**** Unable to load config file.") + log.info("**** Unable to load config file.") + print("**** %s" % e) + log.info("**** %s" % e) + print("**** Nothing done, exiting.") + log.info("**** Nothing done.") + exit(1) + else: + # we have no import config file so display a suitable message followed + # by the help text then exit + print("**** No import config file specified.") + print("**** Nothing done.") print() parser.print_help() exit(1) - except SystemExit as e: - print(e) - exit(0) - except (ValueError, weewx.UnitError) as e: - wlog.printlog(syslog.LOG_INFO, "**** %s" % e) - print("**** Nothing done, exiting.") - wlog.logonly(syslog.LOG_INFO, "**** Nothing done.") - exit(1) - except IOError as e: - wlog.printlog(syslog.LOG_INFO, "**** Unable to load config file.") - wlog.printlog(syslog.LOG_INFO, "**** %s" % e) - print("**** Nothing done, exiting.") - wlog.logonly(syslog.LOG_INFO, "**** Nothing done.") - exit(1) + # execute our main code if __name__ == "__main__": diff --git a/bin/weecfg/tests/expected/weewx40_user_expected.conf b/bin/weecfg/tests/expected/weewx40_user_expected.conf index 5ffc73e3..05ef28a1 100644 --- a/bin/weecfg/tests/expected/weewx40_user_expected.conf +++ b/bin/weecfg/tests/expected/weewx40_user_expected.conf @@ -26,7 +26,7 @@ log_failure = True socket_timeout = 20 # Do not modify this - it is used by setup.py when installing and updating. -version = 4.0.0a8 +version = 4.0.0a9 ############################################################################## diff --git a/bin/weeimport/csvimport.py b/bin/weeimport/csvimport.py index ef3bc9f6..8f999708 100644 --- a/bin/weeimport/csvimport.py +++ b/bin/weeimport/csvimport.py @@ -15,8 +15,8 @@ from __future__ import print_function # Python imports import csv +import logging import os -import syslog # WeeWX imports from . import weeimport @@ -25,6 +25,8 @@ import weewx from weeutil.weeutil import timestamp_to_string, option_as_list from weewx.units import unit_nicknames +log = logging.getLogger(__name__) + # ============================================================================ # class CSVSource @@ -42,13 +44,12 @@ class CSVSource(weeimport.Source): # these details are specified by the user in the wee_import config file. _header_map = None - def __init__(self, config_dict, config_path, csv_config_dict, import_config_path, options, log): + def __init__(self, config_dict, config_path, csv_config_dict, import_config_path, options): # call our parents __init__ super(CSVSource, self).__init__(config_dict, csv_config_dict, - options, - log) + options) # save our import config path self.import_config_path = import_config_path @@ -86,12 +87,17 @@ class CSVSource(weeimport.Source): # tell the user/log what we intend to do _msg = "A CSV import from source file '%s' has been requested." % self.source - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) _msg = "The following options will be used:" - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " config=%s, import-config=%s" % (config_path, self.import_config_path) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) if options.date: _msg = " source=%s, date=%s" % (self.source, options.date) else: @@ -99,37 +105,62 @@ class CSVSource(weeimport.Source): _msg = " source=%s, from=%s, to=%s" % (self.source, options.date_from, options.date_to) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " dry-run=%s, calc_missing=%s, ignore_invalid_data=%s" % (self.dry_run, self.calc_missing, self.ignore_invalid_data) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " tranche=%s, interval=%s, date/time_string_format=%s" % (self.tranche, self.interval, self.raw_datetime_format) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " rain=%s, wind_direction=%s" % (self.rain, self.wind_dir) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " UV=%s, radiation=%s" % (self.UV_sensor, self.solar_sensor) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = "Using database binding '%s', which is bound to database '%s'" % (self.db_binding_wx, self.dbm.database_name) - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) _msg = "Destination table '%s' unit system is '%#04x' (%s)." % (self.dbm.table_name, self.archive_unit_sys, unit_nicknames[self.archive_unit_sys]) - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) if self.calc_missing: - print("Missing derived observations will be calculated.") + _msg = "Missing derived observations will be calculated." + print(_msg) + log.info(_msg) + if not self.UV_sensor: - print("All WeeWX UV fields will be set to None.") + _msg = "All WeeWX UV fields will be set to None." + print(_msg) + log.info(_msg) if not self.solar_sensor: - print("All WeeWX radiation fields will be set to None.") + _msg = "All WeeWX radiation fields will be set to None." + print(_msg) + log.info(_msg) if options.date or options.date_from: - print("Observations timestamped after %s and up to and" % timestamp_to_string(self.first_ts)) - print("including %s will be imported." % timestamp_to_string(self.last_ts)) + _msg = "Observations timestamped after %s and up to and" % timestamp_to_string(self.first_ts) + print(_msg) + log.info(_msg) + _msg = "including %s will be imported." % timestamp_to_string(self.last_ts) + print(_msg) + log.info(_msg) if self.dry_run: - print("This is a dry run, imported data will not be saved to archive.") + _msg = "This is a dry run, imported data will not be saved to archive." + print(_msg) + log.info(_msg) def getRawData(self, period): """Obtain an iterable containing the raw data to be imported. diff --git a/bin/weeimport/cumulusimport.py b/bin/weeimport/cumulusimport.py index e5f93518..4bce8969 100644 --- a/bin/weeimport/cumulusimport.py +++ b/bin/weeimport/cumulusimport.py @@ -15,8 +15,8 @@ from __future__ import print_function # Python imports import csv import glob +import logging import os -import syslog import time # WeeWX imports @@ -26,6 +26,8 @@ import weewx from weeutil.weeutil import timestamp_to_string from weewx.units import unit_nicknames +log = logging.getLogger(__name__) + # Dict to lookup rainRate units given rain units rain_units_dict = {'inch': 'inch_per_hour', 'mm': 'mm_per_hour'} @@ -89,13 +91,12 @@ class CumulusSource(weeimport.Source): 'cur_app_temp': {'map_to': 'appTemp'} } - def __init__(self, config_dict, config_path, cumulus_config_dict, import_config_path, options, log): + def __init__(self, config_dict, config_path, cumulus_config_dict, import_config_path, options): # call our parents __init__ super(CumulusSource, self).__init__(config_dict, cumulus_config_dict, - options, - log) + options) # save our import config path self.import_config_path = import_config_path @@ -207,7 +208,8 @@ class CumulusSource(weeimport.Source): try: self.source = cumulus_config_dict['directory'] except KeyError: - raise weewx.ViolatedPrecondition("Cumulus monthly logs directory not specified in '%s'." % import_config_path) + _msg = "Cumulus monthly logs directory not specified in '%s'." % import_config_path + raise weewx.ViolatedPrecondition(_msg) # property holding the current log file name being processed self.file_name = None @@ -223,34 +225,49 @@ class CumulusSource(weeimport.Source): # tell the user/log what we intend to do _msg = "Cumulus monthly log files in the '%s' directory will be imported" % self.source - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) _msg = "The following options will be used:" - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " config=%s, import-config=%s" % (config_path, self.import_config_path) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) if options.date: _msg = " date=%s" % options.date else: # we must have --from and --to _msg = " from=%s, to=%s" % (options.date_from, options.date_to) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " dry-run=%s, calc_missing=%s, ignore_invalid_data=%s" % (self.dry_run, self.calc_missing, self.ignore_invalid_data) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " tranche=%s, interval=%s" % (self.tranche, self.interval) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " UV=%s, radiation=%s" % (self.UV_sensor, self.solar_sensor) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = "Using database binding '%s', which is bound to database '%s'" % (self.db_binding_wx, self.dbm.database_name) - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) _msg = "Destination table '%s' unit system is '%#04x' (%s)." % (self.dbm.table_name, self.archive_unit_sys, unit_nicknames[self.archive_unit_sys]) - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) if self.calc_missing: print("Missing derived observations will be calculated.") if not self.UV_sensor: diff --git a/bin/weeimport/wdimport.py b/bin/weeimport/wdimport.py index bee7cff1..98f3e0ca 100644 --- a/bin/weeimport/wdimport.py +++ b/bin/weeimport/wdimport.py @@ -17,9 +17,9 @@ import collections import csv import datetime import glob +import logging import operator import os -import syslog import time # WeeWX imports @@ -30,6 +30,7 @@ import weewx from weeutil.weeutil import timestamp_to_string from weewx.units import unit_nicknames +log = logging.getLogger(__name__) # ============================================================================ # class WDSource @@ -183,11 +184,10 @@ class WDSource(weeimport.Source): 'hum7': {'units': 'percent', 'map_to': 'extraHumid7'} } - def __init__(self, config_dict, config_path, wd_config_dict, import_config_path, options, log): + def __init__(self, config_dict, config_path, wd_config_dict, import_config_path, options): # call our parents __init__ - super(WDSource, self).__init__(config_dict, wd_config_dict, - options, log) + super(WDSource, self).__init__(config_dict, wd_config_dict, options) # save the import config path self.import_config_path = import_config_path @@ -397,48 +397,71 @@ class WDSource(weeimport.Source): # tell the user/log what we intend to do _msg = "Weather Display monthly log files in the '%s' directory will be imported" % self.source - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) _msg = "The following options will be used:" - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " config=%s, import-config=%s" % (config_path, self.import_config_path) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) if options.date: _msg = " date=%s" % options.date else: # we must have --from and --to _msg = " from=%s, to=%s" % (options.date_from, options.date_to) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " dry-run=%s, calc_missing=%s, ignore_invalid_data=%s" % (self.dry_run, self.calc_missing, self.ignore_invalid_data) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) if log_unit_sys is not None and log_unit_sys.upper() in ['METRIC', 'US']: # valid unit system specified _msg = " monthly logs are in %s units" % log_unit_sys.upper() - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) else: # group units specified _msg = " monthly logs use the following units:" - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " temperature=%s pressure=%s" % (temp_u, press_u) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " rain=%s speed=%s" % (rain_u, speed_u) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " tranche=%s, interval=%s" % (self.tranche, self.interval) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " UV=%s, radiation=%s ignore extreme temperature and humidity=%s" % (self.UV_sensor, self.solar_sensor, self.ignore_extreme_temp_hum) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = "Using database binding '%s', which is bound to database '%s'" % (self.db_binding_wx, self.dbm.database_name) - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) _msg = "Destination table '%s' unit system is '%#04x' (%s)." % (self.dbm.table_name, self.archive_unit_sys, unit_nicknames[self.archive_unit_sys]) - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) if self.calc_missing: print("Missing derived observations will be calculated.") if not self.UV_sensor: @@ -512,7 +535,8 @@ class WDSource(weeimport.Source): _msg = "Unexpected number of columns found in '%s': %s v %s" % (_fn, len(_row.split(_del)), len(self.logs[lg]['fields'])) - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) # make sure we have full stops as decimal points _clean_data.append(_row.replace(self.decimal, '.')) diff --git a/bin/weeimport/weeimport.py b/bin/weeimport/weeimport.py index 19dc5c7f..53b47f88 100644 --- a/bin/weeimport/weeimport.py +++ b/bin/weeimport/weeimport.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2009-2016 Tom Keffer and +# Copyright (c) 2009-2019 Tom Keffer and # Gary Roderick # # See the file LICENSE.txt for your full rights. @@ -15,9 +15,10 @@ from __future__ import absolute_import # Python imports import datetime +import logging +import numbers import re import sys -import syslog import time from datetime import datetime as dt @@ -32,10 +33,12 @@ import weewx.wxservices from weewx.manager import open_manager_with_config from weewx.units import unit_constants, unit_nicknames, convertStd, to_std_system, ValueTuple -from weeutil.weeutil import timestamp_to_string, option_as_list, to_int, tobool, _get_object +from weeutil.weeutil import timestamp_to_string, option_as_list, to_int, tobool, get_object + +log = logging.getLogger(__name__) # List of sources we support -SUPPORTED_SOURCES = ['CSV', 'WU', 'Cumulus'] +SUPPORTED_SOURCES = ['CSV', 'WU', 'Cumulus', 'WD'] # Minimum requirements in any explicit or implicit WeeWX field-to-import field # map @@ -114,7 +117,7 @@ class Source(object): # reg expression to match any HTML tag of the form <...> _tags = re.compile(r'\<.*\>') - def __init__(self, config_dict, import_config_dict, options, log): + def __init__(self, config_dict, import_config_dict, options): """A generic initialisation. Set some realistic default values for options read from the import @@ -123,8 +126,8 @@ class Source(object): know what records to import. """ - # give our source object some logging abilities - self.wlog = log +# # give our source object some logging abilities +# self.wlog = log # save our WeeWX config dict self.config_dict = config_dict @@ -198,6 +201,7 @@ class Source(object): # Process our command line options self.dry_run = options.dry_run self.verbose = options.verbose + self.suppress = options.suppress # By processing any --date, --from and --to options we need to derive # self.first_ts and self.last_ts; the earliest and latest (inclusive) @@ -216,7 +220,7 @@ class Source(object): _msg = "Invalid --date option specified." raise WeeImportOptionError(_msg) else: - # we have a valid date so do soem date arithmetic + # we have a valid date so do some date arithmetic _last_dt = _first_dt + datetime.timedelta(days=1) self.first_ts = time.mktime(_first_dt.timetuple()) self.last_ts = time.mktime(_last_dt.timetuple()) @@ -293,7 +297,7 @@ class Source(object): self.period_duplicates = set() @staticmethod - def sourceFactory(options, args, log): + def sourceFactory(options, args): """Factory to produce a Source object. Returns an appropriate object depending on the source type. Raises a @@ -302,9 +306,7 @@ class Source(object): # get some key WeeWX parameters # first the config dict to use - config_path, config_dict = weecfg.read_config(None, - args, - file_name=options.config_path) + config_path, config_dict = weecfg.read_config(options.config_path, args) # get wee_import config dict if it exists import_config_path, import_config_dict = weecfg.read_config(None, args, @@ -329,12 +331,11 @@ class Source(object): module_class = '.'.join(['weeimport', source.lower() + 'import', source + 'Source']) - return _get_object(module_class)(config_dict, - config_path, - import_config_dict.get(source, {}), - import_config_path, - options, - log) + return get_object(module_class)(config_dict, + config_path, + import_config_dict.get(source, {}), + import_config_path, + options) def run(self): """Main entry point for importing from an external source. @@ -374,25 +375,40 @@ class Source(object): # get the raw data _msg = 'Obtaining raw import data for period %d...' % self.period_no - self.wlog.verboselog(syslog.LOG_INFO, _msg) + if self.verbose: + print(_msg) + log.info(_msg) _raw_data = self.getRawData(period) _msg = 'Raw import data read successfully for period %d.' % self.period_no - self.wlog.verboselog(syslog.LOG_INFO, _msg) + if self.verbose: + print(_msg) + log.info(_msg) # map the raw data to a WeeWX archive compatible dictionary _msg = 'Mapping raw import data for period %d...' % self.period_no - self.wlog.verboselog(syslog.LOG_INFO, _msg) + if self.verbose: + print(_msg) + log.info(_msg) _mapped_data = self.mapRawData(_raw_data, self.archive_unit_sys) _msg = 'Raw import data mapped successfully for period %d.' % self.period_no - self.wlog.verboselog(syslog.LOG_INFO, _msg) + if self.verbose: + print(_msg) + log.info(_msg) # save the mapped data to archive - _msg = 'Saving mapped data to archive for period %d...' % self.period_no - self.wlog.verboselog(syslog.LOG_INFO, _msg) + # first advise the user and log, but only if its not a dry run + if not self.dry_run: + _msg = 'Saving mapped data to archive for period %d...' % self.period_no + if self.verbose: + print(_msg) + log.info(_msg) self.saveToArchive(archive, _mapped_data) - _msg = 'Mapped data saved to archive successfully for period %d.' % self.period_no - self.wlog.verboselog(syslog.LOG_INFO, _msg) - + # advise the user and log, but only if its not a dry run + if not self.dry_run: + _msg = 'Mapped data saved to archive successfully for period %d.' % self.period_no + if self.verbose: + print(_msg) + log.info(_msg) # increment our period counter self.period_no += 1 # Provide some summary info now that we have finished the import. @@ -401,37 +417,48 @@ class Source(object): if self.total_rec_proc == 0: # nothing imported so say so _msg = 'No records were identified for import. Exiting. Nothing done.' - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) else: # we imported something total_rec = self.total_rec_proc + self.total_duplicate_rec if self.dry_run: # but it was a dry run - self.wlog.printlog(syslog.LOG_INFO, "Finished dry run import") + _msg = "Finished dry run import" + print(_msg) + log.info(_msg) _msg = "%d records were processed and %d unique records would "\ "have been imported." % (total_rec, self.total_rec_proc) - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) if self.total_duplicate_rec > 1: _msg = "%d duplicate records were ignored." % self.total_duplicate_rec - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) elif self.total_duplicate_rec == 1: - self.wlog.printlog(syslog.LOG_INFO, - "1 duplicate record was ignored.") + _msg = "1 duplicate record was ignored." + print(_msg) + log.info(_msg) else: # something should have been saved to database - self.wlog.printlog(syslog.LOG_INFO, "Finished import") + _msg = "Finished import" + print(_msg) + log.info(_msg) _msg = "%d records were processed and %d unique records " \ "imported in %.2f seconds." % (total_rec, self.total_rec_proc, self.tdiff) - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) if self.total_duplicate_rec > 1: _msg = "%d duplicate records were ignored." % self.total_duplicate_rec - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) elif self.total_duplicate_rec == 1: - self.wlog.printlog(syslog.LOG_INFO, - "1 duplicate record was ignored.") + _msg = "1 duplicate record was ignored." + print(_msg) + log.info(_msg) print("Those records with a timestamp already in the archive will not have been") print("imported. Confirm successful import in the WeeWX log file.") @@ -573,9 +600,8 @@ class Source(object): # will use _msg = "The following imported field-to-WeeWX field map will be used:" if self.verbose: - self.wlog.verboselog(syslog.LOG_INFO, _msg) - else: - self.wlog.logonly(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) for _key, _val in six.iteritems(_map): if 'field_name' in _val: _units_msg = "" @@ -585,9 +611,8 @@ class Source(object): _units_msg, _key) if self.verbose: - self.wlog.verboselog(syslog.LOG_INFO, _msg) - else: - self.wlog.logonly(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) else: # no [[FieldMap]] stanza and no _header_map so raise an error as we # don't know what to map @@ -635,7 +660,7 @@ class Source(object): _msg = "Field '%s' not found in source data." % self.map['dateTime']['field_name'] raise WeeImportFieldError(_msg) # now process the raw date time data - if _raw_dateTime.isdigit(): + if isinstance(_raw_dateTime, numbers.Number) or _raw_dateTime.isdigit(): # Our dateTime is a number, is it a timestamp already? # Try to use it and catch the error if there is one and # raise it higher. @@ -647,8 +672,8 @@ class Source(object): _raw_dateTime) raise ValueError(_msg) else: - # it's a string so try to parse it and catch the error if - # there is one and raise it higher + # it's a non-numeric string so try to parse it and catch + # the error if there is one and raise it higher try: _datetm = time.strptime(_raw_dateTime, self.raw_datetime_format) @@ -661,7 +686,7 @@ class Source(object): # if we have a timeframe of concern does our record fall within # it if (self.first_ts is None and self.last_ts is None) or \ - self.first_ts <= _rec_dateTime <= self.last_ts: + self.first_ts < _rec_dateTime <= self.last_ts: # we have no timeframe or if we do it falls within it so # save the dateTime _rec['dateTime'] = _rec_dateTime @@ -735,6 +760,20 @@ class Source(object): # can't catch the error try: _temp = float(_row[self.map[_field]['field_name']].strip()) + except AttributeError: + # the data has not strip() attribute so chances are + # it's a number already + if isinstance(_row[self.map[_field]['field_name']], numbers.Number): + _temp = _row[self.map[_field]['field_name']] + elif _row[self.map[_field]['field_name']] is None: + _temp = None + else: + # we raise the error + _msg = "%s: cannot convert '%s' to float at " \ + "timestamp '%s'." % (_field, + _row[self.map[_field]['field_name']], + timestamp_to_string(_rec['dateTime'])) + raise ValueError(_msg) except TypeError: # perhaps we have a None, so return None for our field _temp = None @@ -813,19 +852,19 @@ class Source(object): if self.map[_field]['field_name'] not in _warned: _msg = "Warning: Import field '%s' is mapped to WeeWX " \ "field '%s' but the" % (self.map[_field]['field_name'], - _field) - self.wlog.printlog(syslog.LOG_INFO, - _msg, - can_suppress=True) + _field) + if not self.suppress: + print(_msg) + log.info(_msg) _msg = " import field '%s' could not be found " \ "in one or more records." % self.map[_field]['field_name'] - self.wlog.printlog(syslog.LOG_INFO, - _msg, - can_suppress=True) + if not self.suppress: + print(_msg) + log.info(_msg) _msg = " WeeWX field '%s' will be set to 'None' in these records." % _field - self.wlog.printlog(syslog.LOG_INFO, - _msg, - can_suppress=True) + if not self.suppress: + print(_msg) + log.info(_msg) # make sure we do this warning once only _warned.append(self.map[_field]['field_name']) # if we have a mapped field for a unit system with a valid value, @@ -865,7 +904,8 @@ class Source(object): # we had more than one unique value for interval, warn the user _msg = "Warning: Records to be imported contain multiple " \ "different 'interval' values." - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) print(" This may mean the imported data is missing some records and it may lead") print(" to data integrity issues. If the raw data has a known, fixed interval") print(" value setting the relevant 'interval' setting in wee_import config to") @@ -887,15 +927,20 @@ class Source(object): print("Import aborted by user. No records saved to archive.") _msg = "User chose to abort import. %d records were processed. " \ "Exiting." % self.total_rec_proc - self.wlog.logonly(syslog.LOG_INFO, _msg) + log.info(_msg) raise SystemExit('Exiting. Nothing done.') - self.wlog.verboselog(syslog.LOG_INFO, - "Mapped %d records." % len(_records)) + _msg = "Mapped %d records." % len(_records) + if self.verbose: + print(_msg) + log.info(_msg) # the user wants to continue or we have only one unique value for # interval so return the records return _records else: - self.wlog.verboselog(syslog.LOG_INFO, "Mapped 0 records.") + _msg = "Mapped 0 records." + if self.verbose: + print(_msg) + log.info(_msg) # we have no records to return so return None return None @@ -945,16 +990,16 @@ class Source(object): if _interval < 0: # so raise an error _msg = "Cannot derive 'interval' for record timestamp: %s." % timestamp_to_string(current_ts) - self.wlog.printlog(syslog.LOG_INFO, _msg) - raise ValueError( - "Raw data is not in ascending date time order.") + print(_msg) + log.info(_msg) + raise ValueError("Raw data is not in ascending date time order.") except TypeError: _interval = None return _interval else: # we don't know what to do so raise an error - raise ValueError( - "Cannot derive 'interval'. Unknown 'interval' setting in %s." % self.import_config_path) + _msg = "Cannot derive 'interval'. Unknown 'interval' setting in %s." % self.import_config_path + raise ValueError(_msg) @staticmethod def getRain(last_rain, current_rain): @@ -1101,7 +1146,7 @@ class Source(object): # tell the user what we have done _msg = "Unique records processed: %d; Last timestamp: %s\r" % (nrecs, timestamp_to_string(_final_rec['dateTime'])) - print(_msg, end=' ', file=sys.stdout) + print(_msg, end='', file=sys.stdout) sys.stdout.flush() _tranche = [] # we have processed all records but do we have any records left @@ -1118,7 +1163,7 @@ class Source(object): # tell the user what we have done _msg = "Unique records processed: %d; Last timestamp: %s\r" % (nrecs, timestamp_to_string(_final_rec['dateTime'])) - print(_msg, end=' ', file=sys.stdout) + print(_msg, end='', file=sys.stdout) print() sys.stdout.flush() # update our counts @@ -1133,11 +1178,14 @@ class Source(object): else: _msg = " %d duplicate records were identified in period %d:" % (num_duplicates, self.period_no) - self.wlog.printlog(syslog.LOG_INFO, _msg, can_suppress=True) + if not self.suppress: + print(_msg) + log.info(_msg) for ts in sorted(self.period_duplicates): _msg = " %s" % timestamp_to_string(ts) - self.wlog.printlog(syslog.LOG_INFO, _msg, - can_suppress=True) + if not self.suppress: + print(_msg) + log.info(_msg) # add the period duplicates to the overall duplicates self.duplicates |= self.period_duplicates # reset the period duplicates @@ -1145,8 +1193,9 @@ class Source(object): elif self.ans == 'n': # user does not want to import so display a message and then # ask to exit - self.wlog.printlog(syslog.LOG_INFO, - 'User chose not to import records. Exiting. Nothing done.') + _msg = "User chose not to import records. Exiting. Nothing done." + print(_msg) + log.info(_msg) raise SystemExit('Exiting. Nothing done.') else: # we have no records to import, advise the user but what we say @@ -1163,74 +1212,6 @@ class Source(object): self.tdiff = time.time() - self.t1 -# ============================================================================ -# class WeeImportLog -# ============================================================================ - - -class WeeImportLog(object): - """Class to handle wee_import logging. - - This class provides a wrapper around the python syslog module to handle - wee_import logging requirements. The --log=- command line option disables - log output otherwise log output is sent to the same log used by WeeWX. - """ - - def __init__(self, opt_logging, opt_verbose, opt_suppress, opt_dry_run): - """Initialise our log environment.""" - - # first check if we are turning off log to file or not - if opt_logging: - log_bool = opt_logging.strip() == '-' - else: - log_bool = False - # Flag to indicate whether we are logging to file or not. Log to file - # every time except when logging is explicitly turned off on the - # command line or its a dry run. - self.log = not (opt_dry_run or log_bool) - # if we are logging then setup our syslog environment - # if --verbose we log up to syslog.LOG_DEBUG - # otherwise just log up to syslog.LOG_INFO - if self.log: - syslog.openlog('wee_import', - logoption=syslog.LOG_PID | syslog.LOG_CONS) - if opt_verbose: - syslog.setlogmask(syslog.LOG_UPTO(syslog.LOG_DEBUG)) - else: - syslog.setlogmask(syslog.LOG_UPTO(syslog.LOG_INFO)) - # logging by other modules (eg WxCalculate) does not use WeeImportLog - # but we can disable most logging by raising the log priority if its a - # dry run - if opt_dry_run: - syslog.setlogmask(syslog.LOG_UPTO(syslog.LOG_CRIT)) - # keep opt_verbose for later - self.verbose = opt_verbose - self.suppress = opt_suppress - - def logonly(self, level, message): - """Log to file only.""" - - # are we logging ? - if self.log: - # add a little preamble to say this is wee_import - _message = 'wee_import: ' + message - syslog.syslog(level, _message) - - def printlog(self, level, message, can_suppress=False): - """Print to screen and log to file.""" - - if not(can_suppress and self.suppress): - print(message) - self.logonly(level, message) - - def verboselog(self, level, message): - """Print to screen if --verbose and log to file always.""" - - if self.verbose: - print(message) - self.logonly(level, message) - - # ============================================================================ # Utility functions # ============================================================================ @@ -1247,4 +1228,3 @@ def get_binding(config_dict): else: db_binding_wx = None return db_binding_wx - diff --git a/bin/weeimport/wuimport.py b/bin/weeimport/wuimport.py index 8014d332..c2189c5b 100644 --- a/bin/weeimport/wuimport.py +++ b/bin/weeimport/wuimport.py @@ -1,24 +1,31 @@ # -# Copyright (c) 2009-2016 Tom Keffer and +# Copyright (c) 2009-2019 Tom Keffer and # Gary Roderick # # See the file LICENSE.txt for your full rights. # -"""Module to interact with Weather Underground PWS history and import raw -observational data for use with weeimport. +"""Module to interact with the Weather Underground API and obtain raw +observational data for use with wee_import. """ # Python imports from __future__ import with_statement from __future__ import absolute_import from __future__ import print_function -import csv + import datetime +import gzip +import json +import logging +import numbers import socket -import syslog +import sys + from datetime import datetime as dt +# python3 compatibility shims +import six from six.moves import urllib # WeeWX imports @@ -28,6 +35,7 @@ import weewx from weeutil.weeutil import timestamp_to_string, option_as_list, startOfDay from weewx.units import unit_nicknames +log = logging.getLogger(__name__) # ============================================================================ # class WUSource @@ -35,53 +43,43 @@ from weewx.units import unit_nicknames class WUSource(weeimport.Source): - """Class to interact with the Weather Underground. + """Class to interact with the Weather Underground API. - Uses WXDailyHistory.asp call via http to obtain historical daily weather - observations for a given PWS. WU uses geolocation of the requester to - determine the units to use when providing historical PWS records. Fields - that can be provided with multiple possible units have the units in use - appended to the returned field name. This means that a request for a user - in a given location for historical data from a given station may well - return different results to the same request being made from another - location. This requires a mechanism to both determine the units in use from - returned data as well as mapping a number of different possible field names - to a given WeeWX archive field name. + Uses PWS history call via http to obtain historical daily weather + observations for a given PWS. Unlike the previous WU import module the use + of the API requires an API key. """ # Dict to map all possible WU field names to WeeWX archive field names and # units - _header_map = {'Time': {'units': 'unix_epoch', 'map_to': 'dateTime'}, - 'TemperatureC': {'units': 'degree_C', 'map_to': 'outTemp'}, - 'TemperatureF': {'units': 'degree_F', 'map_to': 'outTemp'}, - 'DewpointC': {'units': 'degree_C', 'map_to': 'dewpoint'}, - 'DewpointF': {'units': 'degree_F', 'map_to': 'dewpoint'}, - 'PressurehPa': {'units': 'hPa', 'map_to': 'barometer'}, - 'PressureIn': {'units': 'inHg', 'map_to': 'barometer'}, - 'WindDirectionDegrees': {'units': 'degree_compass', - 'map_to': 'windDir'}, - 'WindSpeedKMH': {'units': 'km_per_hour', + _header_map = {'epoch': {'units': 'unix_epoch', 'map_to': 'dateTime'}, + 'tempAvg': {'units': 'degree_F', 'map_to': 'outTemp'}, + 'dewptAvg': {'units': 'degree_F', 'map_to': 'dewpoint'}, + 'heatindexAvg': {'units': 'degree_F', 'map_to': 'heatindex'}, + 'windchillAvg': {'units': 'degree_F', 'map_to': 'windchill'}, + 'pressureAvg': {'units': 'inHg', 'map_to': 'barometer'}, + 'winddirAvg': {'units': 'degree_compass', + 'map_to': 'windDir'}, + 'windspeedAvg': {'units': 'mile_per_hour', 'map_to': 'windSpeed'}, - 'WindSpeedMPH': {'units': 'mile_per_hour', - 'map_to': 'windSpeed'}, - 'WindSpeedGustKMH': {'units': 'km_per_hour', - 'map_to': 'windGust'}, - 'WindSpeedGustMPH': {'units': 'mile_per_hour', - 'map_to': 'windGust'}, - 'Humidity': {'units': 'percent', 'map_to': 'outHumidity'}, - 'dailyrainMM': {'units': 'mm', 'map_to': 'rain'}, - 'dailyrainin': {'units': 'inch', 'map_to': 'rain'}, - 'SolarRadiationWatts/m^2': {'units': 'watt_per_meter_squared', - 'map_to': 'radiation'} + 'windgustHigh': {'units': 'mile_per_hour', + 'map_to': 'windGust'}, + 'humidityAvg': {'units': 'percent', 'map_to': 'outHumidity'}, + 'precipTotal': {'units': 'inch', 'map_to': 'rain'}, + 'precipRate': {'units': 'inch_per_hour', + 'map_to': 'rainRate'}, + 'solarRadiationHigh': {'units': 'watt_per_meter_squared', + 'map_to': 'radiation'}, + 'uvHigh': {'units': 'uv_index', 'map_to': 'UV'} } + _extras = ['pressureMin', 'pressureMax'] - def __init__(self, config_dict, config_path, wu_config_dict, import_config_path, options, log): + def __init__(self, config_dict, config_path, wu_config_dict, import_config_path, options): # call our parents __init__ super(WUSource, self).__init__(config_dict, wu_config_dict, - options, - log) + options) # save our import config path self.import_config_path = import_config_path @@ -92,7 +90,15 @@ class WUSource(weeimport.Source): try: self.station_id = wu_config_dict['station_id'] except KeyError: - raise weewx.ViolatedPrecondition("Weather Underground station ID not specified in '%s'." % import_config_path) + _msg = "Weather Underground station ID not specified in '%s'." % import_config_path + raise weewx.ViolatedPrecondition(_msg) + + # get our WU API key + try: + self.api_key = wu_config_dict['api_key'] + except KeyError: + _msg = "Weather Underground API key not specified in '%s'." % import_config_path + raise weewx.ViolatedPrecondition(_msg) # wind dir bounds _wind_direction = option_as_list(wu_config_dict.get('wind_direction', @@ -130,12 +136,17 @@ class WUSource(weeimport.Source): # tell the user/log what we intend to do _msg = "Observation history for Weather Underground station '%s' will be imported." % self.station_id - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) _msg = "The following options will be used:" - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " config=%s, import-config=%s" % (config_path, self.import_config_path) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) if options.date: _msg = " station=%s, date=%s" % (self.station_id, options.date) else: @@ -143,22 +154,35 @@ class WUSource(weeimport.Source): _msg = " station=%s, from=%s, to=%s" % (self.station_id, options.date_from, options.date_to) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) + _obf_api_key_msg = '='.join([' apiKey', + '*'*(len(self.api_key) - 4) + self.api_key[-4:]]) + if self.verbose: + print(_obf_api_key_msg) + log.debug(_obf_api_key_msg) _msg = " dry-run=%s, calc_missing=%s, ignore_invalid_data=%s" % (self.dry_run, self.calc_missing, self.ignore_invalid_data) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = " tranche=%s, interval=%s, wind_direction=%s" % (self.tranche, self.interval, self.wind_dir) - self.wlog.verboselog(syslog.LOG_DEBUG, _msg) + if self.verbose: + print(_msg) + log.debug(_msg) _msg = "Using database binding '%s', which is bound to database '%s'" % (self.db_binding_wx, self.dbm.database_name) - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) _msg = "Destination table '%s' unit system is '%#04x' (%s)." % (self.dbm.table_name, self.archive_unit_sys, unit_nicknames[self.archive_unit_sys]) - self.wlog.printlog(syslog.LOG_INFO, _msg) + print(_msg) + log.info(_msg) if self.calc_missing: print("Missing derived observations will be calculated.") if options.date or options.date_from: @@ -171,16 +195,9 @@ class WUSource(weeimport.Source): """Get raw observation data and construct a map from WU to WeeWX archive fields. - Obtain raw observational data from WU using a http WXDailyHistory - request. This raw data needs to be cleaned of unnecessary - characters/codes and an iterable returned. - - Since WU geolocates any http request we do not know what units our WU - data will use until we actually receive the data. A further - complication is that WU appends the unit abbreviation to the end of the - returned field name for fields that can have different units. So once - we have the data have received the response we need to determine the - units and create a dict to map the WU fields to WeeWX archive fields. + Obtain raw observational data from WU via the WU API. This raw data + needs some basic processing to place it in a format suitable for + wee_import to ingest. Input parameters: @@ -188,48 +205,129 @@ class WUSource(weeimport.Source): which raw obs data will be read. """ - # the date for which we want the WU data is held in a datetime object, we need to convert it to a timetuple - date_tt = period.timetuple() - # construct our URL using station ID and day, month, year - _url = "http://www.wunderground.com/weatherstation/WXDailyHistory.asp?ID=%s&" \ - "month=%d&day=%d&year=%d&format=1" % (self.station_id, - date_tt[1], - date_tt[2], - date_tt[0]) - # hit the WU site, wrap in a try..except so we can catch any errors + # the date for which we want the WU data is held in a datetime object, + # we need to convert it to a timetuple + day_tt = period.timetuple() + # and then format the date suitable for use in the WU API URL + day = "%4d%02d%02d" % (day_tt.tm_year, + day_tt.tm_mon, + day_tt.tm_mday) + + # construct the URL to be used + url = "https://api.weather.com/v2/pws/history/all?" \ + "stationId=%s&format=json&units=e&numericPrecision=decimal&date=%s&apiKey=%s" \ + % (self.station_id, day, self.api_key) + # create a Request object using the constructed URL + request_obj = urllib.request.Request(url) + # add necessary headers + request_obj.add_header('Cache-Control', 'no-cache') + request_obj.add_header('Accept-Encoding', 'gzip') + # hit the API wrapping in a try..except to catch any errors try: - _wudata = urllib.request.urlopen(_url) + response = urllib.request.urlopen(request_obj) except urllib.error.URLError as e: - self.wlog.printlog(syslog.LOG_ERR, - "Unable to open Weather Underground station %s" % self.station_id) - self.wlog.printlog(syslog.LOG_ERR, " **** %s" % e) + print("Unable to open Weather Underground station " + self.station_id, " or ", e, file=sys.stderr) + log.error("Unable to open Weather Underground station %s or %s" % (self.station_id, e)) raise except socket.timeout as e: - self.wlog.printlog(syslog.LOG_ERR, - "Socket timeout for Weather Underground station %s" % self.station_id) - self.wlog.printlog(syslog.LOG_ERR, " **** %s" % e) + print("Socket timeout for Weather Underground station " + self.station_id, file=sys.stderr) + log.error("Socket timeout for Weather Underground station %s" % self.station_id) + print(" **** %s" % e, file=sys.stderr) + log.error(" **** %s" % e) raise + # check the response code and raise an exception if there was an error + if hasattr(response, 'code') and response.code != 200: + if response.code == 204: + raise IOError("Probably a bad station ID or invalid date") + else: + raise IOError("Bad response code returned: %d" % response.code) - # because the data comes back with lots of HTML tags and whitespace we - # need a bit of logic to clean it up. - _cleanWUdata = [] - for _row in _wudata: - # Convert from byte-string to string - _urow = _row.decode('ascii') - # get rid of any HTML tags - _line = ''.join(WUSource._tags.split(_urow)) - # get rid of any blank lines - if _line != "\n": - # save what's left - _cleanWUdata.append(_line) + # The WU API says that compression is required, but let's be prepared + # if compression is not used + if response.info().get('Content-Encoding') == 'gzip': + buf = six.StringIO(response.read()) + f = gzip.GzipFile(fileobj=buf) + _raw_data = f.read() + # decode the json data + _raw_decoded_data = json.loads(_raw_data) + else: + _raw_data = response + # decode the json data + _raw_decoded_data = json.load(_raw_data) - # now create a dictionary CSV reader, the first line is used as keys to - # the dictionary - _reader = csv.DictReader(_cleanWUdata) + # The raw WU response is not suitable to return as is, we need to + # return an iterable that provides a dict of observational data for each + # available timestamp. In this case a list of dicts is appropriate. + + # initialise a list of dicts + wu_data = [] + # first check we have some observational data + if 'observations' in _raw_decoded_data: + # iterate over each record in the WU data + for record in _raw_decoded_data['observations']: + # initialise a dict to hold the resulting data for this record + _flat_record = {} + # iterate over each WU API response field that we can use + _fields = self._header_map.keys() + self._extras + for obs in _fields: + # The field may appear as a top level field in the WU data + # or it may be embedded in the dict in the WU data that + # contains variable unit data. Look in the top level record + # first. If its there uses it, otherwise look in the + # variable units dict. If it can't be fond then skip it. + if obs in record: + # it's in the top level record + _flat_record[obs] = record[obs] + else: + # it's not in the top level so look in the variable + # units dict + try: + _flat_record[obs] = record['imperial'][obs] + except KeyError: + # it's not there so skip it + pass + if obs == 'epoch': + try: + _date = datetime.date.fromtimestamp(_flat_record['epoch']) + except ValueError: + _flat_record['epoch'] = _flat_record['epoch'] // 1000 + # WU in its wisdom provides min and max pressure but no average + # pressure (unlike other obs) so we need to calculate it. If + # both min and max are numeric use a simple average of the two + # (they will likely be the same anyway for non-RF stations). + # Otherwise use max if numeric, then use min if numeric + # otherwise skip. + self.calc_pressure(_flat_record) + # append the data dict for the current record to the list of + # dicts for this period + wu_data.append(_flat_record) # finally, get our database-source mapping - self.map = self.parseMap('WU', _reader, self.wu_config_dict) - # return our dict reader - return _reader + self.map = self.parseMap('WU', wu_data, self.wu_config_dict) + # return our dict + return wu_data + + @staticmethod + def calc_pressure(record): + """Calculate pressureAvg field. + + The WU API provides min and max pressure but no average pressure. + Calculate an average pressure to be used in the import using one of the + following (in order): + + 1. simple average of min and max pressure + 2. max pressure + 3. min pressure + 4. None + """ + + if 'pressureMin' in record and 'pressureMax' in record and isinstance(record['pressureMin'], numbers.Number) and isinstance(record['pressureMax'], numbers.Number): + record['pressureAvg'] = (record['pressureMin'] + record['pressureMax'])/2.0 + elif 'pressureMax' in record and isinstance(record['pressureMax'], numbers.Number): + record['pressureAvg'] = record['pressureMax'] + elif 'pressureMin' in record and isinstance(record['pressureMin'], numbers.Number): + record['pressureAvg'] = record['pressureMin'] + elif 'pressureMin' in record or 'pressureMax' in record: + record['pressureAvg'] = None def period_generator(self): """Generator function yielding a sequence of datetime objects. diff --git a/bin/weeutil/weeutil.py b/bin/weeutil/weeutil.py index b93050ef..4301f40c 100644 --- a/bin/weeutil/weeutil.py +++ b/bin/weeutil/weeutil.py @@ -389,7 +389,9 @@ def archiveHoursAgoSpan(time_ts, hours_ago=0, grace=1): def archiveSpanSpan(time_ts, time_delta=0, hour_delta=0, day_delta=0, week_delta=0, month_delta=0, year_delta=0): """ Returns a TimeSpan for the last xxx seconds where xxx equals time_delta sec + hour_delta hours + day_delta days + week_delta weeks + month_delta months + year_delta years - Note: For month_delta, 1 month = 30 days, For year_delta, 1 year = 365 days + + NOTE: Use of month_delta and year_delta is deprecated. + See issue #436 (https://github.com/weewx/weewx/issues/436) Example: >>> os.environ['TZ'] = 'Australia/Brisbane' @@ -1095,7 +1097,7 @@ def latlon_string(ll, hemi, which, format_list=None): hemi[0] if ll >= 0 else hemi[1]) -def _get_object(module_class): +def get_object(module_class): """Given a string with a module class name, it imports and returns the class.""" # Split the path into its parts parts = module_class.split('.') @@ -1115,6 +1117,10 @@ def _get_object(module_class): return mod +# For backwards compatibility: +_get_object = get_object + + class GenWithPeek(object): """Generator object which allows a peek at the next object to be returned. diff --git a/bin/weewx/__init__.py b/bin/weewx/__init__.py index 8c302308..90cacc7d 100644 --- a/bin/weewx/__init__.py +++ b/bin/weewx/__init__.py @@ -7,7 +7,7 @@ from __future__ import absolute_import import time -__version__="4.0.0a8" +__version__="4.0.0a9" # Holds the program launch time in unix epoch seconds: # Useful for calculating 'uptime.' diff --git a/bin/weewx/cheetahgenerator.py b/bin/weewx/cheetahgenerator.py index ce39cabe..81f3e8e9 100644 --- a/bin/weewx/cheetahgenerator.py +++ b/bin/weewx/cheetahgenerator.py @@ -186,7 +186,7 @@ class CheetahGenerator(weewx.reportengine.ReportGenerator): x = c.strip() if x: # Get the class - class_ = weeutil.weeutil._get_object(x) + class_ = weeutil.weeutil.get_object(x) # Then instantiate the class, passing self as the sole argument self.search_list_objs.append(class_(self)) diff --git a/bin/weewx/engine.py b/bin/weewx/engine.py index 97b300db..b1d31c92 100644 --- a/bin/weewx/engine.py +++ b/bin/weewx/engine.py @@ -142,7 +142,7 @@ class StdEngine(object): # passing self and the configuration dictionary as the # arguments: log.debug("Loading service %s", svc) - self.service_obj.append(weeutil.weeutil._get_object(svc)(self, config_dict)) + self.service_obj.append(weeutil.weeutil.get_object(svc)(self, config_dict)) log.debug("Finished loading service %s", svc) except Exception: # An exception occurred. Shut down any running services, then @@ -949,8 +949,8 @@ def main(options, args, engine_class=StdEngine): except Terminate: log.info("Terminating weewx version %s", weewx.__version__) weeutil.logger.log_traceback(log.info, " **** ") - # Reraise the exception (this should cause the program to exit) - raise + signal.signal(signal.SIGTERM, signal.SIG_DFL) + os.kill(0, signal.SIGTERM) # Catch any keyboard interrupts and log them except KeyboardInterrupt: diff --git a/bin/weewx/manager.py b/bin/weewx/manager.py index ee6f2b04..187bbf09 100644 --- a/bin/weewx/manager.py +++ b/bin/weewx/manager.py @@ -1015,7 +1015,7 @@ def get_manager_dict_from_config(config_dict, data_binding, manager_dict['schema'] = [(col_name, manager_dict['schema'][col_name]) for col_name in manager_dict['schema']] else: # Schema is a string, with the name of the schema object - manager_dict['schema'] = weeutil.weeutil._get_object(schema_name) + manager_dict['schema'] = weeutil.weeutil.get_object(schema_name) return manager_dict @@ -1031,7 +1031,7 @@ def get_manager_dict(bindings_dict, databases_dict, data_binding, def open_manager(manager_dict, initialize=False): - manager_cls = weeutil.weeutil._get_object(manager_dict['manager']) + manager_cls = weeutil.weeutil.get_object(manager_dict['manager']) if initialize: return manager_cls.open_with_create(manager_dict['database_dict'], manager_dict['table_name'], @@ -1093,7 +1093,7 @@ def drop_database_with_config(config_dict, data_binding, def show_progress(nrec, last_time): """Utility function to show our progress while backfilling""" print("Records processed: %d; Last date: %s\r" % - (nrec, weeutil.weeutil.timestamp_to_string(last_time)), end=' ', file=sys.stdout) + (nrec, weeutil.weeutil.timestamp_to_string(last_time)), end='', file=sys.stdout) sys.stdout.flush() diff --git a/bin/weewx/reportengine.py b/bin/weewx/reportengine.py index bb1d1381..5f7cff7d 100644 --- a/bin/weewx/reportengine.py +++ b/bin/weewx/reportengine.py @@ -182,7 +182,7 @@ class StdReportEngine(threading.Thread): try: # Instantiate an instance of the class. - obj = weeutil.weeutil._get_object(generator)( + obj = weeutil.weeutil.get_object(generator)( self.config_dict, skin_dict, self.gen_ts, diff --git a/bin/weewx/restx.py b/bin/weewx/restx.py index 1926f319..9d378d9b 100644 --- a/bin/weewx/restx.py +++ b/bin/weewx/restx.py @@ -270,7 +270,7 @@ class RESTThread(threading.Thread): if dbmanager is None: # If we don't have a database, we can't do anything - if self.log_failure: + if self.log_failure and weewx.debug >= 2: log.debug("No database specified. Augmentation from database skipped.") return record @@ -1041,8 +1041,8 @@ class WOWThread(AmbientThread): # WOW signals a bad login with a HTML Error 403 code: if e.code == 403: raise BadLogin(e) - elif e.code == 429: - raise FailedPost("Too many requests; data already seen; or too out of date.") + elif e.code >= 400: + raise FailedPost(e) else: raise else: diff --git a/docs/changes.txt b/docs/changes.txt index bac39f76..7f84462e 100644 --- a/docs/changes.txt +++ b/docs/changes.txt @@ -43,10 +43,11 @@ Ported OS uptime to OpenBSD. Fixes issue #428. Thanks to user Jeff Ross! Catch SSL certificate errors in uploaders. Retry after an hour. Fixes issue #413. -Wunderfixer has been ported to the new WU API. This API requires an -API key, which you can get from the WU. Put it in weewx.conf. -Unfortunately, WU is very unreliable whether reposts "stick". -Fixes issue #414 +Wunderfixer has been ported to the new WU API. This API requires an API key, +which you can get from the WU. Put it in weewx.conf. Unfortunately, WU is +very unreliable whether reposts "stick". Fixes issues #414 and #445. + +Wee_import can now import Weather Display monthly log files. Fixed problem where sub-sections DegreeDays and Trend were located under the wrong weewx.conf section. Fixes issue #432. Thanks to user mph for spotting @@ -134,6 +135,9 @@ Fixes issue #431. Thanks to user ls4096! Fixed problem that can cause an exception with restx services that do not use the database manager. See commit 459ccb1. +Sending a SIGTERM signal to weewxd now causes it to exit with status +128 + signal#. PR #442. Thanks to user sshambar! + 3.9.1 02/06/2019 diff --git a/docs/customizing.htm b/docs/customizing.htm index 7356c580..7ae05413 100644 --- a/docs/customizing.htm +++ b/docs/customizing.htm @@ -2436,18 +2436,6 @@ or in foobar units: $day.barometer.min.foobar The maximum barometric pressure over the last immediate 2 weeks. - - $month_delta=months - $span($month_delta=3).outTemp.min - The minimum temperture over the last immediate 3 months (90 days). - - - - $year_delta=years - $span($year_delta=1).windchill.min - The minimum wind chill over the last immediate 1 year (365 days). - - @@ -2838,11 +2826,16 @@ Sunrise, sunset: $almanac.sunrise $almanac.sunset 

$almanac(pressure=0, horizon=-6).sun(use_center=1).rise

The general syntax is:

- 
$almanac(pressure=pressure, horizon=horizon,
-         temperature=temperature_C).heavenly_body(use_center=[01]).attribute
+        
$almanac(almanac_time=time,            ## Unix epoch time
+         lat=latitude, lon=longitude,  ## degrees
+         altitude=altitude,            ## meters
+         pressure=pressure,            ## mbars
+         horizon=horizon,              ## degrees
+         temperature=temperature_C     ## degrees C
+       ).heavenly_body(use_center=[01]).attribute
       

-        
As you can see, in addition to the horizon angle, you can also override atmospheric pressure and temperature
-            (degrees Celsius).
+        

+            As you can see, many other properties can be overridden besides pressure and the horizon angle.
         

 
         

diff --git a/docs/usersguide.htm b/docs/usersguide.htm
index a5125297..d7063fcd 100644
--- a/docs/usersguide.htm
+++ b/docs/usersguide.htm
@@ -107,9 +107,11 @@ Version: 4.0
             consumes about 5% of the CPU, 100 MB of virtual memory, and 20 MB of real memory.
         

 
-        
WeeWX also runs great on a Raspberry Pi, although report generation will take longer. For example, the 12 "To
-            Date" templates take about 5.1 seconds on the RPi, compared to 3.0 seconds on my Fit-PC, and a mere 0.9
-            seconds on my vintage Dell Optiplex 745.
+        

+            WeeWX also runs great on a Raspberry Pi, although report generation will take longer. For example,
+            the 12 "To Date" templates of the "Standard" report take about 5.1 seconds on a RPi B+,
+            compared to 3.0 seconds on my Fit-PC, 0.9 seconds on my vintage Dell Optiplex 745, and 0.3 seconds on
+            a NUC with a 4th gen i5 processor.
         

 
         
Time

diff --git a/docs/utilities.htm b/docs/utilities.htm
index 8c871a64..4f832450 100644
--- a/docs/utilities.htm
+++ b/docs/utilities.htm
@@ -1013,14 +1013,13 @@ No extensions installed
[--dry-run] [--verbose] [--suppress-warnings] - [--log=-] Import observation data into a WeeWX archive. Options: -h, --help show this help message and exit - --config=CONFIG_FILE Use WeeWX configuration file CONFIG_FILE. + --config=CONFIG_FILE Use configuration file CONFIG_FILE. --import-config=IMPORT_CONFIG_FILE Use import configuration file IMPORT_CONFIG_FILE. --dry-run Print what would happen but do not do it. @@ -1031,13 +1030,7 @@ Options: --to=YYYY-mm-dd[THH:MM] Import data up until this date or date-time. Format is YYYY-mm-dd[THH:MM]. - --log=- Control wee_import log output. By default log output - is sent to the WeeWX log file. Log output may be - disabled by using '--log=-'. Some WeeWX API log output - cannot be controlled by wee_import and will be sent to - the default log file irrespective of the '--log' - option. - --verbose Print useful extra output. + --verbose Print and log useful extra output. --suppress-warnings Suppress warnings to stdout. Warnings are still logged. --version Display wee_import version number. @@ -1191,19 +1184,6 @@ summaries using the wee_database utility. importing from Weather Underground or to import all available records when importing from any other source.

-

Option --log

- -

The --log option controls the wee_import log output. - Omitting the option will result in wee_import log output being sent to the WeeWX - log file (nominally the system log, refer to Monitoring - WeeWX and Where to find things to - find it). wee_import log output can be disabled by using --log=-. - The --log option is used as follows: -

- - 
wee_import --import-config=/directory/import.conf --log=-
-
-

Option --verbose

Inclusion of the --verbose option will cause additional information to be printed @@ -1221,7 +1201,7 @@ summaries using the wee_database utility. a given timestamp or there being no data found for a mapped import field. These warnings do not necessarily require action, but they can consist of extensive output and thus make it difficult to follow the import progress. Irrespective of whether --suppress-warnings is used all warnings are sent - to log unless the --log=- option is used. + to log. 

wee_import --import-config=/directory/import.conf --suppress-warnings
@@ -1255,7 +1235,7 @@ summaries using the wee_database utility.
  • CSV to import from a single CSV format file.
    • -
  • WU to import from a Weather Underground PWS daily history. +
  • WU to import from a Weather Underground PWS history.
  • Cumulus to import from one or more Cumulus monthly log files.
    • @@ -1581,22 +1561,33 @@ summaries using the wee_database utility.

    [WU]

    The [WU] section contains the options relating to the import of - observational data from a Weather Underground PWS daily history. + observational data from a Weather Underground PWS history.

    station_id

    -

    The Weather Underground weather station ID of the PWS from which the daily history will be imported. There is +

    The Weather Underground weather station ID of the PWS from which the historical data will be imported. There is no default.

    +

    api_key

    + +

    The Weather Underground API key to be used to obtain the PWS history data. There is no default.

    + +

    + Note
    The API key is a seemingly random string of 32 characters used to access the new + (2019) Weather Underground API. PWS contributors can obtain an API key by logging onto the Weather + Underground internet site and accessing Member Settings. 16 character API keys used with the previous Weather + Underground API are not supported. +

    +

    interval

    Determines how the time interval (WeeWX database field interval) between successive observations is determined. This option is identical in operation to the CSV interval - option but applies to Weather Underground imports only. As Weather Underground often skips observation - records when responding to a daily history query, the use of interval = derive may - give incorrect or inconsistent interval values. Better results may be obtained by using interval = conf + option but applies to Weather Underground imports only. As a Weather Underground PWS history sometimes has + missing records, the use of interval = derive may give incorrect or inconsistent + interval values. Better results may be obtained by using interval = conf if the current WeeWX installation has the same archive_interval as the Weather Underground data, or by using interval = x where x is the time interval in minutes used to upload the Weather Underground data. The most appropriate setting will @@ -2295,7 +2286,8 @@ summaries using the wee_database utility.

    The output should be something like this:

    - 
    Starting wee_import...
+                
    Using WeeWX configuration file /home/weewx/weewx.conf
+Starting wee_import...
 A CSV import from source file '/var/tmp/data.csv' has been requested.
 Using database binding 'wx_binding', which is bound to database 'weewx.sdb'
 Destination table 'archive' unit system is '0x01' (US).
@@ -2312,14 +2304,6 @@ Finished dry run import
                     data will be processed. The import will then be performed but no data will be written to the WeeWX
                     database. Upon completion a brief summary of the records processed is provided.
                 
    
-
-                
    
-                    Note
    As the WeeWX database is not altered when the --dry-run option is used, wee_import log output is
-                    suspended during a dry run import. In effect, the use of --dry-run is
-                    equivalent to --dry-run --log=-. During a dry run import the only wee_import output is that displayed on stdout.
-                
    
             
 
             
  • Once the dry run results are satisfactory the data can be imported using the following command:
@@ -2330,7 +2314,8 @@ Finished dry run import
                     there will be a prompt:
                 
    
 
-                
    Starting wee_import...
+                
    Using WeeWX configuration file /home/weewx/weewx.conf
+Starting wee_import...
 A CSV import from source file '/var/tmp/data.csv' has been requested.
 Using database binding 'wx_binding', which is bound to database 'weewx.sdb'
 Destination table 'archive' unit system is '0x01' (US).
@@ -2385,12 +2370,12 @@ Aug 22 14:38:28 stretch12 weewx[863]: manager: unable to add record 2018-09-04 0
 
         
    Importing from Weather Underground
    
 
-        
    wee_import can import data from the daily history of a Weather Undeground PWS. A
-            Weather Underground daily history provides weather station observations received by Weather Underground for
-            the PWS concerned on a day by day basis. As such, the data is analogous to the WeeWX archive table. When
-            wee_import imports data from a Weather Underground daily history each day is
-            considered a 'period'. wee_import processes one period at a time in chronological
-            order (oldest to newest) and provides import summary data on a per period basis.
+        
    wee_import can import historical observation data for a Weather Underground PWS via
+            the Weather Underground API. The Weather Underground API provides historical weather station observations
+            received by Weather Underground for the PWS concerned on a day by day basis. As such, the data is analogous
+            to the WeeWX archive table. When wee_import imports data from the Weather
+            Underground API each day is considered a 'period'. wee_import processes one period
+            at a time in chronological order (oldest to newest) and provides import summary data on a per period basis.
         
    
 
         
    Mapping data to archive fields
    
@@ -2398,28 +2383,34 @@ Aug 22 14:38:28 stretch12 weewx[863]: manager: unable to add record 2018-09-04 0
         
    A Weather Underground import will populate WeeWX archive fields as follows:
    
 
         
      
-            
    • Provided data exists for each field in the Weather Underground PWS daily history, the following WeeWX
-                archive fields will be directly populated by imported data:
+            
    • Provided data exists for each field returned by the Weather Underground API, the following WeeWX archive
+                fields will be directly populated by imported data:
 
                 
        
                     
      • dateTime
        • 
                     
      • barometer
        • 
                     
      • dewpoint
        • 
+                    
      • heatindex
        • 
                     
      • outHumidity
        • 
                     
      • outTemp
        • 
                     
      • radiation
        • 
                     
      • rain
        • 
+                    
      • rainRate
        • 
+                    
      • UV
        • 
+                    
      • windchill
        • 
                     
      • windDir
        • 
                     
      • windGust
        • 
                     
      • windSpeed
        • 
                 
      
 
                 
      
-                    Note
      If an appropriate field does not exist in the Weather Underground daily
-                    history then the corresponding WeeWX archive field will be set to None/null. For example, if there is no solar radiation sensor then radiation will be null, or if outHumidity was never
-                    uploaded to Weather Undeground then outHumidity will be null.
+                    Note
      If an appropriate field is not returned by the Weather Underground API then
+                    the corresponding WeeWX archive field will contain no data. If the API returns an appropriate field but
+                    with no data, the corresponding WeeWX archive field will be set to None/null.
+                    For example, if the API response has no solar radiation field the WeeWX
+                    radiation archive field will have no data stored. However, if the API
+                    response has a solar radiation field but contains no data, the WeeWX
+                    radiation archive field will be None/null.
                 
      
             
      • 
 
@@ -2437,10 +2428,7 @@ Aug 22 14:38:28 stretch12 weewx[863]: manager: unable to add record 2018-09-04 0
                 
        
                     
      • altimeter
        • 
                     
      • ET
        • 
-                    
      • heatindex
        • 
                     
      • pressure
        • 
-                    
      • rainRate
        • 
-                    
      • windchill
        • 
                 
      
 
                 
      
@@ -2454,7 +2442,7 @@ Aug 22 14:38:28 stretch12 weewx[863]: manager: unable to add record 2018-09-04 0
 
         
      Step-by-step instructions
      
 
-        
      To import observations from the daily history of a Weather Underground PWS:
      
+        
      To import observations from a Weather Underground PWS history:
      
 
         
        
             
      1. Obtain the weather station ID of the Weather Underground PWS from which data is to be imported. The
@@ -2463,6 +2451,11 @@ Aug 22 14:38:28 stretch12 weewx[863]: manager: unable to add record 2018-09-04 0
                 of ISTATION123 will be used.
             
        2. 
 
+            
      2. Obtain the API key to be used to access the Weather Underground API. This will be a seemingly random
+                alphanumeric sequence of 32 characters. API keys are available to Weather Underground PWS contributors
+                by logging on to their Weather Underground account and accessing Member Settings.
+            
        3. 
+
             
      3. Make a backup of the WeeWX database in case the import should go awry.
             
        4. 
 
@@ -2477,7 +2470,6 @@ Aug 22 14:38:28 stretch12 weewx[863]: manager: unable to add record 2018-09-04 0
             
        source = WU
        
             
 
-
             
      4. Confirm that the following options in the [WU] section are correctly set:
 
                 
          
@@ -2485,6 +2477,9 @@ Aug 22 14:38:28 stretch12 weewx[863]: manager: unable to add record 2018-09-04 0
                         character weather station ID of the Weather Underground PWS that will be the source of the
                         imported data.
                     
+                    
        • api_key. The 32 character
+                        API key to be used to access the Weather Underground API.
+                    
          • 
                     
        • interval. Determines how
                         the WeeWX interval field is derived.
                     
          • 
@@ -2556,7 +2551,8 @@ Aug 22 14:38:28 stretch12 weewx[863]: manager: unable to add record 2018-09-04 0
                 
          
                 
          The output should be similar to:
          
 
-                
          Starting wee_import...
+                
          Using WeeWX configuration file /home/weewx/weewx.conf
+Starting wee_import...
 Observation history for Weather Underground station 'ISTATION123' will be imported.
 Using database binding 'wx_binding', which is bound to database 'weewx.sdb'
 Destination table 'archive' unit system is '0x01' (US).
@@ -2577,14 +2573,6 @@ Unique records processed: 71; Last timestamp: 2016-01-23 06:00:00 AEST (14534928
 Finished dry run import
 657 records were processed and 657 unique records would have been imported.
 
          
-
-                
          
-                    Note
          As the WeeWX database is not altered when the --dry-run option is used, wee_import log output is
-                    suspended during a dry run import. In effect, the use of --dry-run is
-                    equivalent to --dry-run --log=-. During a dry run import the only wee_import output is that displayed on stdout(console).
-                
          
             
 
             
        • Once the dry run results are satisfactory the source data can be imported using the following command:
@@ -2596,7 +2584,8 @@ Finished dry run import
                     will be a prompt:
                 
          
 
-                
          Starting wee_import...
+                
          Using WeeWX configuration file /home/weewx/weewx.conf
+Starting wee_import...
 Observation history for Weather Underground station 'ISTATION123' will be imported.
 Using database binding 'wx_binding', which is bound to database 'weewx.sdb'
 Destination table 'archive' unit system is '0x01' (US).
@@ -2611,9 +2600,9 @@ Are you sure you want to proceed (y/n)?
 
          
 
                 
          
-                    Note
          wee_import obtains Weather Underground daily
-                    history data one day at a time via a HTTP request and as such the import of large time spans of data
-                    may take some time. Such imports may be best handled as a series of imports of smaller time spans.
+                    Note
          wee_import obtains Weather Underground data
+                    one day at a time via a HTTP request and as such the import of large time spans of data may take some
+                    time. Such imports may be best handled as a series of imports of smaller time spans.
                 
          
             
          • 
 
@@ -2643,15 +2632,17 @@ imported. Confirm successful import in the WeeWX log file.
 
          
 
                 
          
-                    Note
          It is not unusual to see a Weather Underground import return a different
-                    number of records for the same import performed at different times. If importing the current day
-                    this could be because an additional record may have been added between wee_import runs. For periods before today, this behaviour appears to be a vagary
-                    of Weather Underground. The only solution appears to be to repeat the import with the same --date option setting and observe whether the missing records are imported.
-                    Repeating the import will not adversely affect any existing data as records with timestamps that are
-                    already in the WeeWX archive will be ignored. It may; however, generated many UNIQUE constraint failed: archive.dateTime
-                    messages in the WeeWX log.
+                    Note
          The new (2019) Weather Underground API appears to have an issue when
+                    obtaining historical data for the current day. The first time the API is queried the API returns all
+                    historical data up to and including the most recent record. However, subsequent later API queries
+                    during the same day return the same set of records rather than all records up to and including the
+                    time of the latest API query. Users importing Weather Underground data that includes data from the
+                    current day are advised to carefully check the WeeWX log to ensure that all expected records were
+                    imported. If some records are missing from the current day try running an import for the current day
+                    again using the --date option setting. If this fails then wait until the
+                    following day and perform another import for the day concerned again using the
+                    --date option setting. In all cases confirm what data has been imported by
+                    referring to the WeeWX log.
                 
          
             
 
@@ -2831,7 +2822,8 @@ Aug 22 14:38:28 stretch12 weewx[863]: manager: unable to add record 2018-09-04 0
                 
          
                 
          The output should be similar to:
          
 
-                
          Starting wee_import...
+                
          Using WeeWX configuration file /home/weewx/weewx.conf
+Starting wee_import...
 Cumulus monthly log files in the '/var/tmp/cumulus' directory will be imported
 Using database binding 'wx_binding', which is bound to database 'weewx.sdb'
 Destination table 'archive' unit system is '0x01' (US).
@@ -2876,7 +2868,8 @@ Finished dry run import
                     a prompt:
                 
          
 
-                
          Starting wee_import...
+                
          Using WeeWX configuration file /home/weewx/weewx.conf
+Starting wee_import...
 Cumulus monthly log files in the '/var/tmp/cumulus' directory will be imported
 Using database binding 'wx_binding', which is bound to database 'weewx.sdb'
 Destination table 'archive' unit system is '0x01' (US).
@@ -2895,7 +2888,8 @@ Are you sure you want to proceed (y/n)?
                     preamble may look similar to:
                 
          
 
-                
          Starting wee_import...
+                
          Using WeeWX configuration file /home/weewx/weewx.conf
+Starting wee_import...
 Cumulus monthly log files in the '/var/tmp/cumulus' directory will be imported
 Using database binding 'wx_binding', which is bound to database 'weewx.sdb'
 Destination table 'archive' unit system is '0x01' (US).
@@ -3143,7 +3137,8 @@ Aug 22 14:38:28 stretch12 weewx[863]: manager: unable to add record 2018-09-04 0
                 
          
                 
          The output should be similar to:
          
 
-                
          Starting wee_import...
+                
          Using WeeWX configuration file /home/weewx/weewx.conf
+Starting wee_import...
 Weather Display monthly log files in the '/var/tmp/WD' directory will be imported
 Using database binding 'wx_binding', which is bound to database 'weewx.sdb'
 Destination table 'archive' unit system is '0x01' (US).
@@ -3185,7 +3180,8 @@ Finished dry run import
 
                 
          The output should be similar to:
          
 
-                
          Starting wee_import...
+                
          Using WeeWX configuration file /home/weewx/weewx.conf
+Starting wee_import...
 Weather Display monthly log files in the '/var/tmp/WD' directory will be imported
 Using database binding 'wx_binding', which is bound to database 'weewx.sdb'
 Destination table 'archive' unit system is '0x01' (US).
@@ -3288,7 +3284,8 @@ Finished dry run import
                     a prompt:
                 
          
 
-                
          Starting wee_import...
+                
          Using WeeWX configuration file /home/weewx/weewx.conf
+Starting wee_import...
 Weather Display monthly log files in the '/var/tmp/WD' directory will be imported
 Using database binding 'wx_binding', which is bound to database 'weewx.sdb'
 Destination table 'archive' unit system is '0x01' (US).
@@ -3307,7 +3304,8 @@ Are you sure you want to proceed (y/n)?
                     is ignored. In such cases the preamble may look similar to:
                 
          
 
-                
          SStarting wee_import...
+                
          Using WeeWX configuration file /home/weewx/weewx.conf
+Starting wee_import...
 Weather Display monthly log files in the '/var/tmp/WD' directory will be imported
 Using database binding 'wx_binding', which is bound to database 'weewx.sdb'
 Destination table 'archive' unit system is '0x01' (US).
@@ -3545,10 +3543,7 @@ Options:
                         Socket timeout in seconds. Default is 10.
   -v, --verbose         Print useful extra output.
   -l LOG_FACILITY, --log=LOG_FACILITY
-                        Log selected output to syslog. If omitted no syslog
-                        logging occurs. If LOG_FACILITY is 'weewx' then logs
-                        are written to the same log used by weewx. Any other
-                        parameter will log to syslog.
+                        OBSOLETE. Logging will always occur.
   -t, --test            Test what would happen, but don't do anything.
   -q, --query           For each record, query the user before making a
                         change.
@@ -3646,14 +3641,11 @@ command line, or in the configuration file.
          
 
         
          Option --log
          
 
-        
          Control the wunderfixer log output. The default is no logging. If --log=weewx is used then wunderfixer logs to the same log file
-            as used by WeeWX. Any other setting will result in wunderfixer logs being written
-            to syslog. The --log option is used as follows:
+        
          The --log option is obsolete and no longer performs any function. The
+            --log option has been retained for backwards compatibility with users existing
+            scripts. All wunderfixer log output is always logged via the WeeWX log facility.
         
          
 
-        
          wunderfixer --log=weewx
          
-
         
          Option --test
          
 
         
          The --test option will cause wunderfixer to do everything
@@ -3693,6 +3685,11 @@ command line, or in the configuration file.
          
             options.
         
          
 
+        
          
+            Note
           The data fields shown in the example outputs below are indicative only and the
+            actual output may or may not include other fields.
+        
          
+
         
          To run wunderfixer directly:
          
 
         
            
@@ -3703,10 +3700,10 @@ command line, or in the configuration file.
          
 
                 
          Using configuration file /home/weewx/weewx.conf.
 Using database binding 'wx_binding', which is bound to database 'archive_sqlite'
-2016-09-22 06:30:00 AEST (1474489800); 29.920";  58.9F;  79%; 1.0 mph; 248 deg; 6.0 mph gust;  52.4F; 0.00" rain; 0.01" daily rain  ... skipped.
-2016-09-22 07:35:00 AEST (1474493700); 29.931";  64.9F;  65%; 2.0 mph; 180 deg; 7.0 mph gust;  52.8F; 0.00" rain; 0.01" daily rain  ... skipped.
-2016-09-22 07:55:00 AEST (1474494900); 29.934";  65.8F;  63%; 2.0 mph; 180 deg;10.0 mph gust;  52.8F; 0.00" rain; 0.01" daily rain  ... skipped.
-2016-09-22 08:20:00 AEST (1474496400); 29.938";  66.5F;  59%; 5.0 mph; 180 deg;12.0 mph gust;  51.7F; 0.00" rain; 0.01" daily rain  ... skipped.
+2016-09-22 06:30:00 AEST (1474489800); 29.920";  58.9F;  79%; 1.0 mph; 248 deg; 6.0 mph gust;  52.4F; 0.00" rain; 0.01" ... skipped.
+2016-09-22 07:35:00 AEST (1474493700); 29.931";  64.9F;  65%; 2.0 mph; 180 deg; 7.0 mph gust;  52.8F; 0.00" rain; 0.01" ... skipped.
+2016-09-22 07:55:00 AEST (1474494900); 29.934";  65.8F;  63%; 2.0 mph; 180 deg;10.0 mph gust;  52.8F; 0.00" rain; 0.01" ... skipped.
+2016-09-22 08:20:00 AEST (1474496400); 29.938";  66.5F;  59%; 5.0 mph; 180 deg;12.0 mph gust;  51.7F; 0.00" rain; 0.01" ... skipped.
 
          
                 
          This output indicates that four records were found to be missing. The word 'skipped' at the end of each
                     line indicates that whilst wunderfixer detected the record as missing, the
@@ -3728,10 +3725,10 @@ Using database binding 'wx_binding', which is bound to database 'archive_sqlite'
 
             
          Using configuration file /home/weewx/weewx.conf.
 Using database binding 'wx_binding', which is bound to database 'archive_sqlite'
-2016-09-22 06:30:00 AEST (1474489800); 29.920";  58.9F;  79%; 1.0 mph; 248 deg; 6.0 mph gust;  52.4F; 0.00" rain; 0.01" daily rain  ... published.
-2016-09-22 07:35:00 AEST (1474493700); 29.931";  64.9F;  65%; 2.0 mph; 180 deg; 7.0 mph gust;  52.8F; 0.00" rain; 0.01" daily rain  ... published.
-2016-09-22 07:55:00 AEST (1474494900); 29.934";  65.8F;  63%; 2.0 mph; 180 deg;10.0 mph gust;  52.8F; 0.00" rain; 0.01" daily rain  ... published.
-2016-09-22 08:20:00 AEST (1474496400); 29.938";  66.5F;  59%; 5.0 mph; 180 deg;12.0 mph gust;  51.7F; 0.00" rain; 0.01" daily rain  ... published.
+2016-09-22 06:30:00 AEST (1474489800); 29.920";  58.9F;  79%; 1.0 mph; 248 deg; 6.0 mph gust;  52.4F; 0.00" rain; 0.01" ... published.
+2016-09-22 07:35:00 AEST (1474493700); 29.931";  64.9F;  65%; 2.0 mph; 180 deg; 7.0 mph gust;  52.8F; 0.00" rain; 0.01" ... published.
+2016-09-22 07:55:00 AEST (1474494900); 29.934";  65.8F;  63%; 2.0 mph; 180 deg;10.0 mph gust;  52.8F; 0.00" rain; 0.01" ... published.
+2016-09-22 08:20:00 AEST (1474496400); 29.938";  66.5F;  59%; 5.0 mph; 180 deg;12.0 mph gust;  51.7F; 0.00" rain; 0.01" ... published.
 
          
 
             
          This output indicates that four records were found to be missing. This time word 'skipped' at the end of
@@ -3752,13 +3749,13 @@ Number of WU records:          284
 Number of missing records:     4
 
 Missing records:
-2016-09-22 06:30:00 AEST (1474489800); 29.920";  58.9F;  79%; 1.0 mph; 248 deg; 6.0 mph gust;  52.4F; 0.00" rain; 0.01" daily rain  ...fix? (y/n/a/q):y
+2016-09-22 06:30:00 AEST (1474489800); 29.920";  58.9F;  79%; 1.0 mph; 248 deg; 6.0 mph gust;  52.4F; 0.00" rain; 0.01" ...fix? (y/n/a/q):y
  ...published.
-2016-09-22 07:35:00 AEST (1474493700); 29.931";  64.9F;  65%; 2.0 mph; 180 deg; 7.0 mph gust;  52.8F; 0.00" rain; 0.01" daily rain  ...fix? (y/n/a/q):y
+2016-09-22 07:35:00 AEST (1474493700); 29.931";  64.9F;  65%; 2.0 mph; 180 deg; 7.0 mph gust;  52.8F; 0.00" rain; 0.01" ...fix? (y/n/a/q):y
  ...published.
-2016-09-22 07:55:00 AEST (1474494900); 29.934";  65.8F;  63%; 2.0 mph; 180 deg;10.0 mph gust;  52.8F; 0.00" rain; 0.01" daily rain  ...fix? (y/n/a/q):y
+2016-09-22 07:55:00 AEST (1474494900); 29.934";  65.8F;  63%; 2.0 mph; 180 deg;10.0 mph gust;  52.8F; 0.00" rain; 0.01" ...fix? (y/n/a/q):y
  ...published.
-2016-09-22 08:20:00 AEST (1474496400); 29.938";  66.5F;  59%; 5.0 mph; 180 deg;12.0 mph gust;  51.7F; 0.00" rain; 0.01" daily rain  ...fix? (y/n/a/q):
+2016-09-22 08:20:00 AEST (1474496400); 29.938";  66.5F;  59%; 5.0 mph; 180 deg;12.0 mph gust;  51.7F; 0.00" rain; 0.01" ...fix? (y/n/a/q):
 
          
 
             
@@ -3875,7 +3872,7 @@ Missing records:
                         will depend on the --log option used and your version of Linux.
                     
 
-                    
        • Check the Weather Undeground daily history for the station concerned to ensure that any missing
+                    
        • Check the Weather Underground Weather History for the station concerned to ensure that any missing
                         data was accepted by Weather Underground.
                     
          • 
                 
        
diff --git a/util/import/csv-example.conf b/util/import/csv-example.conf
index 6d20b6cd..21a727d3 100644
--- a/util/import/csv-example.conf
+++ b/util/import/csv-example.conf
@@ -1,6 +1,6 @@
 # EXAMPLE CONFIGURATION FILE FOR IMPORTING FROM CSV FILES
 #
-# Copyright (c) 2009-2016 Tom Keffer  and Gary Roderick.
+# Copyright (c) 2009-2019 Tom Keffer  and Gary Roderick.
 # See the file LICENSE.txt for your rights.
 
 ##############################################################################
diff --git a/util/import/cumulus-example.conf b/util/import/cumulus-example.conf
index 14074b9c..ecae3a4f 100644
--- a/util/import/cumulus-example.conf
+++ b/util/import/cumulus-example.conf
@@ -1,6 +1,6 @@
 # EXAMPLE CONFIGURATION FILE FOR IMPORTING FROM CUMULUS
 #
-# Copyright (c) 2009-2016 Tom Keffer  and Gary Roderick.
+# Copyright (c) 2009-2019 Tom Keffer  and Gary Roderick.
 # See the file LICENSE.txt for your rights.
 
 ##############################################################################
diff --git a/util/import/wu-example.conf b/util/import/wu-example.conf
index 623b2125..2a6d2aa6 100644
--- a/util/import/wu-example.conf
+++ b/util/import/wu-example.conf
@@ -1,6 +1,6 @@
 # EXAMPLE CONFIGURATION FILE FOR IMPORTING FROM THE WEATHER UNDERGROUND
 #
-# Copyright (c) 2009-2016 Tom Keffer  and Gary Roderick.
+# Copyright (c) 2009-2019 Tom Keffer  and Gary Roderick.
 # See the file LICENSE.txt for your rights.
 
 ##############################################################################
@@ -22,6 +22,9 @@ source = WU
     # WU PWS Station ID to be used for import.
     station_id = XXXXXXXX123
 
+    # WU API key to be used for import.
+    api_key = XXXXXXXXXXXXXXXXXXXXXX1234567890
+
     #
     # When importing WU data the following WeeWX database fields will be
     # populated directly by the imported data (provided the corresponding data
@@ -29,13 +32,17 @@ source = WU
     #   barometer
     #   dateTime
     #   dewpoint
+    #   heatindex
     #   outHumidity
     #   outTemp
     #   radiation
     #   rain
+    #   rainRate
+    #   windchill
     #   windDir
     #   windGust
     #   windSpeed
+    #   UV
     #
     # The following WeeWX database fields will be populated from other
     # settings/config files:
@@ -47,10 +54,7 @@ source = WU
     # used during import:
     #   altimeter
     #   ET
-    #   heatindex
     #   pressure
-    #   rainRate
-    #   windchill
     #
     # The following WeeWX fields will be populated with derived values from the
     # imported data provided the --calc-missing command line option is used
diff --git a/weewx.conf b/weewx.conf
index 22b8c3dc..aef33669 100644
--- a/weewx.conf
+++ b/weewx.conf
@@ -23,7 +23,7 @@ log_failure = True
 socket_timeout = 20
 
 # Do not modify this. It is used when installing and updating weewx.
-version = 4.0.0a8
+version = 4.0.0a9
 
 ##############################################################################
 
@@ -118,6 +118,10 @@ version = 4.0.0a8
         station  = replace_me
         password = "replace_me"
 
+        # If you plan on using wunderfixer, set the following
+        # to your API key:
+        api_key = replace_me
+
         # Set the following to True to have weewx use the WU "Rapidfire"
         # protocol. Not all hardware can support it. See the User's Guide.
         rapidfire = False