From 15113e3b29583fd2eb6ae5b33868edf5bd3dd2e6 Mon Sep 17 00:00:00 2001 From: Tom Keffer Date: Sat, 25 Nov 2023 13:45:54 -0800 Subject: [PATCH 01/13] Include [Logging] section in weewx.conf Without it, logging will be as before: to syslog. However, for new installs, it will be to WEEWX_ROOT/log/weewx.log --- docs_src/changes.md | 17 ++++++------- src/weecfg/__init__.py | 12 +++++++++- src/weeutil/logger.py | 18 ++++++++++---- src/weeutil/rsyncupload.py | 2 +- src/weeutil/tests/test_config.py | 2 +- src/weewx/drivers/acurite.py | 2 +- src/weewx/drivers/cc3000.py | 2 +- src/weewx/drivers/te923.py | 2 +- src/weewx/drivers/ultimeter.py | 2 +- src/weewx/drivers/vantage.py | 2 +- src/weewx/drivers/wmr300.py | 2 +- src/weewx/drivers/wmr9x8.py | 2 +- src/weewx/drivers/ws1.py | 2 +- src/weewx/drivers/ws23xx.py | 2 +- src/weewx/tests/test_aggregate.py | 2 +- src/weewx/tests/test_cheetah.py | 2 +- src/weewx/tests/test_daily.py | 2 +- src/weewx/tests/test_database.py | 2 +- src/weewx/tests/test_manager.py | 2 +- src/weewx/tests/test_templates.py | 2 +- src/weewx/tests/test_wxxtypes.py | 2 +- src/weewx/tests/test_xtypes.py | 2 +- .../examples/fileparse/bin/user/fileparse.py | 2 +- src/weewx_data/examples/pmon/bin/user/pmon.py | 2 +- .../examples/tests/test_vaporpressure.py | 2 +- src/weewx_data/weewx.conf | 24 +++++++++++++++++++ 26 files changed, 77 insertions(+), 38 deletions(-) diff --git a/docs_src/changes.md b/docs_src/changes.md index 5516d906..d12fc1e3 100644 --- a/docs_src/changes.md +++ b/docs_src/changes.md @@ -13,8 +13,8 @@ instead of `/home/weewx`. This allows pip installs to be done without root privileges. However, `/home/weewx` can still be used. The new utility [`weectl`](utilities/weectl-about.md) replaces `wee_database`, -`wee_debug`, `wee_device`, `wee_extension`, and `wee_config`. Try `weectl ---help` to see how to use it. +`wee_debug`, `wee_device`, `wee_extension`, `wee_import`, `wee_reports`, +and `wee_config`. Try `weectl --help` to see how to use it. Documentation now uses [MkDocs](https://www.mkdocs.org/). @@ -23,15 +23,12 @@ Package installs now use `systemd` instead of the old System V `/etc/init.d`. Removed all references to the deprecated package `distutils`, which is due to be removed in Python v3.12. -Removed the utility `wunderfixer`. The Weather Underground seems to no longer -allow posting past-dated records. +Removed the utility `wunderfixer`. The Weather Underground no longer +allows posting past-dated records. -Logging handler `rotate` has been removed. Its need to access the privileged -location `/var/log/` on start up would cause crashes, even if it was never -used. Users can opt to add it back in by following the wiki on -[Logging](https://github.com/weewx/weewx/wiki/WeeWX-v4-and-logging). - -MacOS now logs to `syslog` like any other system, rather than `rotate`. +For new installs, logging output is now sent to a file, which rotates at +midnight. Syslog is no longer used. Old installs continue to use whatever they +were using before. Method `ImageDraw.textsize()` and constants `ImageFont.LAYOUT_BASIC`, and `Image.ANTIALIAS` were deprecated in Pillow 9.2 (1-Jul-2022), then removed in diff --git a/src/weecfg/__init__.py b/src/weecfg/__init__.py index c3437ab4..d155d7f0 100644 --- a/src/weecfg/__init__.py +++ b/src/weecfg/__init__.py @@ -116,8 +116,18 @@ def read_config(config_path, args=None, locations=DEFAULT_LOCATIONS, locations for weewx.conf. Returns the filename of the actual configuration file, as well as the ConfigObj. - Args: + Special handling for key "WEEWX_ROOT": + - If it is missing from the config file, then it is added and set to the directory + where the configuration file was found. + - If it is a relative path, then it is converted to an absolute path by + prepending the directory where the configuration file was found. + This version also adds two entries to the returned ConfigObj: + config_path: Location of the actual configuration file that was used. + WEEWX_ROOT_ORIG: The original value of WEEWX_ROOT, or None if there + was no original value + + Args: config_path (str|None): configuration filename. args (list[str]|None): command-line arguments. locations (list[str]): A list of directories to search. diff --git a/src/weeutil/logger.py b/src/weeutil/logger.py index c3a4b1b0..de4b1f1a 100644 --- a/src/weeutil/logger.py +++ b/src/weeutil/logger.py @@ -6,6 +6,7 @@ """WeeWX logging facility""" import logging.config +import os.path import sys from io import StringIO @@ -83,7 +84,7 @@ else: facility = 'user' -def setup(process_name, user_log_dict): +def setup(process_name, config_dict=None): """Set up the weewx logging facility""" global address, facility @@ -96,13 +97,17 @@ def setup(process_name, user_log_dict): # value, then restore later. However, the incoming dictionary may be a simple # Python dictionary and not have interpolation. Hence, the try block. try: - old_interpolation = user_log_dict.interpolation - user_log_dict.interpolation = False + old_interpolation = config_dict.interpolation + config_dict.interpolation = False except AttributeError: old_interpolation = None # Merge in the user additions / changes: - log_config.merge(user_log_dict) + if config_dict: + log_config.merge(config_dict) + weewx_root = config_dict.get("WEEWX_ROOT") + else: + weewx_root = None # Adjust the logging level in accordance to whether the 'debug' flag is on log_level = 'DEBUG' if weewx.debug else 'INFO' @@ -122,6 +127,9 @@ def setup(process_name, user_log_dict): address=address, facility=facility, process_name=process_name) + if key == 'filename' and weewx_root: + section[key] = os.path.join(weewx_root, section[key]) + os.makedirs(os.path.dirname(section[key]), exist_ok=True) # Using the function, walk the 'Logging' part of the structure log_config['Logging'].walk(_fix) @@ -137,7 +145,7 @@ def setup(process_name, user_log_dict): # Restore the old interpolation value if old_interpolation is not None: - user_log_dict.interpolation = old_interpolation + config_dict.interpolation = old_interpolation def log_traceback(log_fn, prefix=''): diff --git a/src/weeutil/rsyncupload.py b/src/weeutil/rsyncupload.py index d484b1cc..0e84e6cb 100644 --- a/src/weeutil/rsyncupload.py +++ b/src/weeutil/rsyncupload.py @@ -148,7 +148,7 @@ if __name__ == '__main__': weewx.debug = 1 - weeutil.logger.setup('rsyncupload', {}) + weeutil.logger.setup('rsyncupload') if len(sys.argv) < 2: print("""Usage: rsyncupload.py path-to-configuration-file [path-to-be-rsync'd]""") diff --git a/src/weeutil/tests/test_config.py b/src/weeutil/tests/test_config.py index 081f9bed..4e11f933 100644 --- a/src/weeutil/tests/test_config.py +++ b/src/weeutil/tests/test_config.py @@ -20,7 +20,7 @@ weewx.debug = 1 log = logging.getLogger(__name__) # Set up logging using the defaults. -weeutil.logger.setup('test_config', {}) +weeutil.logger.setup('test_config') class TestConfigString(unittest.TestCase): diff --git a/src/weewx/drivers/acurite.py b/src/weewx/drivers/acurite.py index a6267cc0..b074cdbe 100644 --- a/src/weewx/drivers/acurite.py +++ b/src/weewx/drivers/acurite.py @@ -958,7 +958,7 @@ if __name__ == '__main__': weewx.debug = 1 - weeutil.logger.setup('acurite', {}) + weeutil.logger.setup('acurite') usage = """%prog [options] [--help]""" diff --git a/src/weewx/drivers/cc3000.py b/src/weewx/drivers/cc3000.py index a4d573f0..abba6f05 100644 --- a/src/weewx/drivers/cc3000.py +++ b/src/weewx/drivers/cc3000.py @@ -1417,7 +1417,7 @@ if __name__ == '__main__': DEBUG_OPENCLOSE = 1 weewx.debug = 1 - weeutil.logger.setup('cc3000', {}) + weeutil.logger.setup('cc3000') if options.testcrc: _check_crc(b'OK') diff --git a/src/weewx/drivers/te923.py b/src/weewx/drivers/te923.py index 30b6929b..5a9d67d8 100644 --- a/src/weewx/drivers/te923.py +++ b/src/weewx/drivers/te923.py @@ -2338,7 +2338,7 @@ if __name__ == '__main__': if options.debug: weewx.debug = 1 - weeutil.logger.setup('te923', {}) + weeutil.logger.setup('te923') if (options.format.lower() != FMT_TE923TOOL and options.format.lower() != FMT_TABLE and diff --git a/src/weewx/drivers/ultimeter.py b/src/weewx/drivers/ultimeter.py index 883e3fe5..b11c8e1c 100644 --- a/src/weewx/drivers/ultimeter.py +++ b/src/weewx/drivers/ultimeter.py @@ -449,7 +449,7 @@ if __name__ == '__main__': if options.debug: weewx.debug = 1 - weeutil.logger.setup('ultimeter', {}) + weeutil.logger.setup('ultimeter') with Station(options.port, debug_serial=options.debug) as station: station.set_logger_mode() diff --git a/src/weewx/drivers/vantage.py b/src/weewx/drivers/vantage.py index dc422e31..7a0bafbb 100644 --- a/src/weewx/drivers/vantage.py +++ b/src/weewx/drivers/vantage.py @@ -3055,7 +3055,7 @@ if __name__ == '__main__': weewx.debug = 1 - weeutil.logger.setup('vantage', {}) + weeutil.logger.setup('vantage') usage = """Usage: python -m weewx.drivers.vantage --help python -m weewx.drivers.vantage --version diff --git a/src/weewx/drivers/wmr300.py b/src/weewx/drivers/wmr300.py index 9d2dec1f..35f49844 100644 --- a/src/weewx/drivers/wmr300.py +++ b/src/weewx/drivers/wmr300.py @@ -1991,7 +1991,7 @@ if __name__ == '__main__': weewx.debug = 1 - weeutil.logger.setup('wmr300', {}) + weeutil.logger.setup('wmr300') usage = """%prog [options] [--help]""" diff --git a/src/weewx/drivers/wmr9x8.py b/src/weewx/drivers/wmr9x8.py index f7acb122..01ee64df 100644 --- a/src/weewx/drivers/wmr9x8.py +++ b/src/weewx/drivers/wmr9x8.py @@ -724,7 +724,7 @@ if __name__ == '__main__': weewx.debug = 2 - weeutil.logger.setup('wmr9x8', {}) + weeutil.logger.setup('wmr9x8') usage = """Usage: %prog --help %prog --version diff --git a/src/weewx/drivers/ws1.py b/src/weewx/drivers/ws1.py index 5460b229..d0918ee3 100644 --- a/src/weewx/drivers/ws1.py +++ b/src/weewx/drivers/ws1.py @@ -513,7 +513,7 @@ if __name__ == '__main__': weewx.debug = 2 DEBUG_READ = 2 - weeutil.logger.setup('ws1', {}) + weeutil.logger.setup('ws1') Station = StationSerial if options.addr is not None: diff --git a/src/weewx/drivers/ws23xx.py b/src/weewx/drivers/ws23xx.py index 618ff3d3..428358c2 100644 --- a/src/weewx/drivers/ws23xx.py +++ b/src/weewx/drivers/ws23xx.py @@ -2107,7 +2107,7 @@ if __name__ == '__main__': if options.debug: weewx.debug = 1 - weeutil.logger.setup('ws23xx', {}) + weeutil.logger.setup('ws23xx') if options.port: port = options.port diff --git a/src/weewx/tests/test_aggregate.py b/src/weewx/tests/test_aggregate.py index 508468cd..4621f760 100644 --- a/src/weewx/tests/test_aggregate.py +++ b/src/weewx/tests/test_aggregate.py @@ -30,7 +30,7 @@ weewx.debug = 1 log = logging.getLogger(__name__) # Set up logging using the defaults. -weeutil.logger.setup('test_aggregate', {}) +weeutil.logger.setup('test_aggregate') # Find the configuration file. It's assumed to be in the same directory as me: config_path = os.path.join(os.path.dirname(__file__), "testgen.conf") diff --git a/src/weewx/tests/test_cheetah.py b/src/weewx/tests/test_cheetah.py index 932420c9..f11b1146 100644 --- a/src/weewx/tests/test_cheetah.py +++ b/src/weewx/tests/test_cheetah.py @@ -19,7 +19,7 @@ weewx.debug = 1 log = logging.getLogger(__name__) # Set up logging using the defaults. -weeutil.logger.setup('test_cheetah', {}) +weeutil.logger.setup('test_cheetah') class RaiseException(object): diff --git a/src/weewx/tests/test_daily.py b/src/weewx/tests/test_daily.py index 17103a66..aee6f195 100644 --- a/src/weewx/tests/test_daily.py +++ b/src/weewx/tests/test_daily.py @@ -29,7 +29,7 @@ weewx.debug = 1 log = logging.getLogger(__name__) # Set up logging using the defaults. -weeutil.logger.setup('test_daily', {}) +weeutil.logger.setup('test_daily') os.environ['TZ'] = 'America/Los_Angeles' time.tzset() diff --git a/src/weewx/tests/test_database.py b/src/weewx/tests/test_database.py index 33dc1d7a..60ff3b72 100644 --- a/src/weewx/tests/test_database.py +++ b/src/weewx/tests/test_database.py @@ -17,7 +17,7 @@ import weedb import weeutil.weeutil import weeutil.logger -weeutil.logger.setup('test_database',{}) +weeutil.logger.setup('test_database') archive_sqlite = {'database_name': '/var/tmp/weewx_test/weedb.sdb', 'driver':'weedb.sqlite'} archive_mysql = {'database_name': 'test_weedb', 'user':'weewx1', 'password':'weewx1', 'driver':'weedb.mysql'} diff --git a/src/weewx/tests/test_manager.py b/src/weewx/tests/test_manager.py index c8ab52fa..9e88bb9b 100644 --- a/src/weewx/tests/test_manager.py +++ b/src/weewx/tests/test_manager.py @@ -29,7 +29,7 @@ import weewx.manager log = logging.getLogger(__name__) -weeutil.logger.setup('test_manager', {}) +weeutil.logger.setup('test_manager') os.environ['TZ'] = 'America/Los_Angeles' time.tzset() diff --git a/src/weewx/tests/test_templates.py b/src/weewx/tests/test_templates.py index c57f99c7..f6afdece 100644 --- a/src/weewx/tests/test_templates.py +++ b/src/weewx/tests/test_templates.py @@ -31,7 +31,7 @@ weewx.debug = 1 log = logging.getLogger(__name__) # Set up logging using the defaults. -weeutil.logger.setup('test_templates', {}) +weeutil.logger.setup('test_templates') os.environ['TZ'] = 'America/Los_Angeles' time.tzset() diff --git a/src/weewx/tests/test_wxxtypes.py b/src/weewx/tests/test_wxxtypes.py index 4d85eb8a..9f7121d5 100644 --- a/src/weewx/tests/test_wxxtypes.py +++ b/src/weewx/tests/test_wxxtypes.py @@ -26,7 +26,7 @@ weewx.debug = 1 log = logging.getLogger(__name__) # Set up logging using the defaults. -weeutil.logger.setup('test_wxxtypes', {}) +weeutil.logger.setup('test_wxxtypes') altitude_vt = weewx.units.ValueTuple(700, "foot", "group_altitude") latitude = 45 diff --git a/src/weewx/tests/test_xtypes.py b/src/weewx/tests/test_xtypes.py index 3a257435..5b505bcc 100644 --- a/src/weewx/tests/test_xtypes.py +++ b/src/weewx/tests/test_xtypes.py @@ -24,7 +24,7 @@ weewx.debug = 1 log = logging.getLogger(__name__) # Set up logging using the defaults. -weeutil.logger.setup('test_xtypes', {}) +weeutil.logger.setup('test_xtypes') os.environ['TZ'] = 'America/Los_Angeles' time.tzset() diff --git a/src/weewx_data/examples/fileparse/bin/user/fileparse.py b/src/weewx_data/examples/fileparse/bin/user/fileparse.py index 7754c629..61f769ec 100644 --- a/src/weewx_data/examples/fileparse/bin/user/fileparse.py +++ b/src/weewx_data/examples/fileparse/bin/user/fileparse.py @@ -126,7 +126,7 @@ if __name__ == "__main__": import weewx weewx.debug = 1 - weeutil.logger.setup('fileparse', {}) + weeutil.logger.setup('fileparse') driver = FileParseDriver() for packet in driver.genLoopPackets(): diff --git a/src/weewx_data/examples/pmon/bin/user/pmon.py b/src/weewx_data/examples/pmon/bin/user/pmon.py index e279f147..0a845dbd 100644 --- a/src/weewx_data/examples/pmon/bin/user/pmon.py +++ b/src/weewx_data/examples/pmon/bin/user/pmon.py @@ -146,7 +146,7 @@ if __name__ == "__main__": import weewx weewx.debug = 1 - weeutil.logger.setup('pmon', {}) + weeutil.logger.setup('pmon') config = { 'Station': { diff --git a/src/weewx_data/examples/tests/test_vaporpressure.py b/src/weewx_data/examples/tests/test_vaporpressure.py index ce517a3a..a4bedf6f 100644 --- a/src/weewx_data/examples/tests/test_vaporpressure.py +++ b/src/weewx_data/examples/tests/test_vaporpressure.py @@ -28,7 +28,7 @@ import weewx.xtypes from weeutil.weeutil import TimeSpan log = logging.getLogger(__name__) -weeutil.logger.setup('test_vaporpressure', {}) +weeutil.logger.setup('test_vaporpressure') weewx.debug = 1 diff --git a/src/weewx_data/weewx.conf b/src/weewx_data/weewx.conf index 49011cc0..f6013063 100644 --- a/src/weewx_data/weewx.conf +++ b/src/weewx_data/weewx.conf @@ -488,6 +488,30 @@ version = 5.0.0b15 ############################################################################## +# This section configures modifications to the default logging. + +[Logging] + + # Root logger + [[root]] + handlers = timed_rotate, + + [[handlers]] + # Log to a set of rotating files + [[[timed_rotate]]] + level = DEBUG + formatter = verbose + class = logging.handlers.TimedRotatingFileHandler + # File to log to, relative to WEEWX_ROOT + filename = log/weewx.log + # When to rotate: + when = midnight + # How many log files to save + backupCount = 7 + + +############################################################################## + # This section configures the internal weewx engine. [Engine] From 6faefb47d5d814f7ef683d51c1889c9fc3f5eb91 Mon Sep 17 00:00:00 2001 From: gary Date: Sun, 26 Nov 2023 09:49:31 +1000 Subject: [PATCH 02/13] move former wee_import docs to weectl import --- ...import-about.md => weectl-import-about.md} | 0 ...mon-opt.md => weectl-import-common-opt.md} | 0 ...fig-opt.md => weectl-import-config-opt.md} | 159 ++++++++---------- ...wee_import-csv.md => weectl-import-csv.md} | 0 ...rt-cumulus.md => weectl-import-cumulus.md} | 0 ...shoot.md => weectl-import-troubleshoot.md} | 0 .../{wee_import-wd.md => weectl-import-wd.md} | 0 ...thercat.md => weectl-import-weathercat.md} | 0 .../{wee_import-wu.md => weectl-import-wu.md} | 0 mkdocs.yml | 20 +-- 10 files changed, 84 insertions(+), 95 deletions(-) rename docs_src/utilities/{wee_import-about.md => weectl-import-about.md} (100%) rename docs_src/utilities/{wee_import-common-opt.md => weectl-import-common-opt.md} (100%) rename docs_src/utilities/{wee_import-config-opt.md => weectl-import-config-opt.md} (88%) rename docs_src/utilities/{wee_import-csv.md => weectl-import-csv.md} (100%) rename docs_src/utilities/{wee_import-cumulus.md => weectl-import-cumulus.md} (100%) rename docs_src/utilities/{wee_import-troubleshoot.md => weectl-import-troubleshoot.md} (100%) rename docs_src/utilities/{wee_import-wd.md => weectl-import-wd.md} (100%) rename docs_src/utilities/{wee_import-weathercat.md => weectl-import-weathercat.md} (100%) rename docs_src/utilities/{wee_import-wu.md => weectl-import-wu.md} (100%) diff --git a/docs_src/utilities/wee_import-about.md b/docs_src/utilities/weectl-import-about.md similarity index 100% rename from docs_src/utilities/wee_import-about.md rename to docs_src/utilities/weectl-import-about.md diff --git a/docs_src/utilities/wee_import-common-opt.md b/docs_src/utilities/weectl-import-common-opt.md similarity index 100% rename from docs_src/utilities/wee_import-common-opt.md rename to docs_src/utilities/weectl-import-common-opt.md diff --git a/docs_src/utilities/wee_import-config-opt.md b/docs_src/utilities/weectl-import-config-opt.md similarity index 88% rename from docs_src/utilities/wee_import-config-opt.md rename to docs_src/utilities/weectl-import-config-opt.md index b638d486..b00f43d6 100644 --- a/docs_src/utilities/wee_import-config-opt.md +++ b/docs_src/utilities/weectl-import-config-opt.md @@ -1,24 +1,10 @@ -`wee_import` requires a second configuration file, the import configuration -file, in addition to the standard WeeWX configuration file. The import -configuration file specifies the import type and various options associated -with each type of import. The import configuration file is specified using the -mandatory `--import-config` option. How you construct the import configuration -file is up to you; however, the recommended method is to copy one of the -example import configuration files located in the `/home/weewx/util/import` or -`/etc/weewx/import` directory as applicable, modify the configuration options -in the newly copied file to suit the import to be performed and then use this -file as the import configuration file. +`wee_import` requires a second configuration file, the import configuration file, in addition to the standard WeeWX configuration file. The import configuration file specifies the import type and various options associated with each type of import. The import configuration file is specified using the mandatory `--import-config` option. How you construct the import configuration file is up to you; however, the recommended method is to copy one of the example import configuration files located in the `util/import` directory as applicable, modify the configuration options in the newly copied file to suit the import to be performed and then use this file as the import configuration file. -Following is the definitive guide to the options available in the import -configuration file. Default values are provided for a number of options, -meaning that if they are not listed in the import configuration file at all -`wee_import` will pick sensible values. When the documentation below gives a -default value this is the value that will be used if the option is omitted. +Following is the definitive guide to the options available in the import configuration file. Default values are provided for a number of options, meaning that if they are not listed in the import configuration file at all `wee_import` will pick sensible values. When the documentation below gives a default value this is the value that will be used if the option is omitted. ### `source`{#import_config_source} -The `source` option determines the type of import to be performed by -`wee_import`. The option is mandatory and must be set to one of the following: +The `source` option determines the type of import to be performed by `wee_import`. The option is mandatory and must be set to one of the following: * `CSV` to import from a single CSV format file. * `WU` to import from a Weather Underground PWS history @@ -30,21 +16,17 @@ There is no default. ## [CSV] -The `[CSV]` section contains the options controlling the import of -observational data from a CSV format file. +The `[CSV]` section contains the options controlling the import of observational data from a CSV format file. ### `file`{#csv_file} -The file containing the CSV format data to be used as the source during the -import. Include full path and filename. +The file containing the CSV format data to be used as the source during the import. Include full path and filename. There is no default. ### `source_encoding`{#csv_encoding} -The source file encoding. This parameter is optional and should only need be -used if the source file uses an encoding other than UTF-8 or an ASCII -compatible encoding. If used, the setting used should be a Python Standard Encoding. +The source file encoding. This parameter is optional and should only need be used if the source file uses an encoding other than UTF-8 or an ASCII compatible encoding. If used, the setting used should be a Python Standard Encoding. The default is `utf-8-sig`. @@ -54,17 +36,13 @@ The character used to separate fields. Default is `,` (comma). ### `decimal`{#csv_decimal} -The character used as the decimal point in the source files. A full stop is -frequently used, but it may be another character. This parameter must be -included in quotation marks. +The character used as the decimal point in the source files. A full stop is frequently used, but it may be another character. This parameter must be included in quotation marks. The default is `'.'`. ### `interval`{#csv_interval} -Determines how the time interval (WeeWX archive table field `interval`) -between successive observations is derived. The interval can be derived by one -of three methods: +Determines how the time interval (WeeWX archive table field `interval`) between successive observations is derived. The interval can be derived by one of three methods: * The interval can be calculated as the time, rounded to the nearest minute, between the date-time of successive records. This method is suitable when the data was recorded at fixed intervals and there are NO missing records in the source data. Use of this method when there are missing records in the source data can compromise the integrity of the WeeWX statistical data. Select this method by setting `interval = derive`. @@ -127,28 +105,7 @@ For example, if the source data uses the format 23 January 2015 15:34 the approp The default is `%Y-%m-%d %H:%M:%S`. !!! Note - `wee_import` does not support the construction of the unique record date - time stamp from separate date and time fields, rather the date-time - information for each imported record must be contained in a single - field. CSV data containing separate date and time fields may require - further manual processing before they can be imported. - -### `rain`{#csv_rain} - -The WeeWX `rain` field records rainfall that was recorded in the preceding archive period, so for a five-minute archive period the `rain` field in each archive record would contain the total rainfall that fell in the previous five minutes. Many weather station applications provide a daily or yearly total. `wee_import` can derive the WeeWX `rain` field in one of two ways: - -* If the imported rainfall data is a running total then `wee_import` can derive the WeeWX `rain` field from successive totals. For this method use `rain = cumulative`. - -* If the imported rainfall data is a discrete value per date-time period then `rain = discrete` should be used. - -!!! Note - `wee_import` only supports cumulative rainfall data that resets on a - midnight boundary, cumulative rainfall data that resets at some other - time; e.g., 9am, is not supported. In such cases the rainfall data will - need to be converted to either reset on a midnight boundary or a discrete - value per date-time period and `rain = discrete` used. The former may be - possible by selecting another rainfall field (if available) in the source - data, otherwise it will require manual manipulation of the source data. + `wee_import` does not support the construction of the unique record date time stamp from separate date and time fields, rather the date-time information for each imported record must be contained in a single field. CSV data containing separate date and time fields may require further manual processing before they can be imported. ### `wind_direction`{#csv_wind_direction} @@ -256,26 +213,24 @@ The default is `0, 360`. ### `[[FieldMap]]`{#csv_fieldmap} -The `[[FieldMap]]` stanza defines the mapping from the source data fields to WeeWX archive fields. The map consists of one row per field being imported using either of the following formats: +The `[[FieldMap]]` stanza defines the mapping from the source data fields to WeeWX archive fields. The map consists of one stanza per WeeWX archive field being populated using the following format: ``` -weewx_archive_field_name = csv_field_name, weewx_unit_name + [[[weewx_archive_field_name]]] + source_field = csv_field_name + unit = weewx_unit_name | text + cumulative = True | False ``` -or +Where `weewx_archive_field_name` is a field name in the in-use WeeWX archive table schema. -``` -weewx_archive_field_name = csv_field_name, text -``` +Each WeeWX archive field stanza supports the following options: -Where: +* `source_field`. The name of the CSV field to be mapped to the WeeWX archive field. Mandatory. +* `unit`. The WeeWX unit name of the units used by `source_field`. Text fields may be imported by setting the unit option to `text`. Mandatory. +* `cumulative`. Whether the `source_field` is a cumulative value or not (e.g, daily rainfall). Optional boolean value. Default is `False`. -* `weewx_archive_field_name` is a field name in the in-use WeeWX archive table schema -* `csv_field_name` is the name of a field from the CSV file -* `weewx_unit_name` is the WeeWX unit name of the units used by `csv_field_name` -* `text` is the literal word text - -This mapping allows `wee_import` to take a source data field, do the appropriate unit conversion and store the resulting value in the appropriate WeeWX archive field. Source data text fields may be mapped to a WeeWX text archive field by using the second form of the field map entry where the literal `text` is used in place of a WeeWX unit name. A mapping is not required for every WeeWX archive field (e.g., the source may not provide inside temperature so no `inTemp` field mapping is required) and neither does every CSV field need to be included in a mapping (e.g., the source data field `monthrain` may have no use if the source data field `rain` provides the data for the WeeWX archive `rain field). Unused field mapping lines will not be used and may be omitted. +This mapping allows `wee_import` to take a source data field, perform the appropriate unit conversion and store the resulting value in the appropriate WeeWX archive field. Source data text fields may be mapped to a WeeWX text archive field by using the second form of the field map entry where the literal `text` is used in place of a WeeWX unit name. A mapping is not required for every WeeWX archive field (e.g., the source may not provide inside temperature so no `inTemp` field mapping is required) and neither does every CSV field need to be included in a mapping (e.g., the source data field `monthrain` may have no use if the source data field `dayrain` provides the data for the WeeWX archive `rain` field). !!! Note Importing of text data into text fields in the WeeWX archive is only supported for WeeWX archive fields that have been configured as text fields. Refer to the Wiki page [Storing text in the database](https://github.com/weewx/weewx/wiki/Storing-text-in-the-database) for details. @@ -285,28 +240,46 @@ If the source data includes a field that contains a WeeWX unit system code (i.e. For example, source CSV data with the following structure: ``` -date_and_time,temp,humid,wind,dir,rainfall,rad,river -23 May 2018 13:00,17.4,56,3.0,45,0.0,956,340 -23 May 2018 13:05,17.6,56,1.0,22.5,0.4,746,341 +date_and_time,temp,humid,wind,dir,dayrain,rad,river,decsription +23 May 2018 13:00,17.4,56,3.0,45,10.0,956,340,'cloudy' +23 May 2018 13:05,17.6,56,1.0,22.5,10.4,746,341,'showers developing' ``` -where `temp` is temperature in Celsius, `humid` is humidity in percent, `wind` is wind speed in km/h, `dir` is wind direction in degrees, `rainfall` is rain in mm, `rad` is radiation in watts per square meter and `river` is river height in mm might use a field map as follows: +where `temp` is temperature in Celsius, `humid` is humidity in percent, `wind` is wind speed in km/h, `dir` is wind direction in degrees, `rainfall` is rain in mm, `rad` is radiation in watts per square meter, `river` is river height in mm and `description` is a text might use a field map as follows: ``` [[FieldMap]] - dateTime = date_and_time, unix_epoch - outTemp = temp, degree_C - outHumidity = humid, percent - windSpeed = wind, km_per_hour - windDir = dir, degree_compass - rain = rainfall, mm - radiation = rad, watt_per_meter_squared + [[[dateTime]]] + source_field = date_and_time + unit = unix_epoch + [[[outTemp]]] + source_field = temp + unit = degree_C + [[[outHumidity]]] + source_field = humid + unit = percent + [[[windSpeed]]] + source = wind + unoit = km_per_hour + [[[windDir]]] + source_field = dir + unit = degree_compass + [[[rain]]] + source_field = dayrain + unit = mm + cumulative = True + [[[radiation]]] + source_field = rad + unit = watt_per_meter_squared + [[[outlook]]] + source_field = description + unit = text ``` If the same source CSV data included a field `unit_info` that contains WeeWX unit system data as follows: ``` -date_and_time,temp,humid,wind,dir,rainfall,rad,river,unit_info +date_and_time,temp,humid,wind,dir,dayrain,rad,river,unit_info 23 May 2018 13:00,17.4,56,3.0,45,0.0,956,340,1 23 May 2018 13:05,17.6,56,1.0,22.5,0.4,746,341,16 ``` @@ -315,14 +288,30 @@ then a field map such as the following might be used: ``` [[FieldMap]] - dateTime = date_and_time, unix_epoch - usUnits = unit_info - outTemp = temp - outHumidity = humid - windSpeed = wind - windDir = dir - rain = rainfall - radiation = rad + [[[dateTime]]] + source_field = date_and_time + unit = unix_epoch + [[[usUnits]]] + source_field = unit_info + [[[outTemp]]] + source_field = temp + unit = degree_C + [[[outHumidity]]] + source_field = humid + unit = percent + [[[windSpeed]]] + source = wind + unoit = km_per_hour + [[[windDir]]] + source_field = dir + unit = degree_compass + [[[rain]]] + source_field = dayrain + unit = mm + cumulative = True + [[[radiation]]] + source_field = rad + unit = watt_per_meter_squared ``` !!! Note diff --git a/docs_src/utilities/wee_import-csv.md b/docs_src/utilities/weectl-import-csv.md similarity index 100% rename from docs_src/utilities/wee_import-csv.md rename to docs_src/utilities/weectl-import-csv.md diff --git a/docs_src/utilities/wee_import-cumulus.md b/docs_src/utilities/weectl-import-cumulus.md similarity index 100% rename from docs_src/utilities/wee_import-cumulus.md rename to docs_src/utilities/weectl-import-cumulus.md diff --git a/docs_src/utilities/wee_import-troubleshoot.md b/docs_src/utilities/weectl-import-troubleshoot.md similarity index 100% rename from docs_src/utilities/wee_import-troubleshoot.md rename to docs_src/utilities/weectl-import-troubleshoot.md diff --git a/docs_src/utilities/wee_import-wd.md b/docs_src/utilities/weectl-import-wd.md similarity index 100% rename from docs_src/utilities/wee_import-wd.md rename to docs_src/utilities/weectl-import-wd.md diff --git a/docs_src/utilities/wee_import-weathercat.md b/docs_src/utilities/weectl-import-weathercat.md similarity index 100% rename from docs_src/utilities/wee_import-weathercat.md rename to docs_src/utilities/weectl-import-weathercat.md diff --git a/docs_src/utilities/wee_import-wu.md b/docs_src/utilities/weectl-import-wu.md similarity index 100% rename from docs_src/utilities/wee_import-wu.md rename to docs_src/utilities/weectl-import-wu.md diff --git a/mkdocs.yml b/mkdocs.yml index 6f5537ef..3e81e2b0 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -94,18 +94,18 @@ nav: - weectl debug: utilities/weectl-debug.md - weectl device: utilities/weectl-device.md - weectl extension: utilities/weectl-extension.md + - weectl import: + - Introduction: utilities/weectl-import-about.md + - Common options: utilities/weectl-import-common-opt.md + - Configuration options: utilities/weectl-import-config-opt.md + - CSV: utilities/weectl-import-csv.md + - Weather Underground: utilities/weectl-import-wu.md + - Cumulus: utilities/weectl-import-cumulus.md + - Weather Display: utilities/weectl-import-wd.md + - WeatherCat: utilities/weectl-import-weathercat.md + - Troubleshooting: utilities/weectl-import-troubleshoot.md - weectl report: utilities/weectl-report.md - weectl station: utilities/weectl-station.md - - wee_import: - - Introduction: utilities/wee_import-about.md - - Common options: utilities/wee_import-common-opt.md - - Configuration options: utilities/wee_import-config-opt.md - - CSV: utilities/wee_import-csv.md - - Weather Underground: utilities/wee_import-wu.md - - Cumulus: utilities/wee_import-cumulus.md - - Weather Display: utilities/wee_import-wd.md - - WeatherCat: utilities/wee_import-weathercat.md - - Troubleshooting: utilities/wee_import-troubleshoot.md - "Hardware guide": - Drivers: hardware/drivers.md From 46196973378d6b18ab02fa385f8825442f7b1713 Mon Sep 17 00:00:00 2001 From: Tom Keffer Date: Sat, 25 Nov 2023 16:22:03 -0800 Subject: [PATCH 03/13] No longer distribute docs. Must access online. --- .gitignore | 1 - docs_src/changes.md | 3 +- docs_src/usersguide/installing.md | 2 +- docs_src/usersguide/where.md | 121 +++++++++++++++--------- makefile | 9 +- mkdocs.yml | 2 +- src/weecfg/station_config.py | 55 +---------- src/weecfg/tests/test_station_config.py | 7 +- src/weectllib/station_cmd.py | 24 ++--- 9 files changed, 97 insertions(+), 127 deletions(-) diff --git a/.gitignore b/.gitignore index 9fe5c1f2..6e6c0fb0 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,3 @@ dist # Ignore database, HTML directories, and built documentation src/weewx_data/archive/ src/weewx_data/public_html/ -src/weewx_data/docs diff --git a/docs_src/changes.md b/docs_src/changes.md index d12fc1e3..6aab68ba 100644 --- a/docs_src/changes.md +++ b/docs_src/changes.md @@ -16,7 +16,8 @@ The new utility [`weectl`](utilities/weectl-about.md) replaces `wee_database`, `wee_debug`, `wee_device`, `wee_extension`, `wee_import`, `wee_reports`, and `wee_config`. Try `weectl --help` to see how to use it. -Documentation now uses [MkDocs](https://www.mkdocs.org/). +Documentation now uses [MkDocs](https://www.mkdocs.org/). It is no longer included in the +distribution, but can always be accessed online at https://weewx.com/docs. Package installs now use `systemd` instead of the old System V `/etc/init.d`. diff --git a/docs_src/usersguide/installing.md b/docs_src/usersguide/installing.md index 233bd7b5..1675b1d0 100644 --- a/docs_src/usersguide/installing.md +++ b/docs_src/usersguide/installing.md @@ -138,7 +138,7 @@ functionality to the core WeeWX capabilities. Use the WeeWX extension management utility to download, install, and manage these extensions. Start by looking in the extensions section of the -[WeeWX Wiki](https;//github.com/weewx/weewx/wiki). When you find an +[WeeWX Wiki](https://github.com/weewx/weewx/wiki). When you find an extension you like, make note of the URL to that extension. The URL will refer to a `zip` or `tgz` file. diff --git a/docs_src/usersguide/where.md b/docs_src/usersguide/where.md index b7e8ee79..879897b5 100644 --- a/docs_src/usersguide/where.md +++ b/docs_src/usersguide/where.md @@ -17,7 +17,8 @@ documentation. | Executables | _`BIN_ROOT`_ | `/usr/share/weewx/` | | SQLite databases | _`SQLITE_ROOT`_ | `/var/lib/weewx/` | | Web pages and images | _`HTML_ROOT`_ | `/var/www/html/weewx/` | - | Documentation | _`DOC_ROOT`_ | `/usr/share/doc/weewx/` | + | Log | | See below | + | Documentation | | https://weewx.com/docs/5.0 | === "RedHat" @@ -30,33 +31,36 @@ documentation. | Executables | _`BIN_ROOT`_ | `/usr/share/weewx/` | | SQLite databases | _`SQLITE_ROOT`_ | `/var/lib/weewx/` | | Web pages and images | _`HTML_ROOT`_ | `/var/www/html/weewx/` | - | Documentation | _`DOC_ROOT`_ | `/usr/share/doc/weewx-x.y.z/` | + | Log | | See below | + | Documentation | | https://weewx.com/docs/5.0 | === "openSUSE" - | Component | Symbolic name | Nominal value | - |-------------------------|------------------|-------------------------------| - | WeeWX root directory | _`WEEWX_ROOT`_ | `/etc/weewx` | - | Skins and templates | _`SKIN_ROOT`_ | `/etc/weewx/skins/` | - | User directory | _`USER_ROOT`_ | `/etc/weewx/bin/user/` | - | Examples | _`EXAMPLE_ROOT`_ | `/etc/weewx/examples/` | - | Executables | _`BIN_ROOT`_ | `/usr/share/weewx/` | - | SQLite databases | _`SQLITE_ROOT`_ | `/var/lib/weewx/` | - | Web pages and images | _`HTML_ROOT`_ | `/var/www/html/weewx/` | - | Documentation | _`DOC_ROOT`_ | `/usr/share/doc/weewx-x.y.z/` | + | Component | Symbolic name | Nominal value | + |-------------------------|------------------|-----------------------------| + | WeeWX root directory | _`WEEWX_ROOT`_ | `/etc/weewx` | + | Skins and templates | _`SKIN_ROOT`_ | `/etc/weewx/skins/` | + | User directory | _`USER_ROOT`_ | `/etc/weewx/bin/user/` | + | Examples | _`EXAMPLE_ROOT`_ | `/etc/weewx/examples/` | + | Executables | _`BIN_ROOT`_ | `/usr/share/weewx/` | + | SQLite databases | _`SQLITE_ROOT`_ | `/var/lib/weewx/` | + | Web pages and images | _`HTML_ROOT`_ | `/var/www/html/weewx/` | + | Log | | See below | + | Documentation | | https://weewx.com/docs/5.0 | === "pip" - | Component | Symbolic name | Nominal value | - |-------------------------|------------------|----------------------| - | WeeWX root directory | _`WEEWX_ROOT`_ | `~/weewx-data` | - | Skins and templates | _`SKIN_ROOT`_ | `skins/` | - | User directory | _`USER_ROOT`_ | `bin/user/` | - | Examples | _`EXAMPLE_ROOT`_ | `examples/` | - | Executables | _`BIN_ROOT`_ | varies | - | SQLite databases | _`SQLITE_ROOT`_ | `archive/` | - | Web pages and images | _`HTML_ROOT`_ | `public_html/` | - | Documentation | _`DOC_ROOT`_ | `docs/` | + | Component | Symbolic name | Nominal value | + |-------------------------|------------------|-----------------------------| + | WeeWX root directory | _`WEEWX_ROOT`_ | `~/weewx-data` | + | Skins and templates | _`SKIN_ROOT`_ | `skins/` | + | User directory | _`USER_ROOT`_ | `bin/user/` | + | Examples | _`EXAMPLE_ROOT`_ | `examples/` | + | Executables | _`BIN_ROOT`_ | varies | + | SQLite databases | _`SQLITE_ROOT`_ | `archive/` | + | Web pages and images | _`HTML_ROOT`_ | `public_html/` | + | Log | | `log/weewx.log` | + | Documentation | | https://weewx.com/docs/5.0 | !!! Note In the locations above, relative paths are *relative to _`WEEWX_ROOT`_*. @@ -64,26 +68,37 @@ documentation. `~` represents the `HOME` directory of the user. -## Location of executables in a pip install - -If you use a pip install, the location of the executables will depend on how -the installation was done. - -| Install method | Commands | Location of executables | -|-----------------------------------------------------|------------------------------------------------------------------------------|-------------------------| -| Virtual environment
(recommended) | `python3 -m venv ~/ve`
`source ~/ve/bin/activate`
`pip3 install weewx` | `~/ve/bin/` | -| pip, no sudo, with `--user` | `pip3 install weewx --user` | `~/.local/bin/` | -| pip, no sudo, no `--user` | `pip3 install weewx` | `~/.local/bin/` | -| pip with sudo
(not recommended) | `sudo pip3 install weewx` | `/usr/local/bin/` (1) | -| Virtual environment with `--user`
(not allowed) | `python3 -m venv ~/ve`
`source ~/ve/bin/activate`
`pip3 install weewx --user` | N/A | - -(1) Checked on Ubuntu 22.02 and Rocky v9.1 - - ## Log files -In the default configuration, WeeWX writes its status to the system log. -This is where to find the system log for each platform. +Logging is configured by the `[Logging]` section in the configuration file, +`weewx.conf`. The section that comes with WeeWX Version 5 looks like this: + +``` ini linenums="1" hl_lines="12 14 16" +[Logging] + [[root]] + handlers = timed_rotate, + + [[handlers]] + # Log to a set of rotating files + [[[timed_rotate]]] + level = DEBUG + formatter = verbose + class = logging.handlers.TimedRotatingFileHandler + # File to log to, relative to WEEWX_ROOT + filename = log/weewx.log + # When to rotate: + when = midnight + # How many log files to save + backupCount = 7 +``` + +This says to use a timed file rotation to log file `log/weewx.log` (line 12). +Note that this is _relative to `WEEWX_ROOT`_, so the actual file is likely to be +`~/weewx-data/log/weewx.log`. The file should be rotated at midnight (line 14). +A week's worth of backups should be saved (line 16). + +If you don't have such a section, then the system log will +be used. This is where to find the system log for each platform. === "Debian" @@ -113,7 +128,25 @@ This is where to find the system log for each platform. !!! note On macOS, the log file is likely to contain only severe log messages. -If the default for your system is inconvenient, or does not provide the logging -fidelity that you require, then you may want to consider logging to a separate, -rotating log file. See the wiki article -[_Logging to rotating files_](https://github.com/weewx/weewx/wiki/WeeWX-v4-and-logging#logging-to-rotating-files). + +See the wiki article [WeeWX V4 and logging](https://github.com/weewx/weewx/wiki/WeeWX-v4-and-logging) +for more information on controlling logging in WeeWX. + + +## Location of executables in a pip install + +This is something you are not likely to need, but can occasionally be useful. +It's included here for completeness. If you use a pip install, the location of +the executables will depend on how the installation was done. + +| Install method | Commands | Location of executables | +|-----------------------------------------------------|------------------------------------------------------------------------------|-------------------------| +| Virtual environment
(recommended) | `python3 -m venv ~/ve`
`source ~/ve/bin/activate`
`pip3 install weewx` | `~/ve/bin/` | +| pip, no sudo, with `--user` | `pip3 install weewx --user` | `~/.local/bin/` | +| pip, no sudo, no `--user` | `pip3 install weewx` | `~/.local/bin/` | +| pip with sudo
(not recommended) | `sudo pip3 install weewx` | `/usr/local/bin/` (1) | +| Virtual environment with `--user`
(not allowed) | `python3 -m venv ~/ve`
`source ~/ve/bin/activate`
`pip3 install weewx --user` | N/A | + +(1) Checked on Ubuntu 22.02 and Rocky v9.1 + + diff --git a/makefile b/makefile index 447ed03f..8dcdd92c 100644 --- a/makefile +++ b/makefile @@ -8,6 +8,9 @@ SIGN=1 # WWW server WEEWX_COM:=weewx.com +BLDDIR=build +DSTDIR=dist + # location of the html documentation WEEWX_HTMLDIR=/var/www/html # location of weewx downloads @@ -17,18 +20,16 @@ WEEWX_STAGING=$(WEEWX_HTMLDIR)/downloads/development_versions # Location of doc sources DOC_SRC=docs_src # Location of built docs -DOC_BUILT=src/weewx_data/docs +DOC_BUILT=$(BLDDIR)/docs # Location of the skins SKINLOC=src/weewx_data/skins +CWD=$(shell pwd) # extract version to be used in package controls and labels VERSION:=$(shell sed -ne 's/^version = "\(.*\)"/\1/p;' pyproject.toml) # just the major.minor part of the version MMVERSION:=$(shell echo "$(VERSION)" | sed -e 's%.[0-9a-z]*$$%%') -CWD=$(shell pwd) -BLDDIR=build -DSTDIR=dist SRCPKG=weewx-$(VERSION).tgz WHEELSRC=weewx-$(VERSION).tar.gz WHEEL=weewx-$(VERSION)-py3-none-any.whl diff --git a/mkdocs.yml b/mkdocs.yml index 3e81e2b0..d702e4ca 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -45,7 +45,7 @@ extra_css: - css/weewx_ui.css docs_dir: 'docs_src' -site_dir: 'src/weewx_data/docs' +site_dir: 'build/docs' nav: - "Overview" : index.md diff --git a/src/weecfg/station_config.py b/src/weecfg/station_config.py index 3b2d41ff..1b889029 100644 --- a/src/weecfg/station_config.py +++ b/src/weecfg/station_config.py @@ -31,7 +31,6 @@ log = logging.getLogger(__name__) def station_create(config_path, *args, dist_config_path=None, weewx_root=None, - docs_root=None, examples_root=None, user_root=None, dry_run=False, @@ -47,8 +46,7 @@ def station_create(config_path, *args, It then: 1. If no_prompt is false, it creates the configuration file by prompting the user. If true, it uses defaults. - 2. It then copies the documentation out of package resources and into WEEWX_ROOT. - 3. Same with the examples and utility files. + 2. Copies the examples and utility files out of package resources and into WEEWX_ROOT. """ if dry_run: @@ -80,7 +78,6 @@ def station_create(config_path, *args, *args, **kwargs) copy_skins(dist_config_dict, dry_run=dry_run) copy_util(config_path, dist_config_dict, dry_run=dry_run) - copy_docs(dist_config_dict, docs_root=docs_root, dry_run=dry_run) copy_examples(dist_config_dict, examples_root=examples_root, dry_run=dry_run) copy_user(dist_config_dict, user_root=user_root, dry_run=dry_run) @@ -514,46 +511,6 @@ def copy_skins(config_dict, dry_run=False): shutil.copytree(src, dest) -def copy_docs(config_dict, docs_root=None, dry_run=False, force=False): - """Copy documentation from package resources to the DOCS_ROOT directory. - - Args: - config_dict (dict): A configuration dictionary - docs_root (str): Path to where the docs should be put, relative to WEEWX_ROOT. - dry_run (bool): True to not actually do anything. Just show what would happen. - force (bool): True to overwrite existing docs. Otherwise, do nothing if they exist. - - Returns: - str|None: Path to the freshly written docs, or None if they already exist and `force` - was False. - """ - - # If the user didn't specify a value, use a default - if not docs_root: - docs_root = 'docs' - - # DOCS_ROOT is relative to WEEWX_PATH. Join them to get the absolute path. - docs_dir = os.path.join(config_dict['WEEWX_ROOT'], docs_root) - - if os.path.isdir(docs_dir) and not force: - print(f"Directory {docs_dir} already exists.") - return None - # If the documentation has not been built, then a FileNotFoundError will be raised. Be - # prepared to catch it. - try: - with weeutil.weeutil.get_resource_path('weewx_data', 'docs') as docs_resources: - print(f"Removing {docs_dir}.") - if not dry_run: - shutil.rmtree(docs_dir, ignore_errors=True) - print(f"Copying new docs into {docs_dir}.") - if not dry_run: - shutil.copytree(docs_resources, docs_dir) - except FileNotFoundError: - print(f"{bcolors.WARNING}Documentation not available. " - f"Try building them using mkdocs.{bcolors.ENDC}", file=sys.stderr) - return docs_dir - - def copy_examples(config_dict, examples_root=None, dry_run=False, force=False): """Copy the examples to the EXAMPLES_ROOT directory. @@ -678,12 +635,12 @@ def copy_util(config_path, config_dict, dry_run=False): return dstdir -def station_upgrade(config_path, dist_config_path=None, docs_root=None, examples_root=None, +def station_upgrade(config_path, dist_config_path=None, examples_root=None, skin_root=None, what=None, no_prompt=False, no_backup=False, dry_run=False): """Upgrade the user data for the configuration file found at config_path""" if what is None: - what = ('config', 'docs', 'examples', 'util') + what = ('config', 'examples', 'util') if dry_run: print("This is a dry run. Nothing will actually be done.") @@ -692,7 +649,6 @@ def station_upgrade(config_path, dist_config_path=None, docs_root=None, examples config_path, config_dict = weecfg.read_config(config_path) abbrev = {'config': 'configuration file', - 'docs': 'documentation', 'util': 'utility files'} choices = ', '.join([abbrev.get(p, p) for p in what]) msg = f"\nUpgrade {choices} in {config_dict['WEEWX_ROOT']}? (Y/n) " @@ -721,11 +677,6 @@ def station_upgrade(config_path, dist_config_path=None, docs_root=None, examples if backup_path: print(f"Saved old configuration file as {backup_path}") - if 'docs' in what: - docs_dir = copy_docs(config_dict, docs_root=docs_root, - dry_run=dry_run, force=True) - print(f"Finished upgrading docs at {docs_dir}") - if 'examples' in what: examples_dir = copy_examples(config_dict, examples_root=examples_root, dry_run=dry_run, force=True) diff --git a/src/weecfg/tests/test_station_config.py b/src/weecfg/tests/test_station_config.py index 70a94c5b..dcf1a31f 100644 --- a/src/weecfg/tests/test_station_config.py +++ b/src/weecfg/tests/test_station_config.py @@ -298,11 +298,8 @@ class TestCreateStation(unittest.TestCase): with tempfile.TemporaryDirectory(dir='/var/tmp') as dirname: config_path = os.path.join(dirname, 'weewx.conf') # We have not run 'pip', so the only copy of weewxd.py is the one in the repository. - # Also, 'docs' are generated by mkdocs, so they do not exist in the repository. - with patch('weecfg.station_config.copy_docs') as mock_copy: - # Create a station using the defaults - weecfg.station_config.station_create(config_path, no_prompt=True) - mock_copy.assert_called_once() + # Create a station using the defaults + weecfg.station_config.station_create(config_path, no_prompt=True) # Retrieve the config file that was created and check it: config_dict = configobj.ConfigObj(config_path, encoding='utf-8') diff --git a/src/weectllib/station_cmd.py b/src/weectllib/station_cmd.py index 4ee214be..78cdfe84 100644 --- a/src/weectllib/station_cmd.py +++ b/src/weectllib/station_cmd.py @@ -22,7 +22,6 @@ station_create_usage = f"""{bcolors.BOLD}weectl station create [--sqlite-root=DIRECTORY] [--html-root=DIRECTORY] [--user-root=DIRECTORY] - [--docs-root=DIRECTORY] [--examples-root=DIRECTORY] [--no-prompt] [--config=FILENAME] @@ -46,10 +45,9 @@ station_reconfigure_usage = f"""{bcolors.BOLD}weectl station reconfigure [--dry-run]{bcolors.ENDC} """ station_upgrade_usage = f"""{bcolors.BOLD}weectl station upgrade - [--docs-root=DIRECTORY] [--examples-root=DIRECTORY] [--skin-root=DIRECTORY] - [--what (docs|examples|util|config|skins)...] + [--what (examples|util|config|skins)...] [--no-backup] [--no-prompt] [--config=FILENAME] @@ -68,7 +66,7 @@ CREATE_DESCRIPTION = """Create a new station data area, including a configuratio + WEEWX_ROOT_DESCRIPTION UPGRADE_DESCRIPTION = """Upgrade an existing station data area, including any combination of the -docs, examples, utility files, configuration file, and skins. """ + WEEWX_ROOT_DESCRIPTION +examples, utility files, configuration file, and skins. """ + WEEWX_ROOT_DESCRIPTION def add_subparser(subparsers): @@ -95,10 +93,6 @@ def add_subparser(subparsers): metavar='DIRECTORY', help='Where to put the "user" directory, relative to ' 'WEEWX_ROOT. Default is "bin/user"') - station_create_parser.add_argument('--docs-root', - metavar='DIRECTORY', - help='Where to put the documentation, relative to ' - 'WEEWX_ROOT. Default is "docs".') station_create_parser.add_argument('--examples-root', metavar='DIRECTORY', help='Where to put the examples, relative to ' @@ -147,13 +141,9 @@ def add_subparser(subparsers): action_parser.add_parser('upgrade', usage=station_upgrade_usage, description=UPGRADE_DESCRIPTION, - help='Upgrade any combination of the docs, examples, utility ' + help='Upgrade any combination of the examples, utility ' 'files, configuration file, and skins.') - station_upgrade_parser.add_argument('--docs-root', - metavar='DIRECTORY', - help='Where to put the new documentation, relative to ' - 'WEEWX_ROOT. Default is "docs".') station_upgrade_parser.add_argument('--examples-root', metavar='DIRECTORY', help='Where to put the new examples, relative to ' @@ -163,11 +153,11 @@ def add_subparser(subparsers): help='Where to put the skins, relative to ' 'WEEWX_ROOT. Default is "skins".') station_upgrade_parser.add_argument('--what', - choices=['docs', 'examples', 'util', 'config', 'skins'], - default=['docs', 'examples', 'util'], + choices=['examples', 'util', 'config', 'skins'], + default=['examples', 'util'], nargs='+', help='What to upgrade. Default is to upgrade the ' - 'documentation, examples, and utility files.') + 'examples, and utility files.') station_upgrade_parser.add_argument('--no-backup', action='store_true', help='Do not backup the old configuration file.') station_upgrade_parser.add_argument('--no-prompt', action='store_true', @@ -211,7 +201,6 @@ def create_station(namespace): skin_root=namespace.skin_root, sqlite_root=namespace.sqlite_root, html_root=namespace.html_root, - docs_root=namespace.docs_root, examples_root=namespace.examples_root, no_prompt=namespace.no_prompt, dry_run=namespace.dry_run) @@ -245,7 +234,6 @@ def reconfigure_station(namespace): def upgrade_station(namespace): weecfg.station_config.station_upgrade(config_path=namespace.config, dist_config_path=namespace.dist_config, - docs_root=namespace.docs_root, examples_root=namespace.examples_root, skin_root=namespace.skin_root, what=namespace.what, From eb68bf261b4b1cfde0a456c701f144fd62eac7f4 Mon Sep 17 00:00:00 2001 From: gjr80 Date: Sun, 26 Nov 2023 12:17:10 +1000 Subject: [PATCH 04/13] word wrap weectl import docs to 78 characters --- docs_src/utilities/weectl-import-about.md | 13 +- .../utilities/weectl-import-common-opt.md | 88 +-- .../utilities/weectl-import-config-opt.md | 572 ++++++++++++++---- docs_src/utilities/weectl-import-csv.md | 178 ++++-- docs_src/utilities/weectl-import-cumulus.md | 236 ++++++-- .../utilities/weectl-import-troubleshoot.md | 20 +- docs_src/utilities/weectl-import-wd.md | 337 ++++++++--- .../utilities/weectl-import-weathercat.md | 271 ++++++--- docs_src/utilities/weectl-import-wu.md | 251 ++++++-- 9 files changed, 1507 insertions(+), 459 deletions(-) diff --git a/docs_src/utilities/weectl-import-about.md b/docs_src/utilities/weectl-import-about.md index 72c31e4a..2fade2b5 100644 --- a/docs_src/utilities/weectl-import-about.md +++ b/docs_src/utilities/weectl-import-about.md @@ -1,11 +1,16 @@ -# wee_import +# weectl import -Some WeeWX users will have historical data from another source (e.g., other weather station software or a manually compiled file) which they wish to import into WeeWX. Such data can, depending upon the source, be imported using the `wee_import` utility. +Some WeeWX users will have historical data from another source (e.g., other +weather station software or a manually compiled file) which they wish to +import into WeeWX. Such data can, depending upon the source, be imported +using the `wee_import` utility. -The `wee_import` utility supports importing observational data from the following sources: +The `wee_import` utility supports importing observational data from the +following sources: * a single [Comma Separated Values (CSV)](wee_import-csv.md) format file -* the historical observations of a [Weather Underground](wee_import-wu.md) personal weather station +* the historical observations of a [Weather Underground](wee_import-wu.md) + personal weather station * one or more [Cumulus](wee_import-cumulus.md) monthly log files * one or more [Weather Display](wee_import-wd.md) monthly log files * one or more [WeatherCat](wee_import-weathercat.md) monthly .cat files diff --git a/docs_src/utilities/weectl-import-common-opt.md b/docs_src/utilities/weectl-import-common-opt.md index aba9e1b2..ce8e3e70 100644 --- a/docs_src/utilities/weectl-import-common-opt.md +++ b/docs_src/utilities/weectl-import-common-opt.md @@ -8,7 +8,8 @@ wee_import --help usage: wee_import --help wee_import --import-config=IMPORT_CONFIG_FILE [--config=CONFIG_FILE] - [--date=YYYY-mm-dd | --from=YYYY-mm-dd[THH:MM] --to=YYYY-mm-dd[THH:MM]] + [--date=YYYY-mm-dd | --from=YYYY-mm-dd[THH:MM] --to=YYYY-mm-dd + [THH:MM]] [--dry-run] [--verbose] [--no-prompt] @@ -52,7 +53,8 @@ but if you have an unusual installation or multiple stations, you may have to tell it explicitly. ``` -wee_import --config=/this/directory/weewx.conf --import-config=/directory/import.conf +wee_import --config=/this/directory/weewx.conf +--import-config=/directory/import.conf ``` ### `--import-config=FILENAME` @@ -71,9 +73,9 @@ wee_import --import-config=/directory/import.conf ### `--dry-run` -The `--dry-run` option will cause the import to proceed but no actual data will -be saved to the database. This is a useful option to use when first importing -data. +The `--dry-run` option will cause the import to proceed but no actual data +will be saved to the database. This is a useful option to use when first +importing data. ``` wee_import --import-config=/directory/import.conf --dry-run @@ -81,14 +83,20 @@ wee_import --import-config=/directory/import.conf --dry-run ### `--date=YYYY-mm-dd` -Records from a single date can be imported by use of the `--date` option. The `--date` option accepts strings of the format `YYYY-mm-dd`. Whilst the use of the `--date` option will limit the imported data to that of a single date, the default action if the `--date` option (and the `--from` and `--to` options) is omitted may vary depending on the source. The operation of the `--date` option is summarised in the following table: +Records from a single date can be imported by use of the `--date` option. +The `--date` option accepts strings of the format `YYYY-mm-dd`. Whilst the +use of the `--date` option will limit the imported data to that of a single +date, the default action if the `--date` option (and the `--from` and `--to` +options) is omitted may vary depending on the source. The operation of the +`--date` option is summarised in the following table: - + @@ -98,8 +106,10 @@ Records from a single date can be imported by use of the `--date` option. The `- - - + +
Option --date
optionRecords imported for a CSV, Cumulus, Weather Display or WeatherCat importRecords imported for a CSV, Cumulus, Weather Display or WeatherCat +import Records imported for a Weather Underground import
--date=2015-12-22All records from 2015-12-22 00:00 (exclusive) to 2015-12-23 00:00 (inclusive)All records from 2015-12-22 00:00 (exclusive) to 2015-12-23 00:00 (inclusive)All records from 2015-12-22 00:00 (exclusive) to 2015-12-23 00:00 +(inclusive)All records from 2015-12-22 00:00 (exclusive) to 2015-12-23 00:00 +(inclusive)
@@ -119,28 +129,30 @@ Records from a single date can be imported by use of the `--date` option. The `- ### `--from` and `--to` -Whilst the `--date` option allows imported data to be limited to a single date, -the `--from` and `--to` options allow finer control by importing only the -records that fall within the date or date-time range specified by the `--from` -and `--to` options. The `--from` option determines the earliest (inclusive), -and the `--to` option determines the latest (exclusive), date or date-time of -the records being imported. The `--from` and `--to` options accept a string of -the format `YYYY-mm-dd[THH:MM]`. The T literal is mandatory if specifying a -date-time. +Whilst the `--date` option allows imported data to be limited to a single +date, the `--from` and `--to` options allow finer control by importing +only the records that fall within the date or date-time range specified by +the `--from` and `--to` options. The `--from` option determines the +earliest (inclusive), and the `--to` option determines the latest +(exclusive), date or date-time of the records being imported. The `--from` +and `--to` options accept a string of the format `YYYY-mm-dd[THH:MM]`. The +T literal is mandatory if specifying a date-time. !!! Note The `--from` and `--to` options must be used as a pair, they cannot be used individually or in conjunction with the `--date`option. -The operation of the `--from` and `--to` options is summarised in the following -table: +The operation of the `--from` and `--to` options is summarised in the +following table: - + - + @@ -152,26 +164,34 @@ table: - - + + - - + + - - + + - - + +
Options --from and --toOptions --from and --to
optionsRecords imported for a CSV, Cumulus, Weather Display or WeatherCat importRecords imported for a CSV, Cumulus, Weather Display or WeatherCat +import Records imported for a Weather Underground import
--from=2015-12-22 --to=2015-12-29All records from 2015-12-22 00:00 (exclusive) to 2015-12-30 00:00 (inclusive)All records from 2015-12-22 00:00 (exclusive) to 2015-12-30 00:00 (inclusive)All records from 2015-12-22 00:00 (exclusive) to 2015-12-30 00:00 +(inclusive)All records from 2015-12-22 00:00 (exclusive) to 2015-12-30 00:00 +(inclusive)
--from=2016-7-18T15:29 --to=2016-7-25All records from 2016-7-18 15:29 (exclusive) to 2016-7-26 00:00 (inclusive)All records from 2016-7-18 15:29 (exclusive) to 2016-7-26 00:00 (inclusive)All records from 2016-7-18 15:29 (exclusive) to 2016-7-26 00:00 +(inclusive)All records from 2016-7-18 15:29 (exclusive) to 2016-7-26 00:00 +(inclusive)
--from=2016-5-12 --to=2016-7-22T22:15All records from 2016-5-12 00:00 (exclusive) to 2016-7-22 22:15 (inclusive)All records from 2016-5-12 00:00 (exclusive) to 2016-7-22 22:15 (inclusive)All records from 2016-5-12 00:00 (exclusive) to 2016-7-22 22:15 +(inclusive)All records from 2016-5-12 00:00 (exclusive) to 2016-7-22 22:15 +(inclusive)
--from=2016-3-18T15:29 --to=2016-6-20T22:00All records from 2016-3-18 15:29 (exclusive) to 2016-6-20 22:00 (inclusive)All records from 2016-3-18 15:29 (exclusive) to 2016-6-20 22:00 (inclusive)All records from 2016-3-18 15:29 (exclusive) to 2016-6-20 22:00 +(inclusive)All records from 2016-3-18 15:29 (exclusive) to 2016-6-20 22:00 +(inclusive)
@@ -209,11 +229,11 @@ wee_import --import-config=/directory/import.conf --no-prompt ``` !!! Warning - Care must be taken when using the `--no-prompt` option as ignoring warnings - during the import process can lead to unexpected results. Whilst existing - data will be protected, the use or acceptance of an incorrect or unexpected - parameter or default may lead to significant amounts of unwanted data being - imported. + Care must be taken when using the `--no-prompt` option as ignoring + warnings during the import process can lead to unexpected results. Whilst + existing data will be protected, the use or acceptance of an + incorrect or unexpected parameter or default may lead to significant + amounts of unwanted data being imported. ### `--suppress-warnings` diff --git a/docs_src/utilities/weectl-import-config-opt.md b/docs_src/utilities/weectl-import-config-opt.md index b00f43d6..47646662 100644 --- a/docs_src/utilities/weectl-import-config-opt.md +++ b/docs_src/utilities/weectl-import-config-opt.md @@ -1,10 +1,24 @@ -`wee_import` requires a second configuration file, the import configuration file, in addition to the standard WeeWX configuration file. The import configuration file specifies the import type and various options associated with each type of import. The import configuration file is specified using the mandatory `--import-config` option. How you construct the import configuration file is up to you; however, the recommended method is to copy one of the example import configuration files located in the `util/import` directory as applicable, modify the configuration options in the newly copied file to suit the import to be performed and then use this file as the import configuration file. +`wee_import` requires a second configuration file, the import configuration +file, in addition to the standard WeeWX configuration file. The import +configuration file specifies the import type and various options associated +with each type of import. The import configuration file is specified using +the mandatory `--import-config` option. How you construct the import +configuration file is up to you; however, the recommended method is to copy +one of the example import configuration files located in the `util/import` +directory as applicable, modify the configuration options in the newly +copied file to suit the import to be performed and then use this file as the +import configuration file. -Following is the definitive guide to the options available in the import configuration file. Default values are provided for a number of options, meaning that if they are not listed in the import configuration file at all `wee_import` will pick sensible values. When the documentation below gives a default value this is the value that will be used if the option is omitted. +Following is the definitive guide to the options available in the import +configuration file. Default values are provided for a number of options, +meaning that if they are not listed in the import configuration file at all +`wee_import` will pick sensible values. When the documentation below gives a +default value this is the value that will be used if the option is omitted. ### `source`{#import_config_source} -The `source` option determines the type of import to be performed by `wee_import`. The option is mandatory and must be set to one of the following: +The `source` option determines the type of import to be performed by +`wee_import`. The option is mandatory and must be set to one of the following: * `CSV` to import from a single CSV format file. * `WU` to import from a Weather Underground PWS history @@ -16,17 +30,23 @@ There is no default. ## [CSV] -The `[CSV]` section contains the options controlling the import of observational data from a CSV format file. +The `[CSV]` section contains the options controlling the import of +observational data from a CSV format file. ### `file`{#csv_file} -The file containing the CSV format data to be used as the source during the import. Include full path and filename. +The file containing the CSV format data to be used as the source during the +import. Include full path and filename. There is no default. ### `source_encoding`{#csv_encoding} -The source file encoding. This parameter is optional and should only need be used if the source file uses an encoding other than UTF-8 or an ASCII compatible encoding. If used, the setting used should be a Python Standard Encoding. +The source file encoding. This parameter is optional and should only need be +used if the source file uses an encoding other than UTF-8 or an ASCII +compatible encoding. If used, the setting used should be a Python Standard Encoding. The default is `utf-8-sig`. @@ -36,80 +56,182 @@ The character used to separate fields. Default is `,` (comma). ### `decimal`{#csv_decimal} -The character used as the decimal point in the source files. A full stop is frequently used, but it may be another character. This parameter must be included in quotation marks. +The character used as the decimal point in the source files. A full stop is +frequently used, but it may be another character. This parameter must be +included in quotation marks. The default is `'.'`. ### `interval`{#csv_interval} -Determines how the time interval (WeeWX archive table field `interval`) between successive observations is derived. The interval can be derived by one of three methods: +Determines how the time interval (WeeWX archive table field `interval`) +between successive observations is derived. The interval can be derived by +one of three methods: -* The interval can be calculated as the time, rounded to the nearest minute, between the date-time of successive records. This method is suitable when the data was recorded at fixed intervals and there are NO missing records in the source data. Use of this method when there are missing records in the source data can compromise the integrity of the WeeWX statistical data. Select this method by setting `interval = derive`. +* The interval can be calculated as the time, rounded to the nearest minute, + between the date-time of successive records. This method is suitable when + the data was recorded at fixed intervals and there are NO missing records + in the source data. Use of this method when there are missing records in + the source data can compromise the integrity of the WeeWX statistical data. + Select this method by setting `interval = derive`. -* The interval can be set to the same value as the `archive_interval` setting under `[StdArchive]` in `weewx.conf`. This setting is useful if the data was recorded at fixed intervals but there are some missing records and the fixed interval is the same as the `archive_interval` setting under `[StdArchive]` in `weewx.conf`. Select this method by setting `interval = conf`. +* The interval can be set to the same value as the `archive_interval` + setting under `[StdArchive]` in `weewx.conf`. This setting is useful if + the data was recorded at fixed intervals but there are some missing + records and the fixed interval is the same as the `archive_interval` + setting under `[StdArchive]` in `weewx.conf`. Select this method by + setting `interval = conf`. -* The interval can be set to a fixed number of minutes. This setting is useful if the source data was recorded at fixed intervals but there are some missing records and the fixed interval is different to the `archive_interval` setting under `[StdArchive]` in `weewx.conf`. Select this method by setting `interval = x` where `x` is an integer number of minutes. +* The interval can be set to a fixed number of minutes. This setting is + useful if the source data was recorded at fixed intervals but there are + some missing records and the fixed interval is different to the + `archive_interval` setting under `[StdArchive]` in `weewx.conf`. Select + this method by setting `interval = x` where `x` is an integer number of + minutes. -If the CSV source data records are equally spaced in time, but some records are missing, then a better result may be achieved using `conf` or a fixed interval setting. +If the CSV source data records are equally spaced in time, but some +records are missing, then a better result may be achieved using `conf` or +a fixed interval setting. The default is `derive`. ### `qc`{#csv_qc} -Determines whether simple quality control checks are applied to imported data. Setting `qc = True` will result in `wee_import` applying the WeeWX `StdQC` minimum and maximum checks to any imported observations. `wee_import` quality control checks use the same configuration settings, and operate in the same manner, as the [_StdQC_](../reference/weewx-options/stdqc.md) service. For example, for minimum/maximum quality checks, if an observation falls outside of the quality control range for that observation, the observation will be set to `None`. In such cases you will be alerted through a short message similar to: +Determines whether simple quality control checks are applied to imported +data. Setting `qc = True` will result in `wee_import` applying the WeeWX +`StdQC` minimum and maximum checks to any imported observations. +`wee_import` quality control checks use the same configuration settings, +and operate in the same manner, as the [_StdQC_](.. +/reference/weewx-options/stdqc.md) service. For example, for +minimum/maximum quality checks, if an observation falls outside of the +quality control range for that observation, the observation will be set to +`None`. In such cases you will be alerted through a short message similar to: ``` -2016-01-12 10:00:00 AEST (1452556800) record value 'outTemp' 194.34 outside limits (0.0, 120.0) +2016-01-12 10:00:00 AEST (1452556800) record value 'outTemp' 194.34 +outside limits (0.0, 120.0) ``` -As derived observations are calculated after the quality control check is applied, derived observations are not subject to quality control checks. Setting `qc = False` will result in `wee_import` not applying quality control checks to imported data. +As derived observations are calculated after the quality control check is +applied, derived observations are not subject to quality control checks. +Setting `qc = False` will result in `wee_import` not applying quality +control checks to imported data. The default is `True`. ### `calc_missing`{#csv_calc_missing} -Determines whether any missing derived observations will be calculated from the imported data. Setting `calc_missing = True` will result in `wee_import` using the WeeWX `StdWXCalculate` service to calculate any missing derived observations from the imported data. Setting `calc_missing = False` will result in WeeWX leaving any missing derived observations as `None`. See [_[StdWXCalculate]_](../reference/weewx-options/stdwxcalculate.md) for details of the observations the `StdWXCalculate` service can calculate. +Determines whether any missing derived observations will be calculated +from the imported data. Setting `calc_missing = True` will result in +`wee_import` using the WeeWX `StdWXCalculate` service to calculate any +missing derived observations from the imported data. Setting `calc_missing += False` will result in WeeWX leaving any missing derived observations as +`None`. See [_[StdWXCalculate]_](../reference/weewx-options/stdwxcalculate. +md) for details of the observations the `StdWXCalculate` service can +calculate. The default is `True`. ### `ignore_invalid_data`{#csv_ignore_invalid_data} -Determines whether invalid data in a source field is ignored or the import aborted. If invalid data is found in a source field and `ignore_invalid_data` is `True` the corresponding WeeWX destination field is set to `None` and the import continues. If invalid data is found in a source field and `ignore_invalid_data` is `False` the import is aborted. +Determines whether invalid data in a source field is ignored or the import +aborted. If invalid data is found in a source field and +`ignore_invalid_data` is `True` the corresponding WeeWX destination field +is set to `None` and the import continues. If invalid data is found in a +source field and `ignore_invalid_data` is `False` the import is aborted. The default is `True`. ### `tranche`{#csv_tranche} -To speed up database operations imported records are committed to database in groups of records rather than individually. The size of the group is set by the `tranche` parameter. Increasing the `tranche` parameter may result in a slight speed increase but at the expense of increased memory usage. Decreasing the `tranche` parameter will result in less memory usage but at the expense of more frequent database access and likely increased time to import. +To speed up database operations imported records are committed to database +in groups of records rather than individually. The size of the group is +set by the `tranche` parameter. Increasing the `tranche` parameter may +result in a slight speed increase but at the expense of increased memory +usage. Decreasing the `tranche` parameter will result in less memory usage +but at the expense of more frequent database access and likely increased +time to import. The default is `250` which should suit most users. ### `UV_sensor`{#csv_UV} -WeeWX records a `None/null` for UV when no UV sensor is installed, whereas some weather station software records a value of 0 for UV index when there is no UV sensor installed. The `UV_sensor` parameter enables `wee_import` to distinguish between the case where a UV sensor is present and the UV index is 0 and the case where no UV sensor is present and UV index is 0. `UV_sensor = False` should be used when no UV sensor was used in producing the source data. `UV_sensor = False` will result in `None/null` being recorded in the WeeWX archive field `UV` irrespective of any UV observations in the source data. `UV_sensor = True` should be used when a UV sensor was used in producing the source data. `UV_sensor = True` will result in UV observations in the source data being stored in the WeeWX archive field `UV`. +WeeWX records a `None/null` for UV when no UV sensor is installed, whereas +some weather station software records a value of 0 for UV index when there +is no UV sensor installed. The `UV_sensor` parameter enables `wee_import` +to distinguish between the case where a UV sensor is present and the UV +index is 0 and the case where no UV sensor is present and UV index is 0. +`UV_sensor = False` should be used when no UV sensor was used in producing +the source data. `UV_sensor = False` will result in `None/null` being +recorded in the WeeWX archive field `UV` irrespective of any UV +observations in the source data. `UV_sensor = True` should be used when a +UV sensor was used in producing the source data. `UV_sensor = True` will +result in UV observations in the source data being stored in the WeeWX +archive field `UV`. The default is `True`. ### `solar_sensor`{#csv_solar} -WeeWX records a `None/null` when no solar radiation sensor is installed, whereas some weather station software records a value of 0 for solar radiation when there is no solar radiation sensor installed. The `solar_sensor` parameter enables `wee_import` to distinguish between the case where a solar radiation sensor is present and solar radiation is 0 and the case where no solar radiation sensor is present and solar radiation is 0. `solar_sensor = False` should be used when no solar radiation sensor was used in producing the source data. `solar_sensor = False` will result in `None/null` being recorded in the WeeWX archive field `radiation` irrespective of any solar radiation observations in the source data. `solar_sensor = True` should be used when a solar radiation sensor was used in producing the source data. `solar_sensor = True` will result in solar radiation observations in the source data being stored in the WeeWX archive field `radiation`. +WeeWX records a `None/null` when no solar radiation sensor is installed, +whereas some weather station software records a value of 0 for solar +radiation when there is no solar radiation sensor installed. The +`solar_sensor` parameter enables `wee_import` to distinguish between the +case where a solar radiation sensor is present and solar radiation is 0 +and the case where no solar radiation sensor is present and solar +radiation is 0. `solar_sensor = False` should be used when no solar +radiation sensor was used in producing the source data. `solar_sensor = +False` will result in `None/null` being recorded in the WeeWX archive +field `radiation` irrespective of any solar radiation observations in the +source data. `solar_sensor = True` should be used when a solar radiation +sensor was used in producing the source data. `solar_sensor = True` will +result in solar radiation observations in the source data being stored in +the WeeWX archive field `radiation`. The default is `True`. ### `raw_datetime_format`{#csv_raw_datetime_format} -WeeWX records each record with a unique unix epoch timestamp, whereas many weather station applications or web sources export observational data with a human-readable date-time. This human-readable date-time is interpreted according to the format set by the `raw_datetime_format` option. This option consists of [Python strptime() format codes](https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior) and literal characters to represent the date-time data being imported. +WeeWX records each record with a unique unix epoch timestamp, whereas many +weather station applications or web sources export observational data with +a human-readable date-time. This human-readable date-time is interpreted +according to the format set by the `raw_datetime_format` option. This +option consists of [Python strptime() format codes](https://docs.python. +org/2/library/datetime.html#strftime-and-strptime-behavior) and literal +characters to represent the date-time data being imported. -For example, if the source data uses the format 23 January 2015 15:34 the appropriate setting for `raw_datetime_format` would be `%d %B %Y %H:%M`, 9:25:00 12/28/16 would use `%H:%M:%S %m/%d/%y`. If the source data provides a unix epoch timestamp as the date-time field the unix epoch timestamp is used directly and the `raw_datetime_format` option is ignored. +For example, if the source data uses the format 23 January 2015 15:34 the +appropriate setting for `raw_datetime_format` would be `%d %B %Y %H:%M`, +9:25:00 12/28/16 would use `%H:%M:%S %m/%d/%y`. If the source data +provides a unix epoch timestamp as the date-time field the unix epoch +timestamp is used directly and the `raw_datetime_format` option is ignored. The default is `%Y-%m-%d %H:%M:%S`. !!! Note - `wee_import` does not support the construction of the unique record date time stamp from separate date and time fields, rather the date-time information for each imported record must be contained in a single field. CSV data containing separate date and time fields may require further manual processing before they can be imported. + `wee_import` does not support the construction of the unique record + date time stamp from separate date and time fields, rather the date-time + information for each imported record must be contained in a single field. + CSV data containing separate date and time fields may require further + manual processing before they can be imported. ### `wind_direction`{#csv_wind_direction} -WeeWX records wind direction in degrees as a number from 0 to 360 inclusive (no wind direction is recorded as `None/null`), whereas some data sources may provide wind direction as number over a different range (e.g., -180 to +180) or may use a particular value when there is no wind direction (e.g., 0 may represent no wind direction and 360 may represent a northerly wind, or -9999 (or some similar clearly invalid number) to represent there being no wind direction). `wee_import` handles such variations in data by defining a range over which imported wind direction values are accepted. Any value outside of this range is treated as there being no wind direction and is recorded as `None/null`. Any value inside the range is normalised to the range 0 to 360 inclusive (e.g., -180 would be normalised to 180). The `wind_direction` option consists of two comma separated numbers of the format lower, upper where lower and upper are inclusive. The operation of the `wind_direction` option is best illustrated through the following table: +WeeWX records wind direction in degrees as a number from 0 to 360 +inclusive (no wind direction is recorded as `None/null`), whereas some +data sources may provide wind direction as number over a different range +(e.g., -180 to +180) or may use a particular value when there is no wind +direction (e.g., 0 may represent no wind direction and 360 may represent a +northerly wind, or -9999 (or some similar clearly invalid number) to +represent there being no wind direction). `wee_import` handles such +variations in data by defining a range over which imported wind direction +values are accepted. Any value outside of this range is treated as there +being no wind direction and is recorded as `None/null`. Any value inside +the range is normalised to the range 0 to 360 inclusive (e.g., -180 would +be normalised to 180). The `wind_direction` option consists of two comma +separated numbers of the format lower, upper where lower and upper are +inclusive. The operation of the `wind_direction` option is best +illustrated through the following table: @@ -213,7 +335,9 @@ The default is `0, 360`. ### `[[FieldMap]]`{#csv_fieldmap} -The `[[FieldMap]]` stanza defines the mapping from the source data fields to WeeWX archive fields. The map consists of one stanza per WeeWX archive field being populated using the following format: +The `[[FieldMap]]` stanza defines the mapping from the source data fields +to WeeWX archive fields. The map consists of one stanza per WeeWX archive +field being populated using the following format: ``` [[[weewx_archive_field_name]]] @@ -222,20 +346,42 @@ The `[[FieldMap]]` stanza defines the mapping from the source data fields to Wee cumulative = True | False ``` -Where `weewx_archive_field_name` is a field name in the in-use WeeWX archive table schema. +Where `weewx_archive_field_name` is a field name in the in-use WeeWX +archive table schema. Each WeeWX archive field stanza supports the following options: -* `source_field`. The name of the CSV field to be mapped to the WeeWX archive field. Mandatory. -* `unit`. The WeeWX unit name of the units used by `source_field`. Text fields may be imported by setting the unit option to `text`. Mandatory. -* `cumulative`. Whether the `source_field` is a cumulative value or not (e.g, daily rainfall). Optional boolean value. Default is `False`. +* `source_field`. The name of the CSV field to be mapped to the WeeWX + archive field. Mandatory. +* `unit`. The WeeWX unit name of the units used by `source_field`. Text + fields may be imported by setting the unit option to `text`. Mandatory. +* `cumulative`. Whether the `source_field` is a cumulative value or not (e. + g, daily rainfall). Optional boolean value. Default is `False`. -This mapping allows `wee_import` to take a source data field, perform the appropriate unit conversion and store the resulting value in the appropriate WeeWX archive field. Source data text fields may be mapped to a WeeWX text archive field by using the second form of the field map entry where the literal `text` is used in place of a WeeWX unit name. A mapping is not required for every WeeWX archive field (e.g., the source may not provide inside temperature so no `inTemp` field mapping is required) and neither does every CSV field need to be included in a mapping (e.g., the source data field `monthrain` may have no use if the source data field `dayrain` provides the data for the WeeWX archive `rain` field). +This mapping allows `wee_import` to take a source data field, perform the +appropriate unit conversion and store the resulting value in the +appropriate WeeWX archive field. Source data text fields may be mapped to +a WeeWX text archive field by using the second form of the field map entry +where the literal `text` is used in place of a WeeWX unit name. A mapping +is not required for every WeeWX archive field (e.g., the source may not +provide inside temperature so no `inTemp` field mapping is required) and +neither does every CSV field need to be included in a mapping (e.g., the +source data field `monthrain` may have no use if the source data field +`dayrain` provides the data for the WeeWX archive `rain` field). !!! Note - Importing of text data into text fields in the WeeWX archive is only supported for WeeWX archive fields that have been configured as text fields. Refer to the Wiki page [Storing text in the database](https://github.com/weewx/weewx/wiki/Storing-text-in-the-database) for details. + Importing of text data into text fields in the WeeWX archive is only + supported for WeeWX archive fields that have been configured as text + fields. Refer to the Wiki page [Storing text in the database] + (https://github.com/weewx/weewx/wiki/Storing-text-in-the-database) for + details. -If the source data includes a field that contains a WeeWX unit system code (i.e. the equivalent of the WeeWX `usUnits` field such as may be obtained from WeeWX or wview data) then this field can be mapped to the WeeWX `usUnits` field and used to set the units used for all fields being imported. In such cases the `weewx_unit_name` portion of the imported fields in the field map is not used and can be omitted. +If the source data includes a field that contains a WeeWX unit system code +(i.e. the equivalent of the WeeWX `usUnits` field such as may be obtained +from WeeWX or wview data) then this field can be mapped to the WeeWX +`usUnits` field and used to set the units used for all fields being +imported. In such cases the `weewx_unit_name` portion of the imported +fields in the field map is not used and can be omitted. For example, source CSV data with the following structure: @@ -245,7 +391,11 @@ date_and_time,temp,humid,wind,dir,dayrain,rad,river,decsription 23 May 2018 13:05,17.6,56,1.0,22.5,10.4,746,341,'showers developing' ``` -where `temp` is temperature in Celsius, `humid` is humidity in percent, `wind` is wind speed in km/h, `dir` is wind direction in degrees, `rainfall` is rain in mm, `rad` is radiation in watts per square meter, `river` is river height in mm and `description` is a text might use a field map as follows: +where `temp` is temperature in Celsius, `humid` is humidity in percent, +`wind` is wind speed in km/h, `dir` is wind direction in degrees, +`rainfall` is rain in mm, `rad` is radiation in watts per square meter, +`river` is river height in mm and `description` is a text might use a +field map as follows: ``` [[FieldMap]] @@ -276,7 +426,8 @@ where `temp` is temperature in Celsius, `humid` is humidity in percent, `wind` i unit = text ``` -If the same source CSV data included a field `unit_info` that contains WeeWX unit system data as follows: +If the same source CSV data included a field `unit_info` that contains +WeeWX unit system data as follows: ``` date_and_time,temp,humid,wind,dir,dayrain,rad,river,unit_info @@ -315,19 +466,30 @@ then a field map such as the following might be used: ``` !!! Note - Any WeeWX archive fields that are derived (e.g., `dewpoint`) and for which there is no field mapping may be calculated during import by use of the [`calc_missing`](#csv_calc_missing) option in the `[CSV]` section of the import configuration file. + Any WeeWX archive fields that are derived (e.g., `dewpoint`) and for + which there is no field mapping may be calculated during import by use of + the [`calc_missing`](#csv_calc_missing) option in the `[CSV]` section of + the import configuration file. !!! Note - The `dateTime` field map entry is a special case. Whereas other field map entries may use any supported WeeWX unit name, or no unit name if the `usUnits` field is populated, the `dateTime` field map entry must include the WeeWX unit name `unix_epoch`. This is because `wee_import` uses the [raw_datetime_format](#csv_raw_datetime_format) config option to convert the supplied date-time field data to a Unix epoch timestamp before the field map is applied. + The `dateTime` field map entry is a special case. Whereas other field + map entries may use any supported WeeWX unit name, or no unit name if the + `usUnits` field is populated, the `dateTime` field map entry must include + the WeeWX unit name `unix_epoch`. This is because `wee_import` uses the + [raw_datetime_format](#csv_raw_datetime_format) config option to convert + the supplied date-time field data to a Unix epoch timestamp before the + field map is applied. ## [WU] -The `[WU]` section contains the options relating to the import of observational data from a Weather Underground PWS history. +The `[WU]` section contains the options relating to the import of +observational data from a Weather Underground PWS history. ### `station_id`{#wu_station_id} -The Weather Underground weather station ID of the PWS from which the historical data will be imported. +The Weather Underground weather station ID of the PWS from which the +historical data will be imported. There is no default. @@ -338,56 +500,90 @@ The Weather Underground API key to be used to obtain the PWS history data. There is no default. !!! Note - The API key is a seemingly random string of 32 characters used to access the new (2019) Weather Underground API. PWS contributors can obtain an API key by logging onto the Weather Underground internet site and accessing Member Settings. 16 character API keys used with the previous Weather Underground API are not supported. + The API key is a seemingly random string of 32 characters used to + access the new (2019) Weather Underground API. PWS contributors can + obtain an API key by logging onto the Weather Underground internet + site and accessing Member Settings. 16 character API keys used with the + previous Weather Underground API are not supported. ### `interval`{#wu_interval} -Determines how the time interval (WeeWX database field `interval`) between successive observations is determined. This option is identical in operation to the CSV [interval](#csv_interval) option but applies to Weather Underground imports only. As a Weather Underground PWS history sometimes has missing records, the use of `interval = derive` may give incorrect or inconsistent interval values. Better results may be obtained by using `interval = conf` if the current WeeWX installation has the same `archive_interval` as the Weather Underground data, or by using `interval = x` where `x` is the time interval in minutes used to upload the Weather Underground data. The most appropriate setting will depend on the completeness and (time) accuracy of the Weather Underground data being imported. +Determines how the time interval (WeeWX database field `interval`) between +successive observations is determined. This option is identical in +operation to the CSV [interval](#csv_interval) option but applies to +Weather Underground imports only. As a Weather Underground PWS history +sometimes has missing records, the use of `interval = derive` may give +incorrect or inconsistent interval values. Better results may be obtained +by using `interval = conf` if the current WeeWX installation has the same +`archive_interval` as the Weather Underground data, or by using `interval += x` where `x` is the time interval in minutes used to upload the Weather +Underground data. The most appropriate setting will depend on the +completeness and (time) accuracy of the Weather Underground data being +imported. The default is `derive`. ### `qc`{#wu_qc} -Determines whether simple quality control checks are applied to imported data. This option is identical in operation to the CSV [qc](#csv_qc) option but applies to Weather Underground imports only. As Weather Underground imports at times contain nonsense values, particularly for fields for which no data was uploaded to Weather Underground by the PWS, the use of quality control checks on imported data can prevent these nonsense values from being imported and contaminating the WeeWX database. +Determines whether simple quality control checks are applied to imported +data. This option is identical in operation to the CSV [qc](#csv_qc) +option but applies to Weather Underground imports only. As Weather +Underground imports at times contain nonsense values, particularly for +fields for which no data was uploaded to Weather Underground by the PWS, +the use of quality control checks on imported data can prevent these +nonsense values from being imported and contaminating the WeeWX database. The default is `True`. ### `calc_missing`{#wu_calc_missing} -Determines whether any missing derived observations will be calculated from the imported data. This option is identical in operation to the CSV [calc_missing](#csv_calc_missing)" option but applies to Weather Underground imports only. +Determines whether any missing derived observations will be calculated +from the imported data. This option is identical in operation to the CSV +[calc_missing](#csv_calc_missing)" option but applies to Weather Underground imports only. The default is `True`. ### `ignore_invalid_data`{#wu_ignore_invalid_data} -Determines whether invalid data in a source field is ignored or the import aborted. This option is identical in operation to the CSV [ignore_invalid_data](#csv_ignore_invalid_data) option but applies to Weather Underground imports only. The default is `True`. +Determines whether invalid data in a source field is ignored or the import +aborted. This option is identical in operation to the CSV +[ignore_invalid_data](#csv_ignore_invalid_data) option but applies to +Weather Underground imports only. The default is `True`. ### `tranche`{#wu_tranche} -The number of records written to the WeeWX database in each transaction. This option is identical in operation to the CSV [tranche](#csv_tranche) option but applies to Weather Underground imports only. +The number of records written to the WeeWX database in each transaction. +This option is identical in operation to the CSV [tranche](#csv_tranche) +option but applies to Weather Underground imports only. The default is `250` which should suit most users. ### `wind_direction`{#wu_wind_direction} -Determines the range of acceptable wind direction values in degrees. This option is identical in operation to the CSV [wind_direction](#csv_wind_direction) option but applies to Weather Underground imports only. +Determines the range of acceptable wind direction values in degrees. This +option is identical in operation to the CSV [wind_direction] +(#csv_wind_direction) option but applies to Weather Underground imports only. The default is `0, 360`. ## [Cumulus] -The `[Cumulus]` section contains the options relating to the import of observational data from Cumulus monthly log files. +The `[Cumulus]` section contains the options relating to the import of +observational data from Cumulus monthly log files. ### `directory`{#cumulus_directory} -The full path to the directory containing the Cumulus monthly log files to be imported. Do not include a trailing `/`. +The full path to the directory containing the Cumulus monthly log files to +be imported. Do not include a trailing `/`. There is no default. ### `source_encoding`{#cumulus_encoding} -The Cumulus monthly log file encoding. This option is identical in operation to the CSV [source_encoding](#csv_encoding) option but applies to Cumulus imports only. +The Cumulus monthly log file encoding. This option is identical in +operation to the CSV [source_encoding](#csv_encoding) option but applies +to Cumulus imports only. The default is `utf-8-sig`. @@ -399,82 +595,120 @@ The default is `derive`. ### `qc`{#cumulus_qc} -Determines whether simple quality control checks are applied to imported data. This option is identical in operation to the CSV [qc](#csv_qc) option but applies to Cumulus imports only. +Determines whether simple quality control checks are applied to imported +data. This option is identical in operation to the CSV [qc](#csv_qc) +option but applies to Cumulus imports only. The default is `>True`. ### `calc_missing`{#cumulus_calc_missing} -Determines whether any missing derived observations will be calculated from the imported data. This option is identical in operation to the CSV [calc_missing](#csv_calc_missing) option but applies to Cumulus imports only. +Determines whether any missing derived observations will be calculated +from the imported data. This option is identical in operation to the CSV +[calc_missing](#csv_calc_missing) option but applies to Cumulus imports only. The default is `True`. ### `separator`{#cumulus_separator} -The character used as the date field separator in the Cumulus monthly log file. A solidus (/) is frequently used, but it may be another character depending on the settings on the machine that produced the Cumulus monthly log files. This parameter must be included in quotation marks. +The character used as the date field separator in the Cumulus monthly log +file. A solidus (/) is frequently used, but it may be another character +depending on the settings on the machine that produced the Cumulus monthly +log files. This parameter must be included in quotation marks. The default is `/`. ### `delimiter`{#cumulus_delimiter} -The character used as the field delimiter in the Cumulus monthly log file. A comma is frequently used, but it may be another character depending on the settings on the machine that produced the Cumulus monthly log files. This parameter must be included in quotation marks. +The character used as the field delimiter in the Cumulus monthly log file. +A comma is frequently used, but it may be another character depending on +the settings on the machine that produced the Cumulus monthly log files. +This parameter must be included in quotation marks. The default is `,`. ### `decimal`{#cumulus_decimal} -The character used as the decimal point in the Cumulus monthly log files. A full stop is frequently used, but it may be another character depending on the settings on the machine that produced the Cumulus monthly log files. This parameter must be included in quotation marks. +The character used as the decimal point in the Cumulus monthly log files. +A full stop is frequently used, but it may be another character depending +on the settings on the machine that produced the Cumulus monthly log files. +This parameter must be included in quotation marks. The default is `.`. ### `ignore_invalid_data`{#cumulus_ignore_invalid_data} -Determines whether invalid data in a source field is ignored or the import aborted. This option is identical in operation to the CSV [ignore_invalid_data](#csv_ignore_invalid_data) option but applies to Cumulus monthly log file imports only. +Determines whether invalid data in a source field is ignored or the import +aborted. This option is identical in operation to the CSV +[ignore_invalid_data](#csv_ignore_invalid_data)option but applies to +Cumulus monthly log file imports only. The default is `True`. ### `tranche`{#cumulus_tranche} -The number of records written to the WeeWX database in each transaction. This option is identical in operation to the CSV [tranche](#csv_tranche) option but applies to Cumulus monthly log file imports only. +The number of records written to the WeeWX database in each transaction. +This option is identical in operation to the CSV [tranche](#csv_tranche) +option but applies to Cumulus monthly log file imports only. The default is `250` which should suit most users. ### `UV_sensor`{#cumulus_UV} -Enables `wee_import` to distinguish between the case where a UV sensor is present and the UV index is 0 and the case where no UV sensor is present and UV index is 0. This option is identical in operation to the CSV [UV_sensor](#csv_UV) option but applies to Cumulus monthly log file imports only. +Enables `wee_import` to distinguish between the case where a UV sensor is +present and the UV index is 0 and the case where no UV sensor is present +and UV index is 0. This option is identical in operation to the CSV +[UV_sensor](#csv_UV) option but applies to Cumulus monthly log file +imports only. The default is `True`. ### `solar_sensor`{#cumulus_solar} -Enables `wee_import` to distinguish between the case where a solar radiation sensor is present and the solar radiation is 0 and the case where no solar radiation sensor is present and solar radiation is 0. This option is identical in operation to the CSV [solar_sensor](#csv_solar) option but applies to Cumulus monthly log file imports only. +Enables `wee_import` to distinguish between the case where a solar +radiation sensor is present and the solar radiation is 0 and the case +where no solar radiation sensor is present and solar radiation is 0. This +option is identical in operation to the CSV [solar_sensor](#csv_solar) +option but applies to Cumulus monthly log file imports only. The default is `True`. ### `[[Units]]`{#cumulus_units} -The `[[Units]]` stanza defines the units used in the Cumulus monthly log files. Units settings are required for `temperature`, `pressure`, `rain` and `speed`. The format for each setting is: +The `[[Units]]` stanza defines the units used in the Cumulus monthly log +files. Units settings are required for `temperature`, `pressure`, `rain` +and `speed`. The format for each setting is: ``` obs_type = weewx_unit_name ``` -Where `obs_type` is one of `temperature`, `pressure`, `rain` or `speed` and `weewx_unit_name` is the WeeWX unit name of the units used by that particular `obs_type`. As Cumulus supports a different suite of possible units only a subset of the available WeeWX unit names can be used for some settings. +Where `obs_type` is one of `temperature`, `pressure`, `rain` or `speed` +and `weewx_unit_name` is the WeeWX unit name of the units used by that +particular `obs_type`. As Cumulus supports a different suite of possible +units only a subset of the available WeeWX unit names can be used for some +settings. ## [WD] -The `[WD]` section contains the options relating to the import of observational data from Weather Display monthly log files. +The `[WD]` section contains the options relating to the import of +observational data from Weather Display monthly log files. ### `directory`{#wd_directory} -The full path to the directory containing the Weather Display monthly log files to be imported. Do not include a trailing `/`. +The full path to the directory containing the Weather Display monthly log +files to be imported. Do not include a trailing `/`. There is no default. ### `logs_to_process`{#wd_logs_to_process} -The Weather Display monthly log files to be processed. Weather Display uses multiple files to record each month of data. Which monthly log files are produced depends on the Weather Display configuration and the capabilities of the weather station. `wee_import` supports the following Weather Display monthly log files: +The Weather Display monthly log files to be processed. Weather Display +uses multiple files to record each month of data. Which monthly log files +are produced depends on the Weather Display configuration and the +capabilities of the weather station. `wee_import` supports the following +Weather Display monthly log files: * MMYYYYlg.txt * MMYYYYlgcsv.csv (csv format version of MMYYYYlg.txt) @@ -487,16 +721,24 @@ where MM is a one or two-digit month and YYYY is a four digit year The format for the `logs_to_process` setting is: ``` -logs_to_process = [lg.txt, | logcsv.csv, | vantagelog.txt, | vantagelogcsv.csv, | vantageextrasensorslog.csv] +logs_to_process = [lg.txt, | logcsv.csv, | vantagelog.txt, | vantagelogcsv. +csv, | vantageextrasensorslog.csv] ``` !!! Note - The leading MMYYYY is omitted when listing the monthly log files to be processed using the `logs_to_process` setting. Inclusion of the leading MMYYYY will cause the import to fail. + The leading MMYYYY is omitted when listing the monthly log files to be + processed using the `logs_to_process` setting. Inclusion of the leading + MMYYYY will cause the import to fail. !!! Note - The MMYYYYlgcsv.csv and MMYYYYvantagelogcsv.csv log files are CSV versions of MMYYYYlg.txt and MMYYYYvantagelog.txt respectively. Either the .txt or .csv version of these files should be used but not both. + The MMYYYYlgcsv.csv and MMYYYYvantagelogcsv.csv log files are CSV + versions of MMYYYYlg.txt and MMYYYYvantagelog.txt respectively. Either + the .txt or .csv version of these files should be used but not both. -The monthly log files selected for processing should be chosen carefully as the selected log files will determine the Weather Display data fields available for import. `wee_import` is able to import the following data from the indicated monthly log files: +The monthly log files selected for processing should be chosen carefully +as the selected log files will determine the Weather Display data fields +available for import. `wee_import` is able to import the following data +from the indicated monthly log files: * MMYYYYlg.txt/MMYYlgcsv.csv: * `average wind speed` @@ -533,110 +775,187 @@ The monthly log files selected for processing should be chosen carefully as the * `extra temperature 6` !!! Note - Whilst the above log files may contain the indicated data the data may only be imported subject to a suitable field map and in-use WeeWX archive table schema (refer to the [[[FieldMap]]](#wd_fieldmap) option). + Whilst the above log files may contain the indicated data the data may + only be imported subject to a suitable field map and in-use WeeWX archive + table schema (refer to the [[[FieldMap]]](#wd_fieldmap) option). The default is `lg.txt, vantagelog.txt, vantageextrasensorslog.csv`. ### `source_encoding`{#wd_encoding} -The Weather Display monthly log file encoding. This option is identical in operation to the CSV [source_encoding](#csv_encoding) option but applies to Weather Display imports only. +The Weather Display monthly log file encoding. This option is identical in +operation to the CSV [source_encoding](#csv_encoding) option but applies +to Weather Display imports only. The default is `utf-8-sig`. ### `interval`{#wd_interval} -Determines how the time interval (WeeWX database field `interval`) between successive observations is determined. This option is identical in operation to the CSV [interval](#csv_interval) option but applies to Weather Display monthly log file imports only. As Weather Display log files nominally have entries at one minute intervals the recommended approach is to set `interval = 1`. As Weather Display monthly log files can, at times, have missing entries, the use of `interval = derive` may give incorrect or inconsistent interval values. If then `archive_interval` for the current WeeWX installation is 1 minute `interval = conf` may be used. In most cases the most appropriate setting will be `interval = 1`. +Determines how the time interval (WeeWX database field `interval`) between +successive observations is determined. This option is identical in +operation to the CSV [interval](#csv_interval) option but applies to +Weather Display monthly log file imports only. As Weather Display log +files nominally have entries at one minute intervals the recommended +approach is to set `interval = 1`. As Weather Display monthly log files +can, at times, have missing entries, the use of `interval = derive` may +give incorrect or inconsistent interval values. If then `archive_interval` +for the current WeeWX installation is 1 minute `interval = conf` may be +used. In most cases the most appropriate setting will be `interval = 1`. The default is `1`. ### `qc`{#wd_qc} -Determines whether simple quality control checks are applied to imported data. This option is identical in operation to the CSV [qc](#csv_qc) option but applies to Weather Display imports only. +Determines whether simple quality control checks are applied to imported +data. This option is identical in operation to the CSV [qc](#csv_qc) +option but applies to Weather Display imports only. The default is `True`. ### `calc_missing`{#wd_calc_missing} -Determines whether any missing derived observations will be calculated from the imported data. This option is identical in operation to the CSV [calc_missing](#csv_calc_missing) option but applies to Weather Display imports only. +Determines whether any missing derived observations will be calculated +from the imported data. This option is identical in operation to the CSV +[calc_missing](#csv_calc_missing) option but applies to Weather Display +imports only. The default is `True`. ### `txt_delimiter`{#wd_txt_delimiter} -The character used as the field delimiter in Weather Display text format monthly log files (.txt files). A space is normally used but another character may be used if necessary. This parameter must be included in quotation marks. +The character used as the field delimiter in Weather Display text format +monthly log files (.txt files). A space is normally used but another +character may be used if necessary. This parameter must be included in +quotation marks. The default is `' '`. ### `csv_delimiter`{#wd_csv_delimiter} -The character used as the field delimiter in Weather Display csv format monthly log files (.csv files). A comma is normally used but another character may be used if necessary. This parameter must be included in quotation marks. +The character used as the field delimiter in Weather Display csv format +monthly log files (.csv files). A comma is normally used but another +character may be used if necessary. This parameter must be included in +quotation marks. The default is `,`. ### `decimal`{#wd_decimal} -The character used as the decimal point in the Weather Display monthly log files. A full stop is frequently used but another character may be used if necessary. This parameter must be included in quotation marks. +The character used as the decimal point in the Weather Display monthly log +files. A full stop is frequently used but another character may be used if +necessary. This parameter must be included in quotation marks. The default is `.`. ### `ignore_missing_log`{#wd_ignore_missing_log} -Determines whether missing log files are to be ignored or the import aborted. Weather Display log files are complete in themselves and a missing log file will have no effect other than there will be no imported data for the period covered by the missing log file. +Determines whether missing log files are to be ignored or the import +aborted. Weather Display log files are complete in themselves and a +missing log file will have no effect other than there will be no imported +data for the period covered by the missing log file. The default is `True`. ### `ignore_invalid_data`{#wd_ignore_invalid_data} -Determines whether invalid data in a source field is ignored or the import aborted. This option is identical in operation to the CSV [ignore_invalid_data](#csv_ignore_invalid_data) option but applies to Weather Display monthly log file imports only. +Determines whether invalid data in a source field is ignored or the import +aborted. This option is identical in operation to the CSV +[ignore_invalid_data](#csv_ignore_invalid_data) option but applies to +Weather Display monthly log file imports only. The default is `True`. ### `tranche`{#wd_tranche} -The number of records written to the WeeWX database in each transaction. This option is identical in operation to the CSV [tranche](#csv_tranche) option but applies to Weather Display monthly log file imports only. +The number of records written to the WeeWX database in each transaction. +This option is identical in operation to the CSV [tranche](#csv_tranche) +option but applies to Weather Display monthly log file imports only. The default is `250` which should suit most users. ### `UV_sensor`{#wd_UV} -Enables `wee_import` to distinguish between the case where a UV sensor is present and the UV index is 0 and the case where no UV sensor is present and UV index is 0. This option is identical in operation to the CSV [UV_sensor](#csv_UV) option but applies to Weather Display monthly log file imports only. +Enables `wee_import` to distinguish between the case where a UV sensor is +present and the UV index is 0 and the case where no UV sensor is present +and UV index is 0. This option is identical in operation to the CSV +[UV_sensor](#csv_UV) option but applies to Weather Display monthly log +file imports only. The default is `True`. ### `solar_sensor`{#wd_solar} -Enables `wee_import` to distinguish between the case where a solar radiation sensor is present and the solar radiation is 0 and the case where no solar radiation sensor is present and solar radiation is 0. This option is identical in operation to the CSV [solar_sensor](#csv_solar) option but applies to Weather Display monthly log file imports only. +Enables `wee_import` to distinguish between the case where a solar +radiation sensor is present and the solar radiation is 0 and the case +where no solar radiation sensor is present and solar radiation is 0. This +option is identical in operation to the CSV [solar_sensor](#csv_solar) +option but applies to Weather Display monthly log file imports only. The default is `True`. ### `ignore_extreme_temp_hum`{#wd_ignore_extreme_temp_hum} -Determines whether extreme temperature and humidity values are ignored. Weather Display log files record the value 255 for temperature and humidity fields if no corresponding sensor is present. Setting `ignore_extreme_temp_hum = True` will cause temperature and humidity values of 255 to be ignored. Setting `ignore_extreme_temp_hum = False` will cause temperature and humidity values of 255 to be treated as valid data to be imported. +Determines whether extreme temperature and humidity values are ignored. +Weather Display log files record the value 255 for temperature and +humidity fields if no corresponding sensor is present. Setting +`ignore_extreme_temp_hum = True` will cause temperature and humidity +values of 255 to be ignored. Setting `ignore_extreme_temp_hum = False` +will cause temperature and humidity values of 255 to be treated as valid +data to be imported. The default is `True`. !!! Note - Setting `ignore_extreme_temp_hum = False` will cause temperature and humidity values of 255 to be imported; however, these values may be rejected by the simple quality control checks implemented if `qc = True` is used. + Setting `ignore_extreme_temp_hum = False` will cause temperature and + humidity values of 255 to be imported; however, these values may be + rejected by the simple quality control checks implemented if `qc = True` + is used. ### `[[Units]]`{#wd_units} -The `[[Units]]` stanza defines the units used in the Weather Display monthly log files. Weather Display monthly log files normally use Metric or US customary units depending on the _Log File_ setting under _Units_ on the _Units/Wind Chill_ tab of the Weather Display _Universal Setup_. In such cases the `units` configuration option may be set to `Metric` or `US` to select either Metric or US customary units. +The `[[Units]]` stanza defines the units used in the Weather Display +monthly log files. Weather Display monthly log files normally use Metric +or US customary units depending on the _Log File_ setting under _Units_ on +the _Units/Wind Chill_ tab of the Weather Display _Universal Setup_. In +such cases the `units` configuration option may be set to `Metric` or `US` +to select either Metric or US customary units. There is no default. -It is also possible to individually specify the log file units used for `temperature`, `pressure`, `rain` and `speed`. The format for each setting is: +It is also possible to individually specify the log file units used for +`temperature`, `pressure`, `rain` and `speed`. The format for each setting is: ``` obs_type = weewx_unit_name ``` -Where `obs_type` is one of `temperature`, `pressure`, `rain` or `speed` and `weewx_unit_name is the WeeWX unit name of the units used by that particular `obs_type. As Weather Display supports a different suite of possible units only a subset of the available WeeWX unit names can be used for some settings. +Where `obs_type` is one of `temperature`, `pressure`, `rain` or `speed` +and `weewx_unit_name` is the WeeWX unit name of the units used by that +particular `obs_type`. As Weather Display supports a different suite of +possible units only a subset of the available WeeWX unit names can be used +for some settings. -The preferred method for defining the Weather Display log file units is through the use of the `units` configuration option. When defining the import log file units either the `units` configuration option should be used or the individual `temperature`, `pressure`, `rain` and `>speed` units defined but not both. If both the `units` configuration option is defined as well as the individual `temperature`, `pressure`, `rain` and `speed` units defined the `units` configuration option takes precedence and all other units settings are ignored. +The preferred method for defining the Weather Display log file units is +through the use of the `units` configuration option. When defining the +import log file units either the `units` configuration option should be +used or the individual `temperature`, `pressure`, `rain` and `>speed` +units defined but not both. If both the `units` configuration option is +defined as well as the individual `temperature`, `pressure`, `rain` and +`speed` units defined the `units` configuration option takes precedence +and all other units settings are ignored. ### `[[FieldMap]]`{#wd_fieldmap} -The `[[FieldMap]]` stanza defines the mapping from the Weather Display monthly log data fields to WeeWX archive fields. By default, imported Weather Display data is mapped to the corresponding WeeWX archive fields using a default field map. The default field map will likely suit most users; however, depending on the station capabilities and the in-use WeeWX database schema, a custom field map may be required if Weather Display monthly logs contain data from additional sensors that cannot be stored in the WeeWX archive using the default field map. A custom field map also makes it possible to limit the Weather Display monthly log data fields that are imported into WeeWX. +The `[[FieldMap]]` stanza defines the mapping from the Weather Display +monthly log data fields to WeeWX archive fields. By default, imported +Weather Display data is mapped to the corresponding WeeWX archive fields +using a default field map. The default field map will likely suit most +users; however, depending on the station capabilities and the in-use WeeWX +database schema, a custom field map may be required if Weather Display +monthly logs contain data from additional sensors that cannot be stored in +the WeeWX archive using the default field map. A custom field map also +makes it possible to limit the Weather Display monthly log data fields +that are imported into WeeWX. The field map consists of one row per field using the format: @@ -644,7 +963,10 @@ The field map consists of one row per field using the format: weewx_archive_field_name = weather_display_field_name ``` -Where `weewx_archive_field_name` is a field name in the in-use WeeWX archive table schema and `weather_display_field_name` is a Weather Display import field name. The available Weather Display import field names are listed in the table below. +Where `weewx_archive_field_name` is a field name in the in-use WeeWX +archive table schema and `weather_display_field_name` is a Weather Display +import field name. The available Weather Display import field names are +listed in the table below.
Option wind_direction
@@ -756,82 +1078,128 @@ Where `weewx_archive_field_name` is a field name in the in-use WeeWX archive tab
Available import field names
-A mapping is not required for every WeeWX archive field (e.g., the Weather Display monthly logs may not provide inside temperature so no `inTemp` field mapping is required) and neither does every Weather Display monthly log field need to be included in a mapping (e.g., the Weather Display monthly log field `soiltemp` may have no data as the station has no soil temperature probe). +A mapping is not required for every WeeWX archive field (e.g., the Weather +Display monthly logs may not provide inside temperature so no `inTemp` +field mapping is required) and neither does every Weather Display monthly +log field need to be included in a mapping (e.g., the Weather Display +monthly log field `soiltemp` may have no data as the station has no soil +temperature probe). !!! Note - Any WeeWX archive fields that are derived (e.g., `dewpoint`) and for which there is no field mapping may be calculated during import by use of the `calc_missing` option in the `[WD]` section of the import configuration file. + Any WeeWX archive fields that are derived (e.g., `dewpoint`) and for + which there is no field mapping may be calculated during import by use of + the `calc_missing` option in the `[WD]` section of the import + configuration file. -The example Weather Display import configuration file located in the `/home/weewx/util/import]` or the `/etc/weewx/import directory contains an example field map in the import configuration file comments. +The example Weather Display import configuration file located in the +`/home/weewx/util/import]` or the `/etc/weewx/import directory contains an +example field map in the import configuration file comments. There is no default. ## [WeatherCat] -The `[WeatherCat]` section contains the options relating to the import of observational data from WeatherCat monthly .cat files. +The `[WeatherCat]` section contains the options relating to the import of +observational data from WeatherCat monthly .cat files. ### `directory`{#wcat_directory} -The full path to the directory containing the year directories that contain the WeatherCat monthly .cat files to be imported. Do not include a trailing `/`. +The full path to the directory containing the year directories that +contain the WeatherCat monthly .cat files to be imported. Do not include a +trailing `/`. There is no default. ### `source_encoding`{#wcat_encoding} -The WeatherCat monthly .cat file encoding. This option is identical in operation to the CSV [source_encoding](#csv_encoding) option but applies to WeatherCat imports only. +The WeatherCat monthly .cat file encoding. This option is identical in +operation to the CSV [source_encoding](#csv_encoding) option but applies +to WeatherCat imports only. The default is `utf-8-sig`. ### `interval`{#wcat_interval} -Determines how the time interval (WeeWX database field `interval`) between successive observations is determined. This option is identical in operation to the CSV [interval](#csv_interval) option but applies to WeatherCat imports only. As WeatherCat monthly .cat files can, at times, have missing entries, the use of `interval = derive` may give incorrect or inconsistent interval values. Better results may be obtained by using `interval = conf` if the `archive_interval` for the current WeeWX installation is the same as the WeatherCat .cat file log interval, or by using `interval = x` where `x` is the time interval in minutes used in the WeatherCat monthly .cat file(s). The most appropriate setting will depend on the completeness and (time) accuracy of the WeatherCat data being imported. +Determines how the time interval (WeeWX database field `interval`) between +successive observations is determined. This option is identical in +operation to the CSV [interval](#csv_interval) option but applies to +WeatherCat imports only. As WeatherCat monthly .cat files can, at times, +have missing entries, the use of `interval = derive` may give incorrect or +inconsistent interval values. Better results may be obtained by using +`interval = conf` if the `archive_interval` for the current WeeWX +installation is the same as the WeatherCat .cat file log interval, or by +using `interval = x` where `x` is the time interval in minutes used in the +WeatherCat monthly .cat file(s). The most appropriate setting will depend +on the completeness and (time) accuracy of the WeatherCat data being imported. The default is `derive`. ### `qc`{#wcat_qc} -Determines whether simple quality control checks are applied to imported data. This option is identical in operation to the CSV [qc](#csv_qc) option but applies to WeatherCat imports only. +Determines whether simple quality control checks are applied to imported +data. This option is identical in operation to the CSV [qc](#csv_qc) +option but applies to WeatherCat imports only. The default is `True`. ### `calc_missing`{#wcat_calc_missing} -Determines whether any missing derived observations will be calculated from the imported data. This option is identical in operation to the CSV [calc_missing](#csv_calc_missing) option but applies to WeatherCat imports only. +Determines whether any missing derived observations will be calculated +from the imported data. This option is identical in operation to the CSV +[calc_missing](#csv_calc_missing) option but applies to WeatherCat imports only. The default is `True`. ### `decimal`{#wcat_decimal} -The character used as the decimal point in the WeatherCat monthly .cat files. This parameter must be included in quotation marks. +The character used as the decimal point in the WeatherCat monthly .cat +files. This parameter must be included in quotation marks. The default is `.`. ### `tranche`{#wcat_tranche} -The number of records written to the WeeWX database in each transaction. This option is identical in operation to the CSV [tranche](#csv_tranche) option but applies to WeatherCat imports only. +The number of records written to the WeeWX database in each transaction. +This option is identical in operation to the CSV [tranche](#csv_tranche) +option but applies to WeatherCat imports only. The default is `250` which should suit most users. ### `UV_sensor`{#wcat_UV} -Enables `wee_import` to distinguish between the case where a UV sensor is present and the UV index is 0 and the case where no UV sensor is present and UV index is 0. This option is identical in operation to the CSV [UV_sensor](#csv_UV) option but applies to WeatherCat imports only. +Enables `wee_import` to distinguish between the case where a UV sensor is +present and the UV index is 0 and the case where no UV sensor is present +and UV index is 0. This option is identical in operation to the CSV +[UV_sensor](#csv_UV) option but applies to WeatherCat imports only. The default is `True`. ### `solar_sensor`{#wcat_solar} -Enables `wee_import` to distinguish between the case where a solar radiation sensor is present and the solar radiation is 0 and the case where no solar radiation sensor is present and solar radiation is 0. This option is identical in operation to the CSV [solar_sensor](#csv_solar) option but applies to WeatherCat imports only. +Enables `wee_import` to distinguish between the case where a solar +radiation sensor is present and the solar radiation is 0 and the case +where no solar radiation sensor is present and solar radiation is 0. This +option is identical in operation to the CSV [solar_sensor](#csv_solar) +option but applies to WeatherCat imports only. The default is `True`. ### `[[Units]]`{#wcat_units} -The `[[Units]]` stanza defines the units used in the WeatherCat monthly .cat files. Unit settings are required for `temperature`, `pressure`, `rain` and `speed`. The format for each setting is: +The `[[Units]]` stanza defines the units used in the WeatherCat monthly . +cat files. Unit settings are required for `temperature`, `pressure`, +`rain` and `speed`. The format for each setting is: ``` obs_type = weewx_unit_name ``` -Where `obs_type` is one of `temperature`, `pressure`, `rain` or `speed` and `weewx_unit_name` is the WeeWX unit name of the units used by that particular `obs_type` (refer to the [_Units_](../reference/units.md) for details of available WeeWX unit names). As WeatherCat supports a different suite of possible units only a subset of the available WeeWX unit names can be used for some settings. +Where `obs_type` is one of `temperature`, `pressure`, `rain` or `speed` +and `weewx_unit_name` is the WeeWX unit name of the units used by that +particular `obs_type` (refer to the [_Units_](../reference/units.md) for +details of available WeeWX unit names). As WeatherCat supports a different +suite of possible units only a subset of the available WeeWX unit names +can be used for some settings. There is no default. diff --git a/docs_src/utilities/weectl-import-csv.md b/docs_src/utilities/weectl-import-csv.md index 51f73477..e371d734 100644 --- a/docs_src/utilities/weectl-import-csv.md +++ b/docs_src/utilities/weectl-import-csv.md @@ -1,18 +1,37 @@ !!! Warning - Running WeeWX during a `wee_import` session can lead to abnormal termination of the import. If WeeWX must remain running (e.g., so that live data is not lost) run the `wee_import` session on another machine or to a second database and merge the in-use and second database once the import is complete. + Running WeeWX during a `wee_import` session can lead to abnormal + termination of the import. If WeeWX must remain running (e.g., so that + live data is not lost) run the `wee_import` session on another machine or + to a second database and merge the in-use and second database once the + import is complete. -`wee_import` can import data from a single CSV file. The CSV source file must be structured as follows: +`wee_import` can import data from a single CSV file. The CSV source file +must be structured as follows: -* The file must have a header row consisting of a comma separated list of field names. The field names can be any valid string as long as each field name is unique within the list. There is no requirement for the field names to be in any particular order as long as the same order is used for the observations on each row in the file. These field names will be mapped to WeeWX field names in the `[CSV]` section of the import configuration file. +* The file must have a header row consisting of a comma separated list of + field names. The field names can be any valid string as long as each + field name is unique within the list. There is no requirement for the + field names to be in any particular order as long as the same order is + used for the observations on each row in the file. These field names + will be mapped to WeeWX field names in the `[CSV]` section of the import + configuration file. -* Observation data for a given date-time must be listed on a single line with individual fields separated by a comma. The fields must be in the same order as the field names in the header row. +* Observation data for a given date-time must be listed on a single line + with individual fields separated by a comma. The fields must be in the + same order as the field names in the header row. -* Blank fields are represented by the use of white space or no space only between commas. +* Blank fields are represented by the use of white space or no space only + between commas. -* Direction data being imported may be represented as numeric degrees or as a string representing the [cardinal, intercardinal and/or secondary intercardinal directions](https://en.wikipedia.org/wiki/Cardinal_direction). +* Direction data being imported may be represented as numeric degrees or + as a string representing the [cardinal, intercardinal and/or secondary + intercardinal directions](https://en.wikipedia.org/wiki/Cardinal_direction). -* There must a field that represents the date-time of the observations on each line. This date-time field must be either a Unix epoch timestamp or any date-time format that can be represented using [Python -strptime() format codes](https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior). +* There must a field that represents the date-time of the observations on + each line. This date-time field must be either a Unix epoch timestamp or + any date-time format that can be represented using [Python strptime() + format codes](https://docs.python.org/2/library/datetime. + html#strftime-and-strptime-behavior). A CSV file suitable for import by `wee_import` may look like this: @@ -41,24 +60,45 @@ Time,Barometer,Temp,Humidity,Windspeed,Dir,Gust,Dayrain,Radiation,Uv ``` !!! Note - [Cardinal, intercardinal and/or secondary intercardinal directions](https://en.wikipedia.org/wiki/Cardinal_direction) may be represented by one, two or three letter abbreviations e.g., N, SE or SSW; by a single word e.g., North, Southwest or Southsouthwest or by hyphenated or spaced words e.g., North West or South-south-west. Capitalisation is ignored as are any spaces, hyphens or other white space. At present only English abbreviations and directions are supported. + [Cardinal, intercardinal and/or secondary intercardinal directions] + (https://en.wikipedia.org/wiki/Cardinal_direction) may be represented by + one, two or three letter abbreviations e.g., N, SE or SSW; by a single + word e.g., North, Southwest or Southsouthwest or by hyphenated or spaced + words e.g., North West or South-south-west. Capitalisation is ignored as + are any spaces, hyphens or other white space. At present only English + abbreviations and directions are supported. ## Mapping data to archive fields -The WeeWX archive fields populated during a CSV import depend on the CSV-to-WeeWX field mappings specified in `[[FieldMap]]` stanza in the import configuration file. If a valid field mapping exists, the WeeWX field exists in the WeeWX archive table schema and provided the mapped CSV field contains valid data, the corresponding WeeWX field will be populated. Note that the CSV import is the only import supported by `wee_import` that allows any WeeWX archive field to be populated. +The WeeWX archive fields populated during a CSV import depend on the +CSV-to-WeeWX field mappings specified in `[[FieldMap]]` stanza in the +import configuration file. If a valid field mapping exists, the WeeWX +field exists in the WeeWX archive table schema and provided the mapped CSV +field contains valid data, the corresponding WeeWX field will be populated. +Note that the CSV import is the only import supported by `wee_import` that +allows any WeeWX archive field to be populated. !!! Note - The use of the [calc_missing](../wee_import-config#csv_calc_missing) option in the import configuration file may result in a number of derived fields being calculated from the imported data. If these derived fields exist in the in-use database schema they will be saved to the database as well. + The use of the [calc_missing](../wee_import-config#csv_calc_missing) + option in the import configuration file may result in a number of derived + fields being calculated from the imported data. If these derived fields + exist in the in-use database schema they will be saved to the database as + well. ## Step-by-step instructions To import observations from a CSV file: -1. Ensure the source data file is in a directory accessible by the machine that will run `wee_import`. For the purposes of the following examples the source data file `data.csv` located in the `/var/tmp` directory will be used. +1. Ensure the source data file is in a directory accessible by the machine + that will run `wee_import`. For the purposes of the following examples + the source data file `data.csv` located in the `/var/tmp` directory + will be used. 1. Make a backup of the WeeWX database in case the import should go awry. -1. Create an import configuration file. In this case we will make a copy of the example CSV import configuration file and save it as `csv.conf` in the `/var/tmp` directory: +1. Create an import configuration file. In this case we will make a copy + of the example CSV import configuration file and save it as `csv.conf` + in the `/var/tmp` directory: ``` $ cp /home/weewx/util/import/csv-example.conf /var/tmp/csv.conf @@ -72,33 +112,51 @@ To import observations from a CSV file: 1. Confirm the following options in the `[CSV]` section are set: - * [file](../wee_import-config#csv_file). The full path and file name of the file containing the CSV formatted data to be imported. + * [file](../wee_import-config#csv_file). The full path and file name + of the file containing the CSV formatted data to be imported. * [delimiter](../wee_import-config#csv_delimiter). The single character used to separate fields. * [interval](../wee_import-config#csv_interval). Determines how the WeeWX interval field is derived. - * [qc](../wee_import-config#csv_qc). Determines whether quality control checks are performed on the imported data. + * [qc](../wee_import-config#csv_qc). Determines whether quality + control checks are performed on the imported data. - * [calc_missing](../wee_import-config#csv_calc_missing). Determines whether missing derived observations will be calculated from the imported data. + * [calc_missing](../wee_import-config#csv_calc_missing). Determines + whether missing derived observations will be calculated from the + imported data. - * [ignore_invalid_data](../wee_import-config#csv_ignore_invalid_data). Determines whether invalid data in a source field is ignored or the import aborted. + * [ignore_invalid_data](../wee_import-config#csv_ignore_invalid_data). + Determines whether invalid data in a source field is ignored or the + import aborted. - * [tranche](../wee_import-config#csv_tranche). The number of records written to the WeeWX database in each transaction. + * [tranche](../wee_import-config#csv_tranche). The number of records + written to the WeeWX database in each transaction. - * [UV_sensor](../wee_import-config#csv_UV). Whether a UV sensor was installed when the source data was produced. + * [UV_sensor](../wee_import-config#csv_UV). Whether a UV sensor was + installed when the source data was produced. - * [solar_sensor](../wee_import-config#csv_solar). Whether a solar radiation sensor was installed when the source data was produced. + * [solar_sensor](../wee_import-config#csv_solar). Whether a solar + radiation sensor was installed when the source data was produced. - * [raw_datetime_format](../wee_import-config#csv_raw_datetime_format). The format of the imported date time field. + * [raw_datetime_format](../wee_import-config#csv_raw_datetime_format). + The format of the imported date time field. - * [rain](../wee_import-config#csv_rain). Determines how the WeeWX rain field is derived. + * [rain](../wee_import-config#csv_rain). Determines how the WeeWX + rain field is derived. + + * [wind_direction](../wee_import-config#csv_wind_direction). + Determines how imported wind direction fields are interpreted. - * [wind_direction](../wee_import-config#csv_wind_direction). Determines how imported wind direction fields are interpreted. + * [[[FieldMap]]](../wee_import-config#csv_fieldmap). Defines the + mapping between imported data fields and WeeWX archive fields. Also + defines the units of measure for each imported field. - * [[[FieldMap]]](../wee_import-config#csv_fieldmap). Defines the mapping between imported data fields and WeeWX archive fields. Also defines the units of measure for each imported field. - -1. When first importing data it is prudent to do a dry run import before any data are actually imported. A dry run import will perform all steps of the import without actually writing imported data to the WeeWX database. In addition, consideration should be given to any additional options such as `--date`. +1. When first importing data it is prudent to do a dry run import before + any data are actually imported. A dry run import will perform all steps + of the import without actually writing imported data to the WeeWX + database. In addition, consideration should be given to any additional + options such as `--date`. To perform a dry run enter the following command: @@ -112,32 +170,42 @@ To import observations from a CSV file: Using WeeWX configuration file /home/weewx/weewx.conf Starting wee_import... A CSV import from source file '/var/tmp/data.csv' has been requested. - Using database binding 'wx_binding', which is bound to database 'weewx.sdb' + Using database binding 'wx_binding', which is bound to database 'weewx. + sdb' Destination table 'archive' unit system is '0x01' (US). Missing derived observations will be calculated. This is a dry run, imported data will not be saved to archive. Starting dry run import ... 27337 records identified for import. - Unique records processed: 27337; Last timestamp: 2018-03-03 06:00:00 AEST (1520020800) + Unique records processed: 27337; Last timestamp: 2018-03-03 06:00:00 + AEST (1520020800) Finished dry run import - 27337 records were processed and 27337 unique records would have been imported. + 27337 records were processed and 27337 unique records would have been + imported. ``` - The output includes details about the data source, the destination of the imported data and some other details on how the data will be processed. The import will then be performed but no data will be written to the WeeWX database. Upon completion a brief summary of the records processed is provided. + The output includes details about the data source, the destination of + the imported data and some other details on how the data will be + processed. The import will then be performed but no data will be + written to the WeeWX database. Upon completion a brief summary of the + records processed is provided. -1. Once the dry run results are satisfactory the data can be imported using the following command: +1. Once the dry run results are satisfactory the data can be imported + using the following command: ``` wee_import --import-config=/var/tmp/csv.conf ``` - This will result in a short preamble similar to that from the dry run. At the end of the preamble there will be a prompt: + This will result in a short preamble similar to that from the dry run. + At the end of the preamble there will be a prompt: ``` Using WeeWX configuration file /home/weewx/weewx.conf Starting wee_import... A CSV import from source file '/var/tmp/data.csv' has been requested. - Using database binding 'wx_binding', which is bound to database 'weewx.sdb' + Using database binding 'wx_binding', which is bound to database 'weewx. + sdb' Destination table 'archive' unit system is '0x01' (US). Missing derived observations will be calculated. Starting import ... @@ -146,36 +214,62 @@ To import observations from a CSV file: Are you sure you want to proceed (y/n)? ``` -1. If the import parameters are acceptable enter `y` to proceed with the import or `n` to abort the import. If the import is confirmed the source data will be imported, processed and saved in the WeeWX database. Information on the progress of the import will be displayed similar to the following: +1. If the import parameters are acceptable enter `y` to proceed with the + import or `n` to abort the import. If the import is confirmed the + source data will be imported, processed and saved in the WeeWX database. + Information on the progress of the import will be displayed similar to + the following: ``` - Unique records processed: 3250; Last timestamp: 2017-12-09 14:45:00 AEST (1512794700) + Unique records processed: 3250; Last timestamp: 2017-12-09 14:45:00 + AEST (1512794700) ``` - The line commencing with `Unique records processed` should update as records are imported with progress information on number of records processed, number of unique records imported and the date time of the latest record processed. Once the initial import is complete `wee_import` will, if requested, calculate any missing derived observations and rebuild the daily summaries. A brief summary should be displayed similar to the following: + The line commencing with `Unique records processed` should update as + records are imported with progress information on number of records + processed, number of unique records imported and the date time of the + latest record processed. Once the initial import is complete + `wee_import` will, if requested, calculate any missing derived + observations and rebuild the daily summaries. A brief summary should be + displayed similar to the following: ``` Calculating missing derived observations... - Processing record: 27337; Last record: 2018-03-03 06:00:00 AEST (1520020800) + Processing record: 27337; Last record: 2018-03-03 06:00:00 AEST + (1520020800) Recalculating daily summaries... Records processed: 27337; Last date: 2018-03-03 06:00:00 AEST (1520020800) Finished recalculating daily summaries Finished calculating missing derived observations ``` - When the import is complete a brief summary is displayed similar to the following: + When the import is complete a brief summary is displayed similar to + the following: ``` Finished import - 27337 records were processed and 27337 unique records imported in 113.91 seconds. + 27337 records were processed and 27337 unique records imported in 113. + 91 seconds. Those records with a timestamp already in the archive will not have been imported. Confirm successful import in the WeeWX log file. ``` -1. Whilst `wee_import` will advise of the number of records processed and the number of unique records found, `wee_import` does know how many, if any, of the imported records were successfully saved to the database. You should look carefully through the WeeWX log file covering the `wee_import` session and take note of any records that were not imported. The most common reason for imported records not being saved to the database is because a record with that timestamp already exists in the database, in such cases something similar to the following will be found in the log: +1. Whilst `wee_import` will advise of the number of records processed and + the number of unique records found, `wee_import` does know how many, if + any, of the imported records were successfully saved to the database. + You should look carefully through the WeeWX log file covering the + `wee_import` session and take note of any records that were not + imported. The most common reason for imported records not being saved + to the database is because a record with that timestamp already exists + in the database, in such cases something similar to the following will + be found in the log: ``` - Aug 22 14:38:28 stretch12 wee_import[1226] ERROR weewx.manager: Unable to add record 2018-09-04 04:20:00 AEST (1535998800) to database 'weewx.sdb': UNIQUE constraint failed: archive.dateTime + Aug 22 14:38:28 stretch12 wee_import[1226] ERROR weewx.manager: + Unable to add record 2018-09-04 04:20:00 AEST (1535998800) to database + 'weewx.sdb': UNIQUE constraint failed: archive.dateTime ``` - In such cases you should take note of the timestamp of the record(s) concerned and make a decision about whether to delete the pre-existing record and re-import the record or retain the pre-existing record. + In such cases you should take note of the timestamp of the record(s) + concerned and make a decision about whether to delete the pre-existing + record and re-import the record or retain the pre-existing record. diff --git a/docs_src/utilities/weectl-import-cumulus.md b/docs_src/utilities/weectl-import-cumulus.md index cbbac455..a321474b 100644 --- a/docs_src/utilities/weectl-import-cumulus.md +++ b/docs_src/utilities/weectl-import-cumulus.md @@ -1,13 +1,26 @@ !!! Warning - Running WeeWX during a `wee_import` session can lead to abnormal termination of the import. If WeeWX must remain running (e.g., so that live data is not lost) run the `wee_import` session on another machine or to a second database and merge the in-use and second database once the import is complete. + Running WeeWX during a `wee_import` session can lead to abnormal + termination of the import. If WeeWX must remain running (e.g., so that + live data is not lost) run the `wee_import` session on another machine or + to a second database and merge the in-use and second database once the + import is complete. -`wee_import` can import observational data from the one or more Cumulus monthly log files. A Cumulus monthly log file records weather station observations for a single month. These files are accumulated over time and can be considered analogous to the WeeWX archive table. When `wee_import` imports data from the Cumulus monthly log files each log file is considered a 'period'. `wee_import` processes one period at a time in chronological order (oldest to newest) and provides import summary data on a per period basis. +`wee_import` can import observational data from the one or more Cumulus +monthly log files. A Cumulus monthly log file records weather station +observations for a single month. These files are accumulated over time and +can be considered analogous to the WeeWX archive table. When `wee_import` +imports data from the Cumulus monthly log files each log file is +considered a 'period'. `wee_import` processes one period at a time in +chronological order (oldest to newest) and provides import summary data on +a per period basis. ## Mapping data to archive fields -A Cumulus monthly log file import will populate the WeeWX archive fields as follows: +A Cumulus monthly log file import will populate the WeeWX archive fields +as follows: -* Provided data exists for each field in the Cumulus monthly logs, the following WeeWX archive fields will be directly populated by imported data: +* Provided data exists for each field in the Cumulus monthly logs, the + following WeeWX archive fields will be directly populated by imported data: * `dateTime` * `barometer` @@ -27,70 +40,105 @@ A Cumulus monthly log file import will populate the WeeWX archive fields as foll * `windchill` !!! Note - If a field in the Cumulus monthly log file has no data the corresponding WeeWX archive field will be set to `None/null`. + If a field in the Cumulus monthly log file has no data the + corresponding WeeWX archive field will be set to `None/null`. -* The following WeeWX archive fields will be populated from other settings or configuration options: +* The following WeeWX archive fields will be populated from other settings + or configuration options: * `interval` * `usUnits` -* The following WeeWX archive fields will be populated with values derived from the imported data provided `calc_missing = True` is included in the `[Cumulus]` section of the import configuration file being used and the field exists in the in-use WeeWX archive table schema. +* The following WeeWX archive fields will be populated with values derived + from the imported data provided `calc_missing = True` is included in the + `[Cumulus]` section of the import configuration file being used and the + field exists in the in-use WeeWX archive table schema. * `altimeter` * `ET` * `pressure` !!! Note - If `calc_missing = False` is included in the `[Cumulus]` section of the import configuration file being used then all of the above fields will be set to `None/null`. The `calc_missing` option default is `True`. + If `calc_missing = False` is included in the `[Cumulus]` section + of the import configuration file being used then all of the above + fields will be set to `None/null`. The `calc_missing` option + default is `True`. ## Step-by-step instructions To import observations from one or more Cumulus monthly log files: -1. Ensure the Cumulus monthly log file(s) to be used for the import are located in a directory accessible by the machine that will run `wee_import`. For the purposes of the following examples, there are nine monthly logs files covering the period October 2016 to June 2017, inclusive, located in the `/var/tmp/cumulus` directory. +1. Ensure the Cumulus monthly log file(s) to be used for the import are + located in a directory accessible by the machine that will run + `wee_import`. For the purposes of the following examples, there are + nine monthly logs files covering the period October 2016 to June 2017, + inclusive, located in the `/var/tmp/cumulus` directory. 1. Make a backup of the WeeWX database in case the import should go awry. -1. Create an import configuration file. In this case we will make a copy of the example Cumulus import configuration file and save it as `cumulus.conf` in the `/var/tmp` directory: +1. Create an import configuration file. In this case we will make a copy + of the example Cumulus import configuration file and save it as + `cumulus.conf` in the `/var/tmp` directory: ``` $ cp /home/weewx/util/import/cumulus-example.conf /var/tmp/cumulus.conf ``` -1. Confirm the [`source`](../wee_import-config#import_config_source) option is set to Cumulus: +1. Confirm the [`source`](../wee_import-config#import_config_source) + option is set to Cumulus: ``` source = Cumulus ``` -1. Confirm that the following options in the `[Cumulus]` section are correctly set: +1. Confirm that the following options in the `[Cumulus]` section are + correctly set: - * [directory](../wee_import-config#cumulus_directory). The full path to the directory containing the Cumulus monthly log files to be used as the source of the imported data. + * [directory](../wee_import-config#cumulus_directory). The full path + to the directory containing the Cumulus monthly log files to be + used as the source of the imported data. - * [interval](../wee_import-config#cumulus_interval). Determines how the WeeWX interval field is derived. + * [interval](../wee_import-config#cumulus_interval). Determines how + the WeeWX interval field is derived. - * [qc](../wee_import-config#cumulus_qc). Determines whether quality control checks are performed on the imported data. + * [qc](../wee_import-config#cumulus_qc). Determines whether quality + control checks are performed on the imported data. - * [calc_missing](../wee_import-config#cumulus_calc_missing). Determines whether missing derived observations will be calculated from the imported data. + * [calc_missing](../wee_import-config#cumulus_calc_missing). + Determines whether missing derived observations will be calculated + from the imported data. - * [separator](../wee_import-config#cumulus_separator). The date field separator used in the Cumulus monthly log files. + * [separator](../wee_import-config#cumulus_separator). The date field + separator used in the Cumulus monthly log files. - * [delimiter](../wee_import-config#cumulus_delimiter). The field delimiter used in the Cumulus monthly log files. + * [delimiter](../wee_import-config#cumulus_delimiter). The field + delimiter used in the Cumulus monthly log files. - * [decimal](../wee_import-config#cumulus_decimal). The decimal point character used in the Cumulus monthly log files. + * [decimal](../wee_import-config#cumulus_decimal). The decimal point + character used in the Cumulus monthly log files. - * [ignore_invalid_data](../wee_import-config#cumulus_ignore_invalid_data). Determines whether invalid data in a source field is ignored or the import aborted. + * [ignore_invalid_data](.. + /wee_import-config#cumulus_ignore_invalid_data). Determines whether + invalid data in a source field is ignored or the import aborted. - * [tranche](../wee_import-config#cumulus_tranche). The number of records written to the WeeWX database in each transaction. + * [tranche](../wee_import-config#cumulus_tranche). The number of + records written to the WeeWX database in each transaction. - * [UV_sensor](../wee_import-config#cumulus_UV). Whether a UV sensor was installed when the source data was produced. + * [UV_sensor](../wee_import-config#cumulus_UV). Whether a UV sensor + was installed when the source data was produced. - * [solar_sensor](../wee_import-config#cumulus_solar). Whether a solar radiation sensor was installed when the source data was produced. + * [solar_sensor](../wee_import-config#cumulus_solar). Whether a solar + radiation sensor was installed when the source data was produced. - * [[[Units]]](../wee_import-config#cumulus_units). Defines the units used in the Cumulus monthly log files. + * [[[Units]]](../wee_import-config#cumulus_units). Defines the units + used in the Cumulus monthly log files. -1. When first importing data it is prudent to do a dry run import before any data is actually imported. A dry run import will perform all steps of the import without actually writing imported data to the WeeWX database. In addition, consideration should be given to any additional options to be used such as `--date`. +1. When first importing data it is prudent to do a dry run import before + any data is actually imported. A dry run import will perform all steps + of the import without actually writing imported data to the WeeWX + database. In addition, consideration should be given to any additional + options to be used such as `--date`. To perform a dry run enter the following command: @@ -98,61 +146,84 @@ To import observations from one or more Cumulus monthly log files: wee_import --import-config=/var/tmp/cumulus.conf --dry-run ``` - This will result in a short preamble with details on the data source, the destination of the imported data and some other details on how the data will be processed. The import will then be performed but no data will be written to the WeeWX database. + This will result in a short preamble with details on the data source, + the destination of the imported data and some other details on how the + data will be processed. The import will then be performed but no data + will be written to the WeeWX database. The output should be similar to: ``` Using WeeWX configuration file /home/weewx/weewx.conf Starting wee_import... - Cumulus monthly log files in the '/var/tmp/cumulus' directory will be imported - Using database binding 'wx_binding', which is bound to database 'weewx.sdb' + Cumulus monthly log files in the '/var/tmp/cumulus' directory will be + imported + Using database binding 'wx_binding', which is bound to database 'weewx. + sdb' Destination table 'archive' unit system is '0x01' (US). Missing derived observations will be calculated. This is a dry run, imported data will not be saved to archive. Starting dry run import ... Records covering multiple periods have been identified for import. Period 1 ... - Unique records processed: 8858; Last timestamp: 2016-10-31 23:55:00 AEST (1477922100) + Unique records processed: 8858; Last timestamp: 2016-10-31 23:55:00 + AEST (1477922100) Period 2 ... - Unique records processed: 8636; Last timestamp: 2016-11-30 23:55:00 AEST (1480514100) + Unique records processed: 8636; Last timestamp: 2016-11-30 23:55:00 + AEST (1480514100) Period 3 ... - Unique records processed: 8925; Last timestamp: 2016-12-31 23:55:00 AEST (1483192500) + Unique records processed: 8925; Last timestamp: 2016-12-31 23:55:00 + AEST (1483192500) Period 4 ... - Unique records processed: 8908; Last timestamp: 2017-01-31 23:55:00 AEST (1485870900) + Unique records processed: 8908; Last timestamp: 2017-01-31 23:55:00 + AEST (1485870900) Period 5 ... - Unique records processed: 8029; Last timestamp: 2017-02-28 23:55:00 AEST (1488290100) + Unique records processed: 8029; Last timestamp: 2017-02-28 23:55:00 + AEST (1488290100) Period 6 ... - Unique records processed: 8744; Last timestamp: 2017-03-31 23:55:00 AEST (1490968500) + Unique records processed: 8744; Last timestamp: 2017-03-31 23:55:00 + AEST (1490968500) Period 7 ... - Unique records processed: 8489; Last timestamp: 2017-04-30 23:02:00 AEST (1493557320) + Unique records processed: 8489; Last timestamp: 2017-04-30 23:02:00 + AEST (1493557320) Period 8 ... - Unique records processed: 8754; Last timestamp: 2017-05-31 23:55:00 AEST (1496238900) + Unique records processed: 8754; Last timestamp: 2017-05-31 23:55:00 + AEST (1496238900) Period 9 ... - Unique records processed: 8470; Last timestamp: 2017-06-30 23:55:00 AEST (1498830900) + Unique records processed: 8470; Last timestamp: 2017-06-30 23:55:00 + AEST (1498830900) Finished dry run import - 77813 records were processed and 77813 unique records would have been imported. + 77813 records were processed and 77813 unique records would have been + imported. ``` !!! Note - The nine periods correspond to the nine monthly log files used for this import. + The nine periods correspond to the nine monthly log files used for + this import. !!! Note - Any periods for which no data could be obtained will be skipped. The lack of data may be due to a missing Cumulus monthly log file. A short explanatory note to this effect will be displayed against the period concerned and an entry included in the log. + Any periods for which no data could be obtained will be skipped. + The lack of data may be due to a missing Cumulus monthly log file. A + short explanatory note to this effect will be displayed against the + period concerned and an entry included in the log. -1. Once the dry run results are satisfactory the data can be imported using the following command: +1. Once the dry run results are satisfactory the data can be imported + using the following command: ``` wee_import --import-config=/var/tmp/cumulus.conf ``` - This will result in a preamble similar to that of a dry run. At the end of the preamble there will be a prompt: + This will result in a preamble similar to that of a dry run. At the + end of the preamble there will be a prompt: ``` Using WeeWX configuration file /home/weewx/weewx.conf Starting wee_import... - Cumulus monthly log files in the '/var/tmp/cumulus' directory will be imported - Using database binding 'wx_binding', which is bound to database 'weewx.sdb' + Cumulus monthly log files in the '/var/tmp/cumulus' directory will be + imported + Using database binding 'wx_binding', which is bound to database 'weewx. + sdb' Destination table 'archive' unit system is '0x01' (US). Missing derived observations will be calculated. Starting import ... @@ -162,13 +233,19 @@ To import observations from one or more Cumulus monthly log files: Are you sure you want to proceed (y/n)? ``` - If there is more than one Cumulus monthly log file then `wee_import` will provide summary information on a per period basis during the import. In addition, if the `--date` option is used then source data that falls outside the date or date range specified with the `--date` option is ignored. In such cases the preamble may look similar to: + If there is more than one Cumulus monthly log file then `wee_import` + will provide summary information on a per period basis during the + import. In addition, if the `--date` option is used then source data + that falls outside the date or date range specified with the `--date` + option is ignored. In such cases the preamble may look similar to: ``` Using WeeWX configuration file /home/weewx/weewx.conf Starting wee_import... - Cumulus monthly log files in the '/var/tmp/cumulus' directory will be imported - Using database binding 'wx_binding', which is bound to database 'weewx.sdb' + Cumulus monthly log files in the '/var/tmp/cumulus' directory will be + imported + Using database binding 'wx_binding', which is bound to database 'weewx. + sdb' Destination table 'archive' unit system is '0x01' (US). Missing derived observations will be calculated. Starting import ... @@ -182,52 +259,87 @@ To import observations from one or more Cumulus monthly log files: Are you sure you want to proceed (y/n)? ``` -1. If the import parameters are acceptable enter `y` to proceed with the import or `n` to abort the import. If the import is confirmed, the source data will be imported, processed and saved in the WeeWX database. Information on the progress of the import will be displayed similar to the following: +1. If the import parameters are acceptable enter `y` to proceed with the + import or `n` to abort the import. If the import is confirmed, the + source data will be imported, processed and saved in the WeeWX database. + Information on the progress of the import will be displayed similar to + the following: ``` - Unique records processed: 2305; Last timestamp: 2016-12-30 00:00:00 AEST (1483020000) + Unique records processed: 2305; Last timestamp: 2016-12-30 00:00:00 + AEST (1483020000) ``` - Again if there is more than one Cumulus monthly log file and if the `--date` option is used the progress information may instead look similar to: + Again if there is more than one Cumulus monthly log file and if the + `--date` option is used the progress information may instead look + similar to: ``` Period 4 ... - Unique records processed: 8908; Last timestamp: 2017-01-31 23:55:00 AEST (1485870900) + Unique records processed: 8908; Last timestamp: 2017-01-31 23:55:00 + AEST;(1485870900) Period 5 ... - Unique records processed: 8029; Last timestamp: 2017-02-28 23:55:00 AEST (1488290100) + Unique;records processed: 8029; Last timestamp: 2017-02-28 23:55:00 + AEST (1488290100) Period 6 ... - Unique records processed: 8744; Last timestamp: 2017-03-31 23:55:00 AEST (1490968500) + Unique;records processed: 8744; Last timestamp: 2017-03-31 23:55:00 + AEST (1490968500) ``` !!! Note - Any periods for which no data could be obtained will be skipped. The lack of data may be due to a missing Cumulus monthly log file. A short explanatory note to this effect will be displayed against the period concerned and an entry included in the log. + Any periods for which no data could be obtained will be skipped. + The lack of data may be due to a missing Cumulus monthly log file. A + short explanatory note to this effect will be displayed against the + period concerned and an entry included in the log. - The line commencing with `Unique records processed` should update as records are imported with progress information on number of records processed, number of unique records imported and the date time of the latest record processed. If the import spans multiple months (ie multiple monthly log files) then a new `Period` line is created for each month. + The line commencing with `Unique records processed` should update as + records are imported with progress information on number of records + processed, number of unique records imported and the date time of the + latest record processed. If the import spans multiple months (ie + multiple monthly log files) then a new `Period` line is created for + each month. - Once the initial import is complete `wee_import` will, if requested, calculate any missing derived observations and rebuild the daily summaries. A brief summary should be displayed similar to the following: + Once the initial import is complete `wee_import` will, if requested, + calculate any missing derived observations and rebuild the daily + summaries. A brief summary should be displayed similar to the following: ``` Calculating missing derived observations ... - Processing record: 77782; Last record: 2017-06-30 00:00:00 AEST (1519826400) + Processing record: 77782; Last record: 2017-06-30 00:00:00 AEST + (1519826400) Recalculating daily summaries... Records processed: 77000; Last date: 2017-06-28 11:45:00 AEST (1519811100) Finished recalculating daily summaries Finished calculating missing derived observations ``` - When the import is complete a brief summary is displayed similar to the following: + When the import is complete a brief summary is displayed similar to + the following: ``` Finished import - 77813 records were processed and 77813 unique records imported in 106.96 seconds. + 77813 records were processed and 77813 unique records imported in 106. + 96 seconds. Those records with a timestamp already in the archive will not have been imported. Confirm successful import in the WeeWX log file. ``` -1. Whilst `wee_import` will advise of the number of records processed and the number of unique records found, `wee_import` does know how many, if any, of the imported records were successfully saved to the database. You should look carefully through the WeeWX log file covering the `wee_import` session and take note of any records that were not imported. The most common reason for imported records not being saved to the database is because a record with that timestamp already exists in the database, in such cases something similar to the following will be found in the log: +1. Whilst `wee_import` will advise of the number of records processed and + the number of unique records found, `wee_import` does know how many, if + any, of the imported records were successfully saved to the database. + You should look carefully through the WeeWX log file covering the + `wee_import` session and take note of any records that were not + imported. The most common reason for imported records not being saved + to the database is because a record with that timestamp already exists + in the database, in such cases something similar to the following will + be found in the log: ``` - Aug 22 14:38:28 stretch12 weewx[863]: manager: unable to add record 2018-09-04 04:20:00 AEST (1535998800) to database 'weewx.sdb': UNIQUE constraint failed: archive.dateTime + Aug 22 14:38:28 stretch12 weewx[863]: manager: unable to add record + 2018-09-04 04:20:00 AEST (1535998800) to database 'weewx.sdb': UNIQUE + constraint failed: archive.dateTime ``` - In such cases take note of the timestamp of the record(s) concerned and make a decision about whether to delete the pre-existing record and re-import the record or retain the pre-existing record. + In such cases take note of the timestamp of the record(s) concerned + and make a decision about whether to delete the pre-existing record and + re-import the record or retain the pre-existing record. diff --git a/docs_src/utilities/weectl-import-troubleshoot.md b/docs_src/utilities/weectl-import-troubleshoot.md index fd038469..c260b77f 100644 --- a/docs_src/utilities/weectl-import-troubleshoot.md +++ b/docs_src/utilities/weectl-import-troubleshoot.md @@ -2,13 +2,23 @@ Sometimes bad things happen during an import. -If errors were encountered, or if you suspect that the WeeWX database has been contaminated with incorrect data, here are some things you can try to fix things up. +If errors were encountered, or if you suspect that the WeeWX database has +been contaminated with incorrect data, here are some things you can try to +fix things up. -* Manually delete the contaminated data. Use SQL commands to manipulate the data in the WeeWX archive database. The simplicity of this process will depend on your ability to use SQL, the amount of data imported, and whether the imported data was dispersed amongst existing. Once contaminated data have been removed the daily summary tables will need to be rebuilt using the `weectl database rebuild-daily` utility. +* Manually delete the contaminated data. Use SQL commands to manipulate + the data in the WeeWX archive database. The simplicity of this process + will depend on your ability to use SQL, the amount of data imported, and + whether the imported data was dispersed amongst existing. Once + contaminated data have been removed the daily summary tables will need + to be rebuilt using the `weectl database rebuild-daily` utility. -* Delete the database and start over. For SQLite, simply delete the database file. For MySQL, drop the database. Then try the import again. +* Delete the database and start over. For SQLite, simply delete the + database file. For MySQL, drop the database. Then try the import again. !!! Warning - Deleting the database file or dropping the database will result in all data in the database being lost. + Deleting the database file or dropping the database will result in + all data in the database being lost. -* If the above steps are not appropriate the database should be restored from backup. You did make a backup before starting the import? +* If the above steps are not appropriate the database should be restored + from backup. You did make a backup before starting the import? diff --git a/docs_src/utilities/weectl-import-wd.md b/docs_src/utilities/weectl-import-wd.md index 35f3c3c7..a558a519 100644 --- a/docs_src/utilities/weectl-import-wd.md +++ b/docs_src/utilities/weectl-import-wd.md @@ -1,7 +1,15 @@ !!! Warning - Running WeeWX during a `wee_import` session can lead to abnormal termination of the import. If WeeWX must remain running (e.g., so that live data is not lost) run the `wee_import` session on another machine or to a second database and merge the in-use and second database once the import is complete. + Running WeeWX during a `wee_import` session can lead to abnormal + termination of the import. If WeeWX must remain running (e.g., so that + live data is not lost) run the `wee_import` session on another machine or + to a second database and merge the in-use and second database once the + import is complete. -`wee_import` can import observational data from the one or more Weather Display monthly log files. Weather Display records observational data on a monthly basis in a number of either space delimited (.txt) and/or comma separated (.csv) text files. `wee_import` can import observational data from the following Weather Display log files: +`wee_import` can import observational data from the one or more Weather +Display monthly log files. Weather Display records observational data on a +monthly basis in a number of either space delimited (.txt) and/or comma +separated (.csv) text files. `wee_import` can import observational data +from the following Weather Display log files: * MMYYYYlg.txt * MMYYYYlgcsv.csv (csv format version of MMYYYYlg.txt) @@ -11,11 +19,21 @@ where MM is a one or two-digit month and YYYY is a four digit year -The Weather Display monthly log files record observational data using a nominal one-minute interval with each file recording various observations for the month and year designated by the MM and YYYY components of the file name. These files are accumulated over time and can be considered analogous to the WeeWX archive table. When `wee_import` imports data from the Weather Display monthly log files each set of log files for a given month and year is considered a 'period'. `wee_import` processes one period at a time in chronological order (oldest to newest) and provides import summary data on a per period basis. +The Weather Display monthly log files record observational data using a +nominal one-minute interval with each file recording various observations +for the month and year designated by the MM and YYYY components of the +file name. These files are accumulated over time and can be considered +analogous to the WeeWX archive table. When `wee_import` imports data from +the Weather Display monthly log files each set of log files for a given +month and year is considered a 'period'. `wee_import` processes one period +at a time in chronological order (oldest to newest) and provides import +summary data on a per period basis. ## Mapping data to archive fields -The WeeWX archive fields populated during the import of Weather Display data depends on the field mapping specified in `[[FieldMap]]` stanza in the import configuration file. A given WeeWX field will be populated if: +The WeeWX archive fields populated during the import of Weather Display +data depends on the field mapping specified in `[[FieldMap]]` stanza in +the import configuration file. A given WeeWX field will be populated if: * a valid field mapping exists, @@ -23,13 +41,17 @@ The WeeWX archive fields populated during the import of Weather Display data dep * the mapped Weather Display field contains valid data. -The following WeeWX archive fields will be populated from other settings or configuration options and need not be included in the field map: +The following WeeWX archive fields will be populated from other settings +or configuration options and need not be included in the field map: * `interval` * `usUnits` -The following WeeWX archive fields will be populated with values derived from the imported data provided `calc_missing = True` is included in the `[WD]` section of the import configuration file being used and the field exists in the in-use WeeWX archive table schema: +The following WeeWX archive fields will be populated with values derived +from the imported data provided `calc_missing = True` is included in the ` +[WD]` section of the import configuration file being used and the field +exists in the in-use WeeWX archive table schema: * `altimeter` @@ -40,18 +62,29 @@ The following WeeWX archive fields will be populated with values derived from th * `windchill` !!! Note - If `calc_missing = False` is included in the `[WD]` section of the import configuration file being used then all of the above fields will be set to `None/null`. The `calc_missing` option default is `True`. + If `calc_missing = False` is included in the `[WD]` section of the + import configuration file being used then all of the above fields will be + set to `None/null`. The `calc_missing` option default is `True`. ## Step-by-step instructions To import observations from one or more Weather Display monthly log files: -1. Ensure the Weather Display monthly log file(s) to be used for the import are located in a directory accessible by the machine that will run `wee_import`. For the purposes of the following examples, there are five months of logs files covering the period September 2018 to January 2019 inclusive located in the `/var/tmp/wd` directory. +1. Ensure the Weather Display monthly log file(s) to be used for the + import are located in a directory accessible by the machine that will + run `wee_import`. For the purposes of the following examples, there are + five months of logs files covering the period September 2018 to January + 2019 inclusive located in the `/var/tmp/wd` directory. 1. Make a backup of the WeeWX database in case the import should go awry. -1. Create an import configuration file, this is easily done by making a copy of the example Weather Display import configuration file located in the `/home/weewx/util/import` or `/etc/weewx/import` directory as applicable. In this case we will make a copy of the example Weather Display import configuration file and save it as `wd.conf` in the `/var/tmp` directory: +1. Create an import configuration file, this is easily done by making a + copy of the example Weather Display import configuration file located + in the `/home/weewx/util/import` or `/etc/weewx/import` directory as + applicable. In this case we will make a copy of the example Weather + Display import configuration file and save it as `wd.conf` in the + `/var/tmp` directory: ``` $ cp /home/weewx/util/import/wd-example.conf /var/tmp/wd.conf @@ -65,42 +98,72 @@ To import observations from one or more Weather Display monthly log files: 1. Confirm that the following options in the `[WD]` section are correctly set: - * [directory](../wee_import-config#wd_directory). The full path to the directory containing the Weather Display monthly log files to be used as the source of the imported data. + * [directory](../wee_import-config#wd_directory). The full path to the + directory containing the Weather Display monthly log files to be + used as the source of the imported data. - * [logs_to_process](../wee_import-config#wd_logs_to_process). Specifies the Weather Display monthly log files to be used to import data. + * [logs_to_process](../wee_import-config#wd_logs_to_process). + Specifies the Weather Display monthly log files to be used to import + data. - * [interval](../wee_import-config#wd_interval). Determines how the WeeWX interval field is derived. + * [interval](../wee_import-config#wd_interval). Determines how the + WeeWX interval field is derived. - * [qc](../wee_import-config#wd_qc). Determines whether quality control checks are performed on the imported data. + * [qc](../wee_import-config#wd_qc). Determines whether quality control + checks are performed on the imported data. - * [calc_missing](../wee_import-config#wd_calc_missing). Determines whether missing derived observations will be calculated from the imported data. + * [calc_missing](../wee_import-config#wd_calc_missing). Determines + whether missing derived observations will be calculated from the + imported data. - * [txt_delimiter](../wee_import-config#wd_txt_delimiter). The field delimiter used in the Weather Display space delimited (*.txt) monthly log files. + * [txt_delimiter](../wee_import-config#wd_txt_delimiter). The field + delimiter used in the Weather Display space delimited (*.txt) + monthly log files. - * [csv_delimiter](../wee_import-config#wd_csv_delimiter). The field delimiter used in the Weather Display monthly comma separated values (*.csv) monthly log files. + * [csv_delimiter](../wee_import-config#wd_csv_delimiter). The field + delimiter used in the Weather Display monthly comma separated values + (*.csv) monthly log files. - * [decimal](../wee_import-config#wd_decimal). The decimal point character used in the Weather Display monthly log files. + * [decimal](../wee_import-config#wd_decimal). The decimal point + character used in the Weather Display monthly log files. - * [ignore_missing_log](../wee_import-config#wd_ignore_missing_log). Determines whether missing log files are to be ignored or the import aborted. + * [ignore_missing_log](../wee_import-config#wd_ignore_missing_log). + Determines whether missing log files are to be ignored or the import + aborted. - * [ignore_invalid_data](../wee_import-config#wd_ignore_invalid_data). Determines whether invalid data in a source field is ignored or the import aborted. + * [ignore_invalid_data](../wee_import-config#wd_ignore_invalid_data). + Determines whether invalid data in a source field is ignored or the + import aborted. - * [tranche](../wee_import-config#wd_tranche). The number of records written to the WeeWX database in each transaction. + * [tranche](../wee_import-config#wd_tranche). The number of records + written to the WeeWX database in each transaction. - * [UV_sensor](../wee_import-config#wd_UV). Whether a UV sensor was installed when the source data was produced. + * [UV_sensor](../wee_import-config#wd_UV). Whether a UV sensor was + installed when the source data was produced. - * [solar_sensor](../wee_import-config#wd_solar). Whether a solar radiation sensor was installed when the source data was produced. + * [solar_sensor](../wee_import-config#wd_solar). Whether a solar + radiation sensor was installed when the source data was produced. - * [ignore_extreme_temp_hum](../wee_import-config#wd_ignore_extreme_temp_hum). Determines whether temperature and humidity values of 255 will be ignored. + * [ignore_extreme_temp_hum](.. + /wee_import-config#wd_ignore_extreme_temp_hum). Determines whether + temperature and humidity values of 255 will be ignored. - * [[[Units]]](../wee_import-config#wd_units). Defines the units used in the Weather Display monthly log files. + * [[[Units]]](../wee_import-config#wd_units). Defines the units used + in the Weather Display monthly log files. - * [[[FieldMap]]](../wee_import-config#wd_fieldmap). Defines the mapping between imported data fields and WeeWX archive fields. + * [[[FieldMap]]](../wee_import-config#wd_fieldmap). Defines the + mapping between imported data fields and WeeWX archive fields. -1. When first importing data it is prudent to do a dry run import before any data is actually imported. A dry run import will perform all steps of the import without actually writing imported data to the WeeWX database. In addition, consideration should be given to any additional options to be used such as `--date`. +1. When first importing data it is prudent to do a dry run import before + any data is actually imported. A dry run import will perform all steps + of the import without actually writing imported data to the WeeWX + database. In addition, consideration should be given to any additional + options to be used such as `--date`. !!! Note - Due to some peculiarities of the Weather Display log structure it may be prudent to use the `--suppress-warnings` option during the initial dry run so the overall progress of the import can be observed. + Due to some peculiarities of the Weather Display log structure it + may be prudent to use the `--suppress-warnings` option during the + initial dry run so the overall progress of the import can be observed. To perform a dry run enter the following command: @@ -108,107 +171,146 @@ To import observations from one or more Weather Display monthly log files: wee_import --import-config=/var/tmp/wd.conf --dry-run --suppress-warnings ``` - This will result in a short preamble with details on the data source, the destination of the imported data and some other details on how the data will be processed. The import will then be performed but no data will be written to the WeeWX database. + This will result in a short preamble with details on the data source, + the destination of the imported data and some other details on how the + data will be processed. The import will then be performed but no data + will be written to the WeeWX database. The output should be similar to: ``` Using WeeWX configuration file /home/weewx/weewx.conf Starting wee_import... - Weather Display monthly log files in the '/var/tmp/WD' directory will be imported - Using database binding 'wx_binding', which is bound to database 'weewx.sdb' + Weather Display monthly log files in the '/var/tmp/WD' directory will + be imported + Using database binding 'wx_binding', which is bound to database 'weewx. + sdb' Destination table 'archive' unit system is '0x01' (US). Missing derived observations will be calculated. This is a dry run, imported data will not be saved to archive. Starting dry run import ... Records covering multiple periods have been identified for import. Period 1 ... - Unique records processed: 43183; Last timestamp: 2018-09-30 23:59:00 AEST (1538315940) + Unique records processed: 43183; Last timestamp: 2018-09-30 23:59:00 + AEST (1538315940) Period 2 ... - Unique records processed: 44620; Last timestamp: 2018-10-31 23:59:00 AEST (1540994340) + Unique records processed: 44620; Last timestamp: 2018-10-31 23:59:00 + AEST (1540994340) Period 3 ... - Unique records processed: 43136; Last timestamp: 2018-11-30 23:59:00 AEST (1543586340) + Unique records processed: 43136; Last timestamp: 2018-11-30 23:59:00 + AEST (1543586340) Period 4 ... - Unique records processed: 44633; Last timestamp: 2018-12-31 23:59:00 AEST (1546264740) + Unique records processed: 44633; Last timestamp: 2018-12-31 23:59:00 + AEST (1546264740) Period 5 ... - Unique records processed: 8977; Last timestamp: 2019-01-07 05:43:00 AEST (1546803780) + Unique records processed: 8977; Last timestamp: 2019-01-07 05:43:00 + AEST (1546803780) Finished dry run import - 184765 records were processed and 184549 unique records would have been imported. + 184765 records were processed and 184549 unique records would have + been imported. 216 duplicate records were ignored. ``` !!! Note - The five periods correspond to the five months of log files used for this import. + The five periods correspond to the five months of log files used + for this import. !!! Note - Any periods for which no data could be obtained will be skipped. The lack of data may be due to a missing Weather Display log file. A short explanatory note to this effect will be displayed against the period concerned and an entry included in the log. + Any periods for which no data could be obtained will be skipped. + The lack of data may be due to a missing Weather Display log file. A + short explanatory note to this effect will be displayed against the + period concerned and an entry included in the log. -1. If the `--suppress-warnings` option was used it may be prudent to do a second dry run this time without the `--suppress-warnings` option. This will allow any warnings generated by the dry run import to be observed: +1. If the `--suppress-warnings` option was used it may be prudent to do a + second dry run this time without the `--suppress-warnings` option. This + will allow any warnings generated by the dry run import to be observed: ``` wee_import --import-config=/var/tmp/wd.conf --dry-run ``` - This will result in a short preamble with details on the data source, the destination of the imported data and some other details on how the data will be processed. The import will then be performed but no data will be written to the WeeWX database. + This will result in a short preamble with details on the data source, + the destination of the imported data and some other details on how the + data will be processed. The import will then be performed but no data + will be written to the WeeWX database. The output should be similar to: ``` Using WeeWX configuration file /home/weewx/weewx.conf Starting wee_import... - Weather Display monthly log files in the '/var/tmp/WD' directory will be imported - Using database binding 'wx_binding', which is bound to database 'weewx.sdb' + Weather Display monthly log files in the '/var/tmp/WD' directory will + be imported + Using database binding 'wx_binding', which is bound to database 'weewx. + sdb' Destination table 'archive' unit system is '0x01' (US). Missing derived observations will be calculated. This is a dry run, imported data will not be saved to archive. Starting dry run import ... Records covering multiple periods have been identified for import. Period 1 ... - Warning: Import field 'radiation' is mapped to WeeWX field 'radiation' but the + Warning: Import field 'radiation' is mapped to WeeWX field 'radiation' + but the import field 'radiation' could not be found in one or more records. WeeWX field 'radiation' will be set to 'None' in these records. - Warning: Import field 'soiltemp' is mapped to WeeWX field 'soilTemp1' but the + Warning: Import field 'soiltemp' is mapped to WeeWX field 'soilTemp1' + but the import field 'soiltemp' could not be found in one or more records. WeeWX field 'soilTemp1' will be set to 'None' in these records. - Warning: Import field 'soilmoist' is mapped to WeeWX field 'soilMoist1' but the + Warning: Import field 'soilmoist' is mapped to WeeWX field + 'soilMoist1' but the import field 'soilmoist' could not be found in one or more records. WeeWX field 'soilMoist1' will be set to 'None' in these records. - Warning: Import field 'humidity' is mapped to WeeWX field 'outHumidity' but the + Warning: Import field 'humidity' is mapped to WeeWX field + 'outHumidity' but the import field 'humidity' could not be found in one or more records. WeeWX field 'outHumidity' will be set to 'None' in these records. - Warning: Import field 'heatindex' is mapped to WeeWX field 'heatindex' but the + Warning: Import field 'heatindex' is mapped to WeeWX field 'heatindex' + but the import field 'heatindex' could not be found in one or more records. WeeWX field 'heatindex' will be set to 'None' in these records. - Warning: Import field 'windspeed' is mapped to WeeWX field 'windSpeed' but the + Warning: Import field 'windspeed' is mapped to WeeWX field 'windSpeed' + but the import field 'windspeed' could not be found in one or more records. WeeWX field 'windSpeed' will be set to 'None' in these records. - Warning: Import field 'barometer' is mapped to WeeWX field 'barometer' but the + Warning: Import field 'barometer' is mapped to WeeWX field 'barometer' + but the import field 'barometer' could not be found in one or more records. WeeWX field 'barometer' will be set to 'None' in these records. - Warning: Import field 'dewpoint' is mapped to WeeWX field 'dewpoint' but the + Warning: Import field 'dewpoint' is mapped to WeeWX field 'dewpoint' + but the import field 'dewpoint' could not be found in one or more records. WeeWX field 'dewpoint' will be set to 'None' in these records. - Warning: Import field 'rainlastmin' is mapped to WeeWX field 'rain' but the + Warning: Import field 'rainlastmin' is mapped to WeeWX field 'rain' + but the import field 'rainlastmin' could not be found in one or more records. WeeWX field 'rain' will be set to 'None' in these records. - Warning: Import field 'direction' is mapped to WeeWX field 'windDir' but the + Warning: Import field 'direction' is mapped to WeeWX field 'windDir' + but the import field 'direction' could not be found in one or more records. WeeWX field 'windDir' will be set to 'None' in these records. - Warning: Import field 'temperature' is mapped to WeeWX field 'outTemp' but the + Warning: Import field 'temperature' is mapped to WeeWX field 'outTemp' + but the import field 'temperature' could not be found in one or more records. WeeWX field 'outTemp' will be set to 'None' in these records. - Warning: Import field 'gustspeed' is mapped to WeeWX field 'windGust' but the + Warning: Import field 'gustspeed' is mapped to WeeWX field 'windGust' + but the import field 'gustspeed' could not be found in one or more records. WeeWX field 'windGust' will be set to 'None' in these records. - Unique records processed: 43183; Last timestamp: 2018-09-30 23:59:00 AEST (1538315940) + Unique records processed: 43183; Last timestamp: 2018-09-30 23:59:00 + AEST (1538315940) Period 2 ... - Unique records processed: 44620; Last timestamp: 2018-10-31 23:59:00 AEST (1540994340) + Unique records processed: 44620; Last timestamp: 2018-10-31 23:59:00 + AEST (1540994340) Period 3 ... - Unique records processed: 43136; Last timestamp: 2018-11-30 23:59:00 AEST (1543586340) + Unique records processed: 43136; Last timestamp: 2018-11-30 23:59:00 + AEST (1543586340) Period 4 ... - Unique records processed: 44633; Last timestamp: 2018-12-31 23:59:00 AEST (1546264740) + Unique records processed: 44633; Last timestamp: 2018-12-31 23:59:00 + AEST (1546264740) Period 5 ... - Unique records processed: 8977; Last timestamp: 2019-01-07 05:43:00 AEST (1546803780) + Unique records processed: 8977; Last timestamp: 2019-01-07 05:43:00 + AEST (1546803780) 6 duplicate records were identified in period 5: 2019-01-04 10:31:00 AEST (1546561860) 2019-01-04 10:32:00 AEST (1546561920) @@ -217,32 +319,51 @@ To import observations from one or more Weather Display monthly log files: 2019-01-04 10:35:00 AEST (1546562100) 2019-01-04 10:36:00 AEST (1546562160) Finished dry run import - 184555 records were processed and 184549 unique records would have been imported. + 184555 records were processed and 184549 unique records would have + been imported. 6 duplicate records were ignored. ``` In this case the following warnings are evident: - * Period one had 12 warnings for import fields that were mapped to WeeWX data fields but for which no data was found. This could be a sign that a complete month of data or a significant portion of the month could be missing, or it could be a case of just the first record of the month is missing (a significant number of Weather Display monthly log files have been found to be missing the first record of the month). In most cases this warning can be ignored. + * Period one had 12 warnings for import fields that were mapped to + WeeWX data fields but for which no data was found. This could be a + sign that a complete month of data or a significant portion of the + month could be missing, or it could be a case of just the first + record of the month is missing (a significant number of Weather + Display monthly log files have been found to be missing the first + record of the month). In most cases this warning can be ignored. - * Period five shows warnings for six entries in the period that have duplicate timestamps. This could be a sign that there is a problem in one or more of the Weather Display monthly log files for that month. However, anecdotally it has been found that duplicate entries often exist in one or more Weather Display monthly log files. If the duplicates are to be ignored then such warnings can be ignored otherwise the incorrect data should be removed from the affected log files before import. + * Period five shows warnings for six entries in the period that have + duplicate timestamps. This could be a sign that there is a problem + in one or more of the Weather Display monthly log files for that + month. However, anecdotally it has been found that duplicate entries + often exist in one or more Weather Display monthly log files. If the + duplicates are to be ignored then such warnings can be ignored + otherwise the incorrect data should be removed from the affected log + files before import. -1. Once the dry run results are satisfactory the data can be imported using the following command: +1. Once the dry run results are satisfactory the data can be imported + using the following command: ``` wee_import --import-config=/var/tmp/wd.conf --suppress-warnings ``` !!! Note - The `--suppress-warnings` option has been used to suppress the previously encountered warnings. + The `--suppress-warnings` option has been used to suppress the + previously encountered warnings. - This will result in a preamble similar to that of a dry run. At the end of the preamble there will be a prompt: + This will result in a preamble similar to that of a dry run. At the + end of the preamble there will be a prompt: ``` Using WeeWX configuration file /home/weewx/weewx.conf Starting wee_import... - Weather Display monthly log files in the '/var/tmp/WD' directory will be imported - Using database binding 'wx_binding', which is bound to database 'weewx.sdb' + Weather Display monthly log files in the '/var/tmp/WD' directory will + be imported + Using database binding 'wx_binding', which is bound to database 'weewx. + sdb' Destination table 'archive' unit system is '0x01' (US). Missing derived observations will be calculated. Starting import ... @@ -252,16 +373,24 @@ To import observations from one or more Weather Display monthly log files: Are you sure you want to proceed (y/n)? ``` - If there is more than one month of Weather Display monthly log files then `wee_import` will provide summary information on a per period basis during the import. In addition, if the `--date` option is used then source data that falls outside the date or date range specified with the `--date` option is ignored. In such cases the preamble may look similar to: + If there is more than one month of Weather Display monthly log files + then `wee_import` will provide summary information on a per period + basis during the import. In addition, if the `--date` option is used + then source data that falls outside the date or date range specified + with the `--date` option is ignored. In such cases the preamble may + look similar to: ``` Using WeeWX configuration file /home/weewx/weewx.conf Starting wee_import... - Weather Display monthly log files in the '/var/tmp/WD' directory will be imported - Using database binding 'wx_binding', which is bound to database 'weewx.sdb' + Weather Display monthly log files in the '/var/tmp/WD' directory will + be imported + Using database binding 'wx_binding', which is bound to database 'weewx. + sdb' Destination table 'archive' unit system is '0x01' (US). Missing derived observations will be calculated. - Observations timestamped after 2018-10-12 00:00:00 AEST (1539266400) and up to and + Observations timestamped after 2018-10-12 00:00:00 AEST (1539266400) + and up to and including 2018-10-13 00:00:00 AEST (1539352800) will be imported. Starting import ... Records covering multiple periods have been identified for import. @@ -272,53 +401,87 @@ To import observations from one or more Weather Display monthly log files: Are you sure you want to proceed (y/n)? ``` -1. If the import parameters are acceptable enter `y` to proceed with the import or `n` to abort the import. If the import is confirmed, the source data will be imported, processed and saved in the WeeWX database. Information on the progress of the import will be displayed similar to the following: +1. If the import parameters are acceptable enter `y` to proceed with the + import or `n` to abort the import. If the import is confirmed, the + source data will be imported, processed and saved in the WeeWX database. + Information on the progress of the import will be displayed similar to + the following: ``` - Unique records processed: 1250; Last timestamp: 2018-12-01 20:49:00 AEST (1543661340) + Unique records processed: 1250; Last timestamp: 2018-12-01 20:49:00 + AEST (1543661340) ``` - Again if there is more than one month of Weather Display monthly log files and if the `--date` option is used the progress information may instead look similar to: + Again if there is more than one month of Weather Display monthly log + files and if the `--date` option is used the progress information may + instead look similar to: ``` Period 2 ... - Unique records processed: 44620; Last timestamp: 2018-10-31 23:59:00 AEST (1540994340) + Unique records processed: 44620; Last timestamp: 2018-10-31 23:59:00 + AEST (1540994340) Period 3 ... - Unique records processed: 43136; Last timestamp: 2018-11-30 23:59:00 AEST (1543586340) + Unique records processed: 43136; Last timestamp: 2018-11-30 23:59:00 + AEST (1543586340) Period 4 ... - Unique records processed: 12000; Last timestamp: 2018-12-09 07:59:00 AEST (1544306340) + Unique records processed: 12000; Last timestamp: 2018-12-09 07:59:00 + AEST (1544306340) ``` !!! Note - Any periods for which no data could be obtained will be skipped. The lack of data may be due to a missing Weather Display log file. A short explanatory note to this effect will be displayed against the period concerned and an entry included in the log. + Any periods for which no data could be obtained will be skipped. + The lack of data may be due to a missing Weather Display log file. A + short explanatory note to this effect will be displayed against the + period concerned and an entry included in the log. - The line commencing with `Unique records processed` should update as records are imported with progress information on number of unique records processed and the date time of the latest record processed. If the import spans multiple months then a new `Period` line is created for each month. + The line commencing with `Unique records processed` should update as + records are imported with progress information on number of unique + records processed and the date time of the latest record processed. If + the import spans multiple months then a new `Period` line is created + for each month. - Once the initial import is complete `wee_import` will, if requested, calculate any missing derived observations and rebuild the daily summaries. A brief summary should be displayed similar to the following: + Once the initial import is complete `wee_import` will, if requested, + calculate any missing derived observations and rebuild the daily + summaries. A brief summary should be displayed similar to the following: ``` Calculating missing derived observations ... - Processing record: 184549; Last record: 2019-01-08 00:00:00 AEST (1546869600) + Processing record: 184549; Last record: 2019-01-08 00:00:00 AEST + (1546869600) Recalculating daily summaries... - Records processed: 184000; Last date: 2019-01-06 20:34:00 AEST (1546770840) + Records processed: 184000; Last date: 2019-01-06 20:34:00 AEST + (1546770840) Finished recalculating daily summaries Finished calculating missing derived observations ``` - When the import is complete a brief summary is displayed similar to the following: + When the import is complete a brief summary is displayed similar to + the following: ``` Finished import - 184765 records were processed and 184549 unique records imported in 699.27 seconds. + 184765 records were processed and 184549 unique records imported in + 699.27 seconds. 216 duplicate records were ignored. Those records with a timestamp already in the archive will not have been imported. Confirm successful import in the WeeWX log file. ``` -1. Whilst `wee_import` will advise of the number of unique records imported, `wee_import` does know how many, if any, of the imported records were successfully saved to the database. You should look carefully through the WeeWX log file covering the `wee_import` session and take note of any records that were not imported. The most common reason for imported records not being saved to the database is because a record with that timestamp already exists in the database, in such cases something similar to the following will be found in the log: +1. Whilst `wee_import` will advise of the number of unique records + imported, `wee_import` does know how many, if any, of the imported + records were successfully saved to the database. You should look + carefully through the WeeWX log file covering the `wee_import` session + and take note of any records that were not imported. The most common + reason for imported records not being saved to the database is because + a record with that timestamp already exists in the database, in such + cases something similar to the following will be found in the log: ``` - Aug 22 14:38:28 stretch12 weewx[863]: manager: unable to add record 2018-09-04 04:20:00 AEST (1535998800) to database 'weewx.sdb': UNIQUE constraint failed: archive.dateTime + Aug 22 14:38:28 stretch12 weewx[863]: manager: unable to add record + 2018-09-04 04:20:00 AEST (1535998800) to database 'weewx.sdb': UNIQUE + constraint failed: archive.dateTime ``` - In such cases take note of the timestamp of the record(s) concerned and make a decision about whether to delete the pre-existing record and re-import the record or retain the pre-existing record. + In such cases take note of the timestamp of the record(s) concerned + and make a decision about whether to delete the pre-existing record and + re-import the record or retain the pre-existing record. diff --git a/docs_src/utilities/weectl-import-weathercat.md b/docs_src/utilities/weectl-import-weathercat.md index c3af81e9..786a2d7f 100644 --- a/docs_src/utilities/weectl-import-weathercat.md +++ b/docs_src/utilities/weectl-import-weathercat.md @@ -1,14 +1,27 @@ !!! Warning - Running WeeWX during a `wee_import` session can lead to abnormal termination of the import. If WeeWX must remain running (e.g., so that live data is not lost) run the `wee_import` session on another machine or to a second database and merge the in-use and second database once the import is complete. + Running WeeWX during a `wee_import` session can lead to abnormal + termination of the import. If WeeWX must remain running (e.g., so that + live data is not lost) run the `wee_import` session on another machine or + to a second database and merge the in-use and second database once the + import is complete. -`wee_import` can import observational data from the one or more WeatherCat monthly .cat files. A WeatherCat monthly .cat file records weather station observations for a single month. These files are accumulated over time and can be considered analogous to the WeeWX archive table. When `wee_import` imports data from the WeatherCat monthly .cat files each file is considered a 'period'. `wee_import` processes one period at a time in chronological order (oldest to newest) and provides import summary data on a per period basis. +`wee_import` can import observational data from the one or more WeatherCat +monthly .cat files. A WeatherCat monthly .cat file records weather station +observations for a single month. These files are accumulated over time and +can be considered analogous to the WeeWX archive table. When `wee_import` +imports data from the WeatherCat monthly .cat files each file is +considered a 'period'. `wee_import` processes one period at a time in +chronological order (oldest to newest) and provides import summary data on +a per period basis. ## Mapping data to archive fields A WeatherCat import will populate the WeeWX archive fields as follows:

-* Provided data exists for each field in the WeatherCat monthly .cat files, the following WeeWX archive fields will be directly populated by imported data: +* Provided data exists for each field in the WeatherCat monthly .cat files, + the following WeeWX archive fields will be directly populated by + imported data: * `dateTime` * `barometer` @@ -28,32 +41,48 @@ A WeatherCat import will populate the WeeWX archive fields as follows:

* `windchill` !!! Note - If a field in the WeatherCat monthly .cat file has no data the corresponding WeeWX archive field will be set to `None/null`. + If a field in the WeatherCat monthly .cat file has no data the + corresponding WeeWX archive field will be set to `None/null`. -* The following WeeWX archive fields will be populated from other settings or configuration options: +* The following WeeWX archive fields will be populated from other settings + or configuration options: * `interval` * `usUnits` -* The following WeeWX archive fields will be populated with values derived from the imported data provided `calc_missing = True` is included in the `[WeatherCat]` section of the import configuration file being used and the field exists in the in-use WeeWX archive table schema: +* The following WeeWX archive fields will be populated with values derived + from the imported data provided `calc_missing = True` is included in the + `[WeatherCat]` section of the import configuration file being used and + the field exists in the in-use WeeWX archive table schema: * `altimeter` * `ET` * `pressure` !!! Note - If `calc_missing = False` is included in the `[WeatherCat]` section of the import configuration file being used then all of the above fields will be set to `None/null`. The `calc_missing` option default is `True`. + If `calc_missing = False` is included in the `[WeatherCat]` + section of the import configuration file being used then all of the + above fields will be set to `None/null`. The `calc_missing` option + default is `True`. ## Step-by-step instructions To import observations from one or more WeatherCat monthly .cat files: -1. Ensure the WeatherCat monthly .cat file(s) to be used for the import are located in year directories with the year directories in turn located in a directory accessible by the machine that will run `wee_import`. For the purposes of the following examples, there are nine monthly logs files covering the period October 2016 to June 2017 inclusive, located in the `/var/tmp/wcat/2016` and `/var/tmp/wcat/2017` directories respectively. +1. Ensure the WeatherCat monthly .cat file(s) to be used for the import + are located in year directories with the year directories in turn + located in a directory accessible by the machine that will run + `wee_import`. For the purposes of the following examples, there are + nine monthly logs files covering the period October 2016 to June 2017 + inclusive, located in the `/var/tmp/wcat/2016` and `/var/tmp/wcat/2017` + directories respectively. 1. Make a backup of the WeeWX database in case the import should go awry. -1. Create an import configuration file. In this case we will make a copy of the example WeatherCat import configuration file and save it as `wcat.conf` in the `/var/tmp` directory: +1. Create an import configuration file. In this case we will make a copy + of the example WeatherCat import configuration file and save it as + `wcat.conf` in the `/var/tmp` directory: ``` $ cp /home/weewx/util/import/weathercat-example.conf /var/tmp/wcat.conf @@ -65,94 +94,143 @@ To import observations from one or more WeatherCat monthly .cat files: source = WeatherCat ``` -1. Confirm the following options in the `[WeatherCat]` section are correctly set: +1. Confirm the following options in the `[WeatherCat]` section are + correctly set: - * [directory](../wee_import-config#wcat_directory). The full path to the directory containing the directories containing the WeatherCat monthly .cat files to be used as the source of the imported data. + * [directory](../wee_import-config#wcat_directory). The full path to + the directory containing the directories containing the WeatherCat + monthly .cat files to be used as the source of the imported data. - * [interval](../wee_import-config#wcat_interval). Determines how the WeeWX interval field is derived. + * [interval](../wee_import-config#wcat_interval). Determines how the + WeeWX interval field is derived. - * [qc](../wee_import-config#wcat_qc). Determines whether quality control checks are performed on the imported data. + * [qc](../wee_import-config#wcat_qc). Determines whether quality + control checks are performed on the imported data. - * [calc_missing](../wee_import-config#wcat_calc_missing). Determines whether missing derived observations will be calculated from the imported data. + * [calc_missing](../wee_import-config#wcat_calc_missing). Determines + whether missing derived observations will be calculated from the + imported data. - * [decimal](../wee_import-config#wcat_decimal). The decimal point character used in the WeatherCat monthly log files. + * [decimal](../wee_import-config#wcat_decimal). The decimal point + character used in the WeatherCat monthly log files. - * [tranche](../wee_import-config#wcat_tranche). The number of records written to the WeeWX database in each transaction. + * [tranche](../wee_import-config#wcat_tranche). The number of records + written to the WeeWX database in each transaction. - * [UV_sensor](../wee_import-config#wcat_UV). Whether a UV sensor was installed when the source data was produced. + * [UV_sensor](../wee_import-config#wcat_UV). Whether a UV sensor was + installed when the source data was produced. - * [solar_sensor](../wee_import-config#wcat_solar). Whether a solar radiation sensor was installed when the source data was produced. + * [solar_sensor](../wee_import-config#wcat_solar). Whether a solar + radiation sensor was installed when the source data was produced. - * [[[Units]]](../wee_import-config#wcat_units). Defines the units used in the WeatherCat monthly .cat files. + * [[[Units]]](../wee_import-config#wcat_units). Defines the units used + in the WeatherCat monthly .cat files. -1. When first importing data it is prudent to do a dry run import before any data is actually imported. A -dry run import will perform all steps of the import without actually writing imported data to the WeeWX -database. In addition, consideration should be given to any additional options to be used such as `--date`. +1. When first importing data it is prudent to do a dry run import before + any data is actually imported. A dry run import will perform all steps + of the import without actually writing imported data to the WeeWX + database. In addition, consideration should be given to any additional + options to be used such as `--date` !!! Note - Whilst WeatherCat monthly .cat files use a fixed set of fields the inclusion of fields other than `t` (timestamp) and `V` (validation) is optional. For this reason the field map used for WeatherCat imports includes fields that may not exist in some WeatherCat monthly .cat files resulting in warnings by `wee_import` that there may be missing data in the import source. These warnings can be extensive and may detract from the ability of the user to monitor the progress of the import. It may be prudent to use the `--suppress-warnings` option during the initial dry run so the overall progress of the import can be more easily observed. + Whilst WeatherCat monthly .cat files use a fixed set of fields the + inclusion of fields other than `t` (timestamp) and `V` (validation) + is optional. For this reason the field map used for WeatherCat + imports includes fields that may not exist in some WeatherCat + monthly .cat files resulting in warnings by `wee_import` that there + may be missing data in the import source. These warnings can be + extensive and may detract from the ability of the user to monitor + the progress of the import. It may be prudent to use the + `--suppress-warnings` option during the initial dry run so the + overall progress of the import can be more easily observed. To perform a dry run enter the following command: ``` - wee_import --import-config=/var/tmp/wcat.conf --dry-run --suppress-warnings + wee_import --import-config=/var/tmp/wcat.conf --dry-run + --suppress-warnings ``` - This will result in a short preamble with details on the data source, the destination of the imported data and some other details on how the data will be processed. The import will then be performed but no data will be written to the WeeWX database. + This will result in a short preamble with details on the data source, + the destination of the imported data and some other details on how the + data will be processed. The import will then be performed but no data + will be written to the WeeWX database. The output should be similar to: ``` Using WeeWX configuration file /home/weewx/weewx.conf Starting wee_import... - WeatherCat monthly .cat files in the '/var/tmp/wcat' directory will be imported - Using database binding 'wx_binding', which is bound to database 'weewx.sdb' + WeatherCat monthly .cat files in the '/var/tmp/wcat' directory will be + imported + Using database binding 'wx_binding', which is bound to database 'weewx. + sdb' Destination table 'archive' unit system is '0x01' (US). Missing derived observations will be calculated. This is a dry run, imported data will not be saved to archive. Starting dry run import ... Records covering multiple periods have been identified for import. Period 1 ... - Unique records processed: 39555; Last timestamp: 2016-10-31 23:59:00 AEST (1477922340) + Unique records processed: 39555; Last timestamp: 2016-10-31 23:59:00 + AEST (1477922340) Period 2 ... - Unique records processed: 38284; Last timestamp: 2016-11-30 23:59:00 AEST (1480514340) + Unique records processed: 38284; Last timestamp: 2016-11-30 23:59:00 + AEST (1480514340) Period 3 ... - Unique records processed: 39555; Last timestamp: 2016-12-31 23:59:00 AEST (1483192740) + Unique records processed: 39555; Last timestamp: 2016-12-31 23:59:00 + AEST (1483192740) Period 4 ... - Unique records processed: 39555; Last timestamp: 2017-01-31 23:59:00 AEST (1485871140) + Unique records processed: 39555; Last timestamp: 2017-01-31 23:59:00 + AEST (1485871140) Period 5 ... - Unique records processed: 35598; Last timestamp: 2017-02-28 23:59:00 AEST (1488290340) + Unique records processed: 35598; Last timestamp: 2017-02-28 23:59:00 + AEST (1488290340) Period 6 ... - Unique records processed: 39555; Last timestamp: 2017-03-31 23:59:00 AEST (1490968740) + Unique records processed: 39555; Last timestamp: 2017-03-31 23:59:00 + AEST (1490968740) Period 7 ... - Unique records processed: 38284; Last timestamp: 2017-04-30 23:59:00 AEST (1493560740) + Unique records processed: 38284; Last timestamp: 2017-04-30 23:59:00 + AEST (1493560740) Period 8 ... - Unique records processed: 38284; Last timestamp: 2017-06-30 23:59:00 AEST (1498831140) + Unique records processed: 38284; Last timestamp: 2017-06-30 23:59:00 + AEST (1498831140) Finished dry run import - 308670 records were processed and 308670 unique records would have been imported. + 308670 records were processed and 308670 unique records would have + been imported. ``` !!! Note - The eight periods correspond to the eight monthly .cat files used for this import. + The eight periods correspond to the eight monthly .cat files used + for this import. !!! Note - Any periods for which no data could be obtained will be skipped. The lack of data may be due to a missing WeatherCat monthly .cat file. A short explanatory note to this effect will be displayed against the period concerned and an entry included in the log. + Any periods for which no data could be obtained will be skipped. + The lack of data may be due to a missing WeatherCat monthly .cat + file. A short explanatory note to this effect will be displayed + against the period concerned and an entry included in the log. -1. If the `--suppress-warnings` option was used it may be prudent to do a second dry run this time without the `--suppress-warnings` option. This will allow any warnings generated by the dry run import to be observed: +1. If the `--suppress-warnings` option was used it may be prudent to do a + second dry run this time without the `--suppress-warnings` option. This + will allow any warnings generated by the dry run import to be observed: ``` wee_import --import-config=/var/tmp/wcat.conf --dry-run ``` - This will result in a short preamble with details on the data source, the destination of the imported data and some other details on how the data will be processed. The import will then be performed but no data will be written to the WeeWX database. + This will result in a short preamble with details on the data source, + the destination of the imported data and some other details on how the + data will be processed. The import will then be performed but no data + will be written to the WeeWX database. The output should be similar to: ``` Using WeeWX configuration file /home/weewx/weewx.conf Starting wee_import... - WeatherCat monthly .cat files in the '/var/tmp/wcat' directory will be imported - Using database binding 'wx_binding', which is bound to database 'weewx.sdb' + WeatherCat monthly .cat files in the '/var/tmp/wcat' directory will be + imported + Using database binding 'wx_binding', which is bound to database 'weewx. + sdb' Destination table 'archive' unit system is '0x01' (US). Missing derived observations will be calculated. This is a dry run, imported data will not be saved to archive. @@ -210,7 +288,8 @@ database. In addition, consideration should be given to any additional options t Warning: Import field 'Lt2' is mapped to WeeWX field 'leafTemp2' but the import field 'Lt2' could not be found in one or more records. WeeWX field 'leafTemp2' will be set to 'None' in these records. - Unique records processed: 39555; Last timestamp: 2016-10-31 23:59:00 AEST (1477922340) + Unique records processed: 39555; Last timestamp: 2016-10-31 23:59:00 + AEST (1477922340) Period 2 ... Warning: Import field 'T1' is mapped to WeeWX field 'extraTemp1' but the import field 'T1' could not be found in one or more records. @@ -263,7 +342,8 @@ database. In addition, consideration should be given to any additional options t Warning: Import field 'Lt2' is mapped to WeeWX field 'leafTemp2' but the import field 'Lt2' could not be found in one or more records. WeeWX field 'leafTemp2' will be set to 'None' in these records. - Unique records processed: 38284; Last timestamp: 2016-11-30 23:59:00 AEST (1480514340) + Unique records processed: 38284; Last timestamp: 2016-11-30 23:59:00 + AEST (1480514340) ... (identical entries for periods 3 to 7 omitted for conciseness) @@ -319,26 +399,37 @@ database. In addition, consideration should be given to any additional options t Warning: Import field 'Lt2' is mapped to WeeWX field 'leafTemp2' but the import field 'Lt2' could not be found in one or more records. WeeWX field 'leafTemp2' will be set to 'None' in these records. - Unique records processed: 38284; Last timestamp: 2017-06-30 23:59:00 AEST (1498831140) + Unique records processed: 38284; Last timestamp: 2017-06-30 23:59:00 + AEST (1498831140) Finished dry run import - 308670 records were processed and 308670 unique records would have been imported. + 308670 records were processed and 308670 unique records would have + been imported. ``` - In this case warnings are evident for numerous import/WeeWX field pairs that are mapped but for which no data could be found. If the warnings relate to fields that are not included in the import source data the warning may be safely ignored. If the warning relate to fields that the user expects to be in the import source data the issue should be investigated further before the import is completed. + In this case warnings are evident for numerous import/WeeWX field + pairs that are mapped but for which no data could be found. If the + warnings relate to fields that are not included in the import source + data the warning may be safely ignored. If the warning relate to fields + that the user expects to be in the import source data the issue should + be investigated further before the import is completed. -1. Once the dry run results are satisfactory the data can be imported using the following command: +1. Once the dry run results are satisfactory the data can be imported + using the following command: ``` wee_import --import-config=/var/tmp/wcat.conf --suppress-warnings ``` - This will result in a preamble similar to that of a dry run. At the end of the preamble there will be a prompt: + This will result in a preamble similar to that of a dry run. At the + end of the preamble there will be a prompt: ``` Using WeeWX configuration file /home/weewx/weewx.conf Starting wee_import... - WeatherCat monthly .cat files in the '/var/tmp/wcat' directory will be imported - Using database binding 'wx_binding', which is bound to database 'weewx.sdb' + WeatherCat monthly .cat files in the '/var/tmp/wcat' directory will + be imported + Using database binding 'wx_binding', which is bound to database 'weewx. + sdb' Destination table 'archive' unit system is '0x01' (US). Missing derived observations will be calculated. Starting import ... @@ -348,13 +439,20 @@ database. In addition, consideration should be given to any additional options t Are you sure you want to proceed (y/n)? ``` - If there is more than one WeatherCat monthly .cat file then `wee_import` will provide summary information on a per period basis during the import. In addition, if the `--date` option is used then source data that falls outside the date or date range specified with the `--date` option is ignored. In such cases the preamble may look similar to: + If there is more than one WeatherCat monthly .cat file then + `wee_import` will provide summary information on a per period basis + during the import. In addition, if the `--date` option is used then + source data that falls outside the date or date range specified with + the `--date` option is ignored. In such cases the preamble may look + similar to: ``` Using WeeWX configuration file /home/weewx/weewx.conf Starting wee_import... - WeatherCat monthly .cat files in the '/var/tmp/wcat' directory will be imported - Using database binding 'wx_binding', which is bound to database 'weewx.sdb' + WeatherCat monthly .cat files in the '/var/tmp/wcat' directory will be + imported + Using database binding 'wx_binding', which is bound to database 'weewx. + sdb' Destination table 'archive' unit system is '0x01' (US). Missing derived observations will be calculated. Starting import ... @@ -368,52 +466,87 @@ database. In addition, consideration should be given to any additional options t Are you sure you want to proceed (y/n)? ``` -1. If the import parameters are acceptable enter `y` to proceed with the import or `n` to abort the import. If the import is confirmed, the source data will be imported, processed and saved in the WeeWX database. Information on the progress of the import will be displayed similar to the following: +1. If the import parameters are acceptable enter `y` to proceed with the + import or `n` to abort the import. If the import is confirmed, the + source data will be imported, processed and saved in the WeeWX database. + Information on the progress of the import will be displayed similar to + the following: ``` - Unique records processed: 2305; Last timestamp: 2016-12-30 00:00:00 AEST (1483020000) + Unique records processed: 2305; Last timestamp: 2016-12-30 00:00:00 + AEST (1483020000) ``` - Again if there is more than one WeatherCat monthly .cat file and if the `--date` option is used the progress information may instead look similar to: + Again if there is more than one WeatherCat monthly .cat file and if + the `--date` option is used the progress information may instead look + similar to: ``` Period 4 ... - Unique records processed: 8908; Last timestamp: 2017-01-31 23:59:00 AEST (1485870900) + Unique records processed: 8908; Last timestamp: 2017-01-31 23:59:00 + AEST (1485870900) Period 5 ... - Unique records processed: 8029; Last timestamp: 2017-02-28 23:59:00 AEST (1488290100) + Unique records processed: 8029; Last timestamp: 2017-02-28 23:59:00 + AEST (1488290100) Period 6 ... - Unique records processed: 8744; Last timestamp: 2017-03-31 23:59:00 AEST (1490968500) + Unique records processed: 8744; Last timestamp: 2017-03-31 23:59:00 + AEST (1490968500) ``` !!! Note - Any periods for which no data could be obtained will be skipped. The lack of data may be due to a missing WeatherCat monthly .cat file. A short explanatory note to this effect will be displayed against the period concerned and an entry included in the log. + Any periods for which no data could be obtained will be skipped. + The lack of data may be due to a missing WeatherCat monthly .cat + file. A short explanatory note to this effect will be displayed + against the period concerned and an entry included in the log. - The line commencing with `Unique records processed` should update as records are imported with progress information on number of records processed, number of unique records imported and the date time of the latest record processed. If the import spans multiple months (ie multiple monthly .cat files) then a new `Period` line is created for each month. + The line commencing with `Unique records processed` should update as + records are imported with progress information on number of records + processed, number of unique records imported and the date time of the + latest record processed. If the import spans multiple months (ie + multiple monthly .cat files) then a new `Period` line is created for + each month. - Once the initial import is complete `wee_import` will, if requested, calculate any missing derived observations and rebuild the daily summaries. A brief summary should be displayed similar to the following: + Once the initial import is complete `wee_import` will, if requested, + calculate any missing derived observations and rebuild the daily + summaries. A brief summary should be displayed similar to the following: ``` Calculating missing derived observations ... - Processing record: 77782; Last record: 2017-06-30 00:00:00 AEST (1519826400) + Processing record: 77782; Last record: 2017-06-30 00:00:00 AEST + (1519826400) Recalculating daily summaries... Records processed: 77000; Last date: 2017-06-28 11:45:00 AEST (1519811100) Finished recalculating daily summaries Finished calculating missing derived observations ``` - When the import is complete a brief summary is displayed similar to the following: + When the import is complete a brief summary is displayed similar to + the following: ``` Finished import - 308670 records were processed and 308670 unique records imported in 1907.61 seconds. + 308670 records were processed and 08670 unique records imported in + 1907.61 seconds. Those records with a timestamp already in the archive will not have been imported. Confirm successful import in the WeeWX log file. ``` -1. Whilst `wee_import` will advise of the number of records processed and the number of unique records found, `wee_import` does know how many, if any, of the imported records were successfully saved to the database. You should look carefully through the WeeWX log file covering the `wee_import` session and take note of any records that were not imported. The most common reason for imported records not being saved to the database is because a record with that timestamp already exists in the database, in such cases something similar to the following will be found in the log: +1. Whilst `wee_import` will advise of the number of records processed and + the number of unique records found, `wee_import` does know how many, if + any, of the imported records were successfully saved to the database. + You should look carefully through the WeeWX log file covering the + `wee_import` session and take note of any records that were not + imported. The most common reason for imported records not being saved + to the database is because a record with that timestamp already exists + in the database, in such cases something similar to the following will + be found in the log: ``` - Aug 22 14:38:28 stretch12 weewx[863]: manager: unable to add record 2018-09-04 04:20:00 AEST (1535998800) to database 'weewx.sdb': UNIQUE constraint failed: archive.dateTime + Aug 22 14:38:28 stretch12 weewx[863]: manager: unable to add record + 2018-09-04 04:20:00 AEST (1535998800) to database 'weewx.sdb': UNIQUE + constraint failed: archive.dateTime ``` - In such cases take note of the timestamp of the record(s) concerned and make a decision about whether to delete the pre-existing record and re-import the record or retain the pre-existing record. + In such cases take note of the timestamp of the record(s) concerned + and make a decision about whether to delete the pre-existing record and + re-import the record or retain the pre-existing record. diff --git a/docs_src/utilities/weectl-import-wu.md b/docs_src/utilities/weectl-import-wu.md index 0a432484..9d1a48ac 100644 --- a/docs_src/utilities/weectl-import-wu.md +++ b/docs_src/utilities/weectl-import-wu.md @@ -1,13 +1,26 @@ !!! Warning - Running WeeWX during a `wee_import` session can lead to abnormal termination of the import. If WeeWX must remain running (e.g., so that live data is not lost) run the `wee_import` session on another machine or to a second database and merge the in-use and second database once the import is complete. + Running WeeWX during a `wee_import` session can lead to abnormal + termination of the import. If WeeWX must remain running (e.g., so that + live data is not lost) run the `wee_import` session on another machine or + to a second database and merge the in-use and second database once the + import is complete. -`wee_import` can import historical observation data for a Weather Underground PWS via the Weather Underground API. The Weather Underground API provides historical weather station observations received by Weather Underground for the PWS concerned on a day by day basis. As such, the data is analogous to the WeeWX archive table. When `wee_import` imports data from the Weather Underground API each day is considered a 'period'. `wee_import` processes one period at a time in chronological order (oldest to newest) and provides import summary data on a per period basis. +`wee_import` can import historical observation data for a Weather +Underground PWS via the Weather Underground API. The Weather Underground +API provides historical weather station observations received by Weather +Underground for the PWS concerned on a day by day basis. As such, the data +is analogous to the WeeWX archive table. When `wee_import` imports data +from the Weather Underground API each day is considered a 'period'. +`wee_import` processes one period at a time in chronological order (oldest +to newest) and provides import summary data on a per period basis. ## Mapping data to archive fields A Weather Underground import will populate WeeWX archive fields as follows: -* Provided data exists for each field returned by the Weather Underground API, the following WeeWX archive fields will be directly populated by imported data: +* Provided data exists for each field returned by the Weather Underground + API, the following WeeWX archive fields will be directly populated by + imported data: * `dateTime` * `barometer` @@ -25,34 +38,57 @@ A Weather Underground import will populate WeeWX archive fields as follows: * `windSpeed` !!! Note - If an appropriate field is not returned by the Weather Underground API the corresponding WeeWX archive field will contain no data. If the API returns an appropriate field but with no data, the corresponding WeeWX archive field will be set to `None/null`. For example, if the API response has no solar radiation field the WeeWX `radiation` archive field will have no data stored. However, if the API response has a solar radiation field but contains no data, the WeeWX `radiation` archive field will be `None/null`. + If an appropriate field is not returned by the Weather Underground + API the corresponding WeeWX archive field will contain no data. If + the API returns an appropriate field but with no data, the + corresponding WeeWX archive field will be set to `None/null`. For + example, if the API response has no solar radiation field the + WeeWX `radiation` archive field will have no data stored. However, + if the API response has a solar radiation field but contains no + data, the WeeWX `radiation` archive field will be `None/null`. -* The following WeeWX archive fields will be populated from other settings or configuration options: +* The following WeeWX archive fields will be populated from other settings + or configuration options: * `interval` * `usUnits` -* The following WeeWX archive fields will be populated with values derived from the imported data provided `calc_missing = True` is included in the `[WU]` section of the import configuration file and the field exists in the in-use WeeWX archive table schema. +* The following WeeWX archive fields will be populated with values derived + from the imported data provided `calc_missing = True` is included in the + `[WU]` section of the import configuration file and the field exists in + the in-use WeeWX archive table schema. * `altimeter` * `ET` * `pressure` !!! Note - If `calc_missing = False` is included in the `[WU]` section of the import configuration file being used then all of the above fields will be set to `None/null`. The `calc_missing` option default is `True`. + If `calc_missing = False` is included in the `[WU]` section of the + import configuration file being used then all of the above fields will be + set to `None/null`. The `calc_missing` option default is `True`. ## Step-by-step instructions To import observations from a Weather Underground PWS history: -1. Obtain the weather station ID of the Weather Underground PWS from which data is to be imported. The station ID will be a sequence of numbers and upper case letters that is usually 11 or 12 characters in length. For the purposes of the following examples a weather station ID of `ISTATION123` will be used. +1. Obtain the weather station ID of the Weather Underground PWS from which + data is to be imported. The station ID will be a sequence of numbers + and upper case letters that is usually 11 or 12 characters in length. + For the purposes of the following examples a weather station ID of + `ISTATION123` will be used. -1. Obtain the API key to be used to access the Weather Underground API. This will be a seemingly random alphanumeric sequence of 32 characters. API keys are available to Weather Underground PWS contributors by logging on to their Weather Underground account and accessing Member Settings. +1. Obtain the API key to be used to access the Weather Underground API. + This will be a seemingly random alphanumeric sequence of 32 characters. + API keys are available to Weather Underground PWS contributors by + logging on to their Weather Underground account and accessing Member + Settings. 1. Make a backup of the WeeWX database in case the import should go awry. -1. Create an import configuration file. In this case we will make a copy of the example Weather Underground import configuration file and save it as `wu.conf` in the `/var/tmp` directory: +1. Create an import configuration file. In this case we will make a copy + of the example Weather Underground import configuration file and save + it as `wu.conf` in the `/var/tmp` directory: ``` $ cp /home/weewx/util/import/wu-example.conf /var/tmp/wu.conf @@ -66,88 +102,147 @@ To import observations from a Weather Underground PWS history: 1. Confirm that the following options in the `[WU]` section are correctly set: - * [station_id](../wee_import-config#wu_station_id). The 11 or 12 character weather station ID of the Weather Underground PWS that will be the source of the imported data. + * [station_id](../wee_import-config#wu_station_id). The 11 or 12 + character weather station ID of the Weather Underground PWS that + will be the source of the imported data. - * [api_key](../wee_import-config#wu_api_key). The 32 character API key to be used to access the Weather Underground API. + * [api_key](../wee_import-config#wu_api_key). The 32 character API key + to be used to access the Weather Underground API. - * [interval](../wee_import-config#wu_interval). Determines how the WeeWX interval field is derived. + * [interval](../wee_import-config#wu_interval). Determines how the + WeeWX interval field is derived. - * [qc](../wee_import-config#wu_qc). Determines whether quality control checks are performed on the imported data. + * [qc](../wee_import-config#wu_qc). Determines whether quality control + checks are performed on the imported data. !!! Note - As Weather Underground imports at times contain nonsense values, particularly for fields for which no data were uploaded to Weather Underground by the PWS, the use of quality control checks on imported data can prevent these nonsense values from being imported and contaminating the WeeWX database. + As Weather Underground imports at times contain nonsense + values, particularly for fields for which no data were + uploaded to Weather Underground by the PWS, the use of quality + control checks on imported data can prevent these nonsense + values from being imported and contaminating the WeeWX database. - * [calc_missing](../wee_import-config#wu_calc_missing). Determines whether missing derived observations will be calculated from the imported data. + * [calc_missing](../wee_import-config#wu_calc_missing). Determines + whether missing derived observations will be calculated from the + imported data. - * [ignore_invalid_data](../wee_import-config#wu_ignore_invalid_data). Determines whether invalid data in a source field is ignored or the import aborted. + * [ignore_invalid_data](../wee_import-config#wu_ignore_invalid_data). + Determines whether invalid data in a source field is ignored or the + import aborted - * [tranche](../wee_import-config#wu_tranche). The number of records written to the WeeWX database in each transaction. + * [tranche](../wee_import-config#wu_tranche). The number of records + written to the WeeWX database in each transaction. - * [wind_direction](../wee_import-config#wu_wind_direction). Determines how imported wind direction fields are interpreted. + * [wind_direction](../wee_import-config#wu_wind_direction). Determines + how imported wind direction fields are interpreted. -1. When first importing data it is prudent to do a dry run import before any data is actually imported. A dry run import will perform all steps of the import without actually writing imported data to the WeeWX database. In addition, consideration should be given to any additional options to be used such as `--date`, `--from` or `--to`. +1. When first importing data it is prudent to do a dry run import before + any data is actually imported. A dry run import will perform all steps + of the import without actually writing imported data to the WeeWX + database. In addition, consideration should be given to any additional + options to be used such as `--date`, `--from` or `--to`. To perform a dry run enter the following command: ``` - wee_import --import-config=/var/tmp/wu.conf --from=2016-01-20T22:30 --to=2016-01-23T06:00 --dry-run + wee_import --import-config=/var/tmp/wu.conf --from=2016-01-20T22:30 + --to=2016-01-23T06:00 --dry-run ``` - In this case the `--from` and `--to` options have been used to import Weather Underground records from 10:30pm on 20 January 2016 to 6:00am on 23 January 2016 inclusive. + In this case the `--from` and `--to` options have been used to import + Weather Underground records from 10:30pm on 20 January 2016 to 6:00am + on 23 January 2016 inclusive. !!! Note - If the `--date` option is omitted, or a date (not date-time) range is specified using the `--from` and `--to` options during a Weather Underground import, then one or more full days of history data will be imported. This includes records timestamped from `00:00` (inclusive) at the start of the day up to but NOT including the `00:00` record at the end of the last day. As the timestamped record refers to observations of the previous interval, such an import actually includes one record with observations from the previous day (the `00:00` record at the start of the day). Whilst this will not present a problem for `wee_import` as any records being imported with a timestamp that already exists in the WeeWX database are ignored, you may wish to use the `--from` and `--to` options with a suitable date-time range to precisely control which records are imported. + If the `--date` option is omitted, or a date (not date-time) range + is specified using the `--from` and `--to` options during a Weather + Underground import, then one or more full days of history data + will be imported. This includes records timestamped from `00:00` + (inclusive) at the start of the day up to but NOT including the + `00:00` record at the end of the last day. As the timestamped + record refers to observations of the previous interval, such an + import actually includes one record with observations from the + previous day (the `00:00` record at the start of the day). Whilst + this will not present a problem for `wee_import` as any records + being imported with a timestamp that already exists in the WeeWX + database are ignored, you may wish to use the `--from` and `--to` + options with a suitable date-time range to precisely control + which records are imported. !!! Note - `wee_import` obtains Weather Underground daily history data one day at a time via a HTTP request and as such the import of large time spans of data may take some time. Such imports may be best handled as a series of imports of smaller time spans. + `wee_import` obtains Weather Underground daily history data one + day at a time via a HTTP request and as such the import of large time + spans of data may take some time. Such imports may be best handled + as a series of imports of smaller time spans. - This will result in a short preamble with details on the data source, the destination of the imported data and some other details on how the data will be processed. The import will then be performed but no data will be written to the WeeWX database. + This will result in a short preamble with details on the data source, + the destination of the imported data and some other details on how the + data will be processed. The import will then be performed but no data + will be written to the WeeWX database. The output should be similar to: ``` Using WeeWX configuration file /home/weewx/weewx.conf Starting wee_import... - Observation history for Weather Underground station 'ISTATION123' will be imported. - Using database binding 'wx_binding', which is bound to database 'weewx.sdb' + Observation history for Weather Underground station 'ISTATION123' will + be imported. + Using database binding 'wx_binding', which is bound to database 'weewx. + sdb' Destination table 'archive' unit system is '0x01' (US). Missing derived observations will be calculated. - Observations timestamped after 2016-01-20 22:30:00 AEST (1453293000) and up to and + Observations timestamped after 2016-01-20 22:30:00 AEST (1453293000) + and up to and including 2016-01-23 06:00:00 AEST (1453492800) will be imported. This is a dry run, imported data will not be saved to archive. Starting dry run import ... Records covering multiple periods have been identified for import. Period 1 ... - Unique records processed: 18; Last timestamp: 2016-01-20 23:55:00 AEST (1453298100) + Unique records processed: 18; Last timestamp: 2016-01-20 23:55:00 AEST + (1453298100) Period 2 ... - Unique records processed: 284; Last timestamp: 2016-01-21 23:55:00 AEST (1453384500) + Unique records processed: 284; Last timestamp: 2016-01-21 23:55:00 + AEST (1453384500) Period 3 ... - Unique records processed: 284; Last timestamp: 2016-01-22 23:55:00 AEST (1453470900) + Unique records processed: 284; Last timestamp: 2016-01-22 23:55:00 + AEST (1453470900) Period 4 ... - Unique records processed: 71; Last timestamp: 2016-01-23 06:00:00 AEST (1453492800) + Unique records processed: 71; Last timestamp: 2016-01-23 06:00:00 AEST + (1453492800) Finished dry run import - 657 records were processed and 657 unique records would have been imported. + 657 records were processed and 657 unique records would have been + imported. ``` !!! Note - Any periods for which no data could be obtained will be skipped. The lack of data may be due to an incorrect station ID, an incorrect date or Weather Underground API problems. A short explanatory note to this effect will be displayed against the period concerned and an entry included in the log. + Any periods for which no data could be obtained will be skipped. + The lack of data may be due to an incorrect station ID, an incorrect + date or Weather Underground API problems. A short explanatory note to + this effect will be displayed against the period concerned and an + entry included in the log. -1. Once the dry run results are satisfactory the source data can be imported using the following command: +1. Once the dry run results are satisfactory the source data can be + imported using the following command: ``` - wee_import --import-config=/var/tmp/wu.conf --from=2016-01-20T22:30 --to=2016-01-23T06:00 + wee_import --import-config=/var/tmp/wu.conf --from=2016-01-20T22:30 + --to=2016-01-23T06:00 ``` - This will result in a short preamble similar to that of a dry run. At the end of the preamble there will be a prompt: + This will result in a short preamble similar to that of a dry run. At + the end of the preamble there will be a prompt: ``` Using WeeWX configuration file /home/weewx/weewx.conf Starting wee_import... - Observation history for Weather Underground station 'ISTATION123' will be imported. - Using database binding 'wx_binding', which is bound to database 'weewx.sdb' + Observation history for Weather Underground station 'ISTATION123' will + be imported. + Using database binding 'wx_binding', which is bound to database 'weewx. + sdb' Destination table 'archive' unit system is '0x01' (US). Missing derived observations will be calculated. - Observations timestamped after 2016-01-20 22:30:00 AEST (1453293000) and up to and + Observations timestamped after 2016-01-20 22:30:00 AEST (1453293000) + and up to and including 2016-01-23 06:00:00 AEST (1453492800) will be imported. Starting import ... Records covering multiple periods have been identified for import. @@ -157,24 +252,44 @@ To import observations from a Weather Underground PWS history: ``` !!! Note - `wee_import` obtains Weather Underground data one day at a time via a HTTP request and as such the import of large time spans of data may take some time. Such imports may be best handled as a series of imports of smaller time spans. + `wee_import` obtains Weather Underground data one day at a time + via a HTTP request and as such the import of large time spans of data + may take some time. Such imports may be best handled as a series of + imports of smaller time spans. -1. If the import parameters are acceptable enter `y` to proceed with the import or `n` to abort the import. If the import is confirmed, the source data will be imported, processed and saved in the WeeWX database. Information on the progress of the import will be displayed similar to the following: +1. If the import parameters are acceptable enter `y` to proceed with the + import or `n` to abort the import. If the import is confirmed, the + source data will be imported, processed and saved in the WeeWX database. + Information on the progress of the import will be displayed similar to + the following: ``` - Unique records processed: 18; Last timestamp: 2016-01-20 23:55:00 AEST (1453298100) + Unique records processed: 18; Last timestamp: 2016-01-20 23:55:00 AEST + (1453298100) Period 2 ... - Unique records processed: 284; Last timestamp: 2016-01-21 23:55:00 AEST (1453384500) + Unique records processed: 284; Last timestamp: 2016-01-21 23:55:00 + AEST (1453384500) Period 3 ... - Unique records processed: 284; Last timestamp: 2016-01-22 23:55:00 AEST (1453470900) + Unique records processed: 284; Last timestamp: 2016-01-22 23:55:00 AEST + (1453470900) ``` !!! Note - Any periods for which no data could be obtained will be skipped. The lack of data may be due to an incorrect station ID, an incorrect date or Weather Underground API problems. A short explanatory note to this effect will be displayed against the period concerned and an entry included in the log. + Any periods for which no data could be obtained will be skipped. + The lack of data may be due to an incorrect station ID, an incorrect + date or Weather Underground API problems. A short explanatory note to + this effect will be displayed against the period concerned and an + entry included in the log. - The line commencing with `Unique records processed` should update as records are imported with progress information on number of records processed, number of unique records imported and the date time of the latest record processed. If the import spans multiple days then a new `Period` line is created for each day. + The line commencing with `Unique records processed` should update as + records are imported with progress information on number of records + processed, number of unique records imported and the date time of the + latest record processed. If the import spans multiple days then a new + `Period` line is created for each day. - Once the initial import is complete `wee_import` will, if requested, calculate any missing derived observations and rebuild the daily summaries. A brief summary should be displayed similar to the following: + Once the initial import is complete `wee_import` will, if requested, + calculate any missing derived observations and rebuild the daily + summaries. A brief summary should be displayed similar to the following: ``` Calculating missing derived observations ... @@ -184,22 +299,50 @@ To import observations from a Weather Underground PWS history: Finished calculating missing derived observations ``` - When the import is complete a brief summary is displayed similar to the following: + When the import is complete a brief summary is displayed similar to + the following: ``` Finished import - 657 records were processed and 657 unique records imported in 78.97 seconds. + 657 records were processed and 657 unique records imported in 78.97 + seconds. Those records with a timestamp already in the archive will not have been imported. Confirm successful import in the WeeWX log file. ``` !!! Note - The new (2019) Weather Underground API appears to have an issue when obtaining historical data for the current day. The first time the API is queried the API returns all historical data up to and including the most recent record. However, subsequent later API queries during the same day return the same set of records rather than all records up to and including the time of the latest API query. Users importing Weather Underground data that includes data from the current day are advised to carefully check the WeeWX log to ensure that all expected records were imported. If some records are missing from the current day try running an import for the current day again using the `--date` option setting. If this fails then wait until the following day and perform another import for the day concerned again using the `--date` option setting. In all cases confirm what data has been imported by referring to the WeeWX log. + The new (2019) Weather Underground API appears to have an issue + when obtaining historical data for the current day. The first time + the API is queried the API returns all historical data up to and + including the most recent record. However, subsequent later API + queries during the same day return the same set of records rather + than all records up to and including the time of the latest API + query. Users importing Weather Underground data that includes data + from the current day are advised to carefully check the WeeWX log + to ensure that all expected records were imported. If some records + are missing from the current day try running an import for the + current day again using the `--date` option setting. If this fails + then wait until the following day and perform another import for + the day concerned again using the `--date` option setting. In all + cases confirm what data has been imported by referring to the + WeeWX log. -1. Whilst `wee_import` will advise of the number of records processed and the number of unique records found, `wee_import` does know how many, if any, of the imported records were successfully saved to the database. You should look carefully through the WeeWX log file covering the `wee_import` session and take note of any records that were not imported. The most common reason for imported records not being saved to the database is because a record with that timestamp already exists in the database, in such cases something similar to the following will be found in the log: +1. Whilst `wee_import` will advise of the number of records processed and + the number of unique records found, `wee_import` does know how many, if + any, of the imported records were successfully saved to the database. + You should look carefully through the WeeWX log file covering the + `wee_import` session and take note of any records that were not + imported. The most common reason for imported records not being saved + to the database is because a record with that timestamp already exists + in the database, in such cases something similar to the following will + be found in the log: ``` - Aug 22 14:38:28 stretch12 weewx[863]: manager: unable to add record 2018-09-04 04:20:00 AEST (1535998800) to database 'weewx.sdb': UNIQUE constraint failed: archive.dateTime + Aug 22 14:38:28 stretch12 weewx[863]: manager: unable to add record + 2018-09-04 04:20:00 AEST (1535998800) to database 'weewx.sdb': UNIQUE + constraint failed: archive.dateTime ``` - In such cases you should take note of the timestamp of the record(s) concerned and make a decision about whether to delete the pre-existing record and re-import the record or retain the pre-existing record. + In such cases you should take note of the timestamp of the record(s) + concerned and make a decision about whether to delete the pre-existing + record and re-import the record or retain the pre-existing record. From 36472cea1a363a9a4482b69bd9d2895221955710 Mon Sep 17 00:00:00 2001 From: gjr80 Date: Sun, 26 Nov 2023 17:29:13 +1000 Subject: [PATCH 05/13] reformat weectl import usage --- src/weectllib/import_cmd.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/weectllib/import_cmd.py b/src/weectllib/import_cmd.py index 0242cb3b..a534df47 100644 --- a/src/weectllib/import_cmd.py +++ b/src/weectllib/import_cmd.py @@ -10,12 +10,12 @@ import weecfg.extension import weectllib.import_actions from weeutil.weeutil import bcolors -import_usage = f"""{bcolors.BOLD}weectl import - --import-config=IMPORT_CONFIG_FILE - [--config=CONFIG_FILE] - [[--date=YYYY-mm-dd] | [[--from=YYYY-mm-dd[THH:MM]] [--to=YYYY-mm-dd[THH:MM]]]] - [--dry-run][--verbose] - [--no-prompt][--suppress-warnings]{bcolors.ENDC} +import_usage = f"""{bcolors.BOLD}weectl import --help + weectl import --import-config=IMPORT_CONFIG_FILE + [--config=CONFIG_FILE] + [[--date=YYYY-mm-dd] | [[--from=YYYY-mm-dd[THH:MM]] [--to=YYYY-mm-dd[THH:MM]]]] + [--dry-run][--verbose] + [--no-prompt][--suppress-warnings]{bcolors.ENDC} """ import_description = """ From 3c8121dbcb6acb19c100b17a1adc80492086e380 Mon Sep 17 00:00:00 2001 From: gjr80 Date: Sun, 26 Nov 2023 17:31:20 +1000 Subject: [PATCH 06/13] revised for changes from wee_import to weectl import --- docs_src/utilities/weectl-import-about.md | 14 ++-- .../utilities/weectl-import-common-opt.md | 74 +++++++++---------- 2 files changed, 40 insertions(+), 48 deletions(-) diff --git a/docs_src/utilities/weectl-import-about.md b/docs_src/utilities/weectl-import-about.md index 2fade2b5..6cde9e11 100644 --- a/docs_src/utilities/weectl-import-about.md +++ b/docs_src/utilities/weectl-import-about.md @@ -3,14 +3,14 @@ Some WeeWX users will have historical data from another source (e.g., other weather station software or a manually compiled file) which they wish to import into WeeWX. Such data can, depending upon the source, be imported -using the `wee_import` utility. +using the `weectl import` utility. -The `wee_import` utility supports importing observational data from the +The `weectl import` utility supports importing observational data from the following sources: -* a single [Comma Separated Values (CSV)](wee_import-csv.md) format file -* the historical observations of a [Weather Underground](wee_import-wu.md) +* a single [Comma Separated Values (CSV)](weectl-import-csv.md) format file +* the historical observations of a [Weather Underground](weectl-import-wu.md) personal weather station -* one or more [Cumulus](wee_import-cumulus.md) monthly log files -* one or more [Weather Display](wee_import-wd.md) monthly log files -* one or more [WeatherCat](wee_import-weathercat.md) monthly .cat files +* one or more [Cumulus](weectl-import-cumulus.md) monthly log files +* one or more [Weather Display](weectl-import-wd.md) monthly log files +* one or more [WeatherCat](weectl-import-weathercat.md) monthly .cat files diff --git a/docs_src/utilities/weectl-import-common-opt.md b/docs_src/utilities/weectl-import-common-opt.md index ce8e3e70..5b92b89c 100644 --- a/docs_src/utilities/weectl-import-common-opt.md +++ b/docs_src/utilities/weectl-import-common-opt.md @@ -1,46 +1,39 @@ Before starting, it's worth running the utility with the `--help` flag to see -how `wee_import` is used: +how `weectl import` is used: ``` -wee_import --help +weectl import --help ``` ``` -usage: wee_import --help - wee_import --import-config=IMPORT_CONFIG_FILE - [--config=CONFIG_FILE] - [--date=YYYY-mm-dd | --from=YYYY-mm-dd[THH:MM] --to=YYYY-mm-dd - [THH:MM]] - [--dry-run] - [--verbose] - [--no-prompt] - [--suppress-warnings] - +usage: weectl import --help + weectl import --import-config=IMPORT_CONFIG_FILE + [--config=CONFIG_FILE] + [[--date=YYYY-mm-dd] | [[--from=YYYY-mm-dd[THH:MM]] [--to=YYYY-mm-dd[THH:MM]]]] + [--dry-run][--verbose] + [--no-prompt][--suppress-warnings] Import observation data into a WeeWX archive. -Options: +optional arguments: -h, --help show this help message and exit - --config CONFIG_FILE Use configuration file CONFIG_FILE. + --config FILENAME Path to configuration file. --import-config IMPORT_CONFIG_FILE - Use import configuration file IMPORT_CONFIG_FILE. + Path to import configuration file. --dry-run Print what would happen but do not do it. --date YYYY-mm-dd Import data for this date. Format is YYYY-mm-dd. --from YYYY-mm-dd[THH:MM] - Import data starting at this date or date-time. Format - is YYYY-mm-dd[THH:MM]. + Import data starting at this date or date-time. Format is YYYY- + mm-dd[THH:MM]. --to YYYY-mm-dd[THH:MM] - Import data up until this date or date-time. Format is - YYYY-mm-dd[THH:MM]. + Import data up until this date or date-time. Format is YYYY-mm- + dd[THH:MM]. --verbose Print and log useful extra output. - --no-prompt Do not prompt. Accept relevant defaults and all y/n - prompts. - --suppress-warnings Suppress warnings to stdout. Warnings are still - logged. + --no-prompt Do not prompt. Accept relevant defaults and all y/n prompts. + --suppress-warnings Suppress warnings to stdout. Warnings are still logged. -wee_import will import data from an external source into a WeeWX -archive. Daily summaries are updated as each archive record is -imported so there should be no need to separately rebuild the daily -summaries. +Import data from an external source into a WeeWX archive. Daily summaries are updated as +each archive record is imported so there should be no need to separately rebuild the +daily summaries. ``` ## Options @@ -53,22 +46,21 @@ but if you have an unusual installation or multiple stations, you may have to tell it explicitly. ``` -wee_import --config=/this/directory/weewx.conf ---import-config=/directory/import.conf +weectl import --config=/this/directory/weewx.conf --import-config=/directory/import.conf ``` ### `--import-config=FILENAME` -`wee_import` uses a secondary configuration file, known as the import +`weectl import` uses a secondary configuration file, known as the import configuration file, to store various import parameters. The `--import-config` option is mandatory for all imports. Example import configuration files for -each type of import supported by `wee_import` are provided in the +each type of import supported by `weectl import` are provided in the `util/import` directory. These example files are best used by making a copy of into a working directory and then modifying the copy to suit your needs. The `--import-config` option is used as follows: ``` -wee_import --import-config=/directory/import.conf +weectl import --import-config=/directory/import.conf ``` ### `--dry-run` @@ -78,7 +70,7 @@ will be saved to the database. This is a useful option to use when first importing data. ``` -wee_import --import-config=/directory/import.conf --dry-run +weectl import --import-config=/directory/import.conf --dry-run ``` ### `--date=YYYY-mm-dd` @@ -212,20 +204,20 @@ import ### `--verbose` Inclusion of the `--verbose` option will cause additional information to be -printed during `wee_import` execution. +printed during `weectl import` execution. ``` -wee_import --import-config=/directory/import.conf --verbose +weectl import --import-config=/directory/import.conf --verbose ``` ### `--no-prompt` -Inclusion of the `--no-prompt` option will run `wee_import` without prompts. +Inclusion of the `--no-prompt` option will run `weectl import` without prompts. Relevant defaults will be used and all y/n prompts are automatically accepted -as 'y'. This may be useful for unattended use of `wee_import`. +as 'y'. This may be useful for unattended use of `weectl import`. ``` -wee_import --import-config=/directory/import.conf --no-prompt +weectl import --import-config=/directory/import.conf --no-prompt ``` !!! Warning @@ -237,8 +229,8 @@ wee_import --import-config=/directory/import.conf --no-prompt ### `--suppress-warnings` -The `--suppress-warnings` option suppresses `wee_import` warning messages from -being displayed on the console during the import. `wee_import` may issue a +The `--suppress-warnings` option suppresses `weectl import` warning messages from +being displayed on the console during the import. `weectl import` may issue a number of warnings during import. These warnings may be due to the source containing more than one entry for a given timestamp or there being no data found for a mapped import field. These warnings do not necessarily require @@ -247,5 +239,5 @@ to follow the import progress. Irrespective of whether `--suppress-warnings` is used all warnings are sent to log. ``` -wee_import --import-config=/directory/import.conf --suppress-warnings +weectl import --import-config=/directory/import.conf --suppress-warnings ``` From 054ac07e06150ddb32569ddd643f77a96f1bf2a7 Mon Sep 17 00:00:00 2001 From: Tom Keffer Date: Sun, 26 Nov 2023 07:41:40 -0800 Subject: [PATCH 07/13] Mention log location in upgrade guide. --- docs_src/upgrade.md | 7 +++++++ docs_src/usersguide/where.md | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs_src/upgrade.md b/docs_src/upgrade.md index 6677a13d..a7bfe294 100644 --- a/docs_src/upgrade.md +++ b/docs_src/upgrade.md @@ -87,6 +87,13 @@ sudo mkdir /etc/weewx/bin cp -r /usr/share/weewx/user/ /etc/weewx/bin/ ``` +### New logging location + +New installs will log to `weewx.log`, whose location depends on the install +method. See the section + +Old installs continue to log to syslog. + ### New utilities Most of the old utilities have been collected and put under one master utility, diff --git a/docs_src/usersguide/where.md b/docs_src/usersguide/where.md index 879897b5..a2deafd6 100644 --- a/docs_src/usersguide/where.md +++ b/docs_src/usersguide/where.md @@ -129,8 +129,8 @@ be used. This is where to find the system log for each platform. On macOS, the log file is likely to contain only severe log messages. -See the wiki article [WeeWX V4 and logging](https://github.com/weewx/weewx/wiki/WeeWX-v4-and-logging) -for more information on controlling logging in WeeWX. +See the wiki article [WeeWX logging](https://github.com/weewx/weewx/wiki/WeeWX-v4-and-logging) +for more information on how to control logging in WeeWX. ## Location of executables in a pip install From 6b2b761930f37174f78b08ee5415588c36ba6e35 Mon Sep 17 00:00:00 2001 From: Tom Keffer Date: Sun, 26 Nov 2023 15:14:19 -0800 Subject: [PATCH 08/13] Revert to logging to syslog. Unfortunately, the Python logging module is not process safe, so we would not be able to have multiple processes writing to a single file. --- TODO.md | 21 ++++++------------- docs_src/upgrade.md | 7 ------- docs_src/usersguide/where.md | 40 +++--------------------------------- src/weeutil/logger.py | 2 ++ src/weewx_data/weewx.conf | 24 ---------------------- 5 files changed, 11 insertions(+), 83 deletions(-) diff --git a/TODO.md b/TODO.md index a42e0c55..c8d91ebd 100644 --- a/TODO.md +++ b/TODO.md @@ -24,7 +24,7 @@ For upgrades: * Copy contents of `/usr/share/weewx/user` to `/etc/weewx/bin/user`, then rename `/usr/share/weewx/user` to `/usr/share/weewx/user-YYmmdd` -Resolved: +## Resolved - no need for loop-on-init arg to weewxd? KEEP IT @@ -46,8 +46,11 @@ Resolved: - for logging, use weewx vs weewxd for log label? WEEWX -- if separate log file, we just do one: ~/weewx-data/log/weewx.log - ONLY weewx.log +- Continue logging to system log. Unfortunately, the Python logging module is + not process safe, so we would be unable to have multiple processes write to + it. So, we stick with syslog. + + Done: @@ -72,11 +75,6 @@ to the system, so a weewx user owns it all. apt/yum/zypper - easiest install, appliance git(src) - developer, minimal system, multiple python configurations -We want /var/log/weewx so that weewx-multi will emit single log for each -weewxd instance. - -We want all logs to /var/log/weewx/weewx.log so that it is clear if someone -is running weewxd and weectl at the same time. For deb/rpm upgrades, the config file is only modified if you select the 'take maintainer changes' option. That takes your old config and 'upgrades' @@ -115,13 +113,6 @@ package install. Change parameter `description`. -## Logging - -Update the logging wiki. - -Log to `~/weewx-data/weewx.log` by default. Use -[TimedRotatingFileHandler](https://docs.python.org/3/library/logging.handlers.html#timedrotatingfilehandler) - ## Wiki diff --git a/docs_src/upgrade.md b/docs_src/upgrade.md index a7bfe294..6677a13d 100644 --- a/docs_src/upgrade.md +++ b/docs_src/upgrade.md @@ -87,13 +87,6 @@ sudo mkdir /etc/weewx/bin cp -r /usr/share/weewx/user/ /etc/weewx/bin/ ``` -### New logging location - -New installs will log to `weewx.log`, whose location depends on the install -method. See the section - -Old installs continue to log to syslog. - ### New utilities Most of the old utilities have been collected and put under one master utility, diff --git a/docs_src/usersguide/where.md b/docs_src/usersguide/where.md index a2deafd6..99d68abd 100644 --- a/docs_src/usersguide/where.md +++ b/docs_src/usersguide/where.md @@ -17,8 +17,7 @@ documentation. | Executables | _`BIN_ROOT`_ | `/usr/share/weewx/` | | SQLite databases | _`SQLITE_ROOT`_ | `/var/lib/weewx/` | | Web pages and images | _`HTML_ROOT`_ | `/var/www/html/weewx/` | - | Log | | See below | - | Documentation | | https://weewx.com/docs/5.0 | + | Documentation | _`DOC_ROOT`_ | `/usr/share/doc/weewx/` | === "RedHat" @@ -31,7 +30,6 @@ documentation. | Executables | _`BIN_ROOT`_ | `/usr/share/weewx/` | | SQLite databases | _`SQLITE_ROOT`_ | `/var/lib/weewx/` | | Web pages and images | _`HTML_ROOT`_ | `/var/www/html/weewx/` | - | Log | | See below | | Documentation | | https://weewx.com/docs/5.0 | === "openSUSE" @@ -45,7 +43,6 @@ documentation. | Executables | _`BIN_ROOT`_ | `/usr/share/weewx/` | | SQLite databases | _`SQLITE_ROOT`_ | `/var/lib/weewx/` | | Web pages and images | _`HTML_ROOT`_ | `/var/www/html/weewx/` | - | Log | | See below | | Documentation | | https://weewx.com/docs/5.0 | === "pip" @@ -59,7 +56,6 @@ documentation. | Executables | _`BIN_ROOT`_ | varies | | SQLite databases | _`SQLITE_ROOT`_ | `archive/` | | Web pages and images | _`HTML_ROOT`_ | `public_html/` | - | Log | | `log/weewx.log` | | Documentation | | https://weewx.com/docs/5.0 | !!! Note @@ -70,35 +66,8 @@ documentation. ## Log files -Logging is configured by the `[Logging]` section in the configuration file, -`weewx.conf`. The section that comes with WeeWX Version 5 looks like this: - -``` ini linenums="1" hl_lines="12 14 16" -[Logging] - [[root]] - handlers = timed_rotate, - - [[handlers]] - # Log to a set of rotating files - [[[timed_rotate]]] - level = DEBUG - formatter = verbose - class = logging.handlers.TimedRotatingFileHandler - # File to log to, relative to WEEWX_ROOT - filename = log/weewx.log - # When to rotate: - when = midnight - # How many log files to save - backupCount = 7 -``` - -This says to use a timed file rotation to log file `log/weewx.log` (line 12). -Note that this is _relative to `WEEWX_ROOT`_, so the actual file is likely to be -`~/weewx-data/log/weewx.log`. The file should be rotated at midnight (line 14). -A week's worth of backups should be saved (line 16). - -If you don't have such a section, then the system log will -be used. This is where to find the system log for each platform. +In the default configuration, WeeWX logs to the system log. This is where to +find the system log for each platform. === "Debian" @@ -128,7 +97,6 @@ be used. This is where to find the system log for each platform. !!! note On macOS, the log file is likely to contain only severe log messages. - See the wiki article [WeeWX logging](https://github.com/weewx/weewx/wiki/WeeWX-v4-and-logging) for more information on how to control logging in WeeWX. @@ -148,5 +116,3 @@ the executables will depend on how the installation was done. | Virtual environment with `--user`
(not allowed) | `python3 -m venv ~/ve`
`source ~/ve/bin/activate`
`pip3 install weewx --user` | N/A | (1) Checked on Ubuntu 22.02 and Rocky v9.1 - - diff --git a/src/weeutil/logger.py b/src/weeutil/logger.py index de4b1f1a..bee8323f 100644 --- a/src/weeutil/logger.py +++ b/src/weeutil/logger.py @@ -128,7 +128,9 @@ def setup(process_name, config_dict=None): facility=facility, process_name=process_name) if key == 'filename' and weewx_root: + # Allow relative file paths, in which case they are relative to WEEWX_ROOT: section[key] = os.path.join(weewx_root, section[key]) + # Create any intervening directories: os.makedirs(os.path.dirname(section[key]), exist_ok=True) # Using the function, walk the 'Logging' part of the structure diff --git a/src/weewx_data/weewx.conf b/src/weewx_data/weewx.conf index f6013063..49011cc0 100644 --- a/src/weewx_data/weewx.conf +++ b/src/weewx_data/weewx.conf @@ -488,30 +488,6 @@ version = 5.0.0b15 ############################################################################## -# This section configures modifications to the default logging. - -[Logging] - - # Root logger - [[root]] - handlers = timed_rotate, - - [[handlers]] - # Log to a set of rotating files - [[[timed_rotate]]] - level = DEBUG - formatter = verbose - class = logging.handlers.TimedRotatingFileHandler - # File to log to, relative to WEEWX_ROOT - filename = log/weewx.log - # When to rotate: - when = midnight - # How many log files to save - backupCount = 7 - - -############################################################################## - # This section configures the internal weewx engine. [Engine] From eb02c018596b62f50d633b5e27fcbfa55fca929a Mon Sep 17 00:00:00 2001 From: Tom Keffer Date: Sun, 26 Nov 2023 16:12:43 -0800 Subject: [PATCH 09/13] On startup, set up a rudimentary logger. To be used until the configuration file has been read. Also, the logging setup now disables existing loggers. This gets rid of the rudimentary logger. --- src/weeutil/logger.py | 2 +- src/weewxd.py | 19 +++++++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/weeutil/logger.py b/src/weeutil/logger.py index bee8323f..6dab2d9b 100644 --- a/src/weeutil/logger.py +++ b/src/weeutil/logger.py @@ -21,7 +21,7 @@ import weewx # LOGGING_STR = """[Logging] version = 1 - disable_existing_loggers = False + disable_existing_loggers = True # Root logger [[root]] diff --git a/src/weewxd.py b/src/weewxd.py index fc5e5e67..e500fa08 100644 --- a/src/weewxd.py +++ b/src/weewxd.py @@ -5,7 +5,6 @@ # """Entry point to the weewx weather system.""" import argparse -import importlib import locale import logging import os @@ -73,9 +72,14 @@ def main(): print(weewx.__version__) sys.exit(0) + # Set up a rudimentary logger until we can read the configuration file. + log.addHandler(logging.handlers.SysLogHandler( address = weeutil.logger.address, + facility = weeutil.logger.facility)) + # User can specify the config file as either a positional argument, or as an option # argument, but not both. if namespace.config_option and namespace.config_arg: + log.critical(epilog) sys.exit(epilog) # Read the configuration file @@ -83,12 +87,14 @@ def main(): config_path, config_dict = weecfg.read_config(namespace.config_arg, [namespace.config_option]) except (IOError, configobj.ConfigObjError) as e: - print("Error parsing config file: %s" % e, file=sys.stderr) + msg = "Error parsing config file: %s" % e + print(msg, file=sys.stderr) + log.critical(msg) weeutil.logger.log_traceback(log.critical, " **** ") sys.exit(weewx.CMD_ERROR) - # Now that we have the configuration dictionary, we can set up logging, debug, and - # other housekeeping chores: + # Now that we have the configuration dictionary, we can set up the user-configured logging and + # debug, as well as perform other housekeeping chores: weewx.initialize(config_dict, namespace.log_label) # Log key bits of information. @@ -117,8 +123,9 @@ def main(): n = 0 while weewx.launchtime_ts < sane: # Log any problems every minute. - if n % 120 == 0: - log.info("Waiting for sane time. Current time is %s", + if n % 10 == 0: + log.info("Waiting for sane time. File time is %s; current time is %s", + weeutil.weeutil.timestamp_to_string(sane), weeutil.weeutil.timestamp_to_string(weewx.launchtime_ts)) n += 1 time.sleep(0.5) From ee3187721585b47c583fd5a1a66eac4f54ebb73c Mon Sep 17 00:00:00 2001 From: Tom Keffer Date: Sun, 26 Nov 2023 16:41:51 -0800 Subject: [PATCH 10/13] V5.0.0b16 --- docs_src/quickstarts/pip.md | 2 +- pyproject.toml | 2 +- src/weecfg/tests/expected/weewx43_user_expected.conf | 2 +- src/weewx/__init__.py | 2 +- src/weewx_data/skins/Ftp/skin.conf | 2 +- src/weewx_data/skins/Mobile/skin.conf | 2 +- src/weewx_data/skins/Rsync/skin.conf | 2 +- src/weewx_data/skins/Seasons/skin.conf | 2 +- src/weewx_data/skins/Smartphone/skin.conf | 2 +- src/weewx_data/skins/Standard/skin.conf | 2 +- src/weewx_data/weewx.conf | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs_src/quickstarts/pip.md b/docs_src/quickstarts/pip.md index 958a5adb..3bd31787 100644 --- a/docs_src/quickstarts/pip.md +++ b/docs_src/quickstarts/pip.md @@ -349,7 +349,7 @@ python3 -m pip install weewx --upgrade Optional: You may want to upgrade your documentation and examples. ``` -weectl station upgrade --what docs examples util +weectl station upgrade --what examples util ``` Optional: You may want to upgrade your skins, although this may break or diff --git a/pyproject.toml b/pyproject.toml index 82764afa..7a0b6053 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "weewx" -version = "5.0.0b15" +version = "5.0.0b16" description = "The WeeWX weather software system. This is an BETA release, and may have many bugs!" authors = [ "Tom Keffer ", diff --git a/src/weecfg/tests/expected/weewx43_user_expected.conf b/src/weecfg/tests/expected/weewx43_user_expected.conf index 851db62a..5e453d0c 100644 --- a/src/weecfg/tests/expected/weewx43_user_expected.conf +++ b/src/weecfg/tests/expected/weewx43_user_expected.conf @@ -26,7 +26,7 @@ log_failure = True socket_timeout = 20 # Do not modify this - it is used by setup.py when installing and updating. -version = 5.0.0b15 +version = 5.0.0b16 ############################################################################## diff --git a/src/weewx/__init__.py b/src/weewx/__init__.py index a99a27a2..552b3393 100644 --- a/src/weewx/__init__.py +++ b/src/weewx/__init__.py @@ -12,7 +12,7 @@ import time import weeutil.logger from weeutil.weeutil import to_int -__version__ = "5.0.0b15" +__version__ = "5.0.0b16" # Holds the program launch time in unix epoch seconds: # Useful for calculating 'uptime.' diff --git a/src/weewx_data/skins/Ftp/skin.conf b/src/weewx_data/skins/Ftp/skin.conf index 96120cfd..13518241 100644 --- a/src/weewx_data/skins/Ftp/skin.conf +++ b/src/weewx_data/skins/Ftp/skin.conf @@ -7,7 +7,7 @@ ############################################################################### SKIN_NAME = Ftp -SKIN_VERSION = 5.0.0b15 +SKIN_VERSION = 5.0.0b16 [Generators] generator_list = weewx.reportengine.FtpGenerator diff --git a/src/weewx_data/skins/Mobile/skin.conf b/src/weewx_data/skins/Mobile/skin.conf index fa23a1ee..cdd03214 100644 --- a/src/weewx_data/skins/Mobile/skin.conf +++ b/src/weewx_data/skins/Mobile/skin.conf @@ -1,7 +1,7 @@ # configuration file for Mobile skin SKIN_NAME = Mobile -SKIN_VERSION = 5.0.0b15 +SKIN_VERSION = 5.0.0b16 [Extras] # Set this URL to display a radar image diff --git a/src/weewx_data/skins/Rsync/skin.conf b/src/weewx_data/skins/Rsync/skin.conf index f98d665c..8e52d5b6 100644 --- a/src/weewx_data/skins/Rsync/skin.conf +++ b/src/weewx_data/skins/Rsync/skin.conf @@ -8,7 +8,7 @@ ############################################################################### SKIN_NAME = Rsync -SKIN_VERSION = 5.0.0b15 +SKIN_VERSION = 5.0.0b16 [Generators] generator_list = weewx.reportengine.RsyncGenerator diff --git a/src/weewx_data/skins/Seasons/skin.conf b/src/weewx_data/skins/Seasons/skin.conf index 9afb0356..edc6ce30 100644 --- a/src/weewx_data/skins/Seasons/skin.conf +++ b/src/weewx_data/skins/Seasons/skin.conf @@ -5,7 +5,7 @@ ############################################################################### SKIN_NAME = Seasons -SKIN_VERSION = 5.0.0b15 +SKIN_VERSION = 5.0.0b16 ############################################################################### diff --git a/src/weewx_data/skins/Smartphone/skin.conf b/src/weewx_data/skins/Smartphone/skin.conf index eab5fc36..03490044 100644 --- a/src/weewx_data/skins/Smartphone/skin.conf +++ b/src/weewx_data/skins/Smartphone/skin.conf @@ -1,7 +1,7 @@ # configuration file for Smartphone skin SKIN_NAME = Smartphone -SKIN_VERSION = 5.0.0b15 +SKIN_VERSION = 5.0.0b16 [Extras] # Set this URL to display a radar image diff --git a/src/weewx_data/skins/Standard/skin.conf b/src/weewx_data/skins/Standard/skin.conf index 0e80daf9..e8bd24f2 100644 --- a/src/weewx_data/skins/Standard/skin.conf +++ b/src/weewx_data/skins/Standard/skin.conf @@ -5,7 +5,7 @@ ############################################################################### SKIN_NAME = Standard -SKIN_VERSION = 5.0.0b15 +SKIN_VERSION = 5.0.0b16 ############################################################################### diff --git a/src/weewx_data/weewx.conf b/src/weewx_data/weewx.conf index 49011cc0..041ac80b 100644 --- a/src/weewx_data/weewx.conf +++ b/src/weewx_data/weewx.conf @@ -20,7 +20,7 @@ log_success = True log_failure = True # This configuration file was created by ... -version = 5.0.0b15 +version = 5.0.0b16 ############################################################################## From 87d3fb338fdc962107eac84409ea9e83b5924e15 Mon Sep 17 00:00:00 2001 From: gjr80 Date: Mon, 27 Nov 2023 17:04:03 +1000 Subject: [PATCH 11/13] update example import config files with current supported sources --- .../util/import/cumulus-example.conf | 24 +++---------- src/weewx_data/util/import/wd-example.conf | 34 +++--------------- .../util/import/weathercat-example.conf | 35 +++---------------- src/weewx_data/util/import/wu-example.conf | 7 ++-- 4 files changed, 17 insertions(+), 83 deletions(-) diff --git a/src/weewx_data/util/import/cumulus-example.conf b/src/weewx_data/util/import/cumulus-example.conf index a4b13b10..d47d4b1e 100644 --- a/src/weewx_data/util/import/cumulus-example.conf +++ b/src/weewx_data/util/import/cumulus-example.conf @@ -1,17 +1,18 @@ # EXAMPLE CONFIGURATION FILE FOR IMPORTING FROM CUMULUS # -# Copyright (c) 2009-2019 Tom Keffer and Gary Roderick. +# Copyright (c) 2009-2023 Tom Keffer and Gary Roderick. # See the file LICENSE.txt for your rights. ############################################################################## -# Specify our source. Available options are: +# Specify the source. Available options are: # CSV - import obs from a single CSV format file # WU - import obs from a Weather Underground PWS history # Cumulus - import obs from a one or more Cumulus monthly log files # WD - import obs from a one or more WD monthly log files +# WeatherCat - import obs from one or more WeatherCat monthly .cat files # Format is: -# source = (CSV | WU | Cumulus) +# source = (CSV | WU | Cumulus | WD | WeatherCat) source = Cumulus ############################################################################## @@ -165,20 +166,3 @@ source = Cumulus # created. Otherwise it may be set to True or omitted. Format is: # solar_sensor = (True | False) solar_sensor = True - - # For correct import of the monthly logs wee_import needs to know what - # units are used in the imported data. The units used for temperature, - # pressure, rain and windspeed related observations in the Cumulus monthly - # logs are set at the Cumulus Station Configuration Screen. The - # [[Units]] settings below should be set to the WeeWX equivalent of the - # units of measure used by Cumulus (eg if Cumulus used 'C' for temperature, - # temperature should be set to 'degree_C'). Note that Cumulus does not - # support all units used by WeeWX (eg 'mmHg') so not all WeeWX unit are - # available options. - [[Units]] - temperature = degree_C # options are 'degree_F' or 'degree_C' - pressure = hPa # options are 'inHg', 'mbar' or 'hPa' - rain = mm # options are 'inch' or 'mm' - speed = km_per_hour # options are 'mile_per_hour', - # 'km_per_hour', 'knot' or - # 'meter_per_second' diff --git a/src/weewx_data/util/import/wd-example.conf b/src/weewx_data/util/import/wd-example.conf index 105345f4..50fe0f27 100644 --- a/src/weewx_data/util/import/wd-example.conf +++ b/src/weewx_data/util/import/wd-example.conf @@ -1,17 +1,18 @@ # EXAMPLE CONFIGURATION FILE FOR IMPORTING FROM WEATHER DISPLAY (WD) # -# Copyright (c) 2009-2019 Tom Keffer and Gary Roderick. +# Copyright (c) 2009-2023 Tom Keffer and Gary Roderick. # See the file LICENSE.txt for your rights. ############################################################################## -# Specify our source. Available options are: +# Specify the source. Available options are: # CSV - import obs from a single CSV format file # WU - import obs from a Weather Underground PWS history # Cumulus - import obs from a one or more Cumulus monthly log files # WD - import obs from a one or more WD monthly log files +# WeatherCat - import obs from one or more WeatherCat monthly .cat files # Format is: -# source = (CSV | WU | Cumulus | WD) +# source = (CSV | WU | Cumulus | WD | WeatherCat) source = WD ############################################################################## @@ -219,33 +220,6 @@ source = WD # The default is True ignore_extreme_temp_hum = True - # For correct import of the monthly logs wee_import needs to know what - # units are used in the imported data. The units used in the WD log files - # are set to either Metric or US in WD via the 'Log File' setting under - # 'Units' on the 'Units/Wind Chill' tab of the WD universal setup. If - # Metric or US units have been used in the log files then it is usually - # sufficient to set the units config option in the [[Units]] stanza to - # Metric or US. For example: - # [[Units]] - # units = metric - # However, in cases where Metric or US may not have been used (eg a custom - # log file) then the units config option should not be used but rather - # units should be specified for temperature, pressure, rainfall and speed - # groups using the temperature, pressure, rain and speed config options. - # In each case the config option should be set to a WeeWX unit option. For - # example: - # [[Units]] - # temperature = degree_C # options are 'degree_F' or 'degree_C' - # pressure = hPa # options are 'inHg', 'mbar' or 'hPa' - # rain = mm # options are 'inch' or 'mm' - # speed = km_per_hour # options are 'mile_per_hour', - # # 'km_per_hour', 'knot' or - # # 'meter_per_second' - # Note that either the units config option should be set or the individual - # unit group config options but not both. - [[Units]] - units = metric - # Map WD log fields to WeeWX archive fields. Format is: # # weewx_archive_field_name = wd_field_name diff --git a/src/weewx_data/util/import/weathercat-example.conf b/src/weewx_data/util/import/weathercat-example.conf index dc65d320..b8c46418 100644 --- a/src/weewx_data/util/import/weathercat-example.conf +++ b/src/weewx_data/util/import/weathercat-example.conf @@ -1,17 +1,18 @@ # EXAMPLE CONFIGURATION FILE FOR IMPORTING FROM WEATHERCAT # -# Copyright (c) 2009-2020 Tom Keffer and Gary Roderick. +# Copyright (c) 2009-2023 Tom Keffer and Gary Roderick. # See the file LICENSE.txt for your rights. ############################################################################## -# Specify our source. Available options are: +# Specify the source. Available options are: # CSV - import obs from a single CSV format file # WU - import obs from a Weather Underground PWS history # Cumulus - import obs from a one or more Cumulus monthly log files -# WeatherCat - import obs from a one or more WeatherCat monthly .cat files +# WD - import obs from a one or more WD monthly log files +# WeatherCat - import obs from one or more WeatherCat monthly .cat files # Format is: -# source = (CSV | WU | Cumulus | WeatherCat) +# source = (CSV | WU | Cumulus | WD | WeatherCat) source = WeatherCat ############################################################################## @@ -134,29 +135,3 @@ source = WeatherCat # tranche = x # where x is an integer tranche = 250 - - # For correct import of the WeatherCat monthly .cat files wee_import needs - # to know what units are used in the imported data. The units used by - # WeatherCat are set via the 'Units/Misc1` tab under the WeatherCat - # 'Preferences' menu. The WeatherCat unit preferences should be mirrored - # below under the [[Units]] stanza and each assigned a WeeWX unit value. As - # WeatherCat allows dewpoint to have different units to other temperatures - # settings for both temperature and dewpoint should be specified; however, - # if only temperature is specified then the temperature units will be - # applied to all temperatures and dewpoint. - # - # An example [[Units]] stanza might be: - # [[Units]] - # temperature = degree_F # options are 'degree_F' or 'degree_C' - # dewpoint = degree_C # options are 'degree_F' or 'degree_C' - # pressure = hPa # options are 'inHg', 'mbar' or 'hPa' - # windspeed = km_per_hour # options are 'mile_per_hour', - # # 'km_per_hour', 'knot' or - # # 'meter_per_second' - # precipitation = mm # options are 'inch' or 'mm' - [[Units]] - temperature = degree_C - dewpoint = degree_C - pressure = hPa - windspeed = km_per_hour - precipitation = mm diff --git a/src/weewx_data/util/import/wu-example.conf b/src/weewx_data/util/import/wu-example.conf index 8a55e4f5..62e7e64d 100644 --- a/src/weewx_data/util/import/wu-example.conf +++ b/src/weewx_data/util/import/wu-example.conf @@ -1,17 +1,18 @@ # EXAMPLE CONFIGURATION FILE FOR IMPORTING FROM THE WEATHER UNDERGROUND # -# Copyright (c) 2009-2019 Tom Keffer and Gary Roderick. +# Copyright (c) 2009-2023 Tom Keffer and Gary Roderick. # See the file LICENSE.txt for your rights. ############################################################################## -# Specify our source. Available options are: +# Specify the source. Available options are: # CSV - import obs from a single CSV format file # WU - import obs from a Weather Underground PWS history # Cumulus - import obs from a one or more Cumulus monthly log files # WD - import obs from a one or more WD monthly log files +# WeatherCat - import obs from one or more WeatherCat monthly .cat files # Format is: -# source = (CSV | WU | Cumulus) +# source = (CSV | WU | Cumulus | WD | WeatherCat) source = WU ############################################################################## From a0294e9c8382e0250936cad341c3a9ce5d436cc1 Mon Sep 17 00:00:00 2001 From: Tom Keffer Date: Mon, 27 Nov 2023 04:24:13 -0800 Subject: [PATCH 12/13] Add -y option to weectl extension uninstall --- docs_src/utilities/weectl-extension.md | 15 ++++++++++----- src/weecfg/extension.py | 10 ++++++++-- src/weectllib/extension_cmd.py | 7 +++++-- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/docs_src/utilities/weectl-extension.md b/docs_src/utilities/weectl-extension.md index b94bb923..f4b78394 100644 --- a/docs_src/utilities/weectl-extension.md +++ b/docs_src/utilities/weectl-extension.md @@ -9,15 +9,16 @@ weectl extension --help ## List installed extensions - weectl extension list - [--config=FILENAME] [--dry-run] + weectl extension list [--config=FILENAME] This action will list all the extensions that you have installed. ## Install an extension - weectl extension install (FILE|DIR|URL) + weectl extension install (FILE|DIR|URL) + [--config=FILENAME] + [--dry-run] [--verbosity=N] This action will install an extension from a zip file, tar file, directory, or URL. @@ -44,6 +45,8 @@ weectl extension install ~/Downloads/windy-0.1.zip ## Uninstall an extension weectl extension uninstall NAME + [--config=FILENAME] + [--dry-run] [--verbosity=N] [-y] This action uninstalls an extension. Use the `list` action to see what to use for `NAME`. @@ -100,10 +103,10 @@ Extension Name Version Description windy 0.7 Upload weather data to Windy. ``` -Uninstall the extension: +Uninstall the extension without asking for confirmation: ``` -% weectl extension uninstall windy +% weectl extension uninstall windy -y Request to remove extension 'windy' Finished removing extension 'windy' ``` @@ -130,4 +133,6 @@ writable changes. How much information to display (0-3). +### -y | --yes +Do not ask for confirmation. Just do it. diff --git a/src/weecfg/extension.py b/src/weecfg/extension.py index a156dab6..d3db09c5 100644 --- a/src/weecfg/extension.py +++ b/src/weecfg/extension.py @@ -364,17 +364,23 @@ class ExtensionEngine(object): except KeyError: pass - def uninstall_extension(self, extension_name): + def uninstall_extension(self, extension_name, noprompt=False): """Uninstall an extension. Args: extension_name(str): The name of the extension. Use 'weectl extension list' to find its name. + noprompt(bool): If False, ask for a confirmation before uninstalling. Otherwise, + just do it. """ - self.printer.out(f"Request to remove extension '{extension_name}'.") if self.dry_run: self.printer.out("This is a dry run. Nothing will actually be done.") + ans = weeutil.weeutil.y_or_n(f"Uninstall extension '{extension_name}'? ", noprompt) + if ans == 'n': + self.printer.out("Nothing done.") + return + # Find the subdirectory containing this extension's installer extension_installer_dir = os.path.join(self.root_dict['EXT_DIR'], extension_name) try: diff --git a/src/weectllib/extension_cmd.py b/src/weectllib/extension_cmd.py index 975a533a..ae1e333e 100644 --- a/src/weectllib/extension_cmd.py +++ b/src/weectllib/extension_cmd.py @@ -20,7 +20,7 @@ extension_install_usage = f""" {bcolors.BOLD}weectl extension install (FILE|DIR """ extension_uninstall_usage = f""" {bcolors.BOLD}weectl extension uninstall NAME [--config=FILENAME] - [--dry-run] [--verbosity=N]{bcolors.ENDC} + [--dry-run] [--verbosity=N] [-y]{bcolors.ENDC} """ extension_usage = '\n '.join((extension_list_usage, extension_install_usage, @@ -97,6 +97,9 @@ def add_subparser(subparsers): 'do it.') uninstall_extension_parser.add_argument('--verbosity', type=int, default=1, metavar='N', help="How much information to display (0|1|2|3).") + uninstall_extension_parser.add_argument('-y', '--yes', action='store_true', + dest="noprompt", + help="Don't ask for confirmation. Just do it.") uninstall_extension_parser.set_defaults(func=uninstall_extension) @@ -112,7 +115,7 @@ def install_extension(namespace): def uninstall_extension(namespace): ext = _get_extension_engine(namespace.config, namespace.dry_run, namespace.verbosity) - ext.uninstall_extension(namespace.name) + ext.uninstall_extension(namespace.name, namespace.noprompt) def _get_extension_engine(config_path, dry_run=False, verbosity=1): From fe752abe7b263592145e3b79e69a1b106a140423 Mon Sep 17 00:00:00 2001 From: Tom Keffer Date: Mon, 27 Nov 2023 04:34:53 -0800 Subject: [PATCH 13/13] Fix test for uninstalling an extension --- src/weecfg/tests/test_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/weecfg/tests/test_config.py b/src/weecfg/tests/test_config.py index dcf701a4..e31d07cc 100644 --- a/src/weecfg/tests/test_config.py +++ b/src/weecfg/tests/test_config.py @@ -421,7 +421,7 @@ class ExtensionInstallTest(unittest.TestCase): # First install... self.engine.install_extension('./pmon.tgz') # ... then uninstall it: - self.engine.uninstall_extension('pmon') + self.engine.uninstall_extension('pmon', noprompt=True) # Assert that everything got removed correctly: self.assertFalse(os.path.exists(os.path.join(self.engine.root_dict['USER_DIR'],