From 3d2288985129d2de088678db5f3363380e250a3f Mon Sep 17 00:00:00 2001 From: Tom Keffer Date: Wed, 19 Jul 2023 15:27:56 -0700 Subject: [PATCH] Remove `wee_database` and references to it --- TODO.md | 1 - bin/wee_database.py | 1170 ------------------------- bin/weectllib/database_cmd.py | 9 +- bin/weectllib/db_actions.py | 3 +- docs_src/custom/database.md | 223 +++-- docs_src/utilities/wee_database.md | 355 -------- docs_src/utilities/wee_import.md | 2 +- docs_src/utilities/weectl-database.md | 32 +- mkdocs.yml | 1 - pkg/debian/rules | 2 +- pkg/weewx.spec.in | 6 +- pyproject.toml | 2 - util/scripts/wee_database | 9 - 13 files changed, 129 insertions(+), 1686 deletions(-) delete mode 100755 bin/wee_database.py delete mode 100644 docs_src/utilities/wee_database.md delete mode 100755 util/scripts/wee_database diff --git a/TODO.md b/TODO.md index 1bf29705..384ec87b 100644 --- a/TODO.md +++ b/TODO.md @@ -20,7 +20,6 @@ When doing `weectl station create`, install everything under the `util` subdirec - Allow rebuild daily summaries for only selected types. - Check semantics of specifying a time for `--from` and/or `--to` in `weectl database calc-missing`. -- Remove the old `wee_database` application and any references to it. - Make sure output of `--help` matches documentation. ## Logging diff --git a/bin/wee_database.py b/bin/wee_database.py deleted file mode 100755 index 94bcfa2d..00000000 --- a/bin/wee_database.py +++ /dev/null @@ -1,1170 +0,0 @@ -# -# Copyright (c) 2009-2023 Tom Keffer -# -# See the file LICENSE.txt for your full rights. -# -"""Configure databases used by WeeWX""" - -# python imports -import datetime -import importlib -import logging -import optparse -import sys -import time - -# weewx imports -import weecfg.database -import weedb -import weeutil.logger -import weewx.manager -import weewx.units -from weeutil.weeutil import timestamp_to_string, y_or_n, to_int - -log = logging.getLogger(__name__) - -usage = """%prog --help - %prog --create - %prog --reconfigure - %prog --transfer --dest-binding=BINDING_NAME [--dry-run] - %prog --add-column=NAME [--type=(REAL|INTEGER)] - %prog --rename-column=NAME --to-name=NEW_NAME - %prog --drop-columns=NAME1,NAME2,... - %prog --check - %prog --update [--dry-run] - %prog --drop-daily - %prog --rebuild-daily [--date=YYYY-mm-dd | - [--from=YYYY-mm-dd] [--to=YYYY-mm-dd]] - [--dry-run] - %prog --reweight [--date=YYYY-mm-dd | - [--from=YYYY-mm-dd] [--to=YYYY-mm-dd]] - [--dry-run] - %prog --calc-missing [--date=YYYY-mm-dd | - [--from=YYYY-mm-dd[THH:MM]] [--to=YYYY-mm-dd[THH:MM]]] - %prog --check-strings - %prog --fix-strings [--dry-run] - -Description: - -Manipulate the WeeWX database. Most of these operations are handled -automatically by WeeWX, but they may be useful in special cases.""" - -epilog = """NOTE: MAKE A BACKUP OF YOUR DATABASE BEFORE USING THIS UTILITY! -Many of its actions are irreversible!""" - - -def main(): - # Create a command line parser: - parser = optparse.OptionParser(usage=usage, epilog=epilog) - - # Add the various verbs... - parser.add_option("--create", action='store_true', - help="Create the WeeWX database and initialize it with the" - " schema.") - parser.add_option("--reconfigure", action='store_true', - help="Create a new database using configuration" - " information found in the configuration file." - " The new database will have the same name as the old" - " database, with a '_new' on the end.") - parser.add_option("--transfer", action='store_true', - help="Transfer the WeeWX archive from source database " - "to destination database.") - parser.add_option("--add-column", type=str, metavar="NAME", - help="Add new column NAME to database.") - parser.add_option("--type", type=str, metavar="TYPE", - help="New database column type (INTEGER|REAL) " - "(option --add-column only). Default is 'REAL'.") - parser.add_option("--rename-column", type=str, metavar="NAME", - help="Rename the column with name NAME.") - parser.add_option("--to-name", type=str, metavar="NEW_NAME", - help="New name of the column (option --rename-column only).") - parser.add_option("--drop-columns", type=str, metavar="NAME1,NAME2,...", - help="Drop one or more columns. Names must be separated by commas, " - "with NO SPACES.") - parser.add_option("--check", action="store_true", - help="Check the calculations in the daily summary tables.") - parser.add_option("--update", action="store_true", - help="Update the daily summary tables if required and" - " recalculate the daily summary maximum windSpeed values.") - parser.add_option("--calc-missing", dest="calc_missing", action="store_true", - help="Calculate and store any missing derived observations.") - parser.add_option("--check-strings", action="store_true", - help="Check the archive table for null strings that may" - " have been introduced by a SQL editing program.") - parser.add_option("--fix-strings", action='store_true', - help="Fix any null strings in a SQLite database.") - parser.add_option("--drop-daily", action='store_true', - help="Drop the daily summary tables from a database.") - parser.add_option("--rebuild-daily", action='store_true', - help="Rebuild the daily summaries from data in the archive table.") - parser.add_option("--reweight", action="store_true", - help="Recalculate the weighted sums in the daily summaries.") - - # ... then add the various options: - parser.add_option("--config", dest="config_path", type=str, - metavar="CONFIG_FILE", - help="Use configuration file CONFIG_FILE.") - parser.add_option("--date", type=str, metavar="YYYY-mm-dd", - help="This date only (options --calc-missing and --rebuild-daily only).") - parser.add_option("--from", dest="from_date", type=str, metavar="YYYY-mm-dd[THH:MM]", - help="Start with this date or date-time" - " (options --calc-missing and --rebuild-daily only).") - parser.add_option("--to", dest="to_date", type=str, metavar="YYYY-mm-dd[THH:MM]", - help="End with this date or date-time" - " (options --calc-missing and --rebuild-daily only).") - parser.add_option("--binding", metavar="BINDING_NAME", default='wx_binding', - help="The data binding to use. Default is 'wx_binding'.") - parser.add_option("--dest-binding", metavar="BINDING_NAME", - help="The destination data binding (option --transfer only).") - parser.add_option('--dry-run', action='store_true', - default=False, - help="Print what would happen but do not do it. Default is False.") - - # Now we are ready to parse the command line: - options, args = parser.parse_args() - - if len(args) > 1: - print("wee_database takes at most a single argument (the path to the configuration file).", - file=sys.stderr) - print("You have %d: %s." % (len(args), args), file=sys.stderr) - sys.exit(2) - - # Do a check to see if the user used more than 1 'verb' - if sum(x is not None for x in [options.create, - options.reconfigure, - options.transfer, - options.add_column, - options.rename_column, - options.drop_columns, - options.check, - options.update, - options.calc_missing, - options.check_strings, - options.fix_strings, - options.drop_daily, - options.rebuild_daily, - options.reweight - ]) != 1: - sys.exit("Must specify one and only one verb.") - - # Check that the various options satisfy our rules - - if options.type and not options.add_column: - sys.exit("Option --type can only be used with option --add-column") - - if options.to_name and not options.rename_column: - sys.exit("Option --to-name can only be used with option --rename-column") - - if options.rename_column and not options.to_name: - sys.exit("Option --rename-column requires option --to-name") - - if options.date and not (options.calc_missing or options.rebuild_daily - or options.reweight): - sys.exit("Option --date can only be used with options " - "--calc-missing, --rebuild-daily, or --reweight") - - if options.from_date and not (options.calc_missing or options.rebuild_daily - or options.reweight): - sys.exit("Option --from can only be used with options " - "--calc-missing, --rebuild-daily, or --reweight") - - if options.to_date and not (options.calc_missing or options.rebuild_daily - or options.reweight): - sys.exit("Option --to can only be used with options " - "--calc-missing, --rebuild-daily, or --reweight") - - if options.dest_binding and not options.transfer: - sys.exit("Option --dest-binding can only be used with option --transfer") - - # get config_dict to use - config_path, config_dict = weecfg.read_config(options.config_path, args) - print("Using configuration file %s" % config_path) - - # Set weewx.debug as necessary: - weewx.debug = to_int(config_dict.get('debug', 0)) - - # Customize the logging with user settings. - weeutil.logger.setup('wee_database', config_dict) - - # Add the 'user' package to PYTHONPATH - weewx.add_user_path(config_dict) - # Now we can import user.extensions - importlib.import_module('user.extensions') - - db_binding = options.binding - # Get the db name, be prepared to catch the error if the binding does not - # exist - try: - database = config_dict['DataBindings'][db_binding]['database'] - except KeyError: - # Couldn't find the database name, maybe the binding does not exist. - # Notify the user and exit. - print("Error obtaining database name from binding '%s'" % db_binding, file=sys.stderr) - sys.exit("Perhaps you need to specify a different binding using --binding") - else: - print("Using database binding '%s', which is bound to database '%s'" % (db_binding, - database)) - if options.dry_run and not (options.check or options.check_strings): - print("This is a dry run. Nothing will actually be done.") - - if options.create: - createMainDatabase(config_dict, db_binding, options.dry_run) - - elif options.reconfigure: - reconfigMainDatabase(config_dict, db_binding, options.dry_run) - - elif options.transfer: - transferDatabase(config_dict, db_binding, options) - - elif options.add_column: - addColumn(config_dict, db_binding, options.add_column, options.type, options.dry_run) - - elif options.rename_column: - renameColumn(config_dict, db_binding, options.rename_column, options.to_name, - options.dry_run) - - elif options.drop_columns: - dropColumns(config_dict, db_binding, options.drop_columns, options.dry_run) - - elif options.check: - check(config_dict, db_binding, options) - - elif options.update: - update(config_dict, db_binding, options) - - elif options.calc_missing: - calc_missing(config_dict, db_binding, options) - - elif options.check_strings: - check_strings(config_dict, db_binding, options, fix=False) - - elif options.fix_strings: - check_strings(config_dict, db_binding, options, fix=True) - - elif options.drop_daily: - dropDaily(config_dict, db_binding, options.dry_run) - - elif options.rebuild_daily: - rebuildDaily(config_dict, db_binding, options) - - elif options.reweight: - reweight(config_dict, db_binding, options) - - if options.dry_run and not (options.check or options.check_strings): - print("This was a dry run. Nothing was actually done.") - - -def createMainDatabase(config_dict, db_binding, dry_run=False): - """Create the WeeWX database""" - - # Try a simple open. If it succeeds, that means the database - # exists and is initialized. Otherwise, an exception will be thrown. - try: - with weewx.manager.open_manager_with_config(config_dict, db_binding) as dbmanager: - print("Database '%s' already exists. Nothing done." % dbmanager.database_name) - except weedb.OperationalError: - if not dry_run: - # Database does not exist. Try again, but allow initialization: - with weewx.manager.open_manager_with_config(config_dict, - db_binding, initialize=True) as dbmanager: - print("Created database '%s'" % dbmanager.database_name) - - -def dropDaily(config_dict, db_binding, dry_run): - """Drop the daily summaries from a WeeWX database""" - - manager_dict = weewx.manager.get_manager_dict_from_config(config_dict, db_binding) - database_name = manager_dict['database_dict']['database_name'] - - print("Proceeding will delete all your daily summaries from database '%s'" % database_name) - ans = y_or_n("Are you sure you want to proceed (y/n)? ") - if ans == 'y': - t1 = time.time() - print("Dropping daily summary tables from '%s' ... " % database_name) - try: - with weewx.manager.open_manager_with_config(config_dict, db_binding) as dbmanager: - try: - if not dry_run: - dbmanager.drop_daily() - except weedb.OperationalError as e: - print("Error '%s'" % e, file=sys.stderr) - print("Drop daily summary tables failed for database '%s'" % database_name) - else: - tdiff = time.time() - t1 - print("Daily summary tables dropped from " - "database '%s' in %.2f seconds" % (database_name, tdiff)) - except weedb.OperationalError: - # No daily summaries. Nothing to be done. - print("No daily summaries found in database '%s'. Nothing done." % database_name) - else: - print("Nothing done.") - - -def rebuildDaily(config_dict, db_binding, options): - """Rebuild the daily summaries.""" - - manager_dict = weewx.manager.get_manager_dict_from_config(config_dict, - db_binding) - database_name = manager_dict['database_dict']['database_name'] - - # get the first and last good timestamps from the archive, these represent - # our bounds for rebuilding - with weewx.manager.Manager.open(manager_dict['database_dict']) as dbmanager: - first_ts = dbmanager.firstGoodStamp() - first_d = datetime.date.fromtimestamp(first_ts) if first_ts is not None else None - last_ts = dbmanager.lastGoodStamp() - last_d = datetime.date.fromtimestamp(last_ts) if first_ts is not None else None - # determine the period over which we are rebuilding from any command line - # date parameters - from_dt, to_dt = _parse_dates(options) - # we have start and stop datetime objects but we work on whole days only, - # so need date object - from_d = from_dt.date() if from_dt is not None else None - to_d = to_dt.date() if to_dt is not None else None - # advise the user/log what we will do - if from_d is None and to_d is None: - _msg = "All daily summaries will be rebuilt." - elif from_d and not to_d: - _msg = "Daily summaries from %s through the end (%s) will be rebuilt." % (from_d, - last_d) - elif not from_d and to_d: - _msg = "Daily summaries from the beginning (%s) through %s will be rebuilt." % (first_d, - to_d) - elif from_d == to_d: - _msg = "Daily summary for %s will be rebuilt." % from_d - else: - _msg = "Daily summaries from %s through %s inclusive will be rebuilt." % (from_d, - to_d) - log.info(_msg) - print(_msg) - ans = y_or_n("Proceed (y/n)? ") - if ans == 'n': - log.info("Nothing done.") - print("Nothing done.") - return - - t1 = time.time() - - # Open up the database. This will create the tables necessary for the daily - # summaries if they don't already exist: - with weewx.manager.open_manager_with_config(config_dict, - db_binding, initialize=True) as dbmanager: - - log.info("Rebuilding daily summaries in database '%s' ..." % database_name) - print("Rebuilding daily summaries in database '%s' ..." % database_name) - if options.dry_run: - return - else: - # now do the actual rebuild - nrecs, ndays = dbmanager.backfill_day_summary(start_d=from_d, - stop_d=to_d, - trans_days=20) - tdiff = time.time() - t1 - # advise the user/log what we did - log.info("Rebuild of daily summaries in database '%s' complete" % database_name) - if nrecs: - sys.stdout.flush() - # fix a bit of formatting inconsistency if less than 1000 records - # processed - if nrecs >= 1000: - print() - if ndays == 1: - _msg = "Processed %d records to rebuild 1 daily summary in %.2f seconds" % (nrecs, - tdiff) - else: - _msg = ("Processed %d records to rebuild %d daily summaries in %.2f seconds" % (nrecs, - ndays, - tdiff)) - print(_msg) - print("Rebuild of daily summaries in database '%s' complete" % database_name) - else: - print("Daily summaries up to date in '%s'" % database_name) - - -def reweight(config_dict, db_binding, options): - """Recalculate the weighted sums in the daily summaries.""" - - # Determine the period over which we are rebuilding from any command line date parameters - from_dt, to_dt = _parse_dates(options) - # Convert from Datetime to Date objects - from_d = from_dt.date() if from_dt is not None else None - to_d = to_dt.date() if to_dt is not None else None - - # advise the user/log what we will do - if from_d is None and to_d is None: - msg = "The weighted sums in all the daily summaries will be recalculated." - elif from_d and not to_d: - msg = "The weighted sums in the daily summaries from %s through the end " \ - "will be recalculated." % from_d - elif not from_d and to_d: - msg = "The weighted sums in the daily summaries from the beginning through %s" \ - "will be recalculated." % to_d - elif from_d == to_d: - msg = "The weighted sums in the daily summary for %s will be recalculated." % from_d - else: - msg = "The weighted sums in the daily summaries from %s through %s, " \ - "inclusive, will be recalculated." % (from_d, to_d) - - log.info(msg) - print(msg) - ans = y_or_n("Proceed (y/n)? ") - if ans == 'n': - log.info("Nothing done.") - print("Nothing done.") - return - - t1 = time.time() - - # Open up the database. - manager_dict = weewx.manager.get_manager_dict_from_config(config_dict, db_binding) - database_name = manager_dict['database_dict']['database_name'] - with weewx.manager.open_manager_with_config(config_dict, db_binding) as dbmanager: - - log.info("Recalculating the weighted summaries in database '%s' ..." % database_name) - print("Recalculating the weighted summaries in database '%s' ..." % database_name) - if not options.dry_run: - # Do the actual recalculations - dbmanager.recalculate_weights(start_d=from_d, stop_d=to_d) - - msg = "Finished reweighting in %.1f seconds." % (time.time() - t1) - log.info(msg) - print(msg) - - -def reconfigMainDatabase(config_dict, db_binding, dry_run): - """Create a new database, then populate it with the contents of an old database""" - - manager_dict = weewx.manager.get_manager_dict_from_config(config_dict, - db_binding) - # Make a copy for the new database (we will be modifying it) - new_database_dict = dict(manager_dict['database_dict']) - - # Now modify the database name - new_database_dict['database_name'] = manager_dict['database_dict']['database_name'] + '_new' - - # First check and see if the new database already exists. If it does, check - # with the user whether it's ok to delete it. - try: - if not dry_run: - weedb.create(new_database_dict) - except weedb.DatabaseExists: - ans = y_or_n("New database '%s' already exists. " - "Delete it first (y/n)? " % new_database_dict['database_name']) - if ans == 'y': - weedb.drop(new_database_dict) - else: - print("Nothing done.") - return - - # Get the unit system of the old archive: - with weewx.manager.Manager.open(manager_dict['database_dict']) as old_dbmanager: - old_unit_system = old_dbmanager.std_unit_system - - if old_unit_system is None: - print("Old database has not been initialized. Nothing to be done.") - return - - # Get the unit system of the new archive: - try: - target_unit_nickname = config_dict['StdConvert']['target_unit'] - except KeyError: - target_unit_system = None - else: - target_unit_system = weewx.units.unit_constants[target_unit_nickname.upper()] - - print("Copying database '%s' to '%s'" % (manager_dict['database_dict']['database_name'], - new_database_dict['database_name'])) - if target_unit_system is None or old_unit_system == target_unit_system: - print("The new database will use the same unit system as the old ('%s')." % - weewx.units.unit_nicknames[old_unit_system]) - else: - print("Units will be converted from the '%s' system to the '%s' system." % - (weewx.units.unit_nicknames[old_unit_system], - weewx.units.unit_nicknames[target_unit_system])) - - ans = y_or_n("Are you sure you wish to proceed (y/n)? ") - if ans == 'y': - t1 = time.time() - weewx.manager.reconfig(manager_dict['database_dict'], - new_database_dict, - new_unit_system=target_unit_system, - new_schema=manager_dict['schema'], - dry_run=dry_run) - tdiff = time.time() - t1 - print("Database '%s' copied to '%s' in %.2f seconds." - % (manager_dict['database_dict']['database_name'], - new_database_dict['database_name'], - tdiff)) - else: - print("Nothing done.") - - -def transferDatabase(config_dict, db_binding, options): - """Transfer 'archive' data from one database to another""" - - # do we have enough to go on, must have a dest binding - if not options.dest_binding: - print("Destination binding not specified. Nothing Done. Aborting.", file=sys.stderr) - return - # get manager dict for our source binding - src_manager_dict = weewx.manager.get_manager_dict_from_config(config_dict, - db_binding) - # get manager dict for our dest binding - try: - dest_manager_dict = weewx.manager.get_manager_dict_from_config(config_dict, - options.dest_binding) - except weewx.UnknownBinding: - # if we can't find the binding display a message then return - print("Unknown destination binding '%s', confirm destination binding." - % options.dest_binding, file=sys.stderr) - print("Nothing Done. Aborting.", file=sys.stderr) - return - except weewx.UnknownDatabase: - # if we can't find the database display a message then return - print("Error accessing destination database, " - "confirm destination binding and/or database.", file=sys.stderr) - print("Nothing Done. Aborting.", file=sys.stderr) - return - except (ValueError, AttributeError): - # maybe a schema issue - print("Error accessing destination database.", file=sys.stderr) - print("Maybe the destination schema is incorrectly specified " - "in binding '%s' in weewx.conf?" % options.dest_binding, file=sys.stderr) - print("Nothing Done. Aborting.", file=sys.stderr) - return - except weewx.UnknownDatabaseType: - # maybe a [Databases] issue - print("Error accessing destination database.", file=sys.stderr) - print("Maybe the destination database is incorrectly defined in weewx.conf?", - file=sys.stderr) - print("Nothing Done. Aborting.", file=sys.stderr) - return - # get a manager for our source - with weewx.manager.Manager.open(src_manager_dict['database_dict']) as src_manager: - # How many source records? - num_recs = src_manager.getSql("SELECT COUNT(dateTime) from %s;" - % src_manager.table_name)[0] - if not num_recs: - # we have no source records to transfer so abort with a message - print("No records found in source database '%s'." - % src_manager.database_name) - print("Nothing done. Aborting.") - return - - if not options.dry_run: # is it a dry run ? - # not a dry run, actually do the transfer - ans = y_or_n("Transfer %s records from source database '%s' " - "to destination database '%s' (y/n)? " - % (num_recs, src_manager.database_name, - dest_manager_dict['database_dict']['database_name'])) - if ans == 'y': - t1 = time.time() - # wrap in a try..except in case we have an error - try: - with weewx.manager.Manager.open_with_create( - dest_manager_dict['database_dict'], - table_name=dest_manager_dict['table_name'], - schema=dest_manager_dict['schema']) as dest_manager: - print("Transferring, this may take a while.... ") - sys.stdout.flush() - - # This could generate a *lot* of log entries. Temporarily disable logging - # for events at or below INFO - logging.disable(logging.INFO) - - # do the transfer, should be quick as it's done as a - # single transaction - nrecs = dest_manager.addRecord(src_manager.genBatchRecords(), - progress_fn=weewx.manager.show_progress) - - # Remove the temporary restriction - logging.disable(logging.NOTSET) - - tdiff = time.time() - t1 - print("\nCompleted.") - if nrecs: - print("%s records transferred from source database '%s' to " - "destination database '%s' in %.2f seconds." - % (nrecs, src_manager.database_name, - dest_manager.database_name, tdiff)) - else: - print("Error. No records were transferred from source " - "database '%s' to destination database '%s'." - % (src_manager.database_name, dest_manager.database_name), - file=sys.stderr) - except ImportError: - # Probably when trying to load db driver - print("Error accessing destination database '%s'." - % (dest_manager_dict['database_dict']['database_name'],), - file=sys.stderr) - print("Nothing done. Aborting.", file=sys.stderr) - raise - except (OSError, weedb.OperationalError): - # probably a weewx.conf typo or MySQL db not created - print("Error accessing destination database '%s'." - % dest_manager_dict['database_dict']['database_name'], file=sys.stderr) - print("Maybe it does not exist (MySQL) or is incorrectly " - "defined in weewx.conf?", file=sys.stderr) - print("Nothing done. Aborting.", file=sys.stderr) - return - - else: - # we decided not to do the transfer - print("Nothing done.") - return - else: - # it's a dry run so say what we would have done then return - print("Transfer %s records from source database '%s' " - "to destination database '%s'." - % (num_recs, src_manager.database_name, - dest_manager_dict['database_dict']['database_name'])) - - -def addColumn(config_dict, db_binding, column_name, column_type, dry_run=False): - """Add a single column to the database. - column_name: The name of the new column. - column_type: The type ("REAL"|"INTEGER|) of the new column. - """ - column_type = column_type or 'REAL' - ans = y_or_n( - "Add new column '%s' of type '%s' to database (y/n)? " % (column_name, column_type)) - if ans == 'y': - dbm = weewx.manager.open_manager_with_config(config_dict, db_binding) - if not dry_run: - dbm.add_column(column_name, column_type) - print(f'New column {column_name} of type {column_type} added to database.') - else: - print("Nothing done.") - - -def renameColumn(config_dict, db_binding, old_column_name, new_column_name, dry_run=False): - """Rename a column in the database. """ - ans = y_or_n("Rename column '%s' to '%s' (y/n)? " % (old_column_name, new_column_name)) - if ans == 'y': - dbm = weewx.manager.open_manager_with_config(config_dict, db_binding) - if not dry_run: - dbm.rename_column(old_column_name, new_column_name) - print("Column '%s' renamed to '%s'." % (old_column_name, new_column_name)) - else: - print("Nothing done.") - - -def dropColumns(config_dict, db_binding, drop_columns, dry_run=False): - """Drop a set of columns from the database""" - drop_list = drop_columns.split(',') - # In case the user ended the list of columns to be dropped with a comma, search for an - # empty column name - try: - drop_list.remove('') - except ValueError: - pass - ans = y_or_n("Drop column(s) '%s' from the database (y/n)? " % ", ".join(drop_list)) - if ans == 'y': - drop_set = set(drop_list) - dbm = weewx.manager.open_manager_with_config(config_dict, db_binding) - # Now drop the columns. If one is missing, a NoColumnError will be raised. Be prepared - # to catch it. - try: - print("This may take a while...") - if not dry_run: - dbm.drop_columns(drop_set) - except weedb.NoColumnError as e: - print(e, file=sys.stderr) - print("Nothing done.") - else: - print("Column(s) '%s' dropped from the database" % ", ".join(drop_list)) - else: - print("Nothing done.") - - -def check(config_dict, db_binding, options): - """Check database and report outstanding fixes/issues. - - Performs the following checks: - - checks database version - - checks for null strings in SQLite database - """ - - t1 = time.time() - - # Check interval weighting - print("Checking daily summary tables version...") - - # Get a database manager object - dbm = weewx.manager.open_manager_with_config(config_dict, db_binding) - - # check the daily summary version - _daily_summary_version = dbm._read_metadata('Version') - msg = "Daily summary tables are at version %s" % _daily_summary_version - log.info(msg) - print(msg) - - if _daily_summary_version is not None and _daily_summary_version >= '2.0': - # interval weighting fix has been applied - msg = "Interval Weighting Fix is not required." - log.info(msg) - print(msg) - else: - print("Recommend running --update to recalculate interval weightings.") - print("Daily summary tables version check completed in %0.2f seconds." % (time.time() - t1)) - - # now check for null strings - check_strings(config_dict, db_binding, options, fix=False) - - -def update(config_dict, db_binding, options): - """Apply any required database fixes. - - Applies the following fixes: - - checks if database version is 3.0, if not interval weighting fix is - applied - - recalculates windSpeed daily summary max and maxtime fields from - archive - """ - - # prompt for confirmation if it is not a dry run - ans = 'y' if options.dry_run else None - while ans not in ['y', 'n']: - ans = input("The update process does not affect archive data, " - "but does alter the database.\nContinue (y/n)? ") - if ans == 'n': - # we decided not to update the summary tables - msg = "Update cancelled" - log.info(msg) - print(msg) - return - - log.info("Preparing interval weighting fix...") - print("Preparing interval weighting fix...") - - # Get a database manager object - dbm = weewx.manager.open_manager_with_config(config_dict, db_binding) - - # check the daily summary version - msg = "Daily summary tables are at version %s" % dbm.version - log.info(msg) - print(msg) - - if dbm.version is not None and dbm.version >= '4.0': - # interval weighting fix has been applied - msg = "Interval weighting fix is not required." - log.info(msg) - print(msg) - else: - # apply the interval weighting - msg = "Calculating interval weights..." - log.info(msg) - print(msg) - t1 = time.time() - if not options.dry_run: - dbm.update() - msg = "Interval Weighting Fix completed in %0.2f seconds." % (time.time() - t1) - log.info(msg) - print(msg) - - # recalc the max/maxtime windSpeed values - _fix_wind(config_dict, db_binding, options) - - -def calc_missing(config_dict, db_binding, options): - """Calculate any missing derived observations and save to database.""" - - msg = "Preparing to calculate missing derived observations..." - log.info(msg) - - # get a db manager dict given the config dict and binding - manager_dict = weewx.manager.get_manager_dict_from_config(config_dict, - db_binding) - # Get the table_name used by the binding, it could be different to the - # default 'archive'. If for some reason it is not specified then fail hard. - table_name = manager_dict['table_name'] - # get the first and last good timestamps from the archive, these represent - # our overall bounds for calculating missing derived obs - with weewx.manager.Manager.open(manager_dict['database_dict'], - table_name=table_name) as dbmanager: - first_ts = dbmanager.firstGoodStamp() - last_ts = dbmanager.lastGoodStamp() - # process any command line options that may limit the period over which - # missing derived obs are calculated - start_dt, stop_dt = _parse_dates(options) - # we now have a start and stop date for processing, we need to obtain those - # as epoch timestamps, if we have no start and/or stop date then use the - # first or last good timestamp instead - start_ts = time.mktime(start_dt.timetuple()) if start_dt is not None else first_ts - 1 - stop_ts = time.mktime(stop_dt.timetuple()) if stop_dt is not None else last_ts - # notify if this is a dry run - if options.dry_run: - msg = "This is a dry run, missing derived observations will be calculated but not saved" - log.info(msg) - print(msg) - _head = "Missing derived observations will be calculated " - # advise the user/log what we will do - if start_dt is None and stop_dt is None: - _tail = "for all records." - elif start_dt and not stop_dt: - _tail = "from %s through to the end (%s)." % (timestamp_to_string(start_ts), - timestamp_to_string(stop_ts)) - elif not start_dt and stop_dt: - _tail = "from the beginning (%s) through to %s." % (timestamp_to_string(start_ts), - timestamp_to_string(stop_ts)) - else: - _tail = "from %s through to %s inclusive." % (timestamp_to_string(start_ts), - timestamp_to_string(stop_ts)) - _msg = "%s%s" % (_head, _tail) - log.info(_msg) - print(_msg) - ans = y_or_n("Proceed (y/n)? ") - if ans == 'n': - _msg = "Nothing done." - log.info(_msg) - print(_msg) - return - - t1 = time.time() - - # construct a CalcMissing config dict - calc_missing_config_dict = {'name': 'Calculate Missing Derived Observations', - 'binding': db_binding, - 'start_ts': start_ts, - 'stop_ts': stop_ts, - 'trans_days': 20, - 'dry_run': options.dry_run} - - # obtain a CalcMissing object - calc_missing_obj = weecfg.database.CalcMissing(config_dict, - calc_missing_config_dict) - log.info("Calculating missing derived observations...") - print("Calculating missing derived observations...") - # Calculate and store any missing observations. Be prepared to - # catch any exceptions from CalcMissing. - try: - calc_missing_obj.run() - except weewx.UnknownBinding as e: - # We have an unknown binding, this could occur if we are using a - # non-default binding and StdWXCalculate has not been told (via - # weewx.conf) to use the same binding. Log it and notify the user then - # exit. - _msg = "Error: '%s'" % e - print(_msg) - log.error(_msg) - print("Perhaps StdWXCalculate is using a different binding. Check " - "configuration file [StdWXCalculate] stanza") - sys.exit("Nothing done. Aborting.") - else: - msg = "Missing derived observations calculated in %0.2f seconds" % (time.time() - t1) - log.info(msg) - print(msg) - - -def _fix_wind(config_dict, db_binding, options): - """Recalculate the windSpeed daily summary max and maxtime fields. - - Create a WindSpeedRecalculation object and call its run() method to - recalculate the max and maxtime fields from archive data. This process is - idempotent so it can be called repeatedly with no ill effect. - """ - - t1 = time.time() - msg = "Preparing maximum windSpeed fix..." - log.info(msg) - print(msg) - - # notify if this is a dry run - if options.dry_run: - print("This is a dry run: maximum windSpeed will be calculated but not saved.") - - # construct a windSpeed recalculation config dict - wind_config_dict = {'name': 'Maximum windSpeed fix', - 'binding': db_binding, - 'trans_days': 100, - 'dry_run': options.dry_run} - - # create a windSpeedRecalculation object - wind_obj = weecfg.database.WindSpeedRecalculation(config_dict, - wind_config_dict) - # perform the recalculation, wrap in a try..except to catch any db errors - try: - wind_obj.run() - except weedb.NoTableError: - msg = "Maximum windSpeed fix applied: no windSpeed found" - log.info(msg) - print(msg) - else: - msg = "Maximum windSpeed fix completed in %0.2f seconds" % (time.time() - t1) - log.info(msg) - print(msg) - - -# These functions are necessary because Python 3 does not allow you to -# parameterize types. So, we use a big if-else. - -def check_type(val, expected): - if expected == 'INTEGER': - return isinstance(val, int) - elif expected == 'REAL': - return isinstance(val, float) - elif expected == 'STR' or expected == 'TEXT': - return isinstance(val, str) - else: - raise ValueError("Unknown type %s" % expected) - - -def set_type(val, target): - if target == 'INTEGER': - return int(val) - elif target == 'REAL': - return float(val) - elif target == 'STR' or target == 'TEXT': - if isinstance(val, bytes): - return val.decode('utf-8') - return str(val) - else: - raise ValueError("Unknown type %s" % target) - - -def check_strings(config_dict, db_binding, options, fix=False): - """Scan the archive table for null strings. - - Identifies and lists any null string occurrences in the archive table. If - fix is True then any null strings that are found are fixed. - """ - - t1 = time.time() - if options.dry_run or not fix: - logging.disable(logging.INFO) - - print("Preparing Null String Fix, this may take a while...") - - if fix: - log.info("Preparing Null String Fix") - # notify if this is a dry run - if options.dry_run: - print("This is a dry run: null strings will be detected but not fixed") - - # open up the main database archive table - with weewx.manager.open_manager_with_config(config_dict, db_binding) as dbmanager: - - obs_list = [] - obs_type_list = [] - - # get the schema and extract the Python type each observation type should be - for column in dbmanager.connection.genSchemaOf('archive'): - # Save the observation name for this column (eg, 'outTemp'): - obs_list.append(column[1]) - # And its type - obs_type_list.append(column[2]) - - records = 0 - _found = [] - # cycle through each row in the database - for record in dbmanager.genBatchRows(): - records += 1 - # now examine each column - for icol in range(len(record)): - # check to see if this column is an instance of the correct - # Python type - if record[icol] is not None and not check_type(record[icol], obs_type_list[icol]): - # Oops. Found a bad one. Print it out. - if fix: - log.info("Timestamp = %s; record['%s']= %r; ... " - % (record[0], obs_list[icol], record[icol])) - - if fix: - # coerce to the correct type. If it can't be done, then - # set it to None. - try: - corrected_value = set_type(record[icol], obs_type_list[icol]) - except ValueError: - corrected_value = None - # update the database with the new value but only if - # it's not a dry run - if not options.dry_run: - dbmanager.updateValue(record[0], obs_list[icol], corrected_value) - _found.append((record[0], obs_list[icol], record[icol], corrected_value)) - # log it - log.info(" changed to %r\n" % corrected_value) - else: - _found.append((record[0], obs_list[icol], record[icol])) - # notify the user of progress - if records % 1000 == 0: - print("Checking record: %d; Timestamp: %s\r" - % (records, timestamp_to_string(record[0])), end=' ') - sys.stdout.flush() - # update our final count now that we have finished - if records: - print("Checking record: %d; Timestamp: %s\r" - % (records, timestamp_to_string(record[0])), end=' ') - print() - else: - print("No records in database.") - tdiff = time.time() - t1 - # now display details of what we found if we found any null strings - if len(_found): - print("The following null strings were found:") - for item in _found: - if len(item) == 4: - print("Timestamp = %s; record['%s'] = %r; ... changed to %r" % item) - else: - print("Timestamp = %s; record['%s'] = %r; ... ignored" % item) - # how many did we fix? - fixed = len([a for a in _found if len(a) == 4]) - # summarise our results - if len(_found) == 0: - # found no null strings, log it and display on screen - log.info("No null strings found.") - print("No null strings found.") - elif fixed == len(_found): - # fixed all we found - if options.dry_run: - # its a dry run so display to screen but not to log - print("%d of %d null strings found would have been fixed." % (fixed, len(_found))) - else: - # really did fix so log and display to screen - log.info("%d of %d null strings found were fixed." % (fixed, len(_found))) - print("%d of %d null strings found were fixed." % (fixed, len(_found))) - elif fix: - # this should never occur - found some but didn't fix them all when we - # should have - if options.dry_run: - # its a dry run so say what would have happened - print("Could not fix all null strings. " - "%d of %d null strings found would have been fixed." % (fixed, - len(_found))) - else: - # really did fix so log and display to screen - log.info("Could not fix all null strings. " - "%d of %d null strings found were fixed." % (fixed, - len(_found))) - print("Could not fix all null strings. " - "%d of %d null strings found were fixed." % (fixed, - len(_found))) - else: - # found some null string but it was only a check not a fix, just - # display to screen - print("%d null strings were found.\r\n" - "Recommend running --fix-strings to fix these strings." % len(_found)) - - # and finally details on time taken - if fix: - log.info("Applied Null String Fix in %0.2f seconds." % tdiff) - print("Applied Null String Fix in %0.2f seconds." % tdiff) - else: - # it was a check not a fix so just display to screen - print("Completed Null String Check in %0.2f seconds." % tdiff) - # just in case, set the syslog level back where we found it - if options.dry_run or not fix: - logging.disable(logging.NOTSET) - - -def _parse_dates(options): - """Parse --date, --from and --to command line options. - - Parses --date or --from and --to to determine a date-time span to be - used. --to and --from in the format y-m-dTHH:MM precisely define a - date-time but anything in the format y-m-d does not. When rebuilding - the daily summaries this imprecision is not import as we merely need a - date-time somewhere within the day being rebuilt. When calculating - missing fields we need date-times for the span over which the - calculations are to be performed. - - Inputs: - options: the optparse options - - Returns: A two-way tuple (from_dt, to_dt) representing the from and to - date-times derived from the --date or --to and --from command line - options where - from_dt: A datetime.datetime object holding the from date-time. May - be None - to_dt: A datetime.datetime object holding the to date-time. May be - None - """ - - # default is None, unless user has specified an option - _from_dt = None - _to_dt = None - - # first look for --date - if options.date: - # we have a --date option, make sure we are not over specified - if options.from_date or options.to_date: - raise ValueError("Specify either --date or a --from and --to combination; not both") - - # there is a --date but is it valid - try: - # this will give a datetime object representing midnight at the - # start of the day - _from_dt = datetime.datetime.strptime(options.date, "%Y-%m-%d") - except ValueError: - raise ValueError("Invalid --date option specified.") - else: - # we have the from date-time, for a --date option our final results - # depend on the action we are to perform - if options.rebuild_daily or options.reweight: - # The daily summaries are stamped with the midnight timestamp - # for each day, so our from and to results need to be within the - # same calendar day else we will rebuild more than just one day. - # For simplicity make them the same. - _to_dt = _from_dt - elif options.calc_missing: - # On the other hand calc missing will be dealing with archive - # records which are epoch timestamped. The midnight stamped - # record is part of the previous day so make our from result - # one second after midnight. THe to result must be midnight at - # the end of the day. - _to_dt = _from_dt + datetime.timedelta(days=1) - _from_dt = _from_dt + datetime.timedelta(seconds=1) - else: - # nothing else uses from and to (yet) but just in case return - # midnight to midnight as the default - _to_dt = _from_dt + datetime.timedelta(days=1) - # we have our results so we can return - return _from_dt, _to_dt - - # we don't have --date so now look for --from and --to - if options.from_date: - # we have a --from but is it valid - try: - if 'T' in options.from_date: - # we have a time so we can precisely determine a date-time - _from_dt = datetime.datetime.strptime(options.from_date, "%Y-%m-%dT%H:%M") - else: - # we have a date only, so use midnight at the start of the day - _from_dt = datetime.datetime.strptime(options.from_date, "%Y-%m-%d") - except ValueError: - raise ValueError("Invalid --from option specified.") - - if options.to_date: - # we have a --to but is it valid - try: - if 'T' in options.to_date: - # we have a time so decode and use that - _to_dt = datetime.datetime.strptime(options.to_date, "%Y-%m-%dT%H:%M") - else: - # we have a date, first obtain a datetime object for midnight - # at the start of the day specified - _to_dt = datetime.datetime.strptime(options.to_date, "%Y-%m-%d") - # since we have a date the result we want depends on what action - # we are to complete - if options.rebuild_daily or options.reweight: - # for a rebuild the to date-time must be within the date - # specified date, which it already is so leave it - pass - elif options.calc_missing: - # for calc missing we want midnight at the end of the day - _to_dt = _to_dt + datetime.timedelta(days=1) - else: - # nothing else uses from and to (yet) but just in case - # return midnight at the end of the day - _to_dt = _to_dt + datetime.timedelta(days=1) - except ValueError: - raise ValueError("Invalid --to option specified.") - - # if we have both from and to date-times make sure from is no later than to - if _from_dt and _to_dt and _to_dt < _from_dt: - raise weewx.ViolatedPrecondition("--from value is later than --to value.") - # we have our results so we can return - return _from_dt, _to_dt - - -if __name__ == "__main__": - main() diff --git a/bin/weectllib/database_cmd.py b/bin/weectllib/database_cmd.py index 8a85c690..364d8dbb 100644 --- a/bin/weectllib/database_cmd.py +++ b/bin/weectllib/database_cmd.py @@ -72,9 +72,8 @@ For example: """ reconfigure_description = """Create a new database using the current configuration information -found in the configuration file. This can be used to change the unit system of a -database. The new database will have the same name as the old database, with a '_new' -on the end.""" +found in the configuration file. This can be used to change the unit system of a database. The new + database will have the same name as the old database, except with a '_new' on the end.""" transfer_description = """Copy a database to a new database. The option "--dest-binding" should hold a database binding @@ -197,7 +196,9 @@ def add_subparser(subparsers): reconfigure_parser = action_parser.add_parser('reconfigure', description=reconfigure_description, usage=reconfigure_usage, - help="Reconfigure a database.", + help="Reconfigure a database, using the current " + "configuration information in the config " + "file.", epilog=epilog) _add_common_args(reconfigure_parser) reconfigure_parser.set_defaults(func=reconfigure_database) diff --git a/bin/weectllib/db_actions.py b/bin/weectllib/db_actions.py index 058f83ac..0c1c4b01 100644 --- a/bin/weectllib/db_actions.py +++ b/bin/weectllib/db_actions.py @@ -233,7 +233,8 @@ def reconfigure_database(config_path, db_binding='wx_binding', dry_run=False): """Create a new database, then populate it with the contents of an old database, but use - the current configuration options.""" + the current configuration options. The reconfigure action will create a new database with the + same name as the old, except with the suffix _new attached to the end.""" config_path, config_dict, database_name = _prepare(config_path, db_binding, dry_run) weectllib.initialize(config_dict) diff --git a/docs_src/custom/database.md b/docs_src/custom/database.md index f86eb419..d70dbd7e 100644 --- a/docs_src/custom/database.md +++ b/docs_src/custom/database.md @@ -143,91 +143,83 @@ schema instead of the default. Thereafter, WeeWX reads the schema directly from the database and your changes will have no effect! -## Modifying an existing database +## Modify the schema of an existing database -The previous section covers the case where you do not have an existing -database, so you modify a starting schema, then use it to initialize the -database. But, what if you already have a database, and you want to -modify it, perhaps by adding a column or two? You cannot create a new -starting schema, because it is only used when the database is first -created. Here is where the tool -[`wee_database`](../../utilities/utilities.htm#wee_database_utility) can be -useful. Be sure to stop WeeWX before attempting to use it. +The previous section covers the case where you do not have an existing database, +so you modify a starting schema, then use it to initialize the database. But, +what if you already have a database, and you want to modify its schema, perhaps +by adding a column or two? Creating a new starting schema is not going to work +because it is only used when the database is first created. Here is where the +command [`weectl database`](/utilities/weectl-database) can be useful. There are two ways to do this. Both are covered below. -1. Modify the database *in situ* by using the tool - `wee_database`. This choice works best for small changes. -2. Transfer the old database to a new one while modifying it along the - way, again by using the tool `wee_database`. This choice is - best for large modifications. +1. Modify the database *in situ*. This choice works best for small changes. +2. Reconfigure the old database to a new one while modifying it along the + way, This choice is best for large modifications. !!! Warning - Before using the tool `wee_database`, MAKE A BACKUP FIRST! + Before using `weectl database`, MAKE A BACKUP FIRST! -### Modify the database *in situ* {#add_archive_type} + Be sure to stop `weewxd` first. -If you want to make some minor modifications to an existing database, -perhaps adding or removing a column, then this can easily be done using -the tool `wee_database` with an appropriate option. We will cover -the cases of adding, removing, and renaming a type. See the -documentation for -[`wee_database`](../../utilities/utilities.htm#wee_database_utility) for more -details. +### Modify the database *in situ* -#### Adding a type +If you want to make some minor modifications to an existing database, perhaps +adding or removing a column, then this can easily be done using the command +`weectl database` with an appropriate action. We will cover the cases of +adding, removing, and renaming a type. See the documentation for [`weectl +database`](/utilities/weectl-database) for more details. -Suppose you have an existing database and you want to add a type, such -as the type `electricity` from the example -[*Adding a second data source*](../service-engine/#Adding_2nd_source). -This can be done in one easy -step using the tool `wee_database` with the option -`--add-column`: +#### Adding a type {#add_archive_type} + +Suppose you have an existing database, to which you want to add a type, such as +the type `electricity` from the example [*Adding a second data +source*](../service-engine/#Adding_2nd_source). This can be done in one easy +step using the action `weectl database add-column`: ``` shell -wee_database --add-column=electricity +weectl database add-column electricity ``` -The tool not only adds `electricity` to the main archive table, -but also to the daily summaries. +The tool not only adds `electricity` to the main archive table, but also to the +daily summaries. #### Removing a type {#remove_archive_type} -In a similar manner, the tool can remove any unneeded types from an -existing database. For example, suppose you are using the -`schemas.wview` schema, but you're pretty sure you're not going -to need to store soil moisture. You can drop the unnecessary types this -way: +In a similar manner, the tool can remove any unneeded types from an existing +database. For example, suppose you are using the `schemas.wview` schema, but +you're pretty sure you're not going to need to store soil moisture. You can drop +the unnecessary types this way: ``` shell -wee_database --drop-columns=soilMoist1,soilMoist2,soilMoist3,soilMoist4 +weectl database drop-columns soilMoist1 soilMoist2 soilMoist3 soilMoist4 ``` -Unlike the option `--add-column`, the option -`--drop-columns` can take more than one type. This is done in -the interest of efficiency: adding new columns is easy and fast with the -SQLite database, but dropping columns requires copying the whole -database. By specifying more than one type, you can amortize the cost -over a single invocation of the utility. +Unlike the action `add-column`, the action `drop-columns` can take more than one +type. This is done in the interest of efficiency: adding new columns is easy and +fast with the SQLite database, but dropping columns requires copying the whole +database. By specifying more than one type, you can amortize the cost over a +single invocation of the utility. !!! Warning Dropping types from a database means *you will lose any data associated with them!* The data cannot be recovered. -#### Renaming a type +#### Renaming a type {#rename_archive_type} -Suppose you just want to rename a type? This can be done using the -option `--to-name`. Here's an example where you rename -`soilMoist1` to `soilMoistGarden`: +Suppose you just want to rename a type? This can be done using the action +`rename-column`. Here's an example where you rename `soilMoist1` to +`soilMoistGarden`: ``` shell -wee_database --rename-column=soilMoist1 --to-name=soilMoistGarden +weectl database rename-column soilMoist1 soilMoistGarden ``` -Note how the option `--rename-column` also requires option -`--to-name`, which specifies the target name. +Note how the action `rename-column` requires _two_ positional arguments: +the column being renamed, and its final name. -### Transfer database using new schema {#transfer_database_using_new_schema} +### Reconfigure database using a new schema {#reconfigure_database_using_new_schema} If you are making major changes to your database, you may find it easier to create a brand-new database using the schema you want, then transfer @@ -242,9 +234,9 @@ Here is the general strategy to do this. 2. Specify this schema as the starting schema for the database. 3. Make sure you have the necessary permissions to create the new database. -4. Use the utility - [`wee_database`](../../utilities/utilities.htm#wee_database_utility) to - create the new database and populate it with data from the old +4. Use the action + [`weectl database reconfigure`](/utilities/weectl-database/#reconfigure-a-database) + to create the new database and populate it with data from the old database. 5. Shuffle databases around so WeeWX will use the new database. @@ -269,23 +261,22 @@ Here are the details: schema = user.myschema.schema ``` -3. **Check permissions.** The reconfiguration utility will create a new - database with the same name as the old, except with the suffix - `_new` attached to the end. Make sure you have the necessary - permissions to do this. In particular, if you are using MySQL, you - will need `CREATE` privileges. +3. **Check permissions.** The transfer action will create a new + database with the same name as the old, except with the suffix `_new` + attached to the end. Make sure you have the necessary permissions to do + this. In particular, if you are using MySQL, you will need `CREATE` + privileges. -4. **Create and populate the new database.** Use the utility - `wee_database` with the `--reconfigure` option. +4. **Create and populate the new database.** Use the command + `weectl database` with the `reconfigure` action. ``` shell - wee_database weewx.conf --reconfigure + weectl database reconfigure ``` - This will create a new database (nominally, `weewx.sdb_new` - if you are using SQLite, `weewx_new` if you are using MySQL), - using the schema found in `user.myschema.schema`, and - populate it with data from the old database. + This will create a new database (nominally, `weewx.sdb_new` if you are using + SQLite, `weewx_new` if you are using MySQL), using the schema found in + `user.myschema.schema`, and populate it with data from the old database. 5. **Shuffle the databases.** Now arrange things so WeeWX can find the new database. @@ -293,47 +284,46 @@ Here are the details: !!! Warning Make a backup of the data before doing any of the next steps! - You can either shuffle the databases around so the new database has - the same name as the old database, or edit `weewx.conf` to - use the new database name. To do the former: + You can either shuffle the databases around so the new database has the same + name as the old database, or edit `weewx.conf` to use the new database name. + To do the former: - For SQLite: + === "SQLite" - ``` shell - cd ~/weewx-data/archive - mv weewx.sdb_new weewx.sdb - ``` + ``` shell + cd ~/weewx-data/archive + mv weewx.sdb_new weewx.sdb + ``` - For MySQL: + === "MySQL" - ``` shell - mysql -u --password= - mysql> DROP DATABASE weewx; # Delete the old database - mysql> CREATE DATABASE weewx; # Create a new one with the same name - mysql> RENAME TABLE weewx_new.archive TO weewx.archive; # Rename to the nominal name - ``` + ``` shell + mysql -u --password= + mysql> DROP DATABASE weewx; # Delete the old database + mysql> CREATE DATABASE weewx; # Create a new one with the same name + mysql> RENAME TABLE weewx_new.archive TO weewx.archive; # Rename to the nominal name + ``` 6. It's worth noting that there's actually a hidden, last step: rebuilding the daily summaries inside the new database. This will be - done automatically by WeeWX at the next startup. Alternatively, it + done automatically by `weewxd` at the next startup. Alternatively, it can be done manually using the - [`wee_database`](../../utilities/utilities.htm#wee_database_utility)utility - and the `--rebuild-daily` option: + [`weectl database rebuild-daily`](/utilities/weectl-database/) action: ``` shell - wee_database --rebuild-daily + weectl database rebuild-daily ``` ## Changing the unit system in an existing database {#Changing_the_unit_system} -Normally, data are stored in the databases using US Customary units, and -you shouldn't care; it is an "implementation detail". Data can always -be displayed using any set of units you want — the section -[*Changing unit systems*](../custom_reports/#changing-unit-systems) explains how to change the -reporting units. Nevertheless, there may be special situations where you -wish to store the data in Metric units. For example, you may need to -allow direct programmatic access to the database from another piece of -software that expects metric units. +Normally, data are stored in the databases using US Customary units, and you +shouldn't care; it is an "implementation detail". Data can always be displayed +using any set of units you want — the section [*Changing unit +systems*](/custom/custom-reports/#changing-unit-systems) explains how to change +the reporting units. Nevertheless, there may be special situations where you +wish to store the data in Metric units. For example, you may need to allow +direct programmatic access to the database from another piece of software that +expects metric units. You should not change the database unit system midstream. That is, do not start with one unit system then, some time later, switch to another. @@ -343,47 +333,38 @@ WeeWX User's Guide. However, you can reconfigure the database by copying it to a new database, performing the unit conversion along the way. You then use this new database. -The general strategy is identical to the strategy outlined above in the -section [*Transfer database using new -schema*](#transfer_database_using_new_schema). The only difference is -that instead of specifying a new starting schema, you specify a -different database unit system. This means that instead of steps 1 and 2 -above, you edit the configuration file and change option -`target_unit` in section -[`[StdConvert]`](../../usersguide/weewx-config-file/stdconvert-config/) to reflect your -choice. For example, if you are switching to metric units, the option -will look like: +The general strategy is identical to the strategy outlined above in the section +[*Reconfigure database using new +schema*](#reconfigure_database_using_new_schema). The only difference is that +instead of specifying a new starting schema, you specify a different database +unit system. This means that instead of steps 1 and 2 above, you edit the +configuration file and change option `target_unit` in section +[`[StdConvert]`](../../usersguide/weewx-config-file/stdconvert-config/) to +reflect your choice. For example, if you are switching to metric units, the +option will look like: ``` ini [StdConvert] target_unit = METRICWX ``` -After changing `target_unit`, you then go ahead with the rest of -the steps. That is run `wee_database` with the -`--reconfigure` option, then shuffle the databases. +After changing `target_unit`, you then go ahead with the rest of the steps. That +is run the action `weectl database reconfigure`, then shuffle the databases. ## Rebuilding the daily summaries -The `wee_database` utility can also be used to rebuild the daily +The `weectl database` command can also be used to rebuild the daily summaries: ``` shell -wee_database weewx.conf --rebuild-daily +weectl database rebuild-daily ``` -In most cases this will be sufficient; however, if anomalies remain in -the daily summaries the daily summary tables may be dropped first before -rebuilding: +In most cases this will be sufficient; however, if anomalies remain in the daily +summaries the daily summary tables may be dropped first before rebuilding: ``` shell -wee_database weewx.conf --drop-daily -``` - -The summaries will automatically be rebuilt the next time WeeWX starts, -or they can be rebuilt with the utility: - -``` shell -wee_database weewx.conf --rebuild-daily +weectl database drop-daily ``` +Then try again with `weectl database rebuild-daily`. diff --git a/docs_src/utilities/wee_database.md b/docs_src/utilities/wee_database.md deleted file mode 100644 index d42fd327..00000000 --- a/docs_src/utilities/wee_database.md +++ /dev/null @@ -1,355 +0,0 @@ -# wee_database - -The database utility simplifies typical database maintenance operations. For -example, it can rebuild the daily summaries or check a SQLite database for -embedded strings where floats are expected. - -Specify `--help` to see how it is used: -``` -wee_database --help -``` -``` -Usage: wee_database --help - wee_database --create - wee_database --reconfigure - wee_database --transfer --dest-binding=BINDING_NAME [--dry-run] - wee_database --add-column=NAME [--type=(REAL|INTEGER)] - wee_database --rename-column=NAME --to-name=NEW_NAME - wee_database --drop-columns=NAME1,NAME2,... - wee_database --check - wee_database --update [--dry-run] - wee_database --drop-daily - wee_database --rebuild-daily [--date=YYYY-mm-dd | - [--from=YYYY-mm-dd] [--to=YYYY-mm-dd]] - [--dry-run] - wee_database --reweight [--date=YYYY-mm-dd | - [--from=YYYY-mm-dd] [--to=YYYY-mm-dd]] - [--dry-run] - wee_database --calc-missing [--date=YYYY-mm-dd | - [--from=YYYY-mm-dd[THH:MM]] [--to=YYYY-mm-dd[THH:MM]]] - wee_database --check-strings - wee_database --fix-strings [--dry-run] - -Description: - -Manipulate the WeeWX database. Most of these operations are handled -automatically by WeeWX, but they may be useful in special cases. - -Options: - -h, --help show this help message and exit - --create Create the WeeWX database and initialize it with the - schema. - --reconfigure Create a new database using configuration information - found in the configuration file. The new database will - have the same name as the old database, with a '_new' - on the end. - --transfer Transfer the WeeWX archive from source database to - destination database. - --add-column=NAME Add new column NAME to database. - --type=TYPE New database column type (INTEGER|REAL) (option --add- - column only). Default is 'REAL'. - --rename-column=NAME Rename the column with name NAME. - --to-name=NEW_NAME New name of the column (option --rename-column only). - --drop-columns=NAME1,NAME2,... - Drop one or more columns. Names must be separated by - commas, with NO SPACES. - --check Check the calculations in the daily summary tables. - --update Update the daily summary tables if required and - recalculate the daily summary maximum windSpeed - values. - --calc-missing Calculate and store any missing derived observations. - --check-strings Check the archive table for null strings that may have - been introduced by a SQL editing program. - --fix-strings Fix any null strings in a SQLite database. - --drop-daily Drop the daily summary tables from a database. - --rebuild-daily Rebuild the daily summaries from data in the archive - table. - --reweight Recalculate the weighted sums in the daily summaries. - --config=CONFIG_FILE Use configuration file CONFIG_FILE. - --date=YYYY-mm-dd This date only (options --calc-missing and --rebuild- - daily only). - --from=YYYY-mm-dd[THH:MM] - Start with this date or date-time (options --calc- - missing and --rebuild-daily only). - --to=YYYY-mm-dd[THH:MM] - End with this date or date-time (options --calc- - missing and --rebuild-daily only). - --binding=BINDING_NAME - The data binding to use. Default is 'wx_binding'. - --dest-binding=BINDING_NAME - The destination data binding (option --transfer only). - --dry-run Print what would happen but do not do it. Default is - False. -``` - -### `--create` - -If the database does not already exist, this action will create it and -initialize it with the schema specified in the WeeWX configuration file. -Because WeeWX does this automatically, this action is rarely needed. - -``` -wee_database --create -``` - -### `--reconfigure` - -This action is useful for changing the schema or unit system in your database. - -It creates a new database with the same name as the old, except with the suffix -`_new` attached at the end (nominally, `weewx.sdb_new` if you are using SQLite, -`weewx_new` if you are using MySQL). It then initializes it with the schema -specified in `weewx.conf`. Finally, it copies over the data from your old -database into the new database. - -``` -wee_database --reconfigure -``` - -See the section Changing -the database unit system in an existing database in the Customization -Guide for step-by-step instructions that use this option. - -### `--transfer` - -This action is useful for moving your database from one type of database to -another, such as from SQLite to MySQL. To use it, you must have two bindings -specified in your `weewx.conf` configuration file. One will serve as the -source, the other as the destination. Specify the source binding with option -`--binding`, the destination binding with option `--dest-binding`. The -`--binding` option may be omitted in which case the default `wx-binding` -will be used. - - -``` -wee_database --transfer --binding=source_binding --dest-binding=dest_binding -wee_database --transfer --dest-binding=dest_binding -``` - -See the Wiki for examples of moving data from SQLite to MySQL, and from MySQL to SQLite, using `wee_database`. - -### `--add-column=NAME` - -This action adds a new database observation type (column) to the database. If -used without the `--type` option, the type will default to `REAL`. - -``` -wee_database --add-column -``` - -Optionally, option `--type` can be used with a SQL type `REAL`, `INTEGER`, -or any other SQL column definition (such as `INTEGER DEFAULT 0`). - -``` -wee_database --add-column=NAME --type=TYPE -``` - -### `--rename-column=NAME` - -Use this action to rename a database observation type (column) to a new name. -It requires the option `--to-name`. - -``` -wee_database --rename-column=NAME --to-name=NEW_NAME -``` - -For example, to rename the column `luminosity` in your schema to `illuminance`: - -``` -wee_database --rename-column=luminosity --to-name=illuminance -``` - -### `--drop-columns=NAME` - -This action will drop one or more observation types (columns) from the -database. If more than one column name is given, they should be separated by -commas and no spaces. - -It is an error to attempt to drop a non-existing column. In this case, nothing -will be done. - -``` -wee_database --drop-columns=NAME1,NAME2 -``` - -!!! Note - When dropping columns from a SQLite database, the entire database must be - copied except for the dropped columns. Because this can be quite slow, if - you are dropping more than one column, it is better to do them all in one - pass. This is why option `--drop-columns` accepts more than one name. - - -### `--check` - -This action will check the calculations in the daily summary tables as well as -checking the archive for null strings (refer to `--check-strings`). If the -daily summary tables contain summaries calculated using an old algorithm, the -user is advised to update the daily summary tables using the `--update` action. -If null strings are found the user is advised to fix them using the -`--fix-strings` action. - -``` -wee_database --check -``` - -FIXME - -### `--update` - -This action updates the daily summary tables to use interval weighted -calculations as well as recalculating the `windSpeed` maximum daily values and -times. Interval weighted calculations are only applied to the daily summaries -if not previously applied. The update process is irreversible and users are -advised to backup their database before performing this action. - -``` -wee_database --update -``` - -For further information on interval weighting and recalculation of daily -`windSpeed` maximum values, see the sections Changes to daily summaries and Recalculation of `windSpeed` maximum values in the Upgrade Guide. - - -### `--drop-daily` - -In addition to the regular archive data, every WeeWX database also includes a -daily summary table for each observation type. Because there can be dozens of -observation types, there can be dozens of these daily summaries. It does not -happen very often, but there can be occasions when it's necessary to drop them -all and then rebuild them. Dropping them by hand would be very tedious! This -action does them all at once. - -``` -wee_database --drop-daily -``` - -### `--rebuild-daily` - -This action is the inverse of action `--drop-daily` in that it rebuilds the -daily summaries from the archive data. In most cases it is not necessary to -drop the daily summary tables using the action `--drop-daily` before rebuilding -them. - -The action `--rebuild-daily` accepts a number of date related options, -`--date`, `--from` and `--to` that allow selective rebuilding of the daily -summaries for one or more days rather than for the entire archive history. -These options may be useful if bogus data has been removed from the archive -covering a single day or a period of few days. The daily summaries can then -be rebuilt for this period only, resulting in a faster rebuild and detailed -low/high values and the associated times being retained for unaffected days. - -The `--date` option limits the daily summary rebuild to the specified date. - -The `--from` and `--to` options may be used together or individually to limit -the daily summary rebuild to a specified period. When used individually the -`--to` option limits the rebuild to the inclusive period from the earliest date -for which there is data in the database through to and including the specified -date. Similarly when used individually the `--from` option limits the rebuild -to the inclusive period from the specified date through until the last date -for which there is data in the database. When the `--from` and `--to` options -are used together the daily summary rebuild is limited to the specified -inclusive period. - -``` -wee_database --rebuild-daily -wee_database --rebuild-daily --date=YYYY-mm-dd -wee_database --rebuild-daily --from=YYYY-mm-dd -wee_database --rebuild-daily --to=YYYY-mm-dd -wee_database --rebuild-daily --from=YYYY-mm-dd --to=YYYY-mm-dd -``` - -!!! Note - Whilst the `--from` and `--to` options will accept optional hour and - minutes in the format `THH:MM`, any such hour and minute options are - ignored by the `--rebuild` action as the daily summaries represent whole - days only and it is not possible to partially rebuild a daily summary. - -!!! Note - When used with the `--rebuild-daily` action the period defined by `--to` - and `--from` is inclusive and the daily summary tables will be rebuild for - the day defined by `--from` and the day defined by `--to` and all days in - between. - - -### `--reweight` - -As an alternative to dropping and rebuilding the daily summaries, this action -simply rebuilds the weighted daily sums (used to calculate averages) from the -archive data. It does not touch the highs and lows. It is much faster than -`--rebuild-daily`, and has the advantage that the highs and lows remain -unchanged. - -Other options are as in `--rebuild-daily`. - - -### `--calc-missing` - -This action calculates derived observations for archive records in the database -and then stores the calculated observations in the database. This can be -useful if erroneous archive data is corrected or some additional observational -data is added to the archive that may alter previously calculated or missing -derived observations. - -The period over which the derived observations are calculated can be limited -through use of the `--date`, `--from` and/or `--to` options. When used without -any of these options `--calc-missing` will calculate derived observations for -all archive records in the database. The `--date` option limits the calculation -of derived observations to the specified date only. The `--from` and `--to` -options can be used together to specify the start and end date-time -respectively of the period over which derived observations will be calculated. - -If `--from` is used by itself the period is fom the date-time specified up to -and including the last archive record in the database. - -If `--to` is used by itself the period is the first archive record in the -database through to the specified date-time. - -``` -wee_database --calc-missing -wee_database --calc-missing --date=YYYY-mm-dd -wee_database --calc-missing --from=YYYY-mm-dd[THH:MM] -wee_database --calc-missing --to=YYYY-mm-dd[THH:MM] -wee_database --calc-missing --from=YYYY-mm-dd[THH:MM] --to=YYYY-mm-dd[THH:MM] -``` - -!!! Note - When a `YYYY-mm-dd` date is used as the `--from` option the period used by - `--calc-missing` includes records after midnight at the start of - `YYYY-mm-dd`, if a `YYYY-mm-ddTHH:MM` format date-time is used as the - `--from` option the period includes records after `YYYY-mm-dd HH:MM`. - When a `YYYY-mm-dd` date is used as the `--to` option the period includes - records up to and including midnight at the end of `YYYY-mm-dd`, if a - `YYYY-mm-ddTHH:MM` format date-time is used as the `--to` option the period - includes records up to and including `YYYY-mm-dd HH:MM`. When using the - `--date` option the period is all records after midnight at the start of - `YYYY-mm-dd` up to and including midnight at the end of `YYYY-mm-dd`, in - effect the same as `--from=YYYY-mm-ddT00:00 --to=YYYY-mm-dd+1T00:00`. - -!!! Note - `--calc-missing` uses the `StdWXCalculate` service to calculate missing - derived observations. The data binding used by the `StdWXCalculate` service - should normally match the data binding of the database being operated on by - `--calc-missing`. Those who use custom or additional data bindings should - take care to ensure the correct data bindings are used by both - `--calc-missing` and the `StdWXCalculate` service. Those who use the - default data binding need take no special precautions. - -### `--check-strings` - -Normally, all entries in the archive are numbers. However, some SQLite -database editors use a null string instead of a null value when deleting -entries. These null strings can cause problems. This action checks the -database to see if it contains any null strings. - -``` -wee_database --check-strings -``` - -### `--fix-strings` - -This action will check for any null strings in a SQLite database and if found -substitute a true null value. - -``` -wee_database --fix-strings -``` diff --git a/docs_src/utilities/wee_import.md b/docs_src/utilities/wee_import.md index b8d1245b..d675ef7b 100644 --- a/docs_src/utilities/wee_import.md +++ b/docs_src/utilities/wee_import.md @@ -3199,7 +3199,7 @@ Aug 22 14:38:28 stretch12 weewx[863]: manager: unable to add record 2018-09-04 0 database. The simplicity of this process will depend on your ability to use SQL, the amount of data imported, and whether the imported data was dispersed amongst existing. Once contaminated data have been removed the daily summary tables will need to be rebuilt using the wee_database utility. + class="code">weectl database rebuild-daily utility.
  • Delete the database and start over. For SQLite, simply delete the database file. For MySQL, drop the diff --git a/docs_src/utilities/weectl-database.md b/docs_src/utilities/weectl-database.md index 5a368755..4fd8e756 100644 --- a/docs_src/utilities/weectl-database.md +++ b/docs_src/utilities/weectl-database.md @@ -6,22 +6,6 @@ Specify `--help` to see the various actions and options: weectl database --help -## Common options - -These are options used by most of the actions. - -### --config - -Path to the configuration file. Default is `~/weewx-data/weewx.conf`. - -### --binding - -The database binding to use. Default is `wx_binding`. - -### --dry-run - -Show what would happen if the action was run, but do not actually make any writable changes. - ## Create a new database weectl database create @@ -240,5 +224,21 @@ highs and lows remain unchanged. Other options are as in `weectl database rebuild-daily`. +## Options + +These are options used by most of the actions. + +### --config + +Path to the configuration file. Default is `~/weewx-data/weewx.conf`. + +### --binding + +The database binding to use. Default is `wx_binding`. + +### --dry-run + +Show what would happen if the action was run, but do not actually make any writable changes. + diff --git a/mkdocs.yml b/mkdocs.yml index 76b5c959..f3e56427 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -108,7 +108,6 @@ nav: - weectl station: utilities/weectl-station.md - weectl extension: utilities/weectl-extension.md - weectl database: utilities/weectl-database.md - - wee_database: utilities/wee_database.md - wee_debug: utilities/wee_debug.md - wee_device: utilities/wee_device.md - wee_import: utilities/wee_import.md diff --git a/pkg/debian/rules b/pkg/debian/rules index ba6c84a7..bdcaf678 100755 --- a/pkg/debian/rules +++ b/pkg/debian/rules @@ -17,7 +17,7 @@ DST_SYSTEMDDIR=$(DST)/etc/systemd/system DST_CFGDIR =$(DST)/etc/weewx # these are the entry points -ENTRIES=weewxd weectl wee_database wee_debug wee_device wee_import wee_reports +ENTRIES=weewxd weectl wee_debug wee_device wee_import wee_reports %: dh $@ --with python3 diff --git a/pkg/weewx.spec.in b/pkg/weewx.spec.in index 6708c8e0..099612ea 100644 --- a/pkg/weewx.spec.in +++ b/pkg/weewx.spec.in @@ -62,7 +62,7 @@ %global cfg_file %{dst_cfg_dir}/weewx.conf %global dst_user_dir %{dst_bin_dir}/user -%define entry_points weewxd wee_config wee_database wee_debug wee_device wee_extension wee_import wee_reports +%define entry_points weewxd weectl wee_debug wee_device wee_import wee_reports Summary: weather software Name: weewx @@ -213,11 +213,9 @@ rm -rf %{buildroot} %{_sysconfdir}/default/weewx %{dst_bin_dir}/ %{_bindir}/weewxd -%{_bindir}/wee_config -%{_bindir}/wee_database +%{_bindir}/weectl %{_bindir}/wee_debug %{_bindir}/wee_device -%{_bindir}/wee_extension %{_bindir}/wee_import %{_bindir}/wee_reports %doc %{dst_doc_dir}/ diff --git a/pyproject.toml b/pyproject.toml index a04655de..4f4c3add 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,6 @@ packages = [ { include = "weeplot", from = "bin" }, { include = "weeutil", from = "bin" }, { include = "weewx", from = "bin" }, - { include = "wee_database.py", from = "bin" }, { include = "wee_debug.py", from = "bin" }, { include = "wee_device.py", from = "bin" }, { include = "wee_import.py", from = "bin" }, @@ -71,7 +70,6 @@ pyserial = "^3.4" pyusb = "^1.0.2" [tool.poetry.scripts] -wee_database = 'wee_database:main' wee_debug = 'wee_debug:main' wee_device = 'wee_device:main' wee_import = 'wee_import:main' diff --git a/util/scripts/wee_database b/util/scripts/wee_database deleted file mode 100755 index 319c6c2d..00000000 --- a/util/scripts/wee_database +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh -app=wee_database.py - -# Get the weewx location and interpreter. Default to something sane, but -# look for overrides from the system defaults. -WEEWX_BINDIR=/usr/share/weewx -WEEWX_PYTHON=python3 -[ -r /etc/default/weewx ] && . /etc/default/weewx -exec "$WEEWX_PYTHON" $WEEWX_PYTHON_ARGS "$WEEWX_BINDIR/$app" "$@"