mirror of
https://github.com/weewx/weewx.git
synced 2026-04-18 00:26:57 -04:00
967 lines
42 KiB
Python
Executable File
967 lines
42 KiB
Python
Executable File
#!/usr/bin/env python
|
|
#
|
|
# Copyright (c) 2009-2019 Tom Keffer <tkeffer@gmail.com>
|
|
#
|
|
# See the file LICENSE.txt for your full rights.
|
|
#
|
|
"""Configure databases used by WeeWX"""
|
|
from __future__ import absolute_import
|
|
from __future__ import print_function
|
|
from __future__ import with_statement
|
|
|
|
# python imports
|
|
import datetime
|
|
import logging
|
|
import optparse
|
|
import sys
|
|
import time
|
|
|
|
import six
|
|
from six.moves import input
|
|
|
|
# weewx imports
|
|
import user.extensions # @UnusedImport
|
|
import weecfg.database
|
|
import weedb
|
|
import weeutil.logger
|
|
import weewx.manager
|
|
import weewx.units
|
|
from weeutil.weeutil import TimeSpan, timestamp_to_string, y_or_n, to_int
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
usage = """wee_database --help
|
|
wee_database --create
|
|
wee_database --reconfigure
|
|
wee_database --transfer --dest-binding=BINDING_NAME [--dry-run]
|
|
wee_database --check
|
|
wee_database --update [--dry-run]
|
|
wee_database --drop-daily
|
|
wee_database --rebuild-daily [--date=YYYY-mm-dd |
|
|
[--from=YYYY-mm-dd] [--to=YYYY-mm-dd]]
|
|
wee_database --calc-missing [--date=YYYY-mm-dd |
|
|
[--from=YYYY-mm-dd[THH:MM]] [--to=YYYY-mm-dd[THH:MM]]]
|
|
wee_database --check-strings
|
|
wee_database --fix-strings [--dry-run]
|
|
|
|
Description:
|
|
|
|
Manipulate the WeeWX database. Most of these operations are handled
|
|
automatically by WeeWX, but they may be useful in special cases."""
|
|
|
|
|
|
def main():
|
|
# Set defaults for logging:
|
|
weeutil.logger.setup('wee_database', {})
|
|
|
|
# Create a command line parser:
|
|
parser = optparse.OptionParser(usage=usage)
|
|
|
|
# Add the various verbs...
|
|
parser.add_option("--create", action='store_true',
|
|
help="Create the WeeWX database and initialize it with the"
|
|
" schema.")
|
|
parser.add_option("--reconfigure", action='store_true',
|
|
help="Create a new database using configuration"
|
|
" information found in the configuration file. In"
|
|
" particular, the new database will use the unit system"
|
|
" found in option [StdConvert][target_unit]."
|
|
" The new database will have the same name as the old"
|
|
" database, with a '_new' on the end.")
|
|
parser.add_option("--transfer", action='store_true',
|
|
help="Transfer the WeeWX archive from source database "
|
|
"to destination database.")
|
|
parser.add_option("--check", action="store_true",
|
|
help="Check the calculations in the daily summary tables.")
|
|
parser.add_option("--update", action="store_true",
|
|
help="Update the daily summary tables if required and"
|
|
" recalculate the daily summary maximum windSpeed values.")
|
|
parser.add_option("--calc-missing", dest="calc_missing", action="store_true",
|
|
help="Calculate and store any missing derived observations.")
|
|
parser.add_option("--check-strings", action="store_true",
|
|
help="Check the archive table for null strings that may"
|
|
" have been introduced by a SQL editing program.")
|
|
parser.add_option("--fix-strings", action='store_true',
|
|
help="Fix any null strings in a SQLite database.")
|
|
parser.add_option("--drop-daily", action='store_true',
|
|
help="Drop the daily summary tables from a database.")
|
|
parser.add_option("--rebuild-daily", action='store_true',
|
|
help="Rebuild the daily summaries from data in the archive table.")
|
|
|
|
# ... then add the various options:
|
|
parser.add_option("--config", dest="config_path", type=str,
|
|
metavar="CONFIG_FILE",
|
|
help="Use configuration file CONFIG_FILE.")
|
|
parser.add_option("--date", type=str, metavar="YYYY-mm-dd",
|
|
help="This date only (options --calc-missing and --rebuild-daily only).")
|
|
parser.add_option("--from", dest="from_date", type=str, metavar="YYYY-mm-dd[THH:MM]",
|
|
help="Start with this date or date-time"
|
|
" (options --calc-missing and --rebuild-daily only).")
|
|
parser.add_option("--to", dest="to_date", type=str, metavar="YYYY-mm-dd[THH:MM]",
|
|
help="End with this date or date-time"
|
|
" (options --calc-missing and --rebuild-daily only).")
|
|
parser.add_option("--binding", metavar="BINDING_NAME", default='wx_binding',
|
|
help="The data binding to use. Default is 'wx_binding'.")
|
|
parser.add_option("--dest-binding", metavar="BINDING_NAME",
|
|
help="The destination data binding (option --transfer only).")
|
|
parser.add_option('--dry-run', action='store_true',
|
|
default=False,
|
|
help="Print what would happen but do not do it. Default is False.")
|
|
|
|
# Now we are ready to parse the command line:
|
|
(options, args) = parser.parse_args()
|
|
|
|
# Do a check to see if the user used more than 1 'verb'
|
|
if sum(1 if x else 0 for x in [options.create,
|
|
options.reconfigure,
|
|
options.transfer,
|
|
options.check,
|
|
options.update,
|
|
options.calc_missing,
|
|
options.check_strings,
|
|
options.fix_strings,
|
|
options.drop_daily,
|
|
options.rebuild_daily]) != 1:
|
|
sys.exit("Must specify one and only one verb.")
|
|
|
|
# get config_dict to use
|
|
config_path, config_dict = weecfg.read_config(options.config_path, args)
|
|
print("Using configuration file %s" % config_path)
|
|
|
|
# Set weewx.debug as necessary:
|
|
weewx.debug = to_int(config_dict.get('debug', 0))
|
|
|
|
# Now we can set up the user customized logging:
|
|
weeutil.logger.setup('wee_database', config_dict)
|
|
|
|
db_binding = options.binding
|
|
database = config_dict['DataBindings'][db_binding]['database']
|
|
print("Using database binding '%s', which is bound to database '%s'" % (db_binding, database))
|
|
|
|
if options.create:
|
|
createMainDatabase(config_dict, db_binding)
|
|
|
|
elif options.reconfigure:
|
|
reconfigMainDatabase(config_dict, db_binding)
|
|
|
|
elif options.transfer:
|
|
transferDatabase(config_dict, db_binding, options)
|
|
|
|
elif options.check:
|
|
check(config_dict, db_binding, options)
|
|
|
|
elif options.update:
|
|
update(config_dict, db_binding, options)
|
|
|
|
elif options.calc_missing:
|
|
calc_missing(config_dict, db_binding, options)
|
|
|
|
elif options.check_strings:
|
|
check_strings(config_dict, db_binding, options, fix=False)
|
|
|
|
elif options.fix_strings:
|
|
check_strings(config_dict, db_binding, options, fix=True)
|
|
|
|
elif options.drop_daily:
|
|
dropDaily(config_dict, db_binding)
|
|
|
|
elif options.rebuild_daily:
|
|
rebuildDaily(config_dict, db_binding, options)
|
|
|
|
|
|
def createMainDatabase(config_dict, db_binding):
|
|
"""Create the WeeWX database"""
|
|
|
|
# Try a simple open. If it succeeds, that means the database
|
|
# exists and is initialized. Otherwise, an exception will be thrown.
|
|
try:
|
|
with weewx.manager.open_manager_with_config(config_dict, db_binding) as dbmanager:
|
|
print("Database '%s' already exists. Nothing done." % dbmanager.database_name)
|
|
except weedb.OperationalError:
|
|
# Database does not exist. Try again, but allow initialization:
|
|
with weewx.manager.open_manager_with_config(config_dict, db_binding, initialize=True) as dbmanager:
|
|
print("Created database '%s'" % dbmanager.database_name)
|
|
|
|
|
|
def dropDaily(config_dict, db_binding):
|
|
"""Drop the daily summaries from a WeeWX database"""
|
|
|
|
manager_dict = weewx.manager.get_manager_dict_from_config(config_dict,
|
|
db_binding)
|
|
database_name = manager_dict['database_dict']['database_name']
|
|
|
|
print("Proceeding will delete all your daily summaries from database '%s'" % database_name)
|
|
ans = y_or_n("Are you sure you want to proceed (y/n)? ")
|
|
if ans == 'y':
|
|
t1 = time.time()
|
|
print("Dropping daily summary tables from '%s' ... " % database_name)
|
|
try:
|
|
with weewx.manager.open_manager_with_config(config_dict, db_binding) as dbmanager:
|
|
try:
|
|
dbmanager.drop_daily()
|
|
except weedb.OperationalError as e:
|
|
print("Error '%s'" % e)
|
|
print("Drop daily summary tables failed for database '%s'" % database_name)
|
|
else:
|
|
tdiff = time.time() - t1
|
|
print("Daily summary tables dropped from "
|
|
"database '%s' in %.2f seconds" % (database_name, tdiff))
|
|
except weedb.OperationalError:
|
|
# No daily summaries. Nothing to be done.
|
|
print("No daily summaries found in database '%s'. Nothing done." % database_name)
|
|
else:
|
|
print("Nothing done.")
|
|
|
|
|
|
def rebuildDaily(config_dict, db_binding, options):
|
|
"""Rebuild the daily summaries."""
|
|
|
|
manager_dict = weewx.manager.get_manager_dict_from_config(config_dict,
|
|
db_binding)
|
|
database_name = manager_dict['database_dict']['database_name']
|
|
|
|
# get the first and last good timestamps from the archive, these represent
|
|
# our bounds for rebuilding
|
|
with weewx.manager.Manager.open(manager_dict['database_dict']) as dbmanager:
|
|
first_ts = dbmanager.firstGoodStamp()
|
|
first_d = datetime.date.fromtimestamp(first_ts) if first_ts is not None else None
|
|
last_ts = dbmanager.lastGoodStamp()
|
|
last_d = datetime.date.fromtimestamp(last_ts) if first_ts is not None else None
|
|
# determine the period over which we are rebuilding from any command line
|
|
# date parameters
|
|
from_dt, to_dt = _parse_dates(options)
|
|
# we have start and stop datetime objects but we work on whole days only,
|
|
# so need date object
|
|
from_d = from_dt.date() if from_dt is not None else None
|
|
to_d = to_dt.date() if to_dt is not None else None
|
|
# advise the user/log what we will do
|
|
if from_d is None and to_d is None:
|
|
_msg = "All daily summaries will be rebuilt."
|
|
elif from_d and not to_d:
|
|
_msg = "Daily summaries from %s through the end (%s) will be rebuilt." % (from_d,
|
|
last_d)
|
|
elif not from_d and to_d:
|
|
_msg = "Daily summaries from the beginning (%s) through %s will be rebuilt." % (first_d,
|
|
to_d)
|
|
elif from_d == to_d:
|
|
_msg = "Daily summary for %s will be rebuilt." % from_d
|
|
else:
|
|
_msg = "Daily summaries from %s through %s inclusive will be rebuilt." % (from_d,
|
|
to_d)
|
|
log.info(_msg)
|
|
print(_msg)
|
|
ans = y_or_n("Proceed (y/n)? ")
|
|
if ans == 'n':
|
|
_msg = "Nothing done."
|
|
log.info(_msg)
|
|
print(_msg)
|
|
return
|
|
|
|
t1 = time.time()
|
|
|
|
# Open up the database. This will create the tables necessary for the daily
|
|
# summaries if they don't already exist:
|
|
with weewx.manager.open_manager_with_config(config_dict, db_binding, initialize=True) as dbmanager:
|
|
|
|
log.info("Rebuilding daily summaries in database '%s' ..." % database_name)
|
|
print("Rebuilding daily summaries in database '%s' ..." % database_name)
|
|
if options.dry_run:
|
|
print("Dry run. Nothing done.")
|
|
return
|
|
else:
|
|
# now do the actual rebuild
|
|
nrecs, ndays = dbmanager.backfill_day_summary(start_d=from_d,
|
|
stop_d=to_d,
|
|
trans_days=20)
|
|
tdiff = time.time() - t1
|
|
# advise the user/log what we did
|
|
log.info("Rebuild of daily summaries in database '%s' complete" % database_name)
|
|
if nrecs:
|
|
sys.stdout.flush()
|
|
# fix a bit of formatting inconsistency if less than 1000 records
|
|
# processed
|
|
if nrecs >= 1000:
|
|
print()
|
|
if ndays == 1:
|
|
_msg = "Processed %d records to rebuild 1 daily summary in %.2f seconds" % (nrecs,
|
|
tdiff)
|
|
else:
|
|
_msg = ("Processed %d records to rebuild %d daily summaries in %.2f seconds" % (nrecs,
|
|
ndays,
|
|
tdiff))
|
|
print(_msg)
|
|
print("Rebuild of daily summaries in database '%s' complete" % database_name)
|
|
else:
|
|
print("Daily summaries up to date in '%s'" % database_name)
|
|
|
|
|
|
def reconfigMainDatabase(config_dict, db_binding):
|
|
"""Create a new database, then populate it with the contents of an old database"""
|
|
|
|
manager_dict = weewx.manager.get_manager_dict_from_config(config_dict,
|
|
db_binding)
|
|
# Make a copy for the new database (we will be modifying it)
|
|
new_database_dict = dict(manager_dict['database_dict'])
|
|
|
|
# Now modify the database name
|
|
new_database_dict['database_name'] = manager_dict['database_dict']['database_name'] + '_new'
|
|
|
|
# First check and see if the new database already exists. If it does, check
|
|
# with the user whether it's ok to delete it.
|
|
try:
|
|
weedb.create(new_database_dict)
|
|
except weedb.DatabaseExists:
|
|
ans = y_or_n("New database '%s' already exists. "
|
|
"Delete it first (y/n)? " % new_database_dict['database_name'])
|
|
if ans == 'y':
|
|
weedb.drop(new_database_dict)
|
|
else:
|
|
print("Nothing done.")
|
|
return
|
|
|
|
# Get the unit system of the old archive:
|
|
with weewx.manager.Manager.open(manager_dict['database_dict']) as old_dbmanager:
|
|
old_unit_system = old_dbmanager.std_unit_system
|
|
|
|
if old_unit_system is None:
|
|
print("Old database has not been initialized. Nothing to be done.")
|
|
return
|
|
|
|
# Get the unit system of the new archive:
|
|
try:
|
|
target_unit_nickname = config_dict['StdConvert']['target_unit']
|
|
except KeyError:
|
|
target_unit_system = None
|
|
else:
|
|
target_unit_system = weewx.units.unit_constants[target_unit_nickname.upper()]
|
|
|
|
print("Copying database '%s' to '%s'" % (manager_dict['database_dict']['database_name'],
|
|
new_database_dict['database_name']))
|
|
if target_unit_system is None or old_unit_system == target_unit_system:
|
|
print("The new database will use the same unit system as the old ('%s')." %
|
|
weewx.units.unit_nicknames[old_unit_system])
|
|
else:
|
|
print("Units will be converted from the '%s' system to the '%s' system." %
|
|
(weewx.units.unit_nicknames[old_unit_system],
|
|
weewx.units.unit_nicknames[target_unit_system]))
|
|
|
|
ans = y_or_n("Are you sure you wish to proceed (y/n)? ")
|
|
if ans == 'y':
|
|
t1 = time.time()
|
|
weewx.manager.reconfig(manager_dict['database_dict'],
|
|
new_database_dict,
|
|
new_unit_system=target_unit_system,
|
|
new_schema=manager_dict['schema'])
|
|
tdiff = time.time() - t1
|
|
print("Database '%s' copied to '%s' in %.2f seconds." % (manager_dict['database_dict']['database_name'],
|
|
new_database_dict['database_name'],
|
|
tdiff))
|
|
else:
|
|
print("Nothing done.")
|
|
|
|
|
|
def transferDatabase(config_dict, db_binding, options):
|
|
"""Transfer 'archive' data from one database to another"""
|
|
|
|
# do we have enough to go on, must have a dest binding
|
|
if not options.dest_binding:
|
|
print("Destination binding not specified. Nothing Done. Aborting.")
|
|
return
|
|
# get manager dict for our source binding
|
|
src_manager_dict = weewx.manager.get_manager_dict_from_config(config_dict,
|
|
db_binding)
|
|
# get manager dict for our dest binding
|
|
try:
|
|
dest_manager_dict = weewx.manager.get_manager_dict_from_config(config_dict,
|
|
options.dest_binding)
|
|
except weewx.UnknownBinding:
|
|
# if we can't find the binding display a message then return
|
|
print("Unknown destination binding '%s', confirm destination binding."
|
|
% options.dest_binding)
|
|
print("Nothing Done. Aborting.")
|
|
return
|
|
except weewx.UnknownDatabase:
|
|
# if we can't find the database display a message then return
|
|
print("Error accessing destination database, confirm destination binding and/or database.")
|
|
print("Nothing Done. Aborting.")
|
|
return
|
|
except (ValueError, AttributeError):
|
|
# maybe a schema issue
|
|
print("Error accessing destination database.")
|
|
print("Maybe the destination schema is incorrectly specified in binding '%s' in weewx.conf?"
|
|
% options.dest_binding)
|
|
print("Nothing Done. Aborting.")
|
|
return
|
|
except weewx.UnknownDatabaseType:
|
|
# maybe a [Databases] issue
|
|
print("Error accessing destination database.")
|
|
print("Maybe the destination database is incorrectly defined in weewx.conf?")
|
|
print("Nothing Done. Aborting.")
|
|
return
|
|
# get a manager for our source
|
|
with weewx.manager.Manager.open(src_manager_dict['database_dict']) as src_manager:
|
|
# get first and last timestamps from the source so we can count the
|
|
# records to transfer and display an appropriate message
|
|
first_ts = src_manager.firstGoodStamp()
|
|
last_ts = src_manager.lastGoodStamp()
|
|
if first_ts is not None and last_ts is not None:
|
|
# we have source records
|
|
num_recs = src_manager.getAggregate(TimeSpan(first_ts, last_ts),
|
|
'dateTime', 'count')[0]
|
|
else:
|
|
# we have no source records to transfer so abort with a message
|
|
print("No records found in source database '%s' for transfer." %
|
|
src_manager.database_name)
|
|
print("Nothing done. Aborting.")
|
|
exit()
|
|
|
|
if not options.dry_run: # is it a dry run ?
|
|
# not a dry run, actually do the transfer
|
|
ans = y_or_n("Transfer %s records from source database '%s' "
|
|
"to destination database '%s' (y/n)? "
|
|
% (num_recs, src_manager.database_name,
|
|
dest_manager_dict['database_dict']['database_name']))
|
|
if ans == 'y':
|
|
t1 = time.time()
|
|
# wrap in a try..except in case we have an error
|
|
try:
|
|
with weewx.manager.Manager.open_with_create(dest_manager_dict['database_dict'],
|
|
table_name=dest_manager_dict['table_name'],
|
|
schema=dest_manager_dict['schema']) as dest_manager:
|
|
sys.stdout.write("transferring, this may take a while.... ")
|
|
sys.stdout.flush()
|
|
# do the transfer, should be quick as it's done as a
|
|
# single transaction
|
|
dest_manager.addRecord(src_manager.genBatchRecords())
|
|
print("complete")
|
|
# get first and last timestamps from the dest so we can
|
|
# count the records transferred and display a message
|
|
first_ts = dest_manager.firstGoodStamp()
|
|
last_ts = dest_manager.lastGoodStamp()
|
|
tdiff = time.time() - t1
|
|
if first_ts is not None and last_ts is not None:
|
|
num_recs = dest_manager.getAggregate(TimeSpan(first_ts, last_ts),
|
|
'dateTime', 'count')[0]
|
|
print("%s records transferred from source database '%s' to" %
|
|
(num_recs, src_manager.database_name))
|
|
print("destination database '%s' in %.2f seconds." %
|
|
(dest_manager.database_name, tdiff))
|
|
else:
|
|
print("Error. No records were transferred from source database '%s' to "
|
|
"destination database '%s'." % (src_manager.database_name,
|
|
dest_manager.database_name))
|
|
except ImportError:
|
|
# Probably when trying to load db driver
|
|
print("Error accessing destination database '%s'." %
|
|
(dest_manager_dict['database_dict']['database_name'],))
|
|
print("Nothing done. Aborting.")
|
|
raise
|
|
except (OSError, weedb.OperationalError):
|
|
# probably a weewx.conf typo or MySQL db not created
|
|
print("Error accessing destination database '%s'."
|
|
% dest_manager_dict['database_dict']['database_name'])
|
|
print("Maybe it does not exist (MySQL) or is incorrectly defined in weewx.conf?")
|
|
print("Nothing done. Aborting.")
|
|
return
|
|
|
|
else:
|
|
# we decided not to do the transfer
|
|
print("Nothing done.")
|
|
return
|
|
else:
|
|
# it's a dry run so say what we would have done then return
|
|
print("Transfer %s records from source database '%s' "
|
|
"to destination database '%s'."
|
|
% (num_recs, src_manager.database_name,
|
|
dest_manager_dict['database_dict']['database_name']))
|
|
print("Dry run, nothing done.")
|
|
|
|
|
|
def check(config_dict, db_binding, options):
|
|
"""Check database and report outstanding fixes/issues.
|
|
|
|
Performs the following checks:
|
|
- checks database version
|
|
- checks for null strings in SQLite database
|
|
"""
|
|
|
|
t1 = time.time()
|
|
|
|
# Check interval weighting
|
|
print("Checking daily summary tables version...")
|
|
|
|
# Get a database manager object
|
|
dbm = weewx.manager.open_manager_with_config(config_dict, db_binding)
|
|
|
|
# check the daily summary version
|
|
_daily_summary_version = dbm._read_metadata('Version')
|
|
msg = "Daily summary tables are at version %s" % _daily_summary_version
|
|
log.info(msg)
|
|
print(msg)
|
|
|
|
if _daily_summary_version is not None and _daily_summary_version >= '2.0':
|
|
# interval weighting fix has been applied
|
|
msg = "Interval Weighting Fix is not required."
|
|
log.info(msg)
|
|
print(msg)
|
|
else:
|
|
print("Recommend running --update to recalculate interval weightings.")
|
|
print("Daily summary tables version check completed in %0.2f seconds." % (time.time() - t1))
|
|
|
|
# now check for null strings
|
|
check_strings(config_dict, db_binding, options, fix=False)
|
|
|
|
|
|
def update(config_dict, db_binding, options):
|
|
"""Apply any required database fixes.
|
|
|
|
Applies the following fixes:
|
|
- checks if database version is 2.0, if not interval weighting fix is
|
|
applied
|
|
- recalculates windSpeed daily summary max and maxtime fields from
|
|
archive
|
|
"""
|
|
|
|
# prompt for confirmation if it is not a dry run
|
|
ans = 'y' if options.dry_run else None
|
|
while ans not in ['y', 'n']:
|
|
ans = input("The update process does not affect archive data, but does alter the database.\nContinue (y/n)? ")
|
|
if ans == 'n':
|
|
# we decided not to update the summary tables
|
|
msg = "Update cancelled"
|
|
log.info(msg)
|
|
print(msg)
|
|
return
|
|
|
|
if options.dry_run:
|
|
logging.disable(logging.INFO)
|
|
|
|
msg = "Preparing Interval Weighting Fix..."
|
|
log.info(msg)
|
|
print(msg)
|
|
|
|
# notify if this is a dry run
|
|
if options.dry_run:
|
|
print("This is a dry run: weighted intervals will be calculated but not saved.")
|
|
|
|
# Get a database manager object
|
|
dbm = weewx.manager.open_manager_with_config(config_dict, db_binding)
|
|
|
|
# Interval weighting
|
|
# first construct an interval weighting config dict
|
|
weighting_config_dict = {'name': 'Interval Weighting Fix',
|
|
'binding': db_binding,
|
|
'trans_days': 100,
|
|
'dry_run': options.dry_run}
|
|
|
|
# create an interval weighting fix object
|
|
weight_obj = weecfg.database.IntervalWeighting(config_dict,
|
|
weighting_config_dict)
|
|
# check the daily summary version
|
|
_daily_summary_version = dbm._read_metadata('Version')
|
|
msg = "Daily summary tables are at version %s" % _daily_summary_version
|
|
log.info(msg)
|
|
print(msg)
|
|
|
|
if _daily_summary_version is not None and _daily_summary_version >= '2.0':
|
|
# interval weighting fix has been applied
|
|
msg = "Interval Weighting Fix is not required."
|
|
log.info(msg)
|
|
print(msg)
|
|
else:
|
|
# apply the interval weighting
|
|
msg = "Calculating interval weights..."
|
|
log.info(msg)
|
|
print(msg)
|
|
t1 = time.time()
|
|
weight_obj.run()
|
|
msg = "Interval Weighting Fix completed in %0.2f seconds." % (time.time() - t1)
|
|
log.info(msg)
|
|
print(msg)
|
|
|
|
# recalc the max/maxtime windSpeed values
|
|
_fix_wind(config_dict, db_binding, options)
|
|
# just in case, set the syslog level back where we found it
|
|
if options.dry_run:
|
|
logging.disable(logging.NOTSET)
|
|
|
|
|
|
def calc_missing(config_dict, db_binding, options):
|
|
"""Calculate any missing derived observations and save to database."""
|
|
|
|
msg = "Preparing to calculate missing derived observations..."
|
|
log.info(msg)
|
|
|
|
manager_dict = weewx.manager.get_manager_dict_from_config(config_dict,
|
|
db_binding)
|
|
# get the first and last good timestamps from the archive, these represent
|
|
# our overall bounds for calculating missing derived obs
|
|
with weewx.manager.Manager.open(manager_dict['database_dict']) as dbmanager:
|
|
first_ts = dbmanager.firstGoodStamp()
|
|
last_ts = dbmanager.lastGoodStamp()
|
|
# process any command line options that may limit the period over which
|
|
# missing derived obs are calculated
|
|
start_dt, stop_dt = _parse_dates(options)
|
|
# we now have a start and stop date for processing, we need to obtain those
|
|
# as epoch timestamps, if we have no start and/or stop date then use the
|
|
# first or last good timestamp instead
|
|
start_ts = time.mktime(start_dt.timetuple()) if start_dt is not None else first_ts - 1
|
|
stop_ts = time.mktime(stop_dt.timetuple()) if stop_dt is not None else last_ts
|
|
# notify if this is a dry run
|
|
if options.dry_run:
|
|
msg = "This is a dry run, missing derived observations will be calculated but not saved"
|
|
log.info(msg)
|
|
print(msg)
|
|
_head = "Missing derived observations will be calculated "
|
|
# advise the user/log what we will do
|
|
if start_dt is None and stop_dt is None:
|
|
_tail = "for all records."
|
|
elif start_dt and not stop_dt:
|
|
_tail = "from %s through to the end (%s)." % (timestamp_to_string(start_ts),
|
|
timestamp_to_string(stop_ts))
|
|
elif not start_dt and stop_dt:
|
|
_tail = "from the beginning (%s) through to %s." % (timestamp_to_string(start_ts),
|
|
timestamp_to_string(stop_ts))
|
|
else:
|
|
_tail = "from %s through to %s inclusive." % (timestamp_to_string(start_ts),
|
|
timestamp_to_string(stop_ts))
|
|
_msg = "%s%s" % (_head, _tail)
|
|
log.info(_msg)
|
|
print(_msg)
|
|
ans = y_or_n("Proceed (y/n)? ")
|
|
if ans == 'n':
|
|
_msg = "Nothing done."
|
|
log.info(_msg)
|
|
print(_msg)
|
|
return
|
|
|
|
t1 = time.time()
|
|
|
|
# construct a CalcMissing config dict
|
|
calc_missing_config_dict = {'name': 'Calculate Missing Derived Observations',
|
|
'binding': db_binding,
|
|
'start_ts': start_ts,
|
|
'stop_ts': stop_ts,
|
|
'trans_days': 20,
|
|
'dry_run': options.dry_run}
|
|
|
|
# obtain a CalcMissing object
|
|
calc_missing_obj = weecfg.database.CalcMissing(config_dict,
|
|
calc_missing_config_dict)
|
|
msg = "Calculating missing derived observations..."
|
|
log.info(msg)
|
|
print(msg)
|
|
# calculate and store any missing observations
|
|
calc_missing_obj.run()
|
|
msg = "Missing derived observations calculated in %0.2f seconds" % (time.time() - t1)
|
|
log.info(msg)
|
|
print(msg)
|
|
|
|
|
|
def _fix_wind(config_dict, db_binding, options):
|
|
"""Recalculate the windSpeed daily summary max and maxtime fields.
|
|
|
|
Create a WindSpeedRecalculation object and call its run() method to
|
|
recalculate the max and maxtime fields from archive data. This process is
|
|
idempotent so it can be called repeatedly with no ill effect.
|
|
"""
|
|
|
|
t1 = time.time()
|
|
msg = "Preparing Maximum windSpeed Fix..."
|
|
log.info(msg)
|
|
print(msg)
|
|
|
|
# notify if this is a dry run
|
|
if options.dry_run:
|
|
print("This is a dry run: maximum windSpeed will be calculated but not saved.")
|
|
|
|
# construct a windSpeed recalculation config dict
|
|
wind_config_dict = {'name': 'Maximum windSpeed Fix',
|
|
'binding': db_binding,
|
|
'trans_days': 100,
|
|
'dry_run': options.dry_run}
|
|
|
|
# create a windSpeedRecalculation object
|
|
wind_obj = weecfg.database.WindSpeedRecalculation(config_dict,
|
|
wind_config_dict)
|
|
# perform the recalculation, wrap in a try..except to catch any db errors
|
|
try:
|
|
wind_obj.run()
|
|
except weedb.NoTableError as e:
|
|
msg = "Maximum windSpeed Fix applied: no windSpeed found"
|
|
log.info(msg)
|
|
print(msg)
|
|
else:
|
|
msg = "Maximum windSpeed Fix completed in %0.2f seconds" % (time.time() - t1)
|
|
log.info(msg)
|
|
print(msg)
|
|
|
|
|
|
# These functions are necessary because Python 3 does not allow you to
|
|
# parameterize types. So, we use a big if-else.
|
|
|
|
def check_type(val, expected):
|
|
if expected == 'INTEGER':
|
|
return isinstance(val, six.integer_types)
|
|
elif expected == 'REAL':
|
|
return isinstance(val, float)
|
|
elif expected == 'STR':
|
|
return isinstance(val, six.string_types)
|
|
else:
|
|
raise ValueError("Unknown type %s" % expected)
|
|
|
|
|
|
def set_type(val, target):
|
|
if target == 'INTEGER':
|
|
return int(val)
|
|
elif target == 'REAL':
|
|
return float(val)
|
|
elif target == 'STR':
|
|
return six.ensure_str(val)
|
|
else:
|
|
raise ValueError("Unknown type %s" % target)
|
|
|
|
|
|
def check_strings(config_dict, db_binding, options, fix=False):
|
|
"""Scan the archive table for null strings.
|
|
|
|
Identifies and lists any null string occurrences in the archive table. If
|
|
fix is True then any null strings that are found are fixed.
|
|
"""
|
|
|
|
t1 = time.time()
|
|
if options.dry_run or not fix:
|
|
logging.disable(logging.INFO)
|
|
|
|
print("Preparing Null String Fix, this may take a while...")
|
|
|
|
if fix:
|
|
log.info("Preparing Null String Fix")
|
|
# notify if this is a dry run
|
|
if options.dry_run:
|
|
print("This is a dry run: null strings will be detected but not fixed")
|
|
|
|
# open up the main database archive table
|
|
with weewx.manager.open_manager_with_config(config_dict, db_binding) as dbmanager:
|
|
|
|
obs_list = []
|
|
obs_type_list = []
|
|
|
|
# get the schema and extract the Python type each observation type should be
|
|
for column in dbmanager.connection.genSchemaOf('archive'):
|
|
# Save the observation name for this column (eg, 'outTemp'):
|
|
obs_list.append(column[1])
|
|
# And its type
|
|
obs_type_list.append(column[2])
|
|
|
|
records = 0
|
|
_found = []
|
|
# cycle through each row in the database
|
|
for record in dbmanager.genBatchRows():
|
|
records += 1
|
|
# now examine each column
|
|
for icol in range(len(record)):
|
|
# check to see if this column is an instance of the correct
|
|
# Python type
|
|
if record[icol] is not None and not check_type(record[icol], obs_type_list[icol]):
|
|
# Oops. Found a bad one. Print it out.
|
|
if fix:
|
|
log.info("Timestamp = %s; record['%s']= %r; ... "
|
|
% (record[0], obs_list[icol], record[icol]))
|
|
|
|
if fix:
|
|
# coerce to the correct type. If it can't be done, then
|
|
# set it to None.
|
|
try:
|
|
corrected_value = set_type(record[icol], obs_type_list[icol])
|
|
except ValueError:
|
|
corrected_value = None
|
|
# update the database with the new value but only if
|
|
# it's not a dry run
|
|
if not options.dry_run:
|
|
dbmanager.updateValue(record[0], obs_list[icol], corrected_value)
|
|
_found.append((record[0], obs_list[icol], record[icol], corrected_value))
|
|
# log it
|
|
log.info(" changed to %r\n" % corrected_value)
|
|
else:
|
|
_found.append((record[0], obs_list[icol], record[icol]))
|
|
# notify the user of progress
|
|
if records % 1000 == 0:
|
|
print("Checking record: %d; Timestamp: %s\r"
|
|
% (records, timestamp_to_string(record[0])), end=' ')
|
|
sys.stdout.flush()
|
|
# update our final count now that we have finished
|
|
print("Checking record: %d; Timestamp: %s\r" % (records, timestamp_to_string(record[0])), end=' ')
|
|
print()
|
|
tdiff = time.time() - t1
|
|
# now display details of what we found if we found any null strings
|
|
if len(_found):
|
|
print("The following null strings were found:")
|
|
for item in _found:
|
|
if len(item) == 4:
|
|
print("Timestamp = %s; record['%s'] = %r; ... changed to %r" % item)
|
|
else:
|
|
print("Timestamp = %s; record['%s'] = %r; ... ignored" % item)
|
|
# how many did we fix?
|
|
fixed = len([a for a in _found if len(a) == 4])
|
|
# summarise our results
|
|
if len(_found) == 0:
|
|
# found no null strings, log it and display on screen
|
|
log.info("No null strings found.")
|
|
print("No null strings found.")
|
|
elif fixed == len(_found):
|
|
# fixed all we found
|
|
if options.dry_run:
|
|
# its a dry run so display to screen but not to log
|
|
print("%d of %d null strings found would have been fixed." % (fixed, len(_found)))
|
|
else:
|
|
# really did fix so log and display to screen
|
|
log.info("%d of %d null strings found were fixed." % (fixed, len(_found)))
|
|
print("%d of %d null strings found were fixed." % (fixed, len(_found)))
|
|
elif fix:
|
|
# this should never occur - found some but didn't fix them all when we
|
|
# should have
|
|
if options.dry_run:
|
|
# its a dry run so say what would have happened
|
|
print("Could not fix all null strings. "
|
|
"%d of %d null strings found would have been fixed." % (fixed,
|
|
len(_found)))
|
|
else:
|
|
# really did fix so log and display to screen
|
|
log.info("Could not fix all null strings. "
|
|
"%d of %d null strings found were fixed." % (fixed,
|
|
len(_found)))
|
|
print("Could not fix all null strings. "
|
|
"%d of %d null strings found were fixed." % (fixed,
|
|
len(_found)))
|
|
else:
|
|
# found some null string but it was only a check not a fix, just
|
|
# display to screen
|
|
print("%d null strings were found.\r\n"
|
|
"Recommend running --fix-strings to fix these strings." % len(_found))
|
|
|
|
# and finally details on time taken
|
|
if fix:
|
|
log.info("Applied Null String Fix in %0.2f seconds." % tdiff)
|
|
print("Applied Null String Fix in %0.2f seconds." % tdiff)
|
|
else:
|
|
# it was a check not a fix so just display to screen
|
|
print("Completed Null String Check in %0.2f seconds." % tdiff)
|
|
# just in case, set the syslog level back where we found it
|
|
if options.dry_run or not fix:
|
|
logging.disable(logging.NOTSET)
|
|
|
|
|
|
def _parse_dates(options):
|
|
"""Parse --date, --from and --to command line options.
|
|
|
|
Parses --date or --from and --to to determine a date-time span to be
|
|
used. --to and --from in the format y-m-dTHH:MM precisely define a
|
|
date-time but anything in the format y-m-d does not. When rebuilding
|
|
the daily summaries this imprecision is not import as we merely need a
|
|
date-time somewhere within the day being rebuilt. When calculating
|
|
missing fields we need date-times for the span over which the
|
|
calculations are to be performed.
|
|
|
|
Inputs:
|
|
options: the optparse options
|
|
|
|
Returns: A two-way tuple (from_dt, to_dt) representing the from and to
|
|
date-times derived from the --date or --to and --from command line
|
|
options where
|
|
from_dt: A datetime.datetime object holding the from date-time. May
|
|
be None
|
|
to_dt: A datetime.datetime object holding the to date-time. May be
|
|
None
|
|
"""
|
|
|
|
# default is None, unless user has specified an option
|
|
_from_dt = None
|
|
_to_dt = None
|
|
|
|
# first look for --date
|
|
if options.date:
|
|
# we have a --date option, make sure we are not over specified
|
|
if options.from_date or options.to_date:
|
|
raise ValueError("Specify either --date or a --from and --to combination; not both")
|
|
|
|
# there is a --date but is it valid
|
|
try:
|
|
# this will give a datetime object representing midnight at the
|
|
# start of the day
|
|
_from_dt = datetime.datetime.strptime(options.date, "%Y-%m-%d")
|
|
except ValueError:
|
|
raise ValueError("Invalid --date option specified.")
|
|
else:
|
|
# we have the from date-time, for a --date option our final results
|
|
# depend on the action we are to perform
|
|
if options.rebuild_daily:
|
|
# The daily summaries are stamped with the midnight timestamp
|
|
# for each day, so our from and to results need to be within the
|
|
# same calendar day else we will rebuild more than just one day.
|
|
# For simplicity make them the same.
|
|
_to_dt = _from_dt
|
|
elif options.calc_missing:
|
|
# On the other hand calc missing will be dealing with archive
|
|
# records which are epoch timestamped. The midnight stamped
|
|
# record is part of the previous day so make our from result
|
|
# one second after midnight. THe to result must be midnight at
|
|
# the end of the day.
|
|
_to_dt = _from_dt + datetime.timedelta(days=1)
|
|
_from_dt = _from_dt + datetime.timedelta(seconds=1)
|
|
else:
|
|
# nothing else uses from and to (yet) but just in case return
|
|
# midnight to midnight as the default
|
|
_to_dt = _from_dt + datetime.timedelta(days=1)
|
|
# we have our results so we can return
|
|
return _from_dt, _to_dt
|
|
|
|
# we don't have --date so now look for --from and --to
|
|
if options.from_date:
|
|
# we have a --from but is it valid
|
|
try:
|
|
if 'T' in options.from_date:
|
|
# we have a time so we can precisely determine a date-time
|
|
_from_dt = datetime.datetime.strptime(options.from_date, "%Y-%m-%dT%H:%M")
|
|
else:
|
|
# we have a date only, so use midnight at the start of the day
|
|
_from_dt = datetime.datetime.strptime(options.from_date, "%Y-%m-%d")
|
|
except ValueError:
|
|
raise ValueError("Invalid --from option specified.")
|
|
|
|
if options.to_date:
|
|
# we have a --to but is it valid
|
|
try:
|
|
if 'T' in options.to_date:
|
|
# we have a time so decode and use that
|
|
_to_dt = datetime.datetime.strptime(options.to_date, "%Y-%m-%dT%H:%M")
|
|
else:
|
|
# we have a date, first obtain a datetime object for midnight
|
|
# at the start of the day specified
|
|
_to_dt = datetime.datetime.strptime(options.to_date, "%Y-%m-%d")
|
|
# since we have a date the result we want depends on what action
|
|
# we are to complete
|
|
if options.rebuild_daily:
|
|
# for a rebuild the to date-time must be within the date
|
|
# specified date, which it already is so leave it
|
|
pass
|
|
elif options.calc_missing:
|
|
# for calc missing we want midnight at the end of the day
|
|
_to_dt = _to_dt + datetime.timedelta(days=1)
|
|
else:
|
|
# nothing else uses from and to (yet) but just in case
|
|
# return midnight at the end of the day
|
|
_to_dt = _to_dt + datetime.timedelta(days=1)
|
|
except ValueError:
|
|
raise ValueError("Invalid --to option specified.")
|
|
|
|
# if we have both from and to date-times make sure from is no later than to
|
|
if _from_dt and _to_dt and _to_dt < _from_dt:
|
|
raise weewx.ViolatedPrecondition("--from value is later than --to value.")
|
|
# we have our results so we can return
|
|
return _from_dt, _to_dt
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|