mirror of
https://github.com/weewx/weewx.git
synced 2026-04-18 00:26:57 -04:00
805 lines
34 KiB
Python
Executable File
805 lines
34 KiB
Python
Executable File
#!/usr/bin/env python
|
|
#
|
|
# Copyright (c) 2009-2019 Tom Keffer <tkeffer@gmail.com>
|
|
#
|
|
# See the file LICENSE.txt for your full rights.
|
|
#
|
|
"""Configure databases used by WeeWX"""
|
|
from __future__ import with_statement
|
|
from __future__ import print_function
|
|
from __future__ import absolute_import
|
|
|
|
# python imports
|
|
import datetime
|
|
import optparse
|
|
import syslog
|
|
import sys
|
|
import time
|
|
|
|
import six
|
|
from six.moves import input
|
|
|
|
# weewx imports
|
|
import user.extensions # @UnusedImport
|
|
import weecfg.database
|
|
import weedb
|
|
import weewx.manager
|
|
import weewx.units
|
|
from weeutil.weeutil import TimeSpan, timestamp_to_string, y_or_n
|
|
|
|
|
|
usage = """wee_database --help
|
|
wee_database --create
|
|
wee_database --reconfigure
|
|
wee_database --transfer --dest-binding=BINDING_NAME [--dry-run]
|
|
wee_database --check
|
|
wee_database --update [--dry-run]
|
|
wee_database --check-strings
|
|
wee_database --fix-strings [--dry-run]
|
|
wee_database --drop-daily
|
|
wee_database --rebuild-daily [--date=YYYY-mm-dd |
|
|
--from=YYYY-mm-dd --to=YYYY-mm-dd]
|
|
|
|
Description:
|
|
|
|
Manipulate the WeeWX database. Most of these operations are handled
|
|
automatically by WeeWX, but they may be useful in special cases."""
|
|
|
|
|
|
def main():
|
|
# Set defaults for the system logger:
|
|
syslog.openlog('wee_database', syslog.LOG_PID | syslog.LOG_CONS)
|
|
|
|
# Create a command line parser:
|
|
parser = optparse.OptionParser(usage=usage)
|
|
|
|
# Add the various verbs...
|
|
parser.add_option("--create", action='store_true',
|
|
help="Create the WeeWX database and initialize it with the"
|
|
" schema.")
|
|
parser.add_option("--reconfigure", action='store_true',
|
|
help="Create a new database using configuration"
|
|
" information found in the configuration file. In"
|
|
" particular, the new database will use the unit system"
|
|
" found in option [StdConvert][target_unit]."
|
|
" The new database will have the same name as the old"
|
|
" database, with a '_new' on the end.")
|
|
parser.add_option("--transfer", action='store_true',
|
|
help="Transfer the WeeWX archive from source database "
|
|
"to destination database.")
|
|
parser.add_option("--check", action="store_true",
|
|
help="Check the calculations in the daily summary tables.")
|
|
parser.add_option("--update", action="store_true",
|
|
help="Update the daily summary tables if required and"
|
|
" recalculate the daily summary maximum windSpeed values.")
|
|
parser.add_option("--check-strings", action="store_true",
|
|
help="Check the archive table for null strings that may"
|
|
" have been introduced by a SQL editing program.")
|
|
parser.add_option("--fix-strings", action='store_true',
|
|
help="Fix any null strings in a SQLite database.")
|
|
parser.add_option("--drop-daily", action='store_true',
|
|
help="Drop the daily summary tables from a database.")
|
|
parser.add_option("--rebuild-daily", action='store_true',
|
|
help="Rebuild the daily summaries from data in the archive table.")
|
|
|
|
# ... then add the various options:
|
|
parser.add_option("--config", dest="config_path", type=str,
|
|
metavar="CONFIG_FILE",
|
|
help="Use configuration file CONFIG_FILE.")
|
|
parser.add_option("--date", type=str, metavar="YYYY-mm-dd",
|
|
help="This date only (option --rebuild-daily only).")
|
|
parser.add_option("--from", dest="from_date", type=str, metavar="YYYY-mm-dd",
|
|
help="Start with this date (option --rebuild-daily only).")
|
|
parser.add_option("--to", dest="to_date", type=str, metavar="YYYY-mm-dd",
|
|
help="End with this date (option --rebuild-daily only).")
|
|
parser.add_option("--binding", metavar="BINDING_NAME", default='wx_binding',
|
|
help="The data binding to use. Default is 'wx_binding'.")
|
|
parser.add_option("--dest-binding", metavar="BINDING_NAME",
|
|
help="The destination data binding (option --transfer only).")
|
|
parser.add_option('--dry-run', action='store_true',
|
|
default=False,
|
|
help="Print what would happen but do not do it. Default is False.")
|
|
|
|
# Now we are ready to parse the command line:
|
|
(options, args) = parser.parse_args()
|
|
|
|
# Do a check to see if the user used more than 1 'verb'
|
|
if sum(1 if x else 0 for x in [options.create,
|
|
options.reconfigure,
|
|
options.transfer,
|
|
options.check,
|
|
options.update,
|
|
options.check_strings,
|
|
options.fix_strings,
|
|
options.drop_daily,
|
|
options.rebuild_daily]) != 1:
|
|
sys.exit("Must specify one and only one verb.")
|
|
|
|
# get config_dict to use
|
|
config_path, config_dict = weecfg.read_config(options.config_path, args)
|
|
print("Using configuration file %s" % config_path)
|
|
|
|
# set syslog level based
|
|
if config_dict.get('debug'):
|
|
syslog.setlogmask(syslog.LOG_UPTO(syslog.LOG_DEBUG))
|
|
else:
|
|
syslog.setlogmask(syslog.LOG_UPTO(syslog.LOG_INFO))
|
|
|
|
db_binding = options.binding
|
|
database = config_dict['DataBindings'][db_binding]['database']
|
|
print("Using database binding '%s', which is bound to database '%s'" % (db_binding, database))
|
|
|
|
if options.create:
|
|
createMainDatabase(config_dict, db_binding)
|
|
|
|
elif options.reconfigure:
|
|
reconfigMainDatabase(config_dict, db_binding)
|
|
|
|
elif options.transfer:
|
|
transferDatabase(config_dict, db_binding, options)
|
|
|
|
elif options.check:
|
|
check(config_dict, db_binding, options)
|
|
|
|
elif options.update:
|
|
update(config_dict, db_binding, options)
|
|
|
|
elif options.check_strings:
|
|
check_strings(config_dict, db_binding, options, fix=False)
|
|
|
|
elif options.fix_strings:
|
|
check_strings(config_dict, db_binding, options, fix=True)
|
|
|
|
elif options.drop_daily:
|
|
dropDaily(config_dict, db_binding)
|
|
|
|
elif options.rebuild_daily:
|
|
rebuildDaily(config_dict, db_binding, options)
|
|
|
|
|
|
def createMainDatabase(config_dict, db_binding):
|
|
"""Create the WeeWX database"""
|
|
|
|
# Try a simple open. If it succeeds, that means the database
|
|
# exists and is initialized. Otherwise, an exception will be thrown.
|
|
try:
|
|
with weewx.manager.open_manager_with_config(config_dict, db_binding) as dbmanager:
|
|
print("Database '%s' already exists. Nothing done." % dbmanager.database_name)
|
|
except weedb.OperationalError:
|
|
# Database does not exist. Try again, but allow initialization:
|
|
with weewx.manager.open_manager_with_config(config_dict, db_binding, initialize=True) as dbmanager:
|
|
print("Created database '%s'" % dbmanager.database_name)
|
|
|
|
|
|
def dropDaily(config_dict, db_binding):
|
|
"""Drop the daily summaries from a WeeWX database"""
|
|
|
|
manager_dict = weewx.manager.get_manager_dict_from_config(config_dict,
|
|
db_binding)
|
|
database_name = manager_dict['database_dict']['database_name']
|
|
|
|
print("Proceeding will delete all your daily summaries from database '%s'" % database_name)
|
|
ans = y_or_n("Are you sure you want to proceed (y/n)? ")
|
|
if ans == 'y':
|
|
t1 = time.time()
|
|
print("Dropping daily summary tables from '%s' ... " % database_name)
|
|
try:
|
|
with weewx.manager.open_manager_with_config(config_dict, db_binding) as dbmanager:
|
|
try:
|
|
dbmanager.drop_daily()
|
|
except weedb.OperationalError as e:
|
|
print("Error '%s'" % e)
|
|
print("Drop daily summary tables failed for database '%s'" % database_name)
|
|
else:
|
|
tdiff = time.time() - t1
|
|
print("Daily summary tables dropped from "
|
|
"database '%s' in %.2f seconds" % (database_name, tdiff))
|
|
except weedb.OperationalError:
|
|
# No daily summaries. Nothing to be done.
|
|
print("No daily summaries found in database '%s'. Nothing done." % database_name)
|
|
else:
|
|
print("Nothing done.")
|
|
|
|
|
|
def rebuildDaily(config_dict, db_binding, options):
|
|
"""Rebuild the daily summaries."""
|
|
|
|
manager_dict = weewx.manager.get_manager_dict_from_config(config_dict,
|
|
db_binding)
|
|
database_name = manager_dict['database_dict']['database_name']
|
|
|
|
# determine the period over which we are rebuilding from any command
|
|
# line date parameters
|
|
start_d, stop_d = _parse_dates(options)
|
|
# advise the user/log what we will do
|
|
if start_d is None and stop_d is None:
|
|
_msg = "All daily summaries will be rebuilt."
|
|
elif start_d and not stop_d:
|
|
_msg = "Daily summaries from %s through the end will be rebuilt." % start_d
|
|
elif not start_d and stop_d:
|
|
_msg = "Daily summaries from the begining through date %s will be rebuilt." % stop_d
|
|
elif start_d == stop_d:
|
|
_msg = "Daily summary for date %s will be rebuilt" % start_d
|
|
else:
|
|
_msg = "Daily summaries from %s through %s inclusive will be rebuilt." % (start_d, stop_d)
|
|
|
|
syslog.syslog(syslog.LOG_INFO, _msg)
|
|
print(_msg)
|
|
|
|
ans = y_or_n("Proceed (y/n)? ")
|
|
if ans == 'n':
|
|
print("Nothing done.")
|
|
return
|
|
|
|
t1 = time.time()
|
|
|
|
# Open up the database. This will create the tables necessary for the daily
|
|
# summaries if they don't already exist:
|
|
with weewx.manager.open_manager_with_config(config_dict, db_binding, initialize=True) as dbmanager:
|
|
|
|
syslog.syslog(syslog.LOG_INFO, "Rebuilding daily summaries in database '%s' ..." % database_name)
|
|
print("Rebuilding daily summaries in database '%s' ..." % database_name)
|
|
if options.dry_run:
|
|
print("Dry run. Nothing done.")
|
|
return
|
|
else:
|
|
# now do the actual rebuild
|
|
nrecs, ndays = dbmanager.backfill_day_summary(start_d=start_d,
|
|
stop_d=stop_d,
|
|
trans_days=20)
|
|
tdiff = time.time() - t1
|
|
# advise the user/log what we did
|
|
syslog.syslog(syslog.LOG_INFO, "Rebuild of daily summaries in database '%s' complete" % database_name)
|
|
if nrecs:
|
|
sys.stdout.flush()
|
|
print()
|
|
print("Processed %d records to rebuild %d day summaries in %.2f seconds " % (nrecs,
|
|
ndays,
|
|
tdiff))
|
|
print("Rebuild of daily summaries in database '%s' complete" % database_name)
|
|
else:
|
|
print("Daily summaries up to date in '%s'" % database_name)
|
|
|
|
|
|
def reconfigMainDatabase(config_dict, db_binding):
|
|
"""Create a new database, then populate it with the contents of an old database"""
|
|
|
|
manager_dict = weewx.manager.get_manager_dict_from_config(config_dict,
|
|
db_binding)
|
|
# Make a copy for the new database (we will be modifying it)
|
|
new_database_dict = dict(manager_dict['database_dict'])
|
|
|
|
# Now modify the database name
|
|
new_database_dict['database_name'] = manager_dict['database_dict']['database_name'] + '_new'
|
|
|
|
# First check and see if the new database already exists. If it does, check
|
|
# with the user whether it's ok to delete it.
|
|
try:
|
|
weedb.create(new_database_dict)
|
|
except weedb.DatabaseExists:
|
|
ans = y_or_n("New database '%s' already exists. "
|
|
"Delete it first (y/n)? " % new_database_dict['database_name'])
|
|
if ans == 'y':
|
|
weedb.drop(new_database_dict)
|
|
else:
|
|
print("Nothing done.")
|
|
return
|
|
|
|
# Get the unit system of the old archive:
|
|
with weewx.manager.Manager.open(manager_dict['database_dict']) as old_dbmanager:
|
|
old_unit_system = old_dbmanager.std_unit_system
|
|
|
|
if old_unit_system is None:
|
|
print("Old database has not been initialized. Nothing to be done.")
|
|
return
|
|
|
|
# Get the unit system of the new archive:
|
|
try:
|
|
target_unit_nickname = config_dict['StdConvert']['target_unit']
|
|
except KeyError:
|
|
target_unit_system = None
|
|
else:
|
|
target_unit_system = weewx.units.unit_constants[target_unit_nickname.upper()]
|
|
|
|
print("Copying database '%s' to '%s'" % (manager_dict['database_dict']['database_name'],
|
|
new_database_dict['database_name']))
|
|
if target_unit_system is None or old_unit_system == target_unit_system:
|
|
print("The new database will use the same unit system as the old ('%s')." %
|
|
weewx.units.unit_nicknames[old_unit_system])
|
|
else:
|
|
print("Units will be converted from the '%s' system to the '%s' system." %
|
|
(weewx.units.unit_nicknames[old_unit_system],
|
|
weewx.units.unit_nicknames[target_unit_system]))
|
|
|
|
ans = y_or_n("Are you sure you wish to proceed (y/n)? ")
|
|
if ans == 'y':
|
|
t1 = time.time()
|
|
weewx.manager.reconfig(manager_dict['database_dict'],
|
|
new_database_dict,
|
|
new_unit_system=target_unit_system,
|
|
new_schema=manager_dict['schema'])
|
|
tdiff = time.time() - t1
|
|
print("Database '%s' copied to '%s' in %.2f seconds." % (manager_dict['database_dict']['database_name'],
|
|
new_database_dict['database_name'],
|
|
tdiff))
|
|
else:
|
|
print("Nothing done.")
|
|
|
|
|
|
def transferDatabase(config_dict, db_binding, options):
|
|
"""Transfer 'archive' data from one database to another"""
|
|
|
|
# do we have enough to go on, must have a dest binding
|
|
if not options.dest_binding:
|
|
print("Destination binding not specified. Nothing Done. Aborting.")
|
|
return
|
|
# get manager dict for our source binding
|
|
src_manager_dict = weewx.manager.get_manager_dict_from_config(config_dict,
|
|
db_binding)
|
|
# get manager dict for our dest binding
|
|
try:
|
|
dest_manager_dict = weewx.manager.get_manager_dict_from_config(config_dict,
|
|
options.dest_binding)
|
|
except weewx.UnknownBinding:
|
|
# if we can't find the binding display a message then return
|
|
print("Unknown destination binding '%s', confirm destination binding."
|
|
% options.dest_binding)
|
|
print("Nothing Done. Aborting.")
|
|
return
|
|
except weewx.UnknownDatabase:
|
|
# if we can't find the database display a message then return
|
|
print("Error accessing destination database, confirm destination binding and/or database.")
|
|
print("Nothing Done. Aborting.")
|
|
return
|
|
except (ValueError, AttributeError):
|
|
# maybe a schema issue
|
|
print("Error accessing destination database.")
|
|
print("Maybe the destination schema is incorrectly specified in binding '%s' in weewx.conf?"
|
|
% options.dest_binding)
|
|
print("Nothing Done. Aborting.")
|
|
return
|
|
except weewx.UnknownDatabaseType:
|
|
# maybe a [Databases] issue
|
|
print("Error accessing destination database.")
|
|
print("Maybe the destination database is incorrectly defined in weewx.conf?")
|
|
print("Nothing Done. Aborting.")
|
|
return
|
|
# get a manager for our source
|
|
with weewx.manager.Manager.open(src_manager_dict['database_dict']) as src_manager:
|
|
# get first and last timestamps from the source so we can count the
|
|
# records to transfer and display an appropriate message
|
|
first_ts = src_manager.firstGoodStamp()
|
|
last_ts = src_manager.lastGoodStamp()
|
|
if first_ts is not None and last_ts is not None:
|
|
# we have source records
|
|
num_recs = src_manager.getAggregate(TimeSpan(first_ts, last_ts),
|
|
'dateTime', 'count')[0]
|
|
else:
|
|
# we have no source records to transfer so abort with a message
|
|
print("No records found in source database '%s' for transfer." %
|
|
src_manager.database_name)
|
|
print("Nothing done. Aborting.")
|
|
exit()
|
|
|
|
if not options.dry_run: # is it a dry run ?
|
|
# not a dry run, actually do the transfer
|
|
ans = y_or_n("Transfer %s records from source database '%s' "
|
|
"to destination database '%s' (y/n)? "
|
|
% (num_recs, src_manager.database_name,
|
|
dest_manager_dict['database_dict']['database_name']))
|
|
if ans == 'y':
|
|
t1 = time.time()
|
|
# wrap in a try..except in case we have an error
|
|
try:
|
|
with weewx.manager.Manager.open_with_create(dest_manager_dict['database_dict'],
|
|
table_name=dest_manager_dict['table_name'],
|
|
schema=dest_manager_dict['schema']) as dest_manager:
|
|
sys.stdout.write("transferring, this may take a while.... ")
|
|
sys.stdout.flush()
|
|
# do the transfer, should be quick as it's done as a
|
|
# single transaction
|
|
dest_manager.addRecord(src_manager.genBatchRecords())
|
|
print("complete")
|
|
# get first and last timestamps from the dest so we can
|
|
# count the records transferred and display a message
|
|
first_ts = dest_manager.firstGoodStamp()
|
|
last_ts = dest_manager.lastGoodStamp()
|
|
tdiff = time.time() - t1
|
|
if first_ts is not None and last_ts is not None:
|
|
num_recs = dest_manager.getAggregate(TimeSpan(first_ts, last_ts),
|
|
'dateTime', 'count')[0]
|
|
print("%s records transferred from source database '%s' to" %
|
|
(num_recs, src_manager.database_name))
|
|
print("destination database '%s' in %.2f seconds." %
|
|
(dest_manager.database_name, tdiff))
|
|
else:
|
|
print(
|
|
"Error. No records were transferred from source database '%s' to destination database '%s'." %
|
|
(src_manager.database_name, dest_manager.database_name))
|
|
except ImportError:
|
|
# Probably when trying to load db driver
|
|
print("Error accessing destination database '%s'." %
|
|
(dest_manager_dict['database_dict']['database_name'],))
|
|
print("Nothing done. Aborting.")
|
|
raise
|
|
except (OSError, weedb.OperationalError):
|
|
# probably a weewx.conf typo or MySQL db not created
|
|
print("Error accessing destination database '%s'."
|
|
% dest_manager_dict['database_dict']['database_name'])
|
|
print("Maybe it does not exist (MySQL) or is incorrectly defined in weewx.conf?")
|
|
print("Nothing done. Aborting.")
|
|
return
|
|
|
|
else:
|
|
# we decided not to do the transfer
|
|
print("Nothing done.")
|
|
return
|
|
else:
|
|
# it's a dry run so say what we would have done then return
|
|
print("Transfer %s records from source database '%s' "
|
|
"to destination database '%s'."
|
|
% (num_recs, src_manager.database_name,
|
|
dest_manager_dict['database_dict']['database_name']))
|
|
print("Dry run, nothing done.")
|
|
|
|
|
|
def check(config_dict, db_binding, options):
|
|
"""Check database and report outstanding fixes/issues.
|
|
|
|
Performs the following checks:
|
|
- checks database version
|
|
- checks for null strings in SQLite database
|
|
"""
|
|
|
|
t1 = time.time()
|
|
|
|
# Check interval weighting
|
|
print("Checking daily summary tables version...")
|
|
|
|
# Get a database manager object
|
|
dbm = weewx.manager.open_manager_with_config(config_dict, db_binding)
|
|
|
|
# check the daily summary version
|
|
_daily_summary_version = dbm._read_metadata('Version')
|
|
msg = "Daily summary tables are at version %s" % _daily_summary_version
|
|
syslog.syslog(syslog.LOG_INFO, msg)
|
|
print(msg)
|
|
|
|
if _daily_summary_version is not None and _daily_summary_version >= '2.0':
|
|
# interval weighting fix has been applied
|
|
msg = "Interval Weighting Fix is not required."
|
|
syslog.syslog(syslog.LOG_INFO, msg)
|
|
print(msg)
|
|
else:
|
|
print("Recommend running --update to recalculate interval weightings.")
|
|
print("Daily summary tables version check completed in %0.2f seconds." % (time.time() - t1))
|
|
|
|
# now check for null strings
|
|
check_strings(config_dict, db_binding, options, fix=False)
|
|
|
|
|
|
def update(config_dict, db_binding, options):
|
|
"""Apply any required database fixes.
|
|
|
|
Applies the following fixes:
|
|
- checks if database version is 2.0, if not interval weighting fix is
|
|
applied
|
|
- recalculates windSpeed daily summary max and maxtime fields from
|
|
archive
|
|
"""
|
|
|
|
# prompt for confirmation if it is not a dry run
|
|
ans = 'y' if options.dry_run else None
|
|
while ans not in ['y', 'n']:
|
|
ans = input("The update process does not affect archive data, but does alter the database.\nContinue (y/n)? ")
|
|
if ans == 'n':
|
|
# we decided not to update the summary tables
|
|
msg = "Update cancelled"
|
|
syslog.syslog(syslog.LOG_INFO, msg)
|
|
print(msg)
|
|
return
|
|
|
|
if options.dry_run:
|
|
_log_level = syslog.setlogmask(syslog.LOG_UPTO(syslog.LOG_WARNING))
|
|
|
|
msg = "Preparing Interval Weighting Fix..."
|
|
syslog.syslog(syslog.LOG_INFO, msg)
|
|
print(msg)
|
|
|
|
# notify if this is a dry run
|
|
if options.dry_run:
|
|
print("This is a dry run: weighted intervals will be calculated but not saved.")
|
|
|
|
# Get a database manager object
|
|
dbm = weewx.manager.open_manager_with_config(config_dict, db_binding)
|
|
database_name = dbm.database_name
|
|
|
|
# Interval weighting
|
|
# first construct an interval weighting config dict
|
|
weighting_config_dict = {'name': 'Interval Weighting Fix',
|
|
'binding': db_binding,
|
|
'trans_days': 100,
|
|
'dry_run': options.dry_run}
|
|
|
|
# create an interval weighting fix object
|
|
weight_obj = weecfg.database.IntervalWeighting(config_dict,
|
|
weighting_config_dict)
|
|
# check the daily summary version
|
|
_daily_summary_version = dbm._read_metadata('Version')
|
|
msg = "Daily summary tables are at version %s" % _daily_summary_version
|
|
syslog.syslog(syslog.LOG_INFO, msg)
|
|
print(msg)
|
|
|
|
if _daily_summary_version is not None and _daily_summary_version >= '2.0':
|
|
# interval weighting fix has been applied
|
|
msg = "Interval Weighting Fix is not required."
|
|
syslog.syslog(syslog.LOG_INFO, msg)
|
|
print(msg)
|
|
else:
|
|
# apply the interval weighting
|
|
msg = "Calculating interval weights..."
|
|
syslog.syslog(syslog.LOG_INFO, msg)
|
|
print(msg)
|
|
t1 = time.time()
|
|
weight_obj.run()
|
|
msg = "Interval Weighting Fix completed in %0.2f seconds." % (time.time() - t1)
|
|
syslog.syslog(syslog.LOG_INFO, msg)
|
|
print(msg)
|
|
|
|
# recalc the max/maxtime windSpeed values
|
|
_fix_wind(config_dict, db_binding, options)
|
|
# just in case, set the syslog level back where we found it
|
|
if options.dry_run:
|
|
syslog.setlogmask(_log_level)
|
|
|
|
|
|
def _fix_wind(config_dict, db_binding, options):
|
|
"""Recalculate the windSpeed daily summary max and maxtime fields.
|
|
|
|
Create a WindSpeedRecalculation object and call its run() method to
|
|
recalculate the max and maxtime fields from archive data. This process is
|
|
idempotent so it can be called repeatedly with no ill effect.
|
|
"""
|
|
|
|
t1 = time.time()
|
|
msg = "Preparing Maximum windSpeed Fix..."
|
|
syslog.syslog(syslog.LOG_INFO, msg)
|
|
print(msg)
|
|
|
|
# notify if this is a dry run
|
|
if options.dry_run:
|
|
print("This is a dry run: maximum windSpeed will be calculated but not saved.")
|
|
|
|
# construct a windSpeed recalculation config dict
|
|
wind_config_dict = {'name': 'Maximum windSpeed Fix',
|
|
'binding': db_binding,
|
|
'trans_days': 100,
|
|
'dry_run': options.dry_run}
|
|
|
|
# create a windSpeedRecalculation object
|
|
wind_obj = weecfg.database.WindSpeedRecalculation(config_dict,
|
|
wind_config_dict)
|
|
# perform the recalculation, wrap in a try..except to catch any db errors
|
|
try:
|
|
wind_obj.run()
|
|
except weedb.NoTableError as e:
|
|
msg = "Maximum windSpeed Fix applied: no windSpeed found"
|
|
syslog.syslog(syslog.LOG_INFO, msg)
|
|
print(msg)
|
|
else:
|
|
msg = "Maximum windSpeed Fix completed in %0.2f seconds" % (time.time() - t1)
|
|
syslog.syslog(syslog.LOG_INFO, msg)
|
|
print(msg)
|
|
|
|
|
|
# These functions are necessary because Python 3 does not allow you to
|
|
# parameterize types. So, we use a big if-else.
|
|
|
|
def check_type(val, expected):
|
|
if expected == 'INTEGER':
|
|
return isinstance(val, six.integer_types)
|
|
elif expected == 'REAL':
|
|
return isinstance(val, float)
|
|
elif expected == 'STR':
|
|
return isinstance(val, six.string_types)
|
|
else:
|
|
raise ValueError("Unknown type %s" % expected)
|
|
|
|
|
|
def set_type(val, target):
|
|
if target == 'INTEGER':
|
|
return int(val)
|
|
elif target == 'REAL':
|
|
return float(val)
|
|
elif target == 'STR':
|
|
return six.ensure_str(val)
|
|
else:
|
|
raise ValueError("Unknown type %s" % target)
|
|
|
|
|
|
def check_strings(config_dict, db_binding, options, fix=False):
|
|
"""Scan the archive table for null strings.
|
|
|
|
Identifies and lists any null string occurrences in the archive table. If
|
|
fix is True then any null strings that are found are fixed.
|
|
"""
|
|
|
|
t1 = time.time()
|
|
if options.dry_run or not fix:
|
|
_log_level = syslog.setlogmask(syslog.LOG_UPTO(syslog.LOG_WARNING))
|
|
|
|
print("Preparing Null String Fix, this may take a while...")
|
|
|
|
if fix:
|
|
syslog.syslog(syslog.LOG_INFO, "Preparing Null String Fix")
|
|
# notify if this is a dry run
|
|
if options.dry_run:
|
|
print("This is a dry run: null strings will be detected but not fixed")
|
|
|
|
# open up the main database archive table
|
|
with weewx.manager.open_manager_with_config(config_dict, db_binding) as dbmanager:
|
|
|
|
obs_list = []
|
|
obs_type_list = []
|
|
|
|
# get the schema and extract the Python type each observation type should be
|
|
for column in dbmanager.connection.genSchemaOf('archive'):
|
|
# Save the observation name for this column (eg, 'outTemp'):
|
|
obs_list.append(column[1])
|
|
# And its type
|
|
obs_type_list.append(column[2])
|
|
|
|
records = 0
|
|
_found = []
|
|
# cycle through each row in the database
|
|
for record in dbmanager.genBatchRows():
|
|
records += 1
|
|
# now examine each column
|
|
for icol in range(len(record)):
|
|
# check to see if this column is an instance of the correct
|
|
# Python type
|
|
if record[icol] is not None and not check_type(record[icol], obs_type_list[icol]):
|
|
# Oops. Found a bad one. Print it out.
|
|
if fix:
|
|
syslog.syslog(syslog.LOG_INFO,
|
|
"Timestamp = %s; record['%s']= %r; ... "
|
|
% (record[0], obs_list[icol], record[icol]))
|
|
|
|
if fix:
|
|
# coerce to the correct type. If it can't be done, then
|
|
# set it to None.
|
|
try:
|
|
corrected_value = set_type(record[icol], obs_type_list[icol])
|
|
except ValueError:
|
|
corrected_value = None
|
|
# update the database with the new value but only if
|
|
# it's not a dry run
|
|
if not options.dry_run:
|
|
dbmanager.updateValue(record[0], obs_list[icol], corrected_value)
|
|
_found.append((record[0], obs_list[icol], record[icol], corrected_value))
|
|
# log it
|
|
syslog.syslog(syslog.LOG_INFO, " changed to %r\n" % corrected_value)
|
|
else:
|
|
_found.append((record[0], obs_list[icol], record[icol]))
|
|
# notify the user of progress
|
|
if records % 1000 == 0:
|
|
print("Checking record: %d; Timestamp: %s\r"
|
|
% (records, timestamp_to_string(record[0])), end=' ')
|
|
sys.stdout.flush()
|
|
# update our final count now that we have finished
|
|
print("Checking record: %d; Timestamp: %s\r" % (records, timestamp_to_string(record[0])), end=' ')
|
|
print()
|
|
tdiff = time.time() - t1
|
|
# now display details of what we found if we found any null strings
|
|
if len(_found):
|
|
print("The following null strings were found:")
|
|
for item in _found:
|
|
if len(item) == 4:
|
|
print("Timestamp = %s; record['%s'] = %r; ... changed to %r" % item)
|
|
else:
|
|
print("Timestamp = %s; record['%s'] = %r; ... ignored" % item)
|
|
# how many did we fix?
|
|
fixed = len([a for a in _found if len(a) == 4])
|
|
# summarise our results
|
|
if len(_found) == 0:
|
|
# found no null strings, log it and display on screen
|
|
syslog.syslog(syslog.LOG_INFO, "No null strings found.")
|
|
print("No null strings found.")
|
|
elif fixed == len(_found):
|
|
# fixed all we found
|
|
if options.dry_run:
|
|
# its a dry run so display to screen but not to log
|
|
print("%d of %d null strings found would have been fixed." % (fixed, len(_found)))
|
|
else:
|
|
# really did fix so log and display to screen
|
|
syslog.syslog(syslog.LOG_INFO,
|
|
"%d of %d null strings found were fixed." % (fixed, len(_found)))
|
|
print("%d of %d null strings found were fixed." % (fixed, len(_found)))
|
|
elif fix:
|
|
# this should never occur - found some but didn't fix them all when we
|
|
# should have
|
|
if options.dry_run:
|
|
# its a dry run so say what would have happened
|
|
print("Could not fix all null strings. "
|
|
"%d of %d null strings found would have been fixed." % (fixed,
|
|
len(_found)))
|
|
else:
|
|
# really did fix so log and display to screen
|
|
syslog.syslog(syslog.LOG_INFO,
|
|
"Could not fix all null strings. "
|
|
"%d of %d null strings found were fixed." % (fixed,
|
|
len(_found)))
|
|
print("Could not fix all null strings. "
|
|
"%d of %d null strings found were fixed." % (fixed,
|
|
len(_found)))
|
|
else:
|
|
# found some null string but it was only a check not a fix, just
|
|
# display to screen
|
|
print("%d null strings were found.\r\n"
|
|
"Recommend running --fix-strings to fix these strings." % len(_found))
|
|
|
|
# and finally details on time taken
|
|
if fix:
|
|
syslog.syslog(syslog.LOG_INFO, "Applied Null String Fix in %0.2f seconds." % tdiff)
|
|
print("Applied Null String Fix in %0.2f seconds." % tdiff)
|
|
else:
|
|
# it was a check not a fix so just display to screen
|
|
print("Completed Null String Check in %0.2f seconds." % tdiff)
|
|
# just in case, set the syslog level back where we found it
|
|
if options.dry_run or not fix:
|
|
syslog.setlogmask(_log_level)
|
|
|
|
|
|
def _parse_dates(options):
|
|
"""Parse --date, --from and --to command line options.
|
|
|
|
Inputs:
|
|
options: the optparse options
|
|
|
|
Returns: A two-way tuple (start_d, stop_d), where
|
|
start_d: A datetime.date object holding the first date. May be None
|
|
stop_d: A dateimte.date object holding the last date. May be None
|
|
"""
|
|
|
|
# Default is None, unless user has specified an option
|
|
_first_d = None
|
|
_last_d = None
|
|
|
|
if options.date:
|
|
# Make sure we are not overspecified
|
|
if options.from_date or options.to_date:
|
|
raise ValueError("Specify either --date or a --from and --to combination; not both")
|
|
|
|
# there is a --date but is it valid
|
|
try:
|
|
_first_d = datetime.datetime.strptime(options.date, "%Y-%m-%d").date()
|
|
except ValueError:
|
|
raise ValueError("Invalid --date option specified.")
|
|
else:
|
|
_last_d = _first_d
|
|
return (_first_d, _last_d)
|
|
|
|
if options.from_date:
|
|
try:
|
|
_first_d = datetime.datetime.strptime(options.from_date, "%Y-%m-%d").date()
|
|
except ValueError:
|
|
raise ValueError("Invalid --from option specified.")
|
|
|
|
if options.to_date:
|
|
try:
|
|
_last_d = datetime.datetime.strptime(options.to_date, "%Y-%m-%d").date()
|
|
except ValueError:
|
|
raise ValueError("Invalid --to option specified.")
|
|
|
|
if _first_d and _last_d:
|
|
# If both --from and --to have been specified, make sure the latter is greater
|
|
# than the former
|
|
if _last_d < _first_d:
|
|
raise weewx.ViolatedPrecondition("--from value is later than --to value.")
|
|
|
|
return (_first_d, _last_d)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|