mirror of
https://github.com/weewx/weewx.git
synced 2026-04-19 00:56:54 -04:00
750 lines
35 KiB
Python
Executable File
750 lines
35 KiB
Python
Executable File
#!/usr/bin/env python
|
|
#
|
|
# Copyright (c) 2009-2015 Tom Keffer <tkeffer@gmail.com>
|
|
#
|
|
# See the file LICENSE.txt for your full rights.
|
|
#
|
|
"""Configure databases used by weeWX"""
|
|
from __future__ import with_statement
|
|
|
|
# python imports
|
|
import datetime
|
|
import optparse
|
|
import syslog
|
|
import sys
|
|
import time
|
|
|
|
# weewx imports
|
|
import user.extensions #@UnusedImport
|
|
import weecfg.database
|
|
import weedb
|
|
import weewx.manager
|
|
import weewx.units
|
|
|
|
from weeutil.weeutil import TimeSpan
|
|
|
|
usage = """wee_database --help
|
|
wee_database --create
|
|
wee_database --reconfigure
|
|
wee_database --transfer --dest-binding=BINDING_NAME [--dry-run]
|
|
wee_database --check
|
|
wee_database --update [--dry-run]
|
|
wee_database --check-strings
|
|
wee_database --fix-strings [--dry-run]
|
|
wee_database --drop-daily
|
|
wee_database --rebuild-daily
|
|
[--date=YYYY-mm-dd | --from=YYYY-mm-dd --to=YYYY-mm-dd]
|
|
|
|
Description:
|
|
|
|
Manipulate the weeWX database. Most of these operations are handled automatically
|
|
by weeWX, but they may be useful in special cases."""
|
|
|
|
# List of 'dest' settings used by our 'verbs', note 'dest' may be explicit or
|
|
# implicit. If adding more 'verbs' need to add corresponding 'dest' here.
|
|
dest_list = ['create', 'drop_daily', 'rebuild_daily', 'reconfigure', 'transfer',
|
|
'check', 'update', 'check_strings', 'fix']
|
|
|
|
def main():
|
|
|
|
# Set defaults for the system logger:
|
|
syslog.openlog('wee_database', syslog.LOG_PID|syslog.LOG_CONS)
|
|
|
|
# Create a command line parser:
|
|
parser = optparse.OptionParser(usage=usage)
|
|
|
|
# Add the various options:
|
|
parser.add_option("--config", dest="config_path", type=str,
|
|
metavar="CONFIG_FILE",
|
|
help="Use configuration file CONFIG_FILE.")
|
|
parser.add_option("--create", dest="create_database", action='store_true',
|
|
help="Create the weeWX database and initialize it with the"
|
|
" schema.")
|
|
parser.add_option("--drop-daily", dest="drop_daily", action='store_true',
|
|
help="Drop the daily summary tables from a database.")
|
|
parser.add_option("--rebuild-daily", dest="rebuild_daily",
|
|
action='store_true',
|
|
help="Rebuild the daily summaries from data in the archive"
|
|
" table.")
|
|
parser.add_option("--date", dest="date", type=str, metavar="YYYY-mm-dd",
|
|
help="This date only (option --rebuild-daily only).")
|
|
parser.add_option("--from", dest="from_date", type=str, metavar="YYYY-mm-dd",
|
|
help="Start with this date (option --rebuild-daily only).")
|
|
parser.add_option("--to", dest="to_date", type=str, metavar="YYYY-mm-dd",
|
|
help="End with this date (option --rebuild-daily only).")
|
|
parser.add_option("--reconfigure", action='store_true',
|
|
help="Create a new database using configuration"
|
|
" information found in the configuration file. In"
|
|
" particular, the new database will use the unit system"
|
|
" found in option [StdConvert][target_unit]."
|
|
" The new database will have the same name as the old"
|
|
" database, with a '_new' on the end.")
|
|
parser.add_option("--transfer", dest="transfer", action='store_true',
|
|
help="Transfer the weeWX archive from source database to"
|
|
" destination database.")
|
|
parser.add_option("--check", dest="check", action="store_true",
|
|
help="Check the calculations in the daily summary tables.")
|
|
parser.add_option("--update", dest="update", action="store_true",
|
|
help="Update the daily summary tables if required and"
|
|
" recalculate the daily summary maximum windSpeed values.")
|
|
parser.add_option("--check-strings", dest="check_strings", action="store_true",
|
|
help="Check the archive table for null strings that may"
|
|
" have been introduced by a SQL editing program.")
|
|
parser.add_option("--fix-strings", dest="fix", action='store_true',
|
|
help="Fix any null strings in a SQLite database.")
|
|
parser.add_option("--binding", dest="binding", metavar="BINDING_NAME",
|
|
default='wx_binding',
|
|
help="The data binding to use. Default is 'wx_binding'.")
|
|
parser.add_option("--dest-binding", dest="dest_binding",
|
|
metavar="BINDING_NAME",
|
|
help="The destination data binding (option --transfer only).")
|
|
parser.add_option('--dry-run', dest="dry_run", action='store_true',
|
|
default=False,
|
|
help='Print what would happen but do not do it. Default'
|
|
' is False.')
|
|
|
|
# Now we are ready to parse the command line:
|
|
(options, args) = parser.parse_args()
|
|
|
|
# Do a check to see if the user used more than 1 'verb' . More than 1 verb
|
|
# should be avoided - what did the user really want to do?
|
|
# first get a list of options actually used
|
|
options_list = [opt for opt, val in options.__dict__.items() if val is not None]
|
|
# now a list of the 'dest' settings used
|
|
dest_used = [opt for opt in options_list if opt in dest_list]
|
|
# If we have more than 1 entry in dest_used then we have more than 1 'verb'
|
|
# used. Then inform the user and exit. If only 1 'verb' then carry on.
|
|
if len(dest_used) > 1:
|
|
# generate the message but we need to speak 'options' (aka verbs)
|
|
# not 'dest'
|
|
# get a dict of {option.dest: option.str} from our universe of options
|
|
opt_dict = dict((x.dest, x.get_opt_string()) for x in parser.option_list[1:])
|
|
# now get a list of 'verbs' used
|
|
verbs_used = [opt_dict[x] for x in dest_used]
|
|
# get rid of the [ and ]
|
|
verbs_str = ', '.join(verbs_used)
|
|
# our exit message
|
|
exit_str = ("Multiple verbs found in command line %s. Only one verb permitted.\nNothing done. Aborting." %
|
|
(verbs_str, ))
|
|
# now exit with our message
|
|
sys.exit(exit_str)
|
|
|
|
# get config_dict to use
|
|
config_path, config_dict = weecfg.read_config(options.config_path, args)
|
|
print "Using configuration file %s" % config_path
|
|
|
|
# set syslog level based
|
|
if int(config_dict.get('debug', 0)):
|
|
syslog.setlogmask(syslog.LOG_UPTO(syslog.LOG_DEBUG))
|
|
else:
|
|
syslog.setlogmask(syslog.LOG_UPTO(syslog.LOG_INFO))
|
|
|
|
db_binding = options.binding
|
|
database = config_dict['DataBindings'][db_binding]['database']
|
|
print "Using database binding '%s', which is bound to database '%s'" % (db_binding, database)
|
|
|
|
if options.create_database:
|
|
createMainDatabase(config_dict, db_binding)
|
|
|
|
if options.drop_daily:
|
|
dropDaily(config_dict, db_binding)
|
|
|
|
if options.rebuild_daily:
|
|
rebuildDaily(config_dict, db_binding, options)
|
|
|
|
if options.reconfigure:
|
|
reconfigMainDatabase(config_dict, db_binding)
|
|
|
|
if options.transfer:
|
|
transferDatabase(config_dict, db_binding, options)
|
|
|
|
if options.check:
|
|
check(config_dict, db_binding, options)
|
|
|
|
if options.update:
|
|
update(config_dict, db_binding, options)
|
|
|
|
if options.check_strings:
|
|
check_strings(config_dict, db_binding, options, fix=False)
|
|
|
|
if options.fix:
|
|
check_strings(config_dict, db_binding, options, fix=True)
|
|
|
|
def createMainDatabase(config_dict, db_binding):
|
|
"""Create the weeWX database"""
|
|
|
|
# Try a simple open. If it succeeds, that means the database
|
|
# exists and is initialized. Otherwise, an exception will be thrown.
|
|
try:
|
|
with weewx.manager.open_manager_with_config(config_dict, db_binding) as dbmanager:
|
|
print "Database '%s' already exists. Nothing done." % (dbmanager.database_name,)
|
|
except weedb.OperationalError:
|
|
# Database does not exist. Try again, but allow initialization:
|
|
with weewx.manager.open_manager_with_config(config_dict, db_binding, initialize=True) as dbmanager:
|
|
print "Created database '%s'" % (dbmanager.database_name,)
|
|
|
|
def dropDaily(config_dict, db_binding):
|
|
"""Drop the daily summaries from a weeWX database"""
|
|
|
|
manager_dict = weewx.manager.get_manager_dict_from_config(config_dict,
|
|
db_binding)
|
|
database_name = manager_dict['database_dict']['database_name']
|
|
|
|
ans = None
|
|
while ans not in ['y', 'n']:
|
|
print "Proceeding will delete all your daily summaries from database '%s'" % database_name
|
|
ans = raw_input("Are you sure you want to proceed (y/n)? ")
|
|
if ans == 'y' :
|
|
t1 = time.time()
|
|
print "Dropping daily summary tables from '%s' ... " % database_name
|
|
try:
|
|
with weewx.manager.open_manager_with_config(config_dict, db_binding) as dbmanager:
|
|
try:
|
|
dbmanager.drop_daily()
|
|
except weedb.OperationalError, e:
|
|
print "Got error '%s'\nPerhaps there was no daily summary?" % e
|
|
else:
|
|
tdiff = time.time() - t1
|
|
print "Daily summary tables dropped from database '%s' in %.2f seconds" % (database_name, tdiff)
|
|
except weedb.OperationalError:
|
|
# No daily summaries. Nothing to be done.
|
|
print "No daily summaries found in database '%s'. Nothing done." % (database_name,)
|
|
|
|
def rebuildDaily(config_dict, db_binding, options):
|
|
"""Rebuild the daily summaries."""
|
|
|
|
manager_dict = weewx.manager.get_manager_dict_from_config(config_dict,
|
|
db_binding)
|
|
database_name = manager_dict['database_dict']['database_name']
|
|
|
|
# determine the period over which we are rebuilding from any command
|
|
# line date parameters
|
|
start_d, stop_d = _parse_dates(options)
|
|
# advise the user/log what we will do
|
|
if start_d is None and stop_d is None:
|
|
_msg = "All daily summaries will be rebuilt."
|
|
elif start_d and not stop_d:
|
|
_msg = "Daily summaries from %s through the end will be rebuilt." % start_d
|
|
elif not start_d and stop_d:
|
|
_msg = "Daily summaries from the begining through date %s will be rebuilt." % stop_d
|
|
elif start_d == stop_d:
|
|
_msg = "Daily summary for date %s will be rebuilt" % start_d
|
|
else:
|
|
_msg = "Daily summaries from %s through %s inclusive will be rebuilt." % (start_d, stop_d)
|
|
|
|
syslog.syslog(syslog.LOG_INFO, _msg)
|
|
print _msg
|
|
|
|
ans = None
|
|
while ans not in ['y', 'n']:
|
|
ans = raw_input("Proceed (y/n)? ")
|
|
if ans == 'y':
|
|
break
|
|
elif ans == 'n':
|
|
print "Nothing done."
|
|
return
|
|
|
|
t1 = time.time()
|
|
|
|
# Open up the database. This will create the tables necessary for the daily
|
|
# summaries if they don't already exist:
|
|
with weewx.manager.open_manager_with_config(config_dict, db_binding, initialize=True) as dbmanager:
|
|
|
|
syslog.syslog(syslog.LOG_INFO, "Rebuilding daily summaries in database '%s' ..." % database_name)
|
|
print "Rebuilding daily summaries in database '%s' ..." % database_name
|
|
if options.dry_run:
|
|
print "Dry run. Nothing done."
|
|
return
|
|
else:
|
|
# now do the actual rebuild
|
|
nrecs, ndays = dbmanager.backfill_day_summary(start_d=start_d,
|
|
stop_d=stop_d,
|
|
trans_days=20)
|
|
tdiff = time.time() - t1
|
|
# advise the user/log what we did
|
|
syslog.syslog(syslog.LOG_INFO, "Rebuild of daily summaries in database '%s' complete" % database_name)
|
|
if nrecs:
|
|
sys.stdout.flush()
|
|
print "Processed %d records to rebuild %d day summaries in %.2f seconds " % (nrecs,
|
|
ndays,
|
|
tdiff)
|
|
print "Rebuild of daily summaries in database '%s' complete" % database_name
|
|
else:
|
|
print "Daily summaries up to date in '%s'" % database_name
|
|
|
|
def reconfigMainDatabase(config_dict, db_binding):
|
|
"""Create a new database, then populate it with the contents of an old database"""
|
|
|
|
manager_dict = weewx.manager.get_manager_dict_from_config(config_dict,
|
|
db_binding)
|
|
# Make a copy for the new database (we will be modifying it)
|
|
new_database_dict = dict(manager_dict['database_dict'])
|
|
|
|
# Now modify the database name
|
|
new_database_dict['database_name'] = manager_dict['database_dict']['database_name'] + '_new'
|
|
|
|
# First check and see if the new database already exists. If it does, check
|
|
# with the user whether it's ok to delete it.
|
|
try:
|
|
weedb.create(new_database_dict)
|
|
except weedb.DatabaseExists:
|
|
ans = None
|
|
while ans not in ['y', 'n']:
|
|
ans = raw_input("New database '%s' already exists. Delete it first (y/n)? " % (new_database_dict['database_name'],))
|
|
if ans == 'y':
|
|
weedb.drop(new_database_dict)
|
|
elif ans == 'n':
|
|
print "Nothing done."
|
|
return
|
|
|
|
# Get the unit system of the old archive:
|
|
with weewx.manager.Manager.open(manager_dict['database_dict']) as old_dbmanager:
|
|
old_unit_system = old_dbmanager.std_unit_system
|
|
|
|
if old_unit_system is None:
|
|
print "Old database has not been initialized. Nothing to be done."
|
|
return
|
|
|
|
# Get the unit system of the new archive:
|
|
try:
|
|
target_unit_nickname = config_dict['StdConvert']['target_unit']
|
|
except KeyError:
|
|
target_unit_system = None
|
|
else:
|
|
target_unit_system = weewx.units.unit_constants[target_unit_nickname.upper()]
|
|
|
|
|
|
ans = None
|
|
while ans not in ['y', 'n']:
|
|
print "Copying database '%s' to '%s'" % (manager_dict['database_dict']['database_name'],
|
|
new_database_dict['database_name'])
|
|
if target_unit_system is None or old_unit_system==target_unit_system:
|
|
print ("The new database will use the same unit system as the old ('%s')." %
|
|
(weewx.units.unit_nicknames[old_unit_system],))
|
|
else:
|
|
print ("Units will be converted from the '%s' system to the '%s' system." %
|
|
(weewx.units.unit_nicknames[old_unit_system],
|
|
weewx.units.unit_nicknames[target_unit_system]))
|
|
ans = raw_input("Are you sure you wish to proceed (y/n)? ")
|
|
if ans == 'y':
|
|
t1 = time.time()
|
|
weewx.manager.reconfig(manager_dict['database_dict'],
|
|
new_database_dict,
|
|
new_unit_system=target_unit_system,
|
|
new_schema=manager_dict['schema'])
|
|
tdiff = time.time() - t1
|
|
print "Database '%s' copied to '%s' in %.2f seconds." % (manager_dict['database_dict']['database_name'],
|
|
new_database_dict['database_name'],
|
|
tdiff)
|
|
elif ans == 'n':
|
|
print "Nothing done."
|
|
|
|
def transferDatabase(config_dict, db_binding, options):
|
|
"""Transfer 'archive' data from one database to another"""
|
|
|
|
# do we have enough to go on, must have a dest binding
|
|
if options.dest_binding is None:
|
|
print "Destination binding not specified. Nothing Done. Aborting."
|
|
return
|
|
# get manager dict for our source binding
|
|
src_manager_dict = weewx.manager.get_manager_dict_from_config(config_dict,
|
|
db_binding)
|
|
# get manager dict for our dest binding
|
|
try:
|
|
dest_manager_dict = weewx.manager.get_manager_dict_from_config(config_dict,
|
|
options.dest_binding)
|
|
except weewx.UnknownBinding:
|
|
# if we can't find the binding display a message then return
|
|
print ("Unknown destination binding '%s', confirm destination binding." %
|
|
(options.dest_binding, ))
|
|
print "Nothing Done. Aborting."
|
|
return
|
|
except weewx.UnknownDatabase:
|
|
# if we can't find the database display a message then return
|
|
print "Error accessing destination database, confirm destination binding and/or database."
|
|
print "Nothing Done. Aborting."
|
|
return
|
|
except (ValueError, AttributeError):
|
|
# maybe a schema issue
|
|
print "Error accessing destination database."
|
|
print ("Maybe the destination schema is incorrectly specified in binding '%s' in weewx.conf?" %
|
|
(options.dest_binding, ))
|
|
print "Nothing Done. Aborting."
|
|
return
|
|
except (weewx.UnknownDatabaseType):
|
|
# maybe a [Databases] issue
|
|
print "Error accessing destination database."
|
|
print "Maybe the destination database is incorrectly defined in weewx.conf?"
|
|
print "Nothing Done. Aborting."
|
|
return
|
|
# get a manager for our source
|
|
with weewx.manager.Manager.open(src_manager_dict['database_dict']) as src_manager:
|
|
# get first and last timestamps from the source so we can count the
|
|
# records to transfer and display an appropriate message
|
|
first_ts = src_manager.firstGoodStamp()
|
|
last_ts = src_manager.lastGoodStamp()
|
|
if first_ts is not None and last_ts is not None:
|
|
# we have source records
|
|
num_recs = src_manager.getAggregate(TimeSpan(first_ts, last_ts),
|
|
'dateTime', 'count')[0]
|
|
else:
|
|
# we have no source records to transfer so abort with a message
|
|
print ("No records found in source database '%s' for transfer." %
|
|
(src_manager.database_name, ))
|
|
print "Nothing done. Aborting."
|
|
exit()
|
|
ans = None
|
|
if not options.dry_run: # is it a dry run ?
|
|
# not a dry run, actually do the transfer
|
|
while ans not in ['y', 'n']:
|
|
ans = raw_input("Transfer %s records from source database '%s' to destination database '%s' (y/n)? " %
|
|
(num_recs, src_manager.database_name, dest_manager_dict['database_dict']['database_name']))
|
|
if ans == 'y':
|
|
t1 = time.time()
|
|
# wrap in a try..except in case we have an error
|
|
try:
|
|
with weewx.manager.Manager.open_with_create(dest_manager_dict['database_dict'],
|
|
table_name=dest_manager_dict['table_name'],
|
|
schema=dest_manager_dict['schema']) as dest_manager:
|
|
sys.stdout.write("transferring, this may take a while.... ")
|
|
sys.stdout.flush()
|
|
# do the transfer, should be quick as it's done as a
|
|
# single transaction
|
|
dest_manager.addRecord(src_manager.genBatchRecords())
|
|
print "complete"
|
|
# get first and last timestamps from the dest so we can
|
|
# count the records transferred and display a message
|
|
first_ts = dest_manager.firstGoodStamp()
|
|
last_ts = dest_manager.lastGoodStamp()
|
|
tdiff = time.time() - t1
|
|
if first_ts is not None and last_ts is not None:
|
|
num_recs = dest_manager.getAggregate(TimeSpan(first_ts, last_ts),
|
|
'dateTime', 'count')[0]
|
|
print ("%s records transferred from source database '%s' to" %
|
|
(num_recs, src_manager.database_name))
|
|
print ("destination database '%s' in %.2f seconds." %
|
|
(dest_manager.database_name, tdiff))
|
|
else:
|
|
print ("Error. No records were transferred from source database '%s' to destination database '%s'." %
|
|
(src_manager.database_name, dest_manager.database_name))
|
|
except ImportError:
|
|
# Probably when trying to load db driver
|
|
print ("Error accessing destination database '%s'." %
|
|
(dest_manager_dict['database_dict']['database_name'], ))
|
|
print "Nothing done. Aborting."
|
|
raise
|
|
except (OSError, weedb.OperationalError):
|
|
# probably a weewx.conf typo or MySQL db not created
|
|
print ("Error accessing destination database '%s'." %
|
|
(dest_manager_dict['database_dict']['database_name'], ))
|
|
print "Maybe it does not exist (MySQL) or is incorrectly defined in weewx.conf?"
|
|
print "Nothing done. Aborting."
|
|
return
|
|
|
|
elif ans == 'n':
|
|
# we decided not to do the transfer
|
|
print "Nothing done."
|
|
return
|
|
else:
|
|
# it's a dry run so say what we would have done then return
|
|
print ("Transfer %s records from source database '%s' to destination database '%s'." %
|
|
(num_recs, src_manager.database_name, dest_manager_dict['database_dict']['database_name']))
|
|
print "Dry run, nothing done."
|
|
|
|
def check(config_dict, db_binding, options):
|
|
"""Check database and report outstanding fixes/issues.
|
|
|
|
Performs the following checks:
|
|
- checks database version
|
|
- checks for null strings in SQLite database
|
|
"""
|
|
|
|
t1 = time.time()
|
|
|
|
# Check interval weighting
|
|
print "Checking daily summary tables version..."
|
|
|
|
# Get a database manager object
|
|
dbm = weewx.manager.open_manager_with_config(config_dict, db_binding)
|
|
|
|
# check the daily summary version
|
|
_daily_summary_version = dbm._read_metadata('Version')
|
|
if _daily_summary_version is not None and _daily_summary_version >= '2.0':
|
|
# interval weighting fix has been applied so advise the user
|
|
print "Daily summary tables are already the latest version. No action required."
|
|
else:
|
|
# interval weighting fix has not been applied so advise the user and
|
|
# then apply the interval weighting fix
|
|
print "Daily summary tables are version %s. Recommend running 'wee_database --update'." % _daily_summary_version
|
|
print "Daily summary tables version check completed in %0.3f seconds." % (time.time() - t1)
|
|
|
|
# now check for null strings
|
|
check_strings(config_dict, db_binding, options, fix=False)
|
|
|
|
def update(config_dict, db_binding, options):
|
|
"""Apply any required database fixes.
|
|
|
|
Applies the following fixes:
|
|
- checks if database version is 2.0, if not interval weighting fix is
|
|
applied
|
|
- recalculates windSpeed daily summary max and maxtime fields from
|
|
archive
|
|
"""
|
|
|
|
t1 = time.time()
|
|
syslog.syslog(syslog.LOG_INFO, "Preparing 'Interval Weighting' fix...")
|
|
print "Preparing 'Interval Weighting' fix..."
|
|
# notify if this is a dry run
|
|
if options.dry_run:
|
|
print "This is a dry run. Interval weighting will be applied but not saved."
|
|
# Get a database manager object
|
|
dbm = weewx.manager.open_manager_with_config(config_dict, db_binding)
|
|
database_name = dbm.database_name
|
|
|
|
# Interval weighting
|
|
# first construct an interval weighting config dict
|
|
weighting_config_dict = {'name': 'Interval Weighting',
|
|
'binding': db_binding,
|
|
'trans_days': 100,
|
|
'dry_run': options.dry_run}
|
|
|
|
# create an interval weighting fix object
|
|
weight_obj = weecfg.database.IntervalWeighting(config_dict,
|
|
weighting_config_dict)
|
|
# check the daily sumamry version
|
|
_daily_summary_version = dbm._read_metadata('Version')
|
|
syslog.syslog(syslog.LOG_INFO,
|
|
"Daily summary version is %s" % _daily_summary_version)
|
|
print "Daily summary version is %s" % _daily_summary_version
|
|
if _daily_summary_version is not None and _daily_summary_version >= '2.0':
|
|
# interval weighting fix has been applied so advise the user
|
|
syslog.syslog(syslog.LOG_INFO,
|
|
"Daily summary tables are already using the latest version.")
|
|
print "Daily summary tables are already using the latest version."
|
|
syslog.syslog(syslog.LOG_INFO, "'Interval Weighting' fix not required.")
|
|
print "'Interval Weighting' fix not required."
|
|
else:
|
|
# interval weighting fix has not been applied so advise the user and
|
|
# then apply the interval weighting fix
|
|
syslog.syslog(syslog.LOG_INFO,
|
|
"Daily summary tables are not using the latest version and will be upgraded.")
|
|
print "Daily summary tables are not using the latest version and will be upgraded."
|
|
# this action is irreversible so check the user really wants to do this
|
|
ans = None
|
|
while ans not in ['y', 'n']:
|
|
ans = raw_input("The update process is not destructive but will irreversibly alter the database.\r\n"
|
|
"You should backup the '%s' database before proceeding. Continue (y/n)? " % database_name)
|
|
if ans == 'y':
|
|
# apply the interval weighting
|
|
weight_obj.run()
|
|
elif ans == 'n':
|
|
# we decided not to update the summary tables
|
|
syslog.syslog(syslog.LOG_INFO,
|
|
"Cancelled by user. No changes were made to the database.")
|
|
print "Cancelled by user. No changes were made to the database."
|
|
return
|
|
print "Applied 'Interval Weighting' fix in %0.2f seconds." % (time.time() - t1)
|
|
|
|
# recalc the max/maxtime windSpeed values
|
|
_fix_wind(config_dict, db_binding, options)
|
|
|
|
def _fix_wind(config_dict, db_binding, options):
|
|
"""Recalculate the windSpeed daily summary max and maxtime fields.
|
|
|
|
Create a WindSpeedRecalculation object and call its run() method to
|
|
recalculate the max and maxtime fields from archive data. This process is
|
|
idempotent so it can be called repeatedly with no ill effect.
|
|
"""
|
|
|
|
t1 = time.time()
|
|
syslog.syslog(syslog.LOG_INFO, "Preparing 'Maximum windSpeed' fix...")
|
|
print "Preparing 'Maximum windSpeed' fix..."
|
|
# notify if this is a dry run
|
|
if options.dry_run:
|
|
print "This is a dry run. Maximum windSpeed will be calculated but not saved."
|
|
# construct a windSpeed recalculation config dict
|
|
wind_config_dict = {'name': 'Maximum windSpeed',
|
|
'binding': db_binding,
|
|
'trans_days': 100,
|
|
'dry_run': options.dry_run}
|
|
|
|
# create a windSpeedRecalculation object
|
|
wind_obj = weecfg.database.WindSpeedRecalculation(config_dict, wind_config_dict)
|
|
# perform the recalculation
|
|
wind_obj.run()
|
|
print "Applied 'Maximum windSpeed' fix in %0.2f seconds." % (time.time() - t1)
|
|
|
|
def check_strings(config_dict, db_binding, options, fix=False):
|
|
"""Scan the archive table for null strings.
|
|
|
|
Identifies and lists any null string occurrences in the archive table. If
|
|
fix is True then any null strings that are found are fixed.
|
|
"""
|
|
|
|
t1 = time.time()
|
|
if fix:
|
|
syslog.syslog(syslog.LOG_INFO, "Preparing 'Null String' fix...")
|
|
print "Preparing 'Null String' fix..."
|
|
# notify if this is a dry run
|
|
if options.dry_run:
|
|
syslog.syslog(syslog.LOG_INFO, "This is a dry run. Null strings will be found but not fixed.")
|
|
print "This is a dry run. Null strings will be found but not fixed."
|
|
else:
|
|
print "Preparing 'Null String' check..."
|
|
|
|
# Open up the main database archive table
|
|
with weewx.manager.open_manager_with_config(config_dict, db_binding) as dbmanager:
|
|
|
|
obs_pytype_list = []
|
|
obs_list = []
|
|
|
|
# Get the schema and extract the Python type each observation type should be
|
|
for column in dbmanager.connection.genSchemaOf('archive'):
|
|
schema_type = column[2]
|
|
if schema_type == 'INTEGER':
|
|
schema_type = int
|
|
elif schema_type == 'REAL':
|
|
schema_type = float
|
|
elif schema_type == 'STR':
|
|
schema_type = str
|
|
# Save the observation type for this column (eg, 'outTemp'):
|
|
obs_list.append(column[1])
|
|
# Save the Python type for this column (eg, 'int'):
|
|
obs_pytype_list.append(schema_type)
|
|
|
|
found = 0
|
|
fixed = 0
|
|
# Cycle through each row in the database
|
|
for record in dbmanager.genBatchRows():
|
|
# Now examine each column
|
|
for icol in range(len(record)):
|
|
# Check to see if this column is an instance of the correct Python type
|
|
if record[icol] is not None and not isinstance(record[icol], obs_pytype_list[icol]):
|
|
# Oops. Found a bad one. Print it out
|
|
found += 1
|
|
if fix:
|
|
syslog.syslog(syslog.LOG_INFO,
|
|
"Timestamp = %s; record['%s']= %r; ... " % (record[0],
|
|
obs_list[icol],
|
|
record[icol]))
|
|
sys.stdout.write("Timestamp = %s; record['%s']= %r; ... " % (record[0],
|
|
obs_list[icol],
|
|
record[icol]))
|
|
|
|
if fix:
|
|
# Cooerce to the correct type. If it can't be done, then set it to None
|
|
try:
|
|
corrected_value = obs_pytype_list[icol](record[icol])
|
|
except ValueError:
|
|
corrected_value = None
|
|
# Update the database with the new value but only if
|
|
# it's not a dry run
|
|
if not options.dry_run:
|
|
dbmanager.updateValue(record[0], obs_list[icol], corrected_value)
|
|
fixed += 1
|
|
# Log it and inform the user
|
|
if options.dry_run:
|
|
syslog.syslog(syslog.LOG_INFO, " would have been changed to %r\n" % corrected_value)
|
|
sys.stdout.write("would have been changed to %r\n" % corrected_value)
|
|
else:
|
|
syslog.syslog(syslog.LOG_INFO, " changed to %r\n" % corrected_value)
|
|
sys.stdout.write("changed to %r\n" % corrected_value)
|
|
else:
|
|
sys.stdout.write("ignored.\n")
|
|
tdiff = time.time() - t1
|
|
if found == 0:
|
|
# found no strings
|
|
if fix:
|
|
syslog.syslog(syslog.LOG_INFO, "No null strings found.")
|
|
print "No null strings found."
|
|
elif fixed == found:
|
|
# fixed all we found
|
|
if options.dry_run:
|
|
syslog.syslog(syslog.LOG_INFO,
|
|
"%d of %d null strings found would have been fixed." % (fixed,
|
|
found))
|
|
print "%d of %d null strings found would hav ebeen fixed." % (fixed,
|
|
found)
|
|
else:
|
|
syslog.syslog(syslog.LOG_INFO,
|
|
"%d of %d null strings found were fixed." % (fixed,
|
|
found))
|
|
print "%d of %d null strings found were fixed." % (fixed, found)
|
|
elif fix:
|
|
# this should never occur - found some but didn't fix them all when we
|
|
# should have.
|
|
if options.dry_run:
|
|
syslog.syslog(syslog.LOG_INFO,
|
|
"Could not fix all null strings. %d of %d null strings would have been fixed." % (fixed,
|
|
found))
|
|
print "Could not fix all null strings. %d of %d null strings found would have been fixed." % (fixed,
|
|
found)
|
|
else:
|
|
syslog.syslog(syslog.LOG_INFO,
|
|
"Could not fix all null strings. %d of %d null strings found were fixed." % (fixed,
|
|
found))
|
|
print "Could not fix all null strings. %d of %d null strings found were fixed." % (fixed,
|
|
found)
|
|
else:
|
|
# this is just a check
|
|
print ("%d null strings were found.\r\n"
|
|
"Recommend running 'wee_database --fix' to fix these strings." % found)
|
|
if fix:
|
|
syslog.syslog(syslog.LOG_INFO, "Applied 'Null String' fix in %0.2f seconds." % tdiff)
|
|
print "Applied 'Null String' fix in %0.2f seconds." % tdiff
|
|
else:
|
|
print "Completed 'Null String' check in %0.3f seconds." % tdiff
|
|
|
|
def _parse_dates(options):
|
|
"""Parse --date, --from and --to command line options.
|
|
|
|
Inputs:
|
|
options: the optparse options
|
|
|
|
Returns: A two-way tuple (start_d, stop_d), where
|
|
start_d: A datetime.date object holding the first date. May be None
|
|
stop_d: A dateimte.date object holding the last date. May be None
|
|
"""
|
|
|
|
# Default is None, unless user has specified an option
|
|
_first_d = None
|
|
_last_d = None
|
|
|
|
if options.date:
|
|
# Make sure we are not overspecified
|
|
if options.from_date or options.to_date:
|
|
raise ValueError("Specify either --date or a --from and --to combination; not both")
|
|
|
|
# there is a --date but is it valid
|
|
try:
|
|
_first_d = datetime.datetime.strptime(options.date, "%Y-%m-%d").date()
|
|
except ValueError:
|
|
raise ValueError("Invalid --date option specified.")
|
|
else:
|
|
_last_d = _first_d
|
|
return (_first_d, _last_d)
|
|
|
|
if options.from_date:
|
|
try:
|
|
_first_d = datetime.datetime.strptime(options.from_date, "%Y-%m-%d").date()
|
|
except ValueError:
|
|
raise ValueError("Invalid --from option specified.")
|
|
|
|
if options.to_date:
|
|
try:
|
|
_last_d = datetime.datetime.strptime(options.to_date, "%Y-%m-%d").date()
|
|
except ValueError:
|
|
raise ValueError("Invalid --to option specified.")
|
|
|
|
if _first_d and _last_d:
|
|
# If both --from and --to have been specified, make sure the latter is greater
|
|
# than the former
|
|
if _last_d < _first_d:
|
|
raise weewx.ViolatedPrecondition("--from value is later than --to value.")
|
|
|
|
return (_first_d, _last_d)
|
|
|
|
|
|
if __name__=="__main__" :
|
|
main()
|