weewx/bin/wee_config_database

#!/usr/bin/env python
#
#    Copyright (c) 2009-2014 Tom Keffer <tkeffer@gmail.com>
#
#    See the file LICENSE.txt for your full rights.
#
#    $Revision$
#    $Author$
#    $Date$
#
"""Configure the databases used by weewx"""
from __future__ import with_statement

import optparse
import syslog
import sys

import user.extensions      #@UnusedImport
import weedb
import weewx.archive
import weewx.stats
import weewx.units
import weeutil.weeutil
import user.schemas

description="""Configure the weewx databases. Most of these functions are
handled automatically by weewx, but they may be useful as a utility in special
cases. In particular, the 'reconfigure' option can be useful if you decide to
add or drop data types from the database schema or change unit systems."""
 
usage="""%prog: [config_path] [--help]
                              [--create-archive] [--drop-daily] 
                              [--reconfigure] [--backfill-daily]
                              [--string-check] [--fix]
                              [--database=DATABASE_NAME]"""

epilog="""If you are using the MySQL database it is assumed that you have the
appropriate permissions for the requested operation."""

def main():

    # Set defaults for the system logger:
    syslog.openlog('wee_config_database', syslog.LOG_PID|syslog.LOG_CONS)

    # Create a command line parser:
    parser = optparse.OptionParser(description=description, usage=usage, epilog=epilog)
    
    # Add the various options:
    parser.add_option("--config", dest="cfgfn", type=str, metavar="FILE",
                      help="use configuration file FILE. Default is /etc/weewx/weewx.conf or /home/weewx/weewx.conf")
    parser.add_option("--create-archive", dest="create_archive", action='store_true',
                      help="Create the archive database.")
    parser.add_option("--drop-daily", dest="drop_daily", action='store_true',
                      help="Drop the daily summary tables from a database.")
    parser.add_option("--reconfigure", action='store_true',
                      help="""Create a new archive database using configuration information found """\
                          """in the configuration file. In particular, the new database will use the """\
                          """unit system found in option [StdConvert][target_unit]. It will use """\
                          """the schema found in './bin/user/schemas.py'. """\
                          """The new database will have the same name as the old database, with a '_new' on the end.""")
    parser.add_option("--backfill-daily", dest="backfill_daily", action='store_true',
                      help="Backfill a database with daily summaries")
    parser.add_option("--string-check", dest="string_check", action="store_true",
                      help="Check a sqlite version of the archive database for embedded strings in it.")
    parser.add_option("--fix", dest="fix", action="store_true",
                      help="If a string is found, fix it.")
    parser.add_option("--database", dest="symname", metavar="DATABASE_NAME",
                      default='wx_database',
                      help="The symbolic name of the database the action is to be applied to. Default is 'wx_database'")

    # Now we are ready to parse the command line:
    (options, args) = parser.parse_args()
    config_fn, config_dict = weeutil.weeutil.read_config(options.cfgfn, args)
    print "Using configuration file %s." % config_fn
    
    db_symname = options.symname
    database_name = config_dict['Databases'][db_symname]
    database_dict, database_cls = weewx.archive.prep_database(config_dict['Databases'], db_symname)
    print "Using database symname '%s', which is bound to database '%s'" % (db_symname, database_name)
    
    if options.create_archive:
        createMainDatabase(database_dict, database_cls, database_name)
    
    if options.drop_daily:
        dropDaily(database_dict, database_cls, database_name)
        
    if options.reconfigure:
        reconfigMainDatabase(database_dict, database_cls, database_name, config_dict)

    if options.backfill_daily:
        backfillDaily(database_dict, database_cls, database_name)
        
    if options.string_check:
        string_check(config_dict, options.fix, db_symname)

def createMainDatabase(database_dict, database_cls, database_name):
    """Create a weewx archive database"""

    # Try a simple open. If it succeeds, that means the database
    # exists and is initialized. Otherwise, an exception will be thrown.
    try:
        archive = database_cls.open(database_dict)
        archive.close()
    except weedb.OperationalError:
        # Database does not exist. Do an open_with_create:
        archive = database_cls.open_with_create(database_dict, user.schemas.defaultArchiveSchema)
        archive.close()
        print "Created database '%s'" % (database_name,)
    else:
        print "Database '%s' already exists. Nothing done." % (database_name,)

def dropDaily(database_dict, database_cls, database_name):
    """Drop the daily summaries from a weewx database"""

    ans = None
    while ans not in ['y', 'n']:
        print "Proceeding will delete all your daily summaries from database '%s'" % database_name
        ans = raw_input("Are you sure you want to proceed (y/n)? ")
        if ans == 'y' :
            
            print "Dropping daily summary tables from '%s' ... " % database_name
            try:
                archive = database_cls.open(database_dict)
            except weedb.OperationalError:
                # No daily summaries. Nothing to be done.
                print "No daily summaries in database '%s'. Nothing done." % (database_name,)
                return
        
            try:
                archive.drop_daily()
            except weedb.OperationalError, e:
                print "Got error '%s'\nPerhaps there was no daily summary?" % e
            else:
                archive.close()
                print "Dropped daily summary tables from database '%s'" % (database_name,)
    
def reconfigMainDatabase(old_database_dict, database_cls, database_name, config_dict):
    """Create a new database, then populate it with the contents of an old database"""

    # For the new database, make a copy of the old database dictionary
    new_database_dict = old_database_dict.dict()
    # Now modify the database name
    new_database_dict['database'] = old_database_dict['database']+'_new'

    # First check and see if the new database already exists. If it does, check
    # with the user whether it's ok to delete it.
    try:
        weedb.create(new_database_dict)
    except weedb.DatabaseExists:
        ans = None
        while ans not in ['y', 'n']:
            ans = raw_input("New database '%s' already exists. Delete it first (y/n)? " % (new_database_dict['database'],))
            if ans == 'y':
                weedb.drop(new_database_dict)
            elif ans == 'n':
                print "Nothing done."
                return

    # Get the unit system of the old archive:
    with weewx.archive.Archive.open(old_database_dict) as old_archive:
        old_unit_system = old_archive.std_unit_system
    
    # Get the unit system of the new archive:
    try:
        target_unit_nickname = config_dict['StdConvert']['target_unit']
    except KeyError:
        target_unit_system = None
    else:
        target_unit_system = weewx.units.unit_constants[target_unit_nickname.upper()]
        
        
    ans = None
    while ans not in ['y', 'n']:
        print "Copying archive database '%s' to '%s'" % (old_database_dict['database'], new_database_dict['database'])
        if target_unit_system is None or old_unit_system==target_unit_system:
            print "The new archive will use the same unit system as the old ('%s')." % (weewx.units.unit_nicknames[old_unit_system],)
        else:
            print "Units will be converted from the '%s' system to the '%s' system." % (weewx.units.unit_nicknames[old_unit_system],
                                                                                        weewx.units.unit_nicknames[target_unit_system])
        ans = raw_input("Are you sure you wish to proceed (y/n)? ")
        if ans == 'y':
            weewx.archive.reconfig(old_database_dict, new_database_dict, target_unit_system)
            print "Done."
        elif ans == 'n':
            print "Nothing done."

def backfillDaily(database_dict, database_cls, database_name):
    """Backfill the daily summaries"""

    print "Backfilling daily summaries in database '%s'" % database_name

    # Open up the archive. This will create the tables necessary for the daily summaries if they
    # don't already exist:
    with weewx.stats.DaySummaryArchive.open_with_create(database_dict, user.schemas.defaultArchiveSchema) as archive:
        nrecs = archive.backfill_day_summary()

    print "Backfilled daily summaries in database '%s', using %d records" % (database_name, nrecs)
    
def string_check(config_dict, fix=False):
    print "Checking archive database for strings..."
    archive_db = config_dict['StdArchive']['archive_database']
    archive_db_dict = config_dict['Databases'][archive_db]

    # Open up the main database archive
    with weewx.archive.Archive.open(archive_db_dict) as archive:
        
        obs_pytype_list = []
        obs_list = []
        
        # Get the schema and extract the Python type each observation type should be
        for column in archive.connection.genSchemaOf('archive'):
            schema_type = column[2]
            if schema_type == 'INTEGER':
                schema_type = int
            elif schema_type == 'REAL':
                schema_type = float
            elif schema_type == 'STR':
                schema_type = str
            # Save the observation type for this column (eg, 'outTemp'):
            obs_list.append(column[1])
            # Save the Python type for this column (eg, 'int'):
            obs_pytype_list.append(schema_type)
        
        # Cycle through each row in the database
        for record in archive.genBatchRows():
            # Now examine each column
            for icol in range(len(record)):
                # Check to see if this column is an instance of the correct Python type
                if record[icol] is not None and not isinstance(record[icol], obs_pytype_list[icol]):
                    # Oops. Found a bad one. Print it out
                    sys.stdout.write("Timestamp = %s; record['%s']= %r; ... " % (record[0], obs_list[icol], record[icol]))
                    
                    if fix:
                        # Cooerce to the correct type. If it can't be done, then set it to None
                        try:
                            corrected_value = obs_pytype_list[icol](record[icol])
                        except ValueError:
                            corrected_value = None
                        # Update the database with the new value
                        archive.updateValue(record[0], obs_list[icol], corrected_value)
                        # Inform the user
                        sys.stdout.write("changed to %r\n" % corrected_value)
                    else:
                        sys.stdout.write("ignored.\n")
                    
if __name__=="__main__" :
    main()