weewx/bin/wee_debug

#!/usr/bin/env python
#
#    Copyright (c) 2009-2015 Tom Keffer <tkeffer@gmail.com> and
#                            Matthew Wall
#
#    See the file LICENSE.txt for your rights.
#
""" Generate weewx debug info """
from __future__ import with_statement

import copy
import optparse
import os
import platform
import sys
import syslog

from configobj import ConfigObj

# weewx imports
import weecfg
import weedb
import weewx.manager
import weewx.units

from weecfg.extension import ExtensionEngine
from weeutil.weeutil import TimeSpan, timestamp_to_string
from weewx.manager import DaySummaryManager

VERSION = weewx.__version__
WEE_DEBUG_VERSION = '0.6'

# keys/setting names to obfuscate in weewx.conf, key value will be obfuscated
# if the key starts any element in the list. Can add additional string elements
# to list if required
OBFUSCATE_MAP = {
    "obfuscate": [
        "apiKey", "api_key", "app_key", "archive_security_key", "id", "key",
        "oauth_token", "password", "raw_security_key", "token", "user",
        "server_url", "station"],
    "do_not_obfuscate": [
        "station_type"]
    }

# weewx archive field to use as the basis of counting number of archive records
# Can be any field but dateTime is preferred as it should be in every weewx
# archive
COUNT_FIELD = 'dateTime'

# Redirect the import of setup (needed to get extension info)
sys.modules['setup'] = weecfg.extension

usage = """wee_debug --help
       wee_debug --info
            [CONFIG_FILE|--config=CONFIG_FILE]
            [--output|--output DEBUG_PATH]
            [--verbosity=0|1|2]
       wee_debug --version

Description:

Generate a standard suite of system/weewx information to aid in remote
debugging. The wee_debug output consists of two parts, the first part containing
a snapshot of relevant system/weewx information and the second part a parsed and
obfuscated copy of weewx.conf. This output can be redirected to file and posted
when seeking assistance via forums or email.

Actions:

--info           Generate a debug report."""

epilog = """wee_debug will attempt to obfuscate obvious personal/private
information in weewx.conf such as user names, passwords and API keys; however,
the user should thoroughly check the generated output for personal/private
information before posting the information publicly."""

def main():

    # Set defaults for the system logger:
    syslog.openlog('wee_debug', syslog.LOG_PID|syslog.LOG_CONS)

    # Create a command line parser:
    parser = optparse.OptionParser(usage = usage,
                                   epilog = epilog)

    # Add the various options:
    parser.add_option("--config", dest="config_path", type=str,
                      metavar="CONFIG_FILE",
                      help="Use configuration file CONFIG_FILE.")
    parser.add_option("--info", dest="info", action='store_true',
                      help="Generate weewx debug output.")
    parser.add_option('--output', action='callback',
                      callback=optional_arg('/var/tmp/weewx.debug'),
                      dest='debug_file', metavar="DEBUG_PATH",
                      help="Write wee_debug output to DEBUG_PATH. DEBUG_PATH "
                      "includes path and file name. Default is "
                      "/var/tmp/weewx.debug.")
    parser.add_option('--verbosity', type=int, default=1,
                      metavar="N", help="How much detail to display, "
                      "0-2, default=1.")
    parser.add_option('--version', dest='version', action='store_true',
                      help='Display wee_debug version number.')

    # Now we are ready to parse the command line:
    (options, args) = parser.parse_args()  # @UnusedVariable

    # check weewx version number for compatibility
    if VERSION[0] < 3:
        # incompatible version of weewx
        print "Incompatible version of weewx detected (%s). Weewx v3.0.0 or greater required." % \
            VERSION
        print "Nothing done, exiting."
        exit(1)

    # get config_dict to use
    config_path, config_dict = weecfg.read_config(options.config_path, args)

    # display wee_debug version info
    if options.version:
        print "wee_debug version: %s" % WEE_DEBUG_VERSION
        exit(1)

    # display debug info
    if options.info:
        # first a message re verbosity
        if options.verbosity == 0:
            print "Using verbosity=0, displaying minimal info"
        elif options.verbosity == 1:
            print "Using verbosity=1, displaying most info"
        else:
            print "Using verbosity=2, displaying all info"
        print
        # then a message re output destination
        if options.debug_file is not None:
            print "wee_debug output will be written to %s" % options.debug_file
        else:
            print "wee_debug output will be sent to stdout(console)"
        print

        # get some key weewx parameters
        db_binding_wx = get_binding(config_dict)
        database_wx = config_dict['DataBindings'][db_binding_wx]['database']

        # generate our debug info sending it to file or console
        if options.debug_file is not None:
            # save stdout for when we clean up
            old_stdout = sys.stdout
            # open our debug file for writing
            _debug_file = open(options.debug_file, 'w')
            # redirect stdout to our file
            sys.stdout = _debug_file
        print "Using configuration file %s" % config_path
        print "Using database binding '%s', which is bound to database '%s'" % (db_binding_wx,
                                                                                database_wx)
        print
        # generate our debug info
        generateDebugInfo(config_dict,
                          config_path,
                          db_binding_wx,
                          options.verbosity)
        print
        print "Parsed and obfuscated weewx.conf"
        # generate our obfuscated weewx.conf
        generateDebugConf(config_dict)
        if options.debug_file is not None:
            # close our file
            _debug_file.close()
            # revert stdout
            sys.stdout = old_stdout
            print "wee_debug output successfully written to %s" % options.debug_file
        else:
            print
            print "wee_debug report successfully generated"
        exit(1)

    # if we have a compatible weewx version but did not use --version or --info
    # then show the wee_debug help
    parser.print_help()

def optional_arg(arg_default):
    """ Callback function to implement optparse command line parameters that
        support default values and optional parameters.

        http://stackoverflow.com/questions/1229146/parsing-empty-options-in-python
    """

    def func(option, opt_str, value, parser):  # @UnusedVariable
        if parser.rargs and not parser.rargs[0].startswith('-'):
            val = parser.rargs[0]
            parser.rargs.pop(0)
        else:
            val = arg_default
        setattr(parser.values, option.dest, val)

    return func

def generateDebugInfo(config_dict, config_path, db_binding_wx, verbosity):
    """ Generate system/weewx debug info """

    # system/OS info
    print "System info"
    _system = platform.system()
    if _system == 'Linux':
        generateLinuxSysInfo(verbosity)
    elif _system == 'BSD':
        print "System info could not be provided, unsupported operating system(BSD)."
    else:
        print "System info could not be provided, unsupported operating system(%s)." % _system
    print

    # weewx version info
    print "General weewx info"
    print "  Weewx version %s detected." % VERSION
    print

    # station info
    print "Station info"
    stationType = config_dict['Station']['station_type']
    print "  Station type: %s" % stationType
    print "  Driver:       %s" % config_dict[stationType]['driver']
    print

    # driver info
    if verbosity > 0:
        print "Driver info"
        driver_dict = {stationType: config_dict[stationType]}
        _config = ConfigObj(driver_dict)
        _config.write(sys.stdout)
        print

    # installed extensions info
    print "Currently installed extensions"
    ext = ExtensionEngine(config_path=config_path,
                          config_dict=config_dict)
    ext.enumerate_extensions()
    print

    # weewx archive info
    try:
        manager_info_dict = getManagerInfo(config_dict, db_binding_wx)
    except weedb.CannotConnect, e:
        print "Unable to connect to database:", e
        print
    except weedb.OperationalError, e:
        print "Error hitting database. It may not be properly initialized:"
        print e
        print
    else:
        if manager_info_dict['units'] in weewx.units.unit_nicknames:
            units_nickname = weewx.units.unit_nicknames[manager_info_dict['units']]
        else:
            units_nickname = "Unknown unit constant"
        print "Archive info"
        print "  Database name:        %s" % manager_info_dict['db_name']
        print "  Table name:           %s" % manager_info_dict['table_name']
        print "  Unit system:          %s (%s)" % (manager_info_dict['units'],
                                                  units_nickname)
        print "  First good timestamp: %s" % timestamp_to_string(manager_info_dict['first_ts'])
        print "  Last good timestamp:  %s" % timestamp_to_string(manager_info_dict['last_ts'])
        if manager_info_dict['ts_count']:
            print "  Number of records:    %s" % manager_info_dict['ts_count'].value
        else:
            print "  Number of records:    %s (no archive records found)" % \
                manager_info_dict['ts_count']
        # if we have a database and a table but no start or stop ts and no records
        # inform the user that the database/table exists but appears empty
        if (manager_info_dict['db_name'] and manager_info_dict['table_name']) and \
        not (manager_info_dict['ts_count'] or manager_info_dict['units'] or \
        manager_info_dict['first_ts'] or manager_info_dict['last_ts']):
            print "                        It is likely that the database (%s) archive table (%s)" % \
                (manager_info_dict['db_name'], manager_info_dict['table_name'])
            print "                        exists but contains no data."
        print "  weewx (weewx.conf) is set to use an archive interval of %s seconds." % \
            config_dict['StdArchive']['archive_interval']
        print "  The station hardware was not interrogated in determining archive interval."
        print

    # weewx database info
    if verbosity > 0:
        print "Databases configured in weewx.conf"
        for db_keys in config_dict['Databases']:
            database_dict = weewx.manager.get_database_dict_from_config(config_dict,
                                                                        db_keys)
            _ = sorted(database_dict.keys())
            print "  Database name:        %s" % database_dict['database_name']
            print "  Database driver:      %s" % database_dict['driver']
            if 'host' in database_dict:
                print "  Database host:        %s" % database_dict['host']
            print

    # sqlkeys/obskeys info
    if verbosity > 1:
        print "Supported SQL keys"
        formatListCols(manager_info_dict['sqlkeys'], 3)
        print
        print "Supported observation keys"
        formatListCols(manager_info_dict['obskeys'], 3)
        print

def generateDebugConf(config_dict):
    """ Generate a parsed and obfuscated weewx.conf and write to sys.stdout """

    # obfuscate config_dict
    _obs_config_dict = obfuscateKey(config_dict,
                                    OBFUSCATE_MAP['obfuscate'],
                                    OBFUSCATE_MAP['do_not_obfuscate'])
    # put obfuscated config_dict into weewx.conf form
    _obs_config_dict.write(sys.stdout)

def generateLinuxSysInfo(verbosity):
    """ Generate system info for a Linux system """

    # get cpu info
    cpuinfo = _readproc_dict('/proc/cpuinfo')
    if verbosity > 0:
        for key in cpuinfo:
            print "  {0: <23}".format(key + ":"), cpuinfo[key]

    # os info
    print
    print "  Operating system:       %s" % ' '.join(platform.linux_distribution())
    print "                          %s" % ' '.join(os.uname())

    # load info
    if verbosity > 0:
        loadstr = _readproc_line('/proc/loadavg')
        (load1, load5, load15, nproc) = loadstr.split()[0:4]  # @UnusedVariable
        print "  1 minute load average:  %s" % load1
        print "  5 minute load average:  %s" % load5
        print "  15 minute load average: %s" % load15

def obfuscateKey(src_dict, obfuscate_list, retain_list):
    """ Obfuscate any dictionary items whose key is contained in passed list.
        Uses recursion to obfuscate any nested keys.
    """

    # make a copy of our source dict as we may need to alter some entries
    _dict = copy.deepcopy(src_dict)
    # step through each key and value pair
    for k, v in src_dict.items():
        # if its a dict then recurse
        if isinstance(v, dict):
            _dict[k] = obfuscateKey(v, obfuscate_list, retain_list)
        # must be a value so obfuscate if in our list or leave it if its not
        else:
            if (any(k.startswith(key) for key in obfuscate_list)
                and not k in retain_list):
                _dict[k] = "XXX obfuscated by wee_debug XXX"
            else:
                _dict[k] = v
    return _dict

def get_binding(config_dict):
    """ Get db_binding for the weewx database """

    # Extract our binding from the StdArchive section of the config file. If
    # it's missing, return None.
    if 'StdArchive' in config_dict:
        db_binding_wx = config_dict['StdArchive'].get('data_binding', 'wx_binding')
    else:
        db_binding_wx = None

    return db_binding_wx

def getManagerInfo(config_dict, db_binding_wx):
    """ Get info from the manager of a weewx archive for inclusion in debug
        report
    """

    with weewx.manager.open_manager_with_config(config_dict, db_binding_wx) as dbmanager_wx:
        db_name = dbmanager_wx.database_name
        table_name = dbmanager_wx.table_name
        units = dbmanager_wx.std_unit_system
        first_ts = dbmanager_wx.first_timestamp
        last_ts = dbmanager_wx.last_timestamp
        # do we have any records in our archive?
        if first_ts and last_ts:
            # We have some records so proceed to count them.
            # Since we are (more than likely) using archive field 'dateTime' for
            # our record count we need to call the getAggregate() method from our
            # parent class. Note that if we change to some other field the 'count'
            # might take a while longer depending on the archive size.
            ts_count = super(DaySummaryManager, dbmanager_wx).getAggregate(TimeSpan(first_ts, last_ts),
                                                              COUNT_FIELD,
                                                              'count')
        else:
            ts_count = None
        sqlkeys = dbmanager_wx.sqlkeys
        obskeys = dbmanager_wx.obskeys
    return {'db_name': db_name,
            'table_name': table_name,
            'units': units,
            'first_ts': first_ts,
            'last_ts': last_ts,
            'ts_count': ts_count,
            'sqlkeys': sqlkeys,
            'obskeys': obskeys
           }

def _readproc_dict(filename):
    """ Read proc file that has 'name:value' format for each line """

    info = {}
    with open(filename) as fp:
        for line in fp:
            if line.find(':') >= 0:
                (n,v) = line.split(':',1)
                info[n.strip()] = v.strip()
    return info

def _readproc_line(filename):
    """ Read single line proc file, return the string """

    with open(filename) as fp:
        info = fp.read()
    return info

def formatListCols(the_list, cols):
    """ Format a list of strings into a given number of columns respecting the
        width of the largest list entry
    """

    max_width = max(map(lambda x: len(x), the_list))
    justifyList = map(lambda x: x.ljust(max_width), the_list)
    lines = (' '.join(justifyList[i:i + cols])
             for i in xrange(0, len(justifyList), cols))
    print "\n".join(lines)

if __name__=="__main__" :
    main()