From 081a8df8bbcd1b8a361e43dfe6b78b6ea5b70524 Mon Sep 17 00:00:00 2001 From: Bill Richter Date: Mon, 20 Mar 2017 09:20:25 -0700 Subject: [PATCH] restore a version of the fix with fixes for freebsd. --- bin/wee_debug | 450 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 450 insertions(+) create mode 100644 bin/wee_debug diff --git a/bin/wee_debug b/bin/wee_debug new file mode 100644 index 00000000..90326c67 --- /dev/null +++ b/bin/wee_debug @@ -0,0 +1,450 @@ +#!/usr/local/bin/python +# +# Copyright (c) 2009-2015 Tom Keffer and +# Matthew Wall +# +# See the file LICENSE.txt for your rights. +# +""" Generate weewx debug info """ +from __future__ import with_statement + +import copy +import optparse +import os +import platform +import sys +import syslog + +from configobj import ConfigObj + +# weewx imports +import weecfg +import weedb +import weewx.manager +import weewx.units + +from weecfg.extension import ExtensionEngine +from weeutil.weeutil import TimeSpan, timestamp_to_string +from weewx.manager import DaySummaryManager + +VERSION = weewx.__version__ +WEE_DEBUG_VERSION = '0.6' + +# keys/setting names to obfuscate in weewx.conf, key value will be obfuscated +# if the key starts any element in the list. Can add additional string elements +# to list if required +OBFUSCATE_MAP = { + "obfuscate": [ + "apiKey", "api_key", "app_key", "archive_security_key", "id", "key", + "oauth_token", "password", "raw_security_key", "token", "user", + "server_url", "station"], + "do_not_obfuscate": [ + "station_type"] + } + +# weewx archive field to use as the basis of counting number of archive records +# Can be any field but dateTime is preferred as it should be in every weewx +# archive +COUNT_FIELD = 'dateTime' + +# Redirect the import of setup (needed to get extension info) +sys.modules['setup'] = weecfg.extension + +usage = """wee_debug --help + wee_debug --info + [CONFIG_FILE|--config=CONFIG_FILE] + [--output|--output DEBUG_PATH] + [--verbosity=0|1|2] + wee_debug --version + +Description: + +Generate a standard suite of system/weewx information to aid in remote +debugging. The wee_debug output consists of two parts, the first part containing +a snapshot of relevant system/weewx information and the second part a parsed and +obfuscated copy of weewx.conf. This output can be redirected to file and posted +when seeking assistance via forums or email. + +Actions: + +--info Generate a debug report.""" + +epilog = """wee_debug will attempt to obfuscate obvious personal/private +information in weewx.conf such as user names, passwords and API keys; however, +the user should thoroughly check the generated output for personal/private +information before posting the information publicly.""" + +def main(): + + # Set defaults for the system logger: + syslog.openlog('wee_debug', syslog.LOG_PID|syslog.LOG_CONS) + + # Create a command line parser: + parser = optparse.OptionParser(usage = usage, + epilog = epilog) + + # Add the various options: + parser.add_option("--config", dest="config_path", type=str, + metavar="CONFIG_FILE", + help="Use configuration file CONFIG_FILE.") + parser.add_option("--info", dest="info", action='store_true', + help="Generate weewx debug output.") + parser.add_option('--output', action='callback', + callback=optional_arg('/var/tmp/weewx.debug'), + dest='debug_file', metavar="DEBUG_PATH", + help="Write wee_debug output to DEBUG_PATH. DEBUG_PATH " + "includes path and file name. Default is " + "/var/tmp/weewx.debug.") + parser.add_option('--verbosity', type=int, default=1, + metavar="N", help="How much detail to display, " + "0-2, default=1.") + parser.add_option('--version', dest='version', action='store_true', + help='Display wee_debug version number.') + + # Now we are ready to parse the command line: + (options, args) = parser.parse_args() # @UnusedVariable + + # check weewx version number for compatibility + if VERSION[0] < 3: + # incompatible version of weewx + print "Incompatible version of weewx detected (%s). Weewx v3.0.0 or greater required." % \ + VERSION + print "Nothing done, exiting." + exit(1) + + # get config_dict to use + config_path, config_dict = weecfg.read_config(options.config_path, args) + + # display wee_debug version info + if options.version: + print "wee_debug version: %s" % WEE_DEBUG_VERSION + exit(1) + + # display debug info + if options.info: + # first a message re verbosity + if options.verbosity == 0: + print "Using verbosity=0, displaying minimal info" + elif options.verbosity == 1: + print "Using verbosity=1, displaying most info" + else: + print "Using verbosity=2, displaying all info" + print + # then a message re output destination + if options.debug_file is not None: + print "wee_debug output will be written to %s" % options.debug_file + else: + print "wee_debug output will be sent to stdout(console)" + print + + # get some key weewx parameters + db_binding_wx = get_binding(config_dict) + database_wx = config_dict['DataBindings'][db_binding_wx]['database'] + + # generate our debug info sending it to file or console + if options.debug_file is not None: + # save stdout for when we clean up + old_stdout = sys.stdout + # open our debug file for writing + _debug_file = open(options.debug_file, 'w') + # redirect stdout to our file + sys.stdout = _debug_file + print "Using configuration file %s" % config_path + print "Using database binding '%s', which is bound to database '%s'" % (db_binding_wx, + database_wx) + print + # generate our debug info + generateDebugInfo(config_dict, + config_path, + db_binding_wx, + options.verbosity) + print + print "Parsed and obfuscated weewx.conf" + # generate our obfuscated weewx.conf + generateDebugConf(config_dict) + if options.debug_file is not None: + # close our file + _debug_file.close() + # revert stdout + sys.stdout = old_stdout + print "wee_debug output successfully written to %s" % options.debug_file + else: + print + print "wee_debug report successfully generated" + exit(1) + + # if we have a compatible weewx version but did not use --version or --info + # then show the wee_debug help + parser.print_help() + +def optional_arg(arg_default): + """ Callback function to implement optparse command line parameters that + support default values and optional parameters. + + http://stackoverflow.com/questions/1229146/parsing-empty-options-in-python + """ + + def func(option, opt_str, value, parser): # @UnusedVariable + if parser.rargs and not parser.rargs[0].startswith('-'): + val = parser.rargs[0] + parser.rargs.pop(0) + else: + val = arg_default + setattr(parser.values, option.dest, val) + + return func + +def generateDebugInfo(config_dict, config_path, db_binding_wx, verbosity): + """ Generate system/weewx debug info """ + + # system/OS info + print "System info" + _system = platform.system() + if _system == 'Linux': + generateLinuxSysInfo(verbosity) + elif _system.find ("BSD") != -1: + generateFreeBSDSysInfo(verbosity) + else: + print "System info could not be provided, unsupported operating system(%s)." % _system + print + + # weewx version info + print "General Weewx info" + print " Weewx version %s detected." % VERSION + print + + # station info + print "Station info" + stationType = config_dict['Station']['station_type'] + print " Station type: %s" % stationType + print " Driver: %s" % config_dict[stationType]['driver'] + print + + # driver info + if verbosity > 0: + print "Driver info" + driver_dict = {stationType: config_dict[stationType]} + _config = ConfigObj(driver_dict) + _config.write(sys.stdout) + print + + # installed extensions info + print "Currently installed extensions" + ext = ExtensionEngine(config_path=config_path, + config_dict=config_dict) + ext.enumerate_extensions() + print + + # weewx archive info + try: + manager_info_dict = getManagerInfo(config_dict, db_binding_wx) + except weedb.CannotConnect, e: + print "Unable to connect to database:", e + print + except weedb.OperationalError, e: + print "Error hitting database. It may not be properly initialized:" + print e + print + else: + if manager_info_dict['units'] in weewx.units.unit_nicknames: + units_nickname = weewx.units.unit_nicknames[manager_info_dict['units']] + else: + units_nickname = "Unknown unit constant" + print "Archive info" + print " Database name: %s" % manager_info_dict['db_name'] + print " Table name: %s" % manager_info_dict['table_name'] + print " Unit system: %s (%s)" % (manager_info_dict['units'], + units_nickname) + print " First good timestamp: %s" % timestamp_to_string(manager_info_dict['first_ts']) + print " Last good timestamp: %s" % timestamp_to_string(manager_info_dict['last_ts']) + if manager_info_dict['ts_count']: + print " Number of records: %s" % manager_info_dict['ts_count'].value + else: + print " Number of records: %s (no archive records found)" % \ + manager_info_dict['ts_count'] + # if we have a database and a table but no start or stop ts and no records + # inform the user that the database/table exists but appears empty + if (manager_info_dict['db_name'] and manager_info_dict['table_name']) and \ + not (manager_info_dict['ts_count'] or manager_info_dict['units'] or \ + manager_info_dict['first_ts'] or manager_info_dict['last_ts']): + print " It is likely that the database (%s) archive table (%s)" % \ + (manager_info_dict['db_name'], manager_info_dict['table_name']) + print " exists but contains no data." + print " weewx (weewx.conf) is set to use an archive interval of %s seconds." % \ + config_dict['StdArchive']['archive_interval'] + print " The station hardware was not interrogated in determining archive interval." + print + + # weewx database info + if verbosity > 0: + print "Databases configured in weewx.conf" + for db_keys in config_dict['Databases']: + database_dict = weewx.manager.get_database_dict_from_config(config_dict, + db_keys) + _ = sorted(database_dict.keys()) + print " Database name: %s" % database_dict['database_name'] + print " Database driver: %s" % database_dict['driver'] + if 'host' in database_dict: + print " Database host: %s" % database_dict['host'] + print + + # sqlkeys/obskeys info + if verbosity > 1: + print "Supported SQL keys" + formatListCols(manager_info_dict['sqlkeys'], 3) + print + print "Supported observation keys" + formatListCols(manager_info_dict['obskeys'], 3) + print + +def generateDebugConf(config_dict): + """ Generate a parsed and obfuscated weewx.conf and write to sys.stdout """ + + # obfuscate config_dict + _obs_config_dict = obfuscateKey(config_dict, + OBFUSCATE_MAP['obfuscate'], + OBFUSCATE_MAP['do_not_obfuscate']) + # put obfuscated config_dict into weewx.conf form + _obs_config_dict.write(sys.stdout) + +def generateFreeBSDSysInfo(verbosity): + # % string operator deprecated Python 3.1 and up. + """ Generate system info for a *BSD system """ + if verbosity > 0: + print "\n Operating system: " + str(platform.uname()) + print " Release: " + platform.release() + print " Platform: " + platform.platform() + print " Processor: " + platform.processor() + print " System: " + platform.system() + print " Python Version: " + platform.python_version() + + # load info + if verbosity > 0: + loadavg = '%.2f %.2f %.2f' % os.getloadavg() + (load1, load5, load15) = loadavg.split(" ") + + print " 1 minute load average: " + load1 + print " 5 minute load average: " + load5 + print " 15 minute load average: " + load15 + +def generateLinuxSysInfo(verbosity): + """ Generate system info for a Linux system """ + + # get cpu info + cpuinfo = _readproc_dict('/proc/cpuinfo') + if verbosity > 0: + for key in cpuinfo: + print " {0: <23}".format(key + ":"), cpuinfo[key] + + # os info + print + print " Operating system: %s" % ' '.join(platform.linux_distribution()) + print " %s" % ' '.join(os.uname()) + + # load info + if verbosity > 0: + loadstr = _readproc_line('/proc/loadavg') + (load1, load5, load15, nproc) = loadstr.split()[0:4] # @UnusedVariable + print " 1 minute load average: %s" % load1 + print " 5 minute load average: %s" % load5 + print " 15 minute load average: %s" % load15 + +def obfuscateKey(src_dict, obfuscate_list, retain_list): + """ Obfuscate any dictionary items whose key is contained in passed list. + Uses recursion to obfuscate any nested keys. + """ + + # make a copy of our source dict as we may need to alter some entries + _dict = copy.deepcopy(src_dict) + # step through each key and value pair + for k, v in src_dict.items(): + # if its a dict then recurse + if isinstance(v, dict): + _dict[k] = obfuscateKey(v, obfuscate_list, retain_list) + # must be a value so obfuscate if in our list or leave it if its not + else: + if (any(k.startswith(key) for key in obfuscate_list) + and not k in retain_list): + _dict[k] = "XXX obfuscated by wee_debug XXX" + else: + _dict[k] = v + return _dict + +def get_binding(config_dict): + """ Get db_binding for the weewx database """ + + # Extract our binding from the StdArchive section of the config file. If + # it's missing, return None. + if 'StdArchive' in config_dict: + db_binding_wx = config_dict['StdArchive'].get('data_binding', 'wx_binding') + else: + db_binding_wx = None + + return db_binding_wx + +def getManagerInfo(config_dict, db_binding_wx): + """ Get info from the manager of a weewx archive for inclusion in debug + report + """ + + with weewx.manager.open_manager_with_config(config_dict, db_binding_wx) as dbmanager_wx: + db_name = dbmanager_wx.database_name + table_name = dbmanager_wx.table_name + units = dbmanager_wx.std_unit_system + first_ts = dbmanager_wx.first_timestamp + last_ts = dbmanager_wx.last_timestamp + # do we have any records in our archive? + if first_ts and last_ts: + # We have some records so proceed to count them. + # Since we are (more than likely) using archive field 'dateTime' for + # our record count we need to call the getAggregate() method from our + # parent class. Note that if we change to some other field the 'count' + # might take a while longer depending on the archive size. + ts_count = super(DaySummaryManager, dbmanager_wx).getAggregate(TimeSpan(first_ts, last_ts), + COUNT_FIELD, + 'count') + else: + ts_count = None + sqlkeys = dbmanager_wx.sqlkeys + obskeys = dbmanager_wx.obskeys + return {'db_name': db_name, + 'table_name': table_name, + 'units': units, + 'first_ts': first_ts, + 'last_ts': last_ts, + 'ts_count': ts_count, + 'sqlkeys': sqlkeys, + 'obskeys': obskeys + } + +def _readproc_dict(filename): + """ Read proc file that has 'name:value' format for each line """ + + info = {} + with open(filename) as fp: + for line in fp: + if line.find(':') >= 0: + (n,v) = line.split(':',1) + info[n.strip()] = v.strip() + return info + +def _readproc_line(filename): + """ Read single line proc file, return the string """ + + with open(filename) as fp: + info = fp.read() + return info + +def formatListCols(the_list, cols): + """ Format a list of strings into a given number of columns respecting the + width of the largest list entry + """ + + max_width = max(map(lambda x: len(x), the_list)) + justifyList = map(lambda x: x.ljust(max_width), the_list) + lines = (' '.join(justifyList[i:i + cols]) + for i in xrange(0, len(justifyList), cols)) + print "\n".join(lines) + +if __name__=="__main__" : + main()