#
#    Copyright (c) 2009-2015 Tom Keffer <tkeffer@gmail.com>
#
#    See the file LICENSE.txt for your rights.
#
"""Utilities used by the setup and configure programs"""

from __future__ import with_statement

import errno
import glob
import os.path
import shutil
import sys
import StringIO
import tempfile

import configobj

import weeutil.weeutil
from weewx import all_service_groups

minor_comment_block = [""]
major_comment_block = ["", "##############################################################################", ""]

#==============================================================================
#                  Section tuples
# Each ConfigObj section is recursively described by a "section tuple." This is
# a 3-way tuple with elements:
#
#   0: The name of the section;
#   1: A list of any subsection tuples;
#   2: A list of any scalar names.

canonical_order = ('',
[('Station', [], ['location', 'latitude', 'longitude', 'altitude',
                  'station_type', 'rain_year_start', 'week_start']), 
 ('AcuRite', [], []),
 ('CC3000', [], []),
 ('FineOffsetUSB', [], []),
 ('Simulator', [], []),
 ('TE923', [], []),
 ('Ultimeter', [], []),
 ('Vantage', [], []),
 ('WMR100', [], []),
 ('WMR200', [], []),
 ('WMR300', [], []),
 ('WMR9x8', [], []),
 ('WS1', [], []),
 ('WS23xx', [], []),
 ('WS28xx', [], []),
 ('StdRESTful', [('StationRegistry', [], ['register_this_station']), 
                 ('AWEKAS', [], ['enable', 'username', 'password']), 
                 ('CWOP', [], ['enable', 'station']), 
                 ('PWSweather', [], ['enable', 'station', 'password']), 
                 ('WOW', [], ['enable', 'station', 'password']), 
                 ('Wunderground', [], ['enable', 'station', 'password', 'rapidfire'])], []), 
 ('StdReport', [('StandardReport', [('Units', [('Groups', [], ['group_altitude', 'group_speed2', 'group_pressure', 'group_rain', 'group_rainrate', 'group_temperature', 'group_degree_day', 'group_speed'])], [])], ['skin']), 
                ('FTP', [], ['skin', 'secure_ftp', 'port', 'passive']), 
                ('RSYNC', [], ['skin', 'delete'])], 
  ['SKIN_ROOT', 'HTML_ROOT', 'data_binding']), 
 ('StdConvert', [], ['target_unit']), ('StdCalibrate', [('Corrections', [], [])], []), 
 ('StdQC', [('MinMax', [], ['barometer', 'outTemp', 'inTemp',
                            'outHumidity', 'inHumidity', 'windSpeed'])], []),
 ('StdWXCalculate', [('Calculations', [], ['pressure', 'barometer', 'altimeter', 'windchill',
                                           'heatindex', 'dewpoint', 'inDewpoint', 'rainRate']),
                     ('Algorithms', [], ['altimeter', 'maxSolarRad'])],
  ['ignore_zero_wind', 'rain_period', 'et_period', 'wind_height', 'atc', 'nfac', 'max_delta_12h']),
 ('StdTimeSynch', [], ['clock_check', 'max_drift']), 
 ('StdArchive', [], ['archive_interval', 'archive_delay', 'record_generation',
                     'loop_hilo', 'data_binding']), 
 ('DataBindings', [('wx_binding', [], ['database', 'table_name', 'manager',
                                       'schema'])], []), 
 ('Databases', [('archive_sqlite', [], ['database_type', 'database_name']), 
                ('archive_mysql',  [], ['database_type', 'database_name'])], []),
 ('DatabaseTypes', [('SQLite', [], ['driver', 'SQLITE_ROOT']),
                    ('MySQL',  [], ['driver', 'host', 'user', 'password'])], []),
 ('Engine', [('Services', [], ['prep_services', 'data_services',
                               'process_services', 'archive_services',
                               'restful_services', 'report_services'])], [])], 
['debug', 'WEEWX_ROOT', 'socket_timeout', 'version'])

def get_section_tuple(c_dict, section_name=''):
    """ The above "canonical" ordering can be  generated from a config file
    by using this function:
    c = configobj.ConfigObj('weewx.conf')
    print get_section_tuple(c)"""

    subsections = [get_section_tuple(c_dict[ss], ss) for ss in c_dict.sections]
    section_tuple = (section_name, subsections, c_dict.scalars)
    return section_tuple
#==============================================================================

us_group = {'group_altitude': 'foot',
            'group_degree_day': 'degree_F_day',
            'group_pressure': 'inHg',
            'group_rain': 'inch',
            'group_rainrate': 'inch_per_hour',
            'group_speed': 'mile_per_hour',
            'group_speed2': 'mile_per_hour2',
            'group_temperature': 'degree_F'}

metric_group = {'group_altitude': 'meter',
                'group_degree_day': 'degree_C_day',
                'group_pressure': 'mbar',
                'group_rain': 'cm',
                'group_rainrate': 'cm_per_hour',
                'group_speed': 'km_per_hour',
                'group_speed2': 'km_per_hour2',
                'group_temperature': 'degree_C'}

metricwx_group = {'group_altitude': 'meter',
                  'group_degree_day': 'degree_C_day',
                  'group_pressure': 'mbar',
                  'group_rain': 'mm',
                  'group_rainrate': 'mm_per_hour',
                  'group_speed': 'meter_per_second',
                  'group_speed2': 'meter_per_second2',
                  'group_temperature': 'degree_C'}

class ExtensionError(IOError):
    """Errors when installing or uninstalling an extension"""
    
class Logger(object):
    def __init__(self, verbosity=0):
        self.verbosity = verbosity
    def log(self, msg, level=0):
        if self.verbosity >= level:
            print "%s%s" % ('  ' * (level - 1), msg)
    def set_verbosity(self, verbosity):
        self.verbosity = verbosity

#==============================================================================
#              Utilities that find and save ConfigObj objects
#==============================================================================

DEFAULT_LOCATIONS = ['../..', '/etc/weewx', '/home/weewx']

def find_file(file_path=None, args=None, locations=DEFAULT_LOCATIONS,
              file_name='weewx.conf'):
    """Find and return a path to a file, looking in "the usual places."
    
    General strategy:

    First, file_path is tried. If not found there, then the first element of
    args is tried.

    If those fail, try a path based on where the application is running.
    
    If that fails, then the list of directory locations is searched,
    looking for a file with file name file_name. 
    
    If after all that, the file still cannot be found, then an IOError
    exception will be raised.
    
    Parameters:

    file_path: A candidate path to the file.

    args: command-line arguments. If the file cannot be found in file_path,
    then the first element in args will be tried.
    
    locations: A list of directories to be searched. If they do not
    start with a slash ('/'), then they will be treated as relative to
    this file (bin/weecfg/__init__.py). 
    Default is ['../..', '/etc/weewx', '/home/weewx'].

    file_name: The name of the file to be found. This is used
    only if the directories must be searched. Default is 'weewx.conf'.

    returns: full path to the file
    """

    # Start by searching args (if available)
    if file_path is None and args:
        for i in range(len(args)):
            if not args[i].startswith('-'):
                file_path = args[i]
                del args[i]
                break

    if file_path is None:
        for directory in locations:
            # If this is a relative path, then prepend with the
            # directory this file is in:
            if not directory.startswith('/'):
                directory = os.path.join(os.path.dirname(__file__), directory)
            candidate = os.path.abspath(os.path.join(directory, file_name))
            if os.path.isfile(candidate):
                return candidate

    if file_path is None:
        raise IOError("Unable to find file '%s'. Tried directories %s" %
                      (file_name, locations))
    elif not os.path.isfile(file_path):
        raise IOError("%s is not a file" % file_path)

    return file_path

def read_config(config_path, args=None, locations=DEFAULT_LOCATIONS,
                file_name='weewx.conf'):
    """Read the specified configuration file, return an instance of ConfigObj
    with the file contents. If no file is specified, look in the standard
    locations for weewx.conf. Returns the filename of the actual configuration
    file, as well as the ConfigObj.

    config_path: configuration filename

    args: command-line arguments

    return: path-to-file, instance-of-ConfigObj
    
    Raises:
    
    SyntaxError: If there is a syntax error in the file
    
    IOError: If the file cannot be found
    """
    # Find and open the config file:
    config_path = find_file(config_path, args,
                            locations=locations, file_name=file_name)
    # Now open it up and parse it.
    config_dict = configobj.ConfigObj(config_path, file_error=True)
    return config_path, config_dict

def save_with_backup(config_dict, config_path):
    return save(config_dict, config_path, backup=True)

def save(config_dict, config_path, backup=False):
    """Save the config file, backing up as necessary."""
    
    # Check to see if the file exists and we are supposed to make backup:
    if os.path.exists(config_path) and backup:
        
        # Yes. We'll have to back it up.
        backup_path = weeutil.weeutil.move_with_timestamp(config_path)

        # Now we can save the file. Get a temporary file:
        tmpfile = tempfile.NamedTemporaryFile("w")

        # Write the configuration dictionary to it:
        config_dict.write(tmpfile)
        tmpfile.flush()

        # Now move the temporary file into the proper place:
        shutil.copyfile(tmpfile.name, config_path)

    else:
        
        # No existing file or no backup required. Just write.
        with open(config_path, 'w') as fd:
            config_dict.write(fd)
        backup_path = None

    return backup_path

#==============================================================================
#              Utilities that modify ConfigObj objects
#==============================================================================

def modify_config(config_dict, stn_info, logger, debug=False):
    """If a driver has a configuration editor, then use that to insert the
    stanza for the driver in the config_dict.  If there is no configuration
    editor, then inject a generic configuration, i.e., just the driver name
    with a single 'driver' element that points to the driver file.
    """
    driver_editor = None
    driver_name = None
    driver_version = None

    # Get the driver editor, name, and version:
    driver = stn_info.get('driver')
    if driver:
        try:
            # Look up driver info:
            driver_editor, driver_name, driver_version = \
                load_driver_editor(driver)
        except Exception, e:
            sys.exit("Driver %s failed to load: %s" % (driver, e))
        stn_info['station_type'] = driver_name
        if debug:
            logger.log('Using %s version %s (%s)' %
                       (driver_name, driver_version, driver), level=1)

    # Get a driver stanza, if possible
    stanza = None
    if driver_name is not None:
        if driver_editor is not None:
            orig_stanza_text = None

            # if a previous stanza exists for this driver, grab it
            if driver_name in config_dict:
                orig_stanza = configobj.ConfigObj(interpolation=False)
                orig_stanza[driver_name] = config_dict[driver_name]
                orig_stanza_text = '\n'.join(orig_stanza.write())

            # let the driver process the stanza or give us a new one
            stanza_text = driver_editor.get_conf(orig_stanza_text)
            stanza = configobj.ConfigObj(stanza_text.splitlines())

            # let the driver modify other parts of the configuration
            driver_editor.modify_config(config_dict)
        else:
            stanza = configobj.ConfigObj(interpolation=False)
            if driver_name in config_dict:
                stanza[driver_name] = config_dict[driver_name]
            else:
                stanza[driver_name] = {}

    # If we have a stanza, inject it into the configuration dictionary
    if stanza is not None and driver_name is not None:
        # Ensure that the driver field matches the path to the actual driver
        stanza[driver_name]['driver'] = driver
        # Insert the stanza in the configuration dictionary:
        config_dict[driver_name] = stanza[driver_name]
        # Add a major comment deliminator:
        config_dict.comments[driver_name] = major_comment_block
        # If we have a [Station] section, move the new stanza to just after it
        if 'Station' in config_dict:
            reorder_sections(config_dict, driver_name, 'Station', after=True)
            # make the stanza the station type
            config_dict['Station']['station_type'] = driver_name

    # Apply any overrides from the stn_info
    if stn_info:
        # Update driver stanza with any overrides from stn_info
        if driver_name is not None and driver_name in stn_info:
            for k in stn_info[driver_name]:
                config_dict[driver_name][k] = stn_info[driver_name][k]
        # Update station information with stn_info overrides
        for p in ['location', 'latitude', 'longitude', 'altitude']:
            if stn_info.get(p) is not None:
                if debug:
                    logger.log("Using %s for %s" % (stn_info[p], p), level=2)
                config_dict['Station'][p] = stn_info[p]
        # Update units display with any stn_info overrides
        if (stn_info.get('units') is not None and
            'StdReport' in config_dict and
            'StandardReport' in config_dict['StdReport']):
            if stn_info.get('units') in ['metric', 'metricwx']:
                if debug:
                    logger.log("Using Metric units for display", level=2)
                config_dict['StdReport']['StandardReport'].update({
                        'Units': {
                            'Groups': metricwx_group}})
            elif stn_info.get('units') == 'us':
                if debug:
                    logger.log("Using US units for display", level=2)
                config_dict['StdReport']['StandardReport'].update({
                        'Units': {
                            'Groups': us_group}})

#==============================================================================
#              Utilities that update and merge ConfigObj objects
#==============================================================================

def update_and_merge(config_dict, template_dict):
    
    update_config(config_dict)
    merge_config(config_dict, template_dict)
    
    # We use the number of comment lines for the 'Station' section as a
    # heuristic of whether the config dict has been updated to the new
    # comment structure
    if len(config_dict.comments['Station']) <= 3:
        transfer_comments(config_dict, template_dict)
    
def update_config(config_dict):
    """Update a (possibly old) configuration dictionary to the latest format.

    Raises exception of type ValueError if it cannot be done.
    """

    # Get the version number. If it does not appear at all, then
    # assume a very old version:
    config_version = config_dict.get('version') or '1.0.0'

    # Updates only care about the major and minor numbers
    parts = config_version.split('.')
    major = parts[0]
    minor = parts[1]

    # Take care of the collation problem when comparing things like
    # version '1.9' to '1.10' by prepending a '0' to the former:
    if len(minor) < 2:
        minor = '0' + minor

    # I don't know how to merge older, V1.X configuration files, only
    # newer V2.X ones.
    if major == '1':
        raise ValueError("Cannot merge version %s. Too old" % config_version)

    if major == '2' and minor < '07':
        update_to_v27(config_dict)

    if major < '3':
        update_to_v30(config_dict)
        
    update_to_v32(config_dict)
    
    update_to_v36(config_dict)

def merge_config(config_dict, template_dict):
    """Merge the configuration dictionary into the template dictionary,
    overriding any options. Return the results.
    
    config_dict: An existing, older configuration dictionary.
    
    template_dict: A newer dictionary supplied by the installer.
    """

    # Turn off interpolation so what gets merged is the symbolic name
    # (such as WEEWX_ROOT), and not its interpolated value. 
    csave, config_dict.interpolation = config_dict.interpolation, False
    tsave, template_dict.interpolation = template_dict.interpolation, False

    # Merge new stuff from the template:
    weeutil.weeutil.conditional_merge(config_dict, template_dict)
    
    config_dict.interpolation = csave
    template_dict.interpolation = tsave

    # Finally, update the version number:
    config_dict['version'] = template_dict['version']

    return config_dict

def update_to_v27(config_dict):
    """Updates a configuration file to the latest V2.X version.
    Since V2.7 was the last 2.X version, that's our target"""

    service_map_v2 = {'weewx.wxengine.StdTimeSynch' : 'prep_services',
                      'weewx.wxengine.StdConvert'   : 'process_services',
                      'weewx.wxengine.StdCalibrate' : 'process_services',
                      'weewx.wxengine.StdQC'        : 'process_services',
                      'weewx.wxengine.StdArchive'   : 'archive_services',
                      'weewx.wxengine.StdPrint'     : 'report_services',
                      'weewx.wxengine.StdReport'    : 'report_services'}

    # webpath is now station_url
    webpath = config_dict['Station'].get('webpath', None)
    station_url = config_dict['Station'].get('station_url', None)
    if webpath is not None and station_url is None:
        config_dict['Station']['station_url'] = webpath
    config_dict['Station'].pop('webpath', None)

    if 'StdArchive' in config_dict:
        # Option stats_types is no longer used. Get rid of it.
        config_dict['StdArchive'].pop('stats_types', None)

    # --- Davis Vantage series ---
    if 'Vantage' in config_dict:
        try:
            if config_dict['Vantage']['driver'].strip() == 'weewx.VantagePro':
                config_dict['Vantage']['driver'] = 'weewx.drivers.vantage'
        except KeyError:
            pass

    # --- Oregon Scientific WMR100 ---

    # The section name has changed from WMR-USB to WMR100
    if 'WMR-USB' in config_dict:
        if 'WMR100' in config_dict:
            sys.exit("\n*** Configuration file has both a 'WMR-USB' "
                     "section and a 'WMR100' section. Aborting ***\n\n")
        config_dict.rename('WMR-USB', 'WMR100')
    # If necessary, reflect the section name in the station type:
    try:
        if config_dict['Station']['station_type'].strip() == 'WMR-USB':
            config_dict['Station']['station_type'] = 'WMR100'
    except KeyError:
        pass
    # Finally, the name of the driver has been changed
    try:
        if config_dict['WMR100']['driver'].strip() == 'weewx.wmrx':
            config_dict['WMR100']['driver'] = 'weewx.drivers.wmr100'
    except KeyError:
        pass

    # --- Oregon Scientific WMR9x8 series ---

    # The section name has changed from WMR-918 to WMR9x8
    if 'WMR-918' in config_dict:
        if 'WMR9x8' in config_dict:
            sys.exit("\n*** Configuration file has both a 'WMR-918' "
                     "section and a 'WMR9x8' section. Aborting ***\n\n")
        config_dict.rename('WMR-918', 'WMR9x8')
    # If necessary, reflect the section name in the station type:
    try:
        if config_dict['Station']['station_type'].strip() == 'WMR-918':
            config_dict['Station']['station_type'] = 'WMR9x8'
    except KeyError:
        pass
    # Finally, the name of the driver has been changed
    try:
        if config_dict['WMR9x8']['driver'].strip() == 'weewx.WMR918':
            config_dict['WMR9x8']['driver'] = 'weewx.drivers.wmr9x8'
    except KeyError:
        pass

    # --- Fine Offset instruments ---

    try:
        if config_dict['FineOffsetUSB']['driver'].strip() == 'weewx.fousb':
            config_dict['FineOffsetUSB']['driver'] = 'weewx.drivers.fousb'
    except KeyError:
        pass

    #--- The weewx Simulator ---

    try:
        if config_dict['Simulator']['driver'].strip() == 'weewx.simulator':
            config_dict['Simulator']['driver'] = 'weewx.drivers.simulator'
    except KeyError:
        pass

    # See if the engine configuration section has the old-style "service_list":
    if 'Engines' in config_dict and 'service_list' in config_dict['Engines']['WxEngine']:
        # It does. Break it up into five, smaller lists. If a service
        # does not appear in the dictionary "service_map_v2", meaning we
        # do not know what it is, then stick it in the last group we
        # have seen. This should get its position about right.
        last_group = 'prep_services'

        # Set up a bunch of empty groups in the right order
        for group in ['prep_services', 'process_services', 'archive_services',
                      'restful_services', 'report_services']:
            config_dict['Engines']['WxEngine'][group] = list()

        # Now map the old service names to the right group
        for _svc_name in config_dict['Engines']['WxEngine']['service_list']:
            svc_name = _svc_name.strip()
            # Skip the no longer needed StdRESTful service:
            if svc_name == 'weewx.wxengine.StdRESTful':
                continue
            # Do we know about this service?
            if svc_name in service_map_v2:
                # Yes. Get which group it belongs to, and put it there
                group = service_map_v2[svc_name]
                config_dict['Engines']['WxEngine'][group].append(svc_name)
                last_group = group
            else:
                # No. Put it in the last group.
                config_dict['Engines']['WxEngine'][last_group].append(svc_name)

        # Now add the restful services, using the old driver name to help us
        for section in config_dict['StdRESTful'].sections:
            svc = config_dict['StdRESTful'][section]['driver']
            # weewx.restful has changed to weewx.restx
            if svc.startswith('weewx.restful'):
                svc = 'weewx.restx.Std' + section
            # awekas is in weewx.restx since 2.6
            if svc.endswith('AWEKAS'):
                svc = 'weewx.restx.AWEKAS'
            config_dict['Engines']['WxEngine']['restful_services'].append(svc)

        # Depending on how old a version the user has, the station registry
        # may have to be included:
        if 'weewx.restx.StdStationRegistry' not in config_dict['Engines']['WxEngine']['restful_services']:
            config_dict['Engines']['WxEngine']['restful_services'].append('weewx.restx.StdStationRegistry')

        # Get rid of the no longer needed service_list:
        config_dict['Engines']['WxEngine'].pop('service_list')

    # Clean up the CWOP configuration
    if 'StdRESTful' in config_dict and 'CWOP' in config_dict['StdRESTful']:
        # Option "interval" has changed to "post_interval"
        if 'interval' in config_dict['StdRESTful']['CWOP']:
            config_dict['StdRESTful']['CWOP']['post_interval'] = \
                config_dict['StdRESTful']['CWOP']['interval']
            config_dict['StdRESTful']['CWOP'].pop('interval')
        # Option "server" has become "server_list". It is also no longer
        # included in the default weewx.conf, so just pop it.
        if 'server' in config_dict['StdRESTful']['CWOP']:
            config_dict['StdRESTful']['CWOP'].pop('server')

    # Remove the no longer needed "driver" from all the RESTful services:
    if 'StdRESTful' in config_dict:
        for section in config_dict['StdRESTful'].sections:
            config_dict['StdRESTful'][section].pop('driver', None)

    config_dict['version'] = '2.7.0'

def update_to_v30(config_dict):
    """Update a configuration file to V3.0"""
    old_database = None

    v3_additions = """[DataBindings]
    # This section binds a data store to a database

    [[wx_binding]]
        # The database must match one of the sections in [Databases] 
        database = archive_sqlite
        # The name of the table within the database
        table_name = archive
        # The manager handles aggregation of data for historical summaries
        manager = weewx.wxmanager.WXDaySummaryManager
        # The schema defines the structure of the database.
        # It is *only* used when the database is created.
        schema = schemas.wview.schema

"""

    if 'Databases' in config_dict:
        # The stats database no longer exists. Remove it from the [Databases]
        # section:
        config_dict['Databases'].pop('stats_sqlite', None)
        config_dict['Databases'].pop('stats_mysql', None)
        # The key "database" changed to "database_name"
        for stanza in config_dict['Databases']:
            if 'database' in config_dict['Databases'][stanza]:
                config_dict['Databases'][stanza].rename('database',
                                                        'database_name')

    if 'StdReport' in config_dict:
        # The key "data_binding" is now used instead of these:
        config_dict['StdReport'].pop('archive_database', None)
        config_dict['StdReport'].pop('stats_database', None)

    if 'StdArchive' in config_dict:
        old_database = config_dict['StdArchive'].pop('archive_database', None)
        config_dict['StdArchive'].pop('stats_database', None)
        config_dict['StdArchive'].pop('archive_schema', None)
        config_dict['StdArchive'].pop('stats_schema', None)

    # Section ['Engines'] got renamed to ['Engine']
    if 'Engine' not in config_dict and 'Engines' in config_dict:
        config_dict.rename('Engines', 'Engine')
        # Subsection [['WxEngine']] got renamed to [['Services']]
        if 'WxEngine' in config_dict['Engine']:
            config_dict['Engine'].rename('WxEngine', 'Services')

            # Finally, module "wxengine" got renamed to "engine". Go through
            # each of the service lists, making the change
            for list_name in config_dict['Engine']['Services']:
                service_list = config_dict['Engine']['Services'][list_name]
                # If service_list is not already a list (it could be just a
                # single name), then make it a list:
                if not hasattr(service_list, '__iter__'):
                    service_list = [service_list]
                config_dict['Engine']['Services'][list_name] = \
                    [this_item.replace('wxengine', 'engine') for this_item in service_list]
        try:
            # Finally, make sure the new StdWXCalculate service is in the list:
            if 'weewx.wxservices.StdWXCalculate' not in config_dict['Engine']['Services']['process_services']:
                config_dict['Engine']['Services']['process_services'].append('weewx.wxservices.StdWXCalculate')
        except KeyError:
            pass

    if 'DataBindings' not in config_dict:
        # Insert a [DataBindings] section. First create it
        c = configobj.ConfigObj(StringIO.StringIO(v3_additions))
        # Now merge it in:
        config_dict.merge(c)
        # For some reason, ConfigObj strips any leading comments. Put them back:
        config_dict.comments['DataBindings'] = major_comment_block
        # Move the new section to just before [Databases]
        reorder_sections(config_dict, 'DataBindings', 'Databases')
        # No comments between the [DataBindings] and [Databases] sections:
        config_dict.comments['Databases'] = [""]
        config_dict.inline_comments['Databases'] = []

        # If there was an old database, add it in the new, correct spot:
        if old_database:
            try:
                config_dict['DataBindings']['wx_binding']['database'] = old_database
            except KeyError:
                pass

    config_dict['version'] = '3.0.0'

def update_to_v32(config_dict):
    """Update a configuration file to V3.2"""
    
    # For interpolation to work, it's critical that WEEWX_ROOT not end
    # with a trailing slash ('/'). Convert it to the normative form:
    config_dict['WEEWX_ROOT'] = os.path.normpath(config_dict['WEEWX_ROOT'])
    
    # Add a default database-specific top-level stanzas if necessary
    if 'DatabaseTypes' not in config_dict:
        # Do SQLite first. Start with a sanity check:
        try:
            assert(config_dict['Databases']['archive_sqlite']['driver'] == 'weedb.sqlite')
        except KeyError:
            pass
        # Set the default [[SQLite]] section. Turn off interpolation first, so the
        # symbol for WEEWX_ROOT does not get lost.
        save, config_dict.interpolation = config_dict.interpolation, False
        config_dict['DatabaseTypes'] = {
            'SQLite': {'driver': 'weedb.sqlite',
                       'SQLITE_ROOT': '%(WEEWX_ROOT)s/archive'}}
        config_dict.interpolation = save
        try:
            root = config_dict['Databases']['archive_sqlite']['root']
            database_name = config_dict['Databases']['archive_sqlite']['database_name']
            fullpath = os.path.join(root, database_name)
            dirname = os.path.dirname(fullpath)
            # By testing to see if they end up resolving to the same thing,
            # we can keep the interpolation used to specify SQLITE_ROOT above.
            if dirname != config_dict['DatabaseTypes']['SQLite']['SQLITE_ROOT']:
                config_dict['DatabaseTypes']['SQLite']['SQLITE_ROOT'] = dirname
            config_dict['Databases']['archive_sqlite']['database_name'] = os.path.basename(fullpath)
            config_dict['Databases']['archive_sqlite']['database_type'] = 'SQLite'
            config_dict['Databases']['archive_sqlite'].pop('root', None)
            config_dict['Databases']['archive_sqlite'].pop('driver', None)
        except KeyError:
            pass
    
        # Now do MySQL. Start with a sanity check:
        try:
            assert(config_dict['Databases']['archive_mysql']['driver'] == 'weedb.mysql')
        except KeyError:
            pass
        config_dict['DatabaseTypes']['MySQL'] = {'driver': 'weedb.mysql',
                                                 'host': 'localhost',
                                                 'user': 'weewx',
                                                 'password': 'weewx'}
        try:
            config_dict['DatabaseTypes']['MySQL']['host'] = config_dict['Databases']['archive_mysql']['host']
            config_dict['DatabaseTypes']['MySQL']['user'] = config_dict['Databases']['archive_mysql']['user']
            config_dict['DatabaseTypes']['MySQL']['password'] = config_dict['Databases']['archive_mysql']['password']
            config_dict['Databases']['archive_mysql'].pop('host', None)
            config_dict['Databases']['archive_mysql'].pop('user', None)
            config_dict['Databases']['archive_mysql'].pop('password', None)
            config_dict['Databases']['archive_mysql'].pop('driver', None)
            config_dict['Databases']['archive_mysql']['database_type'] = 'MySQL'
        except KeyError:
            pass
            
    # Version 3.2 introduces the 'enable' keyword for RESTful protocols. Set
    # it appropriately
    def set_enable(c, service, keyword):
        # Check to see whether this config file has the service listed
        try:
            c['StdRESTful'][service]
        except KeyError:
            # It does not. Nothing to do.
            return

        # Now check to see whether it already has the option 'enable':
        if 'enable' in c['StdRESTful'][service]:
            # It does. No need to proceed
            return

        # The option 'enable' is not present. Add it,
        # and set based on whether the keyword is present:
        if keyword in c['StdRESTful'][service]:
            c['StdRESTful'][service]['enable'] = 'true'
        else:
            c['StdRESTful'][service]['enable'] = 'false'

    set_enable(config_dict, 'AWEKAS', 'username')
    set_enable(config_dict, 'CWOP', 'station')
    set_enable(config_dict, 'PWSweather', 'station')
    set_enable(config_dict, 'WOW', 'station')
    set_enable(config_dict, 'Wunderground', 'station')
    
    config_dict['version'] = '3.2.0'
        
def update_to_v36(config_dict):
    """Update a configuration file to V3.6"""
    
    # Perform the following only if the dictionary has a StdWXCalculate section
    if config_dict.get('StdWXCalculate'):
        # No need to update if it already has a 'Calculations' section:
        if not config_dict['StdWXCalculate'].get('Calculations'):
            # Save the comment attached to the first scalar
            try:
                first = config_dict['StdWXCalculate'].scalars[0]
                comment = config_dict['StdWXCalculate'].comments[first]
                config_dict['StdWXCalculate'].comments[first] = ''
            except IndexError:
                comment = """    # Derived quantities are calculated by this service. Possible values are:
    #  hardware        - use the value provided by hardware
    #  software        - use the value calculated by weewx
    #  prefer_hardware - use value provide by hardware if available,
    #                      otherwise use value calculated by weewx"""
            # Create a new 'Calculations' section:
            config_dict['StdWXCalculate']['Calculations'] = {}
            # Now transfer over the options. Make a copy of them first: we will be 
            # deleting some of them.
            scalars = list(config_dict['StdWXCalculate'].scalars)
            for scalar in scalars:
                # These scalars don't get moved:
                if not scalar in ['ignore_zero_wind', 'rain_period', 
                                  'et_period', 'wind_height', 'atc', 
                                  'nfac', 'max_delta_12h']:
                    config_dict['StdWXCalculate']['Calculations'][scalar] = config_dict['StdWXCalculate'][scalar]
                    config_dict['StdWXCalculate'].pop(scalar)
            # Insert the old comment at the top of the new stanza:
            try:
                first = config_dict['StdWXCalculate']['Calculations'].scalars[0]
                config_dict['StdWXCalculate']['Calculations'].comments[first] = comment
            except IndexError:
                pass

def transfer_comments(config_dict, template_dict):
    
    # If this is the top-level, transfer the initial comments
    if config_dict.parent is config_dict:
        config_dict.initial_comment = template_dict.initial_comment
    
    # Now go through each section, transferring its comments
    for section in config_dict.sections:
        try:
            config_dict.comments[section] = template_dict.comments[section]
            # Recursively transfer the subsection comments:
            transfer_comments(config_dict[section], template_dict[section])
        except KeyError:
            pass

    # Finally, do the section's scalars:
    for scalar in config_dict.scalars:
        try:
            config_dict.comments[scalar] = template_dict.comments[scalar]
        except KeyError:
            pass

#==============================================================================
#              Utilities that extract from ConfigObj objects
#==============================================================================

def get_station_info(config_dict):
    """Extract station info from config dictionary."""
    stn_info = dict()
    if config_dict is not None:
        if 'Station' in config_dict:
            stn_info['location'] = weeutil.weeutil.list_as_string(config_dict['Station'].get('location'))
            stn_info['latitude'] = config_dict['Station'].get('latitude')
            stn_info['longitude'] = config_dict['Station'].get('longitude')
            stn_info['altitude'] = config_dict['Station'].get('altitude')
            if 'station_type' in config_dict['Station']:
                stn_info['station_type'] = config_dict['Station']['station_type']
                if stn_info['station_type'] in config_dict:
                    stn_info['driver'] = config_dict[stn_info['station_type']]['driver']
        if 'StdReport' in config_dict:
            stn_info['units'] = get_unit_info(config_dict)

    return stn_info

def get_unit_info(config_dict):
    """Intuit what unit system the reports are in."""
    try:
        group_dict = config_dict['StdReport']['StandardReport']['Units']['Groups']
        # Look for a strict superset of the group settings:
        if all(group_dict[group] == us_group[group] for group in us_group):
            return 'us'
        elif all(group_dict[group] == metric_group[group] for group in metric_group):
            return 'metric'
        elif all(group_dict[group] == metricwx_group[group] for group in metricwx_group):
            return 'metricwx'
    except KeyError:
        return None

#==============================================================================
#                Utilities that manipulate ConfigObj objects
#==============================================================================

# The following utility is probably not necessary any longer.
# reorder_to_ref() should be used instead.
def reorder_sections(config_dict, src, dst, after=False):
    """Move the section with key src to just before (after=False) or after
    (after=True) the section with key dst. """
    bump = 1 if after else 0
    # We need both keys to procede:
    if src not in config_dict.sections or dst not in config_dict.sections:
        return
    # If index raises an exception, we want to fail hard.
    # Find the source section (the one we intend to move):
    src_idx = config_dict.sections.index(src)
    # Remove it
    config_dict.sections.pop(src_idx)
    # Find the destination
    dst_idx = config_dict.sections.index(dst)
    # Now reorder the attribute 'sections', putting src just before dst:
    config_dict.sections = config_dict.sections[:dst_idx + bump] + [src] + \
                           config_dict.sections[dst_idx + bump:]

def reorder_to_ref(config_dict, section_tuple=canonical_order):
    """Reorder any sections in concordance with a reference ordering.
    
    See the definition for canonical_ordering for the details of the tuple
    used to describe a section.
    """
    if not len(section_tuple):
        return
    # Get the names of any subsections in the order they should be in:
    subsection_order = [x[0] for x in section_tuple[1]]
    # Reorder the subsections, then the scalars
    config_dict.sections = reorder(config_dict.sections, subsection_order)
    config_dict.scalars = reorder(config_dict.scalars, section_tuple[2])
    
    # Now recursively go through each of my subsections,
    # allowing them to reorder their contents
    for ss_tuple in section_tuple[1]:
        ss_name = ss_tuple[0]
        if ss_name in config_dict:
            reorder_to_ref(config_dict[ss_name], ss_tuple)
    
def reorder(name_list, ref_list):
    """Reorder the names in name_list, according to a reference list."""
    result = []
    # Use the ordering in ref_list, to reassemble the name list:
    for name in ref_list:
        # These always come at the end
        if name in ['FTP', 'RSYNC']:
            continue
        if name in name_list:
            result.append(name)
    # For any that were not in the reference list and are left over, tack
    # them on to the end:
    for name in name_list:
        if name not in ref_list:
            result.append(name)
    # Finally, add these, so they are at the very end
    for name in ref_list:
        if name in name_list and name in ['FTP', 'RSYNC']:
            result.append(name)
            
    # Make sure I have the same number I started with
    assert(len(name_list) == len(result))
    return result
    
def remove_and_prune(a_dict, b_dict):
    """Remove fields from a_dict that are present in b_dict"""
    for k in b_dict:
        if isinstance(b_dict[k], dict):
            if k in a_dict and type(a_dict[k]) is configobj.Section:
                remove_and_prune(a_dict[k], b_dict[k])
                if not a_dict[k].sections:
                    a_dict.pop(k)
        elif k in a_dict:
            a_dict.pop(k)

def prepend_path(a_dict, label, value):
    """Prepend the value to every instance of the label in dict a_dict"""
    for k in a_dict:
        if isinstance(a_dict[k], dict):
            prepend_path(a_dict[k], label, value)
        elif k == label:
            a_dict[k] = os.path.join(value, a_dict[k])

# def replace_string(a_dict, label, value):
#    for k in a_dict:
#        if isinstance(a_dict[k], dict):
#            replace_string(a_dict[k], label, value)
#        else:
#            a_dict[k] = a_dict[k].replace(label, value)

#==============================================================================
#                Utilities that work on drivers
#==============================================================================

def get_all_driver_infos():
    # first look in the drivers directory
    infos = get_driver_infos()
    # then add any drivers in the user directory
    infos.update(get_driver_infos('user'))
    return infos

def get_driver_infos(driver_pkg_name='weewx.drivers', excludes=['__init__.py']):
    """Scan the drivers folder, extracting information about each available
    driver. Return as a dictionary, keyed by the driver module name.
    
    Valid drivers must be importable, and must have attribute "DRIVER_NAME"
    defined.
    """

    __import__(driver_pkg_name)
    driver_package = sys.modules[driver_pkg_name]
    driver_pkg_directory = os.path.dirname(os.path.abspath(driver_package.__file__))
    driver_list = [os.path.basename(f) for f in glob.glob(os.path.join(driver_pkg_directory, "*.py"))]

    driver_info_dict = {}
    for filename in driver_list:
        if filename in excludes:
            continue

        # Get the driver module name. This will be something like
        # 'weewx.drivers.fousb'
        driver_module_name = os.path.splitext("%s.%s" % (driver_pkg_name,
                                                         filename))[0]
        
        try:
            # Try importing the module
            __import__(driver_module_name)
            driver_module = sys.modules[driver_module_name]

            # A valid driver will define the attribute "DRIVER_NAME"
            if hasattr(driver_module, 'DRIVER_NAME'):
                # A driver might define the attribute DRIVER_VERSION
                driver_module_version = driver_module.DRIVER_VERSION \
                    if hasattr(driver_module, 'DRIVER_VERSION') else '?'
                # Create an entry for it, keyed by the driver module name
                driver_info_dict[driver_module_name] = {
                    'module_name': driver_module_name,
                    'driver_name': driver_module.DRIVER_NAME,
                    'version': driver_module_version,
                    'status': ''}
        except ImportError, e:
            # If the import fails, report it in the status
            driver_info_dict[driver_module_name] = {
                'module_name': driver_module_name,
                'driver_name': '?',
                'version': '?',
                'status': e}
        except Exception, e:
            # Ignore anything else.  This might be a python file that is not
            # a driver, a python file with errors, or who knows what.
            pass

    return driver_info_dict

def print_drivers():
    """Get information about all the available drivers, then print it out."""
    driver_info_dict = get_all_driver_infos()
    keys = sorted(driver_info_dict)
    print "%-25s%-15s%-9s%-25s" % (
        "Module name", "Driver name", "Version", "Status")
    for d in keys:
        print "  %(module_name)-25s%(driver_name)-15s%(version)-9s%(status)-25s" % driver_info_dict[d]

def load_driver_editor(driver_module_name):
    """Load the configuration editor from the driver file
    
    driver_module_name: A string holding the driver name.
                        E.g., 'weewx.drivers.fousb'
    """
    __import__(driver_module_name)
    driver_module = sys.modules[driver_module_name]
    editor = None
    driver_name = None
    driver_version = 'undefined'
    if hasattr(driver_module, 'confeditor_loader'):
        loader_function = getattr(driver_module, 'confeditor_loader')
        editor = loader_function()
    if hasattr(driver_module, 'DRIVER_NAME'):
        driver_name = driver_module.DRIVER_NAME
    if hasattr(driver_module, 'DRIVER_VERSION'):
        driver_version = driver_module.DRIVER_VERSION
    return editor, driver_name, driver_version


#==============================================================================
#                Utilities that seek info from the command line
#==============================================================================

def prompt_for_info(location=None, latitude='90.000', longitude='0.000',
                    altitude=['0', 'meter'], units='metric', **kwargs):
    #
    #  Description
    #
    print "Enter a brief description of the station, such as its location.  For example:"
    print "Santa's Workshop, North Pole"
    loc = prompt_with_options("description", location)

    #
    #  Altitude
    #
    print "Specify altitude, with units 'foot' or 'meter'.  For example:"
    print "35, foot"
    print "12, meter"
    msg = "altitude [%s]: " % weeutil.weeutil.list_as_string(altitude) if altitude else "altitude: "
    alt = None
    while alt is None:
        ans = raw_input(msg).strip()
        if ans:
            parts = ans.split(',')
            if len(parts) == 2:
                try:
                    # Test whether the first token can be converted into a
                    # number. If not, an exception will be raised.
                    float(parts[0])
                    if parts[1].strip() in ['foot', 'meter']:
                        alt = [parts[0].strip(), parts[1].strip()]
                except (ValueError, TypeError):
                    pass
        elif altitude:
            alt = altitude

        if not alt:
            print "Unrecognized response. Try again."

    #
    # Latitude & Longitude
    #
    print "Specify latitude in decimal degrees, negative for south."
    lat = prompt_with_limits("latitude", latitude, -90, 90)
    print "Specify longitude in decimal degrees, negative for west."
    lon = prompt_with_limits("longitude", longitude, -180, 180)

    #
    # Display units
    #
    print "Indicate the preferred units for display: 'metric' or 'us'"
    uni = prompt_with_options("units", units, ['us', 'metric'])

    return {'location': loc,
            'altitude': alt,
            'latitude': lat,
            'longitude': lon,
            'units': uni}


def prompt_for_driver(dflt_driver=None):
    """Get the information about each driver, return as a dictionary."""
    infos = get_all_driver_infos()
    keys = sorted(infos)
    dflt_idx = None
    print "Installed drivers include:"
    for i, d in enumerate(keys):
        print " %2d) %-15s %-25s %s" % (i, infos[d].get('driver_name', '?'),
                                        "(%s)" % d, infos[d].get('status', ''))
        if dflt_driver == d:
            dflt_idx = i
    msg = "choose a driver [%d]: " % dflt_idx if dflt_idx is not None else "choose a driver: "
    idx = 0
    ans = None
    while ans is None:
        ans = raw_input(msg).strip()
        if not ans:
            ans = dflt_idx
        try:
            idx = int(ans)
            if not 0 <= idx < len(keys):
                ans = None
        except (ValueError, TypeError):
            ans = None
    return keys[idx]

def prompt_for_driver_settings(driver):
    """Let the driver prompt for any required settings.  If the driver does
    not define a method for prompting, return an empty dictionary."""
    settings = dict()
    try:
        __import__(driver)
        driver_module = sys.modules[driver]
        loader_function = getattr(driver_module, 'confeditor_loader')
        editor = loader_function()
        settings[driver_module.DRIVER_NAME] = editor.prompt_for_settings()
    except AttributeError:
        pass
    return settings

def prompt_with_options(prompt, default=None, options=None):
    """Ask the user for an input with an optional default value.
    
    prompt: A string to be used for a prompt.
    
    default: A default value. If the user simply hits <enter>, this
    is the value returned. Optional.
    
    options: A list of possible choices. The returned value must be in
    this list. Optional."""

    msg = "%s [%s]: " % (prompt, default) if default is not None else "%s: " % prompt
    value = None
    while value is None:
        value = raw_input(msg).strip()
        if value:
            if options and value not in options:
                value = None
        elif default is not None:
            value = default

    return value

def prompt_with_limits(prompt, default=None, low_limit=None, high_limit=None):
    """Ask the user for an input with an optional default value. The
    returned value must lie between optional upper and lower bounds.
    
    prompt: A string to be used for a prompt.
    
    default: A default value. If the user simply hits <enter>, this
    is the value returned. Optional.
    
    low_limit: The value must be equal to or greater than this value.
    Optional.
    
    high_limit: The value must be less than or equal to this value.
    Optional.
    """
    msg = "%s [%s]: " % (prompt, default) if default is not None else "%s: " % prompt
    value = None
    while value is None:
        value = raw_input(msg).strip()
        if value:
            try:
                v = float(value)
                if (low_limit is not None and v < low_limit) or \
                   (high_limit is not None and v > high_limit):
                    value = None
            except (ValueError, TypeError):
                value = None
        elif default is not None:
            value = default

    return value

#==============================================================================
#            Miscellaneous utilities
#==============================================================================

def extract_roots(config_path, config_dict, bin_root):
    """Get the location of the various root directories used by weewx."""
    
    root_dict = {'WEEWX_ROOT': config_dict['WEEWX_ROOT'],
                 'CONFIG_ROOT': os.path.dirname(config_path)}
    # If bin_root has not been defined, then figure out where it is using
    # the location of this file:
    if bin_root:
        root_dict['BIN_ROOT'] = bin_root
    else:
        root_dict['BIN_ROOT'] = os.path.abspath(os.path.join(
                os.path.dirname(__file__), '..'))
    # The user subdirectory:
    root_dict['USER_ROOT'] = os.path.join(root_dict['BIN_ROOT'], 'user')
    # The extensions directory is in the user directory:
    root_dict['EXT_ROOT'] = os.path.join(root_dict['USER_ROOT'], 'installer')
    # Add SKIN_ROOT if it can be found:
    try:
        root_dict['SKIN_ROOT'] = os.path.abspath(os.path.join(
                root_dict['WEEWX_ROOT'],
                config_dict['StdReport']['SKIN_ROOT']))
    except KeyError:
        pass
    
    return root_dict

def extract_tar(filename, target_dir, logger=None):
    """Extract files from a tar archive into a given directory
    
    Returns: A list of the extracted files
    """
    logger = logger or Logger()
    import tarfile
    logger.log("Extracting from tar archive %s" % filename, level=1)
    tar_archive = None
    try:
        tar_archive = tarfile.open(filename, mode='r')
        tar_archive.extractall(target_dir)
        member_names = [os.path.normpath(x.name) for x in tar_archive.getmembers()]
        return member_names
    finally:
        if tar_archive is not None:
            tar_archive.close()

def extract_zip(filename, target_dir, logger=None):
    """Extract files from a zip archive into the specified directory.

    Returns: a list of the extracted files
    """
    logger = logger or Logger()
    import zipfile
    logger.log("Extracting from zip archive %s" % filename, level=1)
    zip_archive = None
    try:
        zip_archive = zipfile.ZipFile(open(filename, mode='r'))
        member_names = zip_archive.namelist()
        # manually extract files since extractall is only in python 2.6+
#        zip_archive.extractall(target_dir)
        for f in member_names:
            if f.endswith('/'):
                dst = "%s/%s" % (target_dir, f)
                mkdir_p(dst)
        for f in member_names:
            if not f.endswith('/'):
                path = "%s/%s" % (target_dir, f)
                with open(path, 'wb') as dest_file:
                    dest_file.write(zip_archive.read(f))
        return member_names
    finally:
        if zip_archive is not None:
            zip_archive.close()

def mkdir_p(path):
    """equivalent to 'mkdir -p'"""
    try:
        os.makedirs(path)
    except OSError, e:
        if e.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise

def get_extension_installer(extension_installer_dir):
    """Get the installer in the given extension installer subdirectory"""
    old_path = sys.path
    try:
        # Inject the location of the installer directory into the path
        sys.path.insert(0, extension_installer_dir)
        try:
            # Now I can import the extension's 'install' module:
            __import__('install')
        except ImportError:
            raise ExtensionError("Cannot find 'install' module in %s" % extension_installer_dir)
        install_module = sys.modules['install']
        loader = getattr(install_module, 'loader')
        # Get rid of the module:
        sys.modules.pop('install', None)
        installer = loader()
    finally:
        # Restore the path
        sys.path = old_path

    return (install_module.__file__, installer)
