From 6a3fef7d1cd8fd44b628b8a1e156ddcfc4d55e28 Mon Sep 17 00:00:00 2001 From: Matthew Wall Date: Mon, 30 Dec 2013 14:53:05 +0000 Subject: [PATCH] merge util changes from trunk. merge catchup-on-startup changes from trunk. merge fousb, ws23xx, te923, and wmr200 driver changes from trunk. --- bin/user/forecast.py | 557 ++++++++++++++------- bin/user/test/test_forecast.py | 109 ++-- bin/wee_config_ws23xx | 6 +- bin/weewx/abstractstation.py | 3 + bin/weewx/cheetahgenerator.py | 6 +- bin/weewx/drivers/fousb.py | 68 ++- bin/weewx/drivers/te923.py | 79 ++- bin/weewx/drivers/wmr200.py | 517 +++++++++++-------- bin/weewx/drivers/ws23xx.py | 724 +++++++++++++++++---------- bin/weewx/imagegenerator.py | 2 +- bin/weewx/station.py | 19 +- bin/weewx/wxengine.py | 26 +- docs/changes.txt | 15 + util/init.d/weewx.debian | 4 +- util/logrotate.d/weewx | 8 +- util/logwatch/scripts/services/weewx | 214 +++++--- 16 files changed, 1471 insertions(+), 886 deletions(-) diff --git a/bin/user/forecast.py b/bin/user/forecast.py index 294f944e..f6e55fc6 100644 --- a/bin/user/forecast.py +++ b/bin/user/forecast.py @@ -4,6 +4,12 @@ API_VERSION: 2 +Compatibility: + + US National Weather Service (NWS) point forecasts as of July 2013 + Weather Underground (WU) forecast10day and hourly10day as of July 2013 + XTide 2.10 (possibly earlier versions as well) + Design The forecasting module supports various forecast methods for weather and @@ -123,8 +129,6 @@ Configuration #interval = 10800 [Databases] - ... - # a typical installation will use either sqlite or mysql [[forecast_sqlite]] root = %(WEEWX_ROOT)s @@ -313,7 +317,13 @@ Skin Configuration Variables for Templates - Here are the variables that can be used in template files. + This section shows some of the variables that can be used in template files. + +Labels + + Labels are grouped by module and identified by key. For example, specifying + $forecast.label('Zambretti', 'C') returns 'Becoming fine', and specifying + $forecast.label('Weather', 'temp') returns 'Temperature'. $forecast.label(module, key) @@ -390,9 +400,32 @@ $summary.obvis array # http://www.surf-forecast.com/ # http://ocean.peterbrueggeman.com/tidepredict.html -# FIXME: 'method' should be called 'source' +# FIXME: WU defines the following: +# maxhumidity +# minhumidity +# feelslike +# mslp - mean sea level pressure +# FIXME: ensure compatibility with uk met office +# http://www.metoffice.gov.uk/datapoint/product/uk-3hourly-site-specific-forecast +# forecast in 3-hour increments for up to 5 days in the future +# UVIndex (1-11) +# feels like temperature +# weather type (0-30) +# visibility (UN, VP, PO, MO, GO, VG, EX) +# textual description +# wind direction is 16-point compass +# air quality index +# also see icons used for uk metoffice + +# FIXME: add support for openweathermap if/when the api stabilizes +# FIXME: 'method' should be called 'source' +# FIXME: obvis should be an array? +# FIXME: add field for tornado, hurricane/cyclone? + +import hashlib import httplib +import os, errno import socket import string import subprocess @@ -402,8 +435,8 @@ import time import urllib2 import weewx -from weewx.wxengine import StdService import weeutil.weeutil +from weewx.wxengine import StdService from weewx.cheetahgenerator import SearchList try: @@ -433,8 +466,8 @@ def loginf(msg): def logerr(msg): logmsg(syslog.LOG_ERR, msg) -def get_int(config_dict, label, default_value): - value = config_dict.get(label, default_value) +def toint(label, value, default_value): + """convert to integer but also permit a value of None""" if isinstance(value, str) and value.lower() == 'none': value = None if value is not None: @@ -445,29 +478,28 @@ def get_int(config_dict, label, default_value): value = default_value return value -# FIXME: WU defines the following: -# maxhumidity -# minhumidity -# feelslike -# uvi - uv index -# mslp - mean sea level pressure -# condition -# wx - imported from us nws forecast -# fctcode - forecast code -# there is overlap between condition, wx, and fctcode. also, each may contain -# any combination of precip, obvis, and sky cover. +def mkdir_p(path): + """equivalent to 'mkdir -p'""" + try: + os.makedirs(path) + except OSError, e: + if e.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + +def save_fc_data(fc, dirname, basename='forecast-data', msgs=None): + """save raw forecast data to disk, typically for diagnostics""" + ts = int(time.time()) + tstr = time.strftime('%Y%m%d%H%M', time.localtime(ts)) + mkdir_p(dirname) + fn = '%s/%s-%s' % (dirname, basename, tstr) + with open(fn, 'w') as f: + if msgs is not None: + for m in msgs: + f.write("%s\n" % m) + f.write(fc) -# FIXME: ensure compatibility with uk met office -# http://www.metoffice.gov.uk/datapoint/product/uk-3hourly-site-specific-forecast -# forecast in 3-hour increments for up to 5 days in the future -# UVIndex (1-11) -# feels like temperature -# weather type (0-30) -# visibility (UN, VP, PO, MO, GO, VG, EX) -# textual description -# wind direction is 16-point compass -# air quality index -# also see icons used for uk metoffice """Database Schema @@ -539,9 +571,6 @@ def get_int(config_dict, label, default_value): zcode used only by zambretti forecast """ -# FIXME: obvis should be an array? -# FIXME: add field for tornado, hurricane/cyclone? - defaultForecastSchema = [('method', 'VARCHAR(10) NOT NULL'), ('usUnits', 'INTEGER NOT NULL'), # weewx.US ('dateTime', 'INTEGER NOT NULL'), # epoch @@ -649,7 +678,7 @@ weather_label_dict = { # types of precipitation 'rain' : 'Rain', 'rainshwrs' : 'Rain Showers', - 'sprinkles' : 'Rain Sprinkles', # FIXME: no db field + 'sprinkles' : 'Rain Sprinkles', # FIXME: no db field for this 'tstms' : 'Thunderstorms', 'drizzle' : 'Drizzle', 'snow' : 'Snow', @@ -728,97 +757,148 @@ class Forecast(StdService): interval=1800, max_age=604800): super(Forecast, self).__init__(engine, config_dict) + # single database for all forecasts d = config_dict.get('Forecast', {}) - self.interval = int(d.get('interval', interval)) - self.max_age = get_int(d, 'max_age', max_age) - - dd = config_dict['Forecast'].get(fid, {}) - self.interval = int(dd.get('interval', self.interval)) - self.max_age = get_int(dd, 'max_age', self.max_age) - + self.database = d['database'] + self.table = d.get('table', 'archive') schema_str = d.get('schema', None) self.schema = weeutil.weeutil._get_object(schema_str) \ if schema_str is not None else defaultForecastSchema - self.database = d['database'] - self.table = d.get('table', 'archive') + # these options can be different for each forecast method + # how often to do the forecast + self.interval = self._get_opt(d, fid, 'interval', interval) + self.interval = int(self.interval) + # how long to keep forecast records + self.max_age = self._get_opt(d, fid, 'max_age', max_age) + self.max_age = toint('max_age', self.max_age, None) + # option to vacuum the sqlite database + self.vacuum = self._get_opt(d, fid, 'vacuum', False) + self.vacuum = weeutil.weeutil.tobool(self.vacuum) + # how often to retry database failures + self.db_max_tries = self._get_opt(d, fid, 'database_max_tries', 3) + self.db_max_tries = int(self.db_max_tries) + # how long to wait between retries, in seconds + self.db_retry_wait = self._get_opt(d, fid, 'database_retry_wait', 10) + self.db_retry_wait = int(self.db_retry_wait) # use single_thread for debugging - self.single_thread = d.get('single_thread', False) - self.updating = False - - # option to vacuum the sqlite database when pruning - self.vacuum = d.get('vacuum', False) + self.single_thread = self._get_opt(d, fid, 'single_thread', False) + self.single_thread = weeutil.weeutil.tobool(self.single_thread) + # option to save raw forecast to disk + self.save_raw = self._get_opt(d, fid, 'save_raw', False) + self.save_raw = weeutil.weeutil.tobool(self.save_raw) + # option to save failed foreast to disk for diagnosis + self.save_failed = self._get_opt(d, fid, 'save_failed', False) + self.save_failed = weeutil.weeutil.tobool(self.save_failed) + # where to save the raw forecasts + self.diag_dir = self._get_opt(d, fid, 'diagnostic_dir', '/var/tmp/fc') self.method_id = fid self.last_ts = 0 + self.updating = False + self.last_raw_digest = None + self.last_fail_digest = None - # do the database setup here, as a way to check the schema - # compatibility between database and software. - archive = Forecast.setup_database(self.database, - self.table, self.method_id, - self.config_dict, self.schema) - dbcol = archive.connection.columnsOf(self.table) - memcol = [x[0] for x in self.schema] - if dbcol != memcol: - raise Exception('%s: schema mismatch: %s != %s' % (self.method_id, - dbcol, memcol)) - - # find out when the last forecast happened - self.last_ts = Forecast.get_last_forecast_ts(archive, - self.table, - self.method_id) + # setup database + with Forecast.setup_database(self.database, + config_dict['Databases'][self.database], + self.table, self.schema, + self.method_id) as archive: + # ensure schema on disk matches schema in memory + dbcol = archive.connection.columnsOf(self.table) + memcol = [x[0] for x in self.schema] + if dbcol != memcol: + raise Exception('%s: schema mismatch: %s != %s' % + (self.method_id, dbcol, memcol)) + # find out when the last forecast happened + self.last_ts = Forecast.get_last_forecast_ts(archive, + self.table, + self.method_id) # ensure that the forecast has a chance to update on each new record self.bind(weewx.NEW_ARCHIVE_RECORD, self.update_forecast) + def _get_opt(self, d, fid, label, default_v): + """get an option from dict, prefer specialized value if one exists""" + v = d.get(label, default_v) + dd = d.get(fid, {}) + v = dd.get(label, v) + return v + + def save_raw_forecast(self, fc, basename='raw', msgs=None): + m = hashlib.md5() + m.update(fc) + digest = m.hexdigest() + if self.last_raw_digest == digest: + return + save_fc_data(fc, self.diag_dir, basename=basename, msgs=msgs) + self.last_raw_digest = digest + + def save_failed_forecast(self, fc, basename='fail', msgs=None): + m = hashlib.md5() + m.update(fc) + digest = m.hexdigest() + if self.last_fail_digest == digest: + return + save_fc_data(fc, self.diag_dir, basename=basename, msgs=msgs) + self.last_fail_digest = digest + def update_forecast(self, event): if self.single_thread: self.do_forecast(event) - else: - if self.updating: - logdbg('%s: update thread already running' % self.method_id) - else: - t = ForecastThread(self.do_forecast, event) - t.setName(self.method_id + 'Thread') - logdbg('%s: starting thread' % self.method_id) - t.start() - - def do_forecast(self, event): - """do the forecast if it is time, then save to database and prune.""" - self.updating = True - now = time.time() - if self.last_ts is None or now - self.interval > self.last_ts: - try: - fcast = self.get_forecast(event) - if fcast is not None: - archive = Forecast.setup_database(self.database, - self.table, - self.method_id, - self.config_dict, - self.schema) - Forecast.save_forecast(archive, fcast) - self.last_ts = now - if self.max_age is not None: - Forecast.prune_forecasts(archive, - self.table, - self.method_id, - now - self.max_age, - vacuum=self.vacuum) - except Exception, e: - logerr('%s: forecast failure: %s' % (self.method_id, e)) + elif self.updating: + logdbg('%s: update thread already running' % self.method_id) + elif time.time() - self.interval > self.last_ts: + t = ForecastThread(self.do_forecast, event) + t.setName(self.method_id + 'Thread') + logdbg('%s: starting thread' % self.method_id) + t.start() else: logdbg('%s: not yet time to do the forecast' % self.method_id) - logdbg('%s: terminating thread' % self.method_id) - self.updating = False + + def do_forecast(self, event): + self.updating = True + archive = None + try: + records = self.get_forecast(event) + if records is None: + return + archive = Forecast.setup_database(self.database, + self.config_dict['Databases'][self.database], + self.table, self.schema, + self.method_id) + Forecast.save_forecast(archive, records, self.method_id, + self.db_max_tries, self.db_retry_wait) + self.last_ts = int(time.time()) + if self.max_age is not None: + Forecast.prune_forecasts(archive, self.table, self.method_id, + self.last_ts - self.max_age, + self.db_max_tries, self.db_retry_wait) + if self.vacuum: + Forecast.vacuum_database(archive, self.method_id) + except Exception, e: + logerr('%s: forecast failure: %s' % (self.method_id, e)) + finally: + logdbg('%s: terminating thread' % self.method_id) + if archive is not None: + archive.close() + self.updating = False def get_forecast(self, event): - """get the forecast, return a forecast record or array of records.""" + """get the forecast, return an array of forecast records.""" return None + @staticmethod + def setup_database(dbname, dbcfg, table, schema, method_id='Unknown'): + archive = weewx.archive.Archive.open_with_create(dbcfg, schema, table) + logdbg("%s: using table '%s' in database '%s'" % + (method_id, table, dbname)) + return archive + @staticmethod def get_last_forecast_ts(archive, table, method_id): - sql = "select dateTime,issued_ts from %s where method = '%s' and dateTime = (select dateTime from %s where method = '%s' order by dateTime desc limit 1)" % (table, method_id, table, method_id) + sql = "select dateTime,issued_ts from %s where method = '%s' and dateTime = (select dateTime from %s where method = '%s' order by dateTime desc limit 1) limit 1" % (table, method_id, table, method_id) r = archive.getSql(sql) if r is None: return None @@ -829,36 +909,60 @@ class Forecast(StdService): return int(r[0]) @staticmethod - def save_forecast(archive, record): - """add a forecast record or array of records to the database. - - record - dictionary with keys corresponding to database fields - """ - if record is None: - return - archive.addRecord(record) + def save_forecast(archive, records, + method_id='Unknown', max_tries=3, retry_wait=10): + for count in range(max_tries): + try: + logdbg('%s: saving %d forecast records' % + (method_id, len(records))) + archive.addRecord(records, log_level=syslog.LOG_DEBUG) + loginf('%s: saved %d forecast records' % + (method_id, len(records))) + break + except Exception, e: + logerr('%s: save failed (attempt %d of %d): %s' % + (method_id, (count+1), max_tries, e)) + logdbg('%s: waiting %d seconds before retry' % + (method_id, retry_wait)) + time.sleep(retry_wait) + else: + raise Exception('save failed after %d attempts' % max_tries) @staticmethod - def prune_forecasts(archive, table, method_id, ts, vacuum=False): - """remove old forecasts from the database + def prune_forecasts(archive, table, method_id, ts, + max_tries=3, retry_wait=10): + """remove forecasts older than ts from the database""" - ts - timestamp, in seconds. records older than this will be deleted. - """ - sql = "delete from %s where method = '%s' and dateTime < %d" % (table, method_id, ts) - archive.getSql(sql) - loginf('%s: deleted forecasts prior to %d' % (method_id, ts)) + sql = "delete from %s where method = '%s' and dateTime < %d" % ( + table, method_id, ts) + for count in range(max_tries): + try: + logdbg('%s: deleting forecasts prior to %d' % (method_id, ts)) + archive.getSql(sql) + loginf('%s: deleted forecasts prior to %d' % (method_id, ts)) + break + except Exception, e: + logerr('%s: prune failed (attempt %d of %d): %s' % + (method_id, (count+1), max_tries, e)) + logdbg('%s: waiting %d seconds before retry' % + (method_id, retry_wait)) + time.sleep(retry_wait) + else: + raise Exception('prune failed after %d attemps' % max_tries) + @staticmethod + def vacuum_database(archive, method_id='Unknown'): # vacuum will only work on sqlite databases. it will compact the # database file. if we do not do this, the file grows even though # we prune records from the database. it should be ok to run this # on a mysql database - it will silently fail. - if vacuum: - try: - archive.getSql('vacuum') - except Exception, e: - logdbg('vacuuming failed: %s' % e) - pass + try: + logdbg('%s: vacuuming the database' % method_id) + archive.getSql('vacuum') + except Exception, e: + logdbg('%s: vacuuming failed: %s' % (method_id, e)) + # this method is used only by the unit tests @staticmethod def get_saved_forecasts(archive, table, method_id, since_ts=None): """return saved forecasts since the indicated timestamp @@ -873,13 +977,6 @@ class Forecast(StdService): records.append(r) return records - @staticmethod - def setup_database(database, table, method_id, config_dict, schema): - archive = weewx.archive.Archive.open_with_create(config_dict['Databases'][database], schema, table) - loginf("%s: using table '%s' in database '%s'" % - (method_id, table, database)) - return archive - # ----------------------------------------------------------------------------- # Zambretti Forecaster @@ -900,7 +997,7 @@ class ZambrettiForecast(Forecast): def __init__(self, engine, config_dict): super(ZambrettiForecast, self).__init__(engine, config_dict, Z_KEY, interval=600) - d = config_dict['Forecast'].get(Z_KEY, {}) + d = config_dict.get('Forecast', {}).get(Z_KEY, {}) self.hemisphere = d.get('hemisphere', 'NORTH') self.lower_pressure = float(d.get('lower_pressure', 950.0)) self.upper_pressure = float(d.get('upper_pressure', 1050.0)) @@ -945,7 +1042,7 @@ class ZambrettiForecast(Forecast): record['event_ts'] = ts record['zcode'] = code loginf('%s: generated 1 forecast record' % Z_KEY) - return record + return [record] zambretti_label_dict = { 'A' : "Settled fine", @@ -1152,7 +1249,7 @@ class NWSForecast(Forecast): def __init__(self, engine, config_dict): super(NWSForecast, self).__init__(engine, config_dict, NWS_KEY, interval=10800) - d = config_dict['Forecast'].get(NWS_KEY, {}) + d = config_dict.get('Forecast', {}).get(NWS_KEY, {}) self.url = d.get('url', NWS_DEFAULT_PFM_URL) self.max_tries = int(d.get('max_tries', 3)) self.lid = d.get('lid', None) @@ -1176,6 +1273,8 @@ class NWSForecast(Forecast): logerr('%s: no PFM data for %s from %s' % (NWS_KEY, self.foid, self.url)) return None + if self.save_raw: + self.save_raw_forecast(text, basename='nws-raw') matrix = NWSParseForecast(text, self.lid) if matrix is None: logerr('%s: no PFM found for %s in forecast from %s' % @@ -1183,6 +1282,8 @@ class NWSForecast(Forecast): return None logdbg('%s: forecast matrix: %s' % (NWS_KEY, matrix)) records = NWSProcessForecast(self.foid, self.lid, matrix) + if len(records) == 0 and self.save_failed: + self.save_failed_forecast(text, basename='nws-fail') msg = 'got %d forecast records' % len(records) if 'desc' in matrix or 'location' in matrix: msg += ' for %s %s' % (matrix.get('desc',''), @@ -1435,7 +1536,15 @@ def NWSProcessForecast(foid, lid, matrix): # # There are two WU forecasts - daily (forecast10day) and hourly (hourly10day) # -# forecast10day +# A forecast from WU contains a number of fields whose contents may overlap +# with other fields. These include: +# condition - not well defined +# wx - imported from us nws forecast +# fctcode - forecast code +# There is overlap between condition, wx, and fctcode. Also, each may contain +# any combination of precip, obvis, and sky cover. +# +# forecast10day --------------------------------------------------------------- # # date # period @@ -1458,7 +1567,7 @@ def NWSProcessForecast(foid, lid, matrix): # maxhumidity # minhumidity # -# hourly10day +# hourly10day ----------------------------------------------------------------- # # fcttime # dewpoint @@ -1571,7 +1680,7 @@ WU_SKY_DICT = { 'sunny':'CL', 'mostlysunny':'FW', 'partlysunny':'SC', - 'FIXME':'BK', + 'FIXME':'BK', # FIXME: NWS defines BK, but WU has nothing equivalent 'partlycloudy':'B1', 'mostlycloudy':'B2', 'cloudy':'OV', @@ -1585,7 +1694,7 @@ class WUForecast(Forecast): def __init__(self, engine, config_dict): super(WUForecast, self).__init__(engine, config_dict, WU_KEY, interval=10800) - d = config_dict['Forecast'].get(WU_KEY, {}) + d = config_dict.get('Forecast', {}).get(WU_KEY, {}) self.url = d.get('url', WU_DEFAULT_URL) self.max_tries = int(d.get('max_tries', 3)) self.api_key = d.get('api_key', None) @@ -1609,7 +1718,8 @@ class WUForecast(Forecast): raise Exception, '\n'.join(errmsg) loginf('%s: interval=%s max_age=%s api_key=%s location=%s fc=%s' % - (WU_KEY, self.interval, self.max_age, self.api_key, + (WU_KEY, self.interval, self.max_age, + 'X'*(len(self.api_key)-4) + self.api_key[-4:], self.location, self.forecast_type)) def get_forecast(self, event): @@ -1620,7 +1730,11 @@ class WUForecast(Forecast): logerr('%s: no forecast data for %s from %s' % (WU_KEY, self.location, self.url)) return None - records = WUParseForecast(text, location=self.location) + if self.save_raw: + self.save_raw_forecast(text, basename='wu-raw') + records,msgs = WUParseForecast(text, location=self.location) + if self.save_failed and len(msgs) > 0: + self.save_failed_forecast(text, basename='wu-fail', msgs=msgs) loginf('%s: got %d forecast records' % (WU_KEY, len(records))) return records @@ -1643,7 +1757,12 @@ def WUDownloadForecast(api_key, location, u = '%s/%s/%s/q/%s.json' % (url, api_key, fc_type, location) \ if url == WU_DEFAULT_URL else url - loginf("%s: downloading forecast from '%s'" % (WU_KEY, u)) + masked = list(u) + idx = u.find(api_key) + if idx >= 0: + for i in range(len(api_key)-4): + masked[idx+i] = 'X' + loginf("%s: downloading forecast from '%s'" % (WU_KEY, ''.join(masked))) for count in range(max_tries): try: response = urllib2.urlopen(u) @@ -1660,14 +1779,15 @@ def WUDownloadForecast(api_key, location, def WUParseForecast(text, issued_ts=None, now=None, location=None): obj = json.loads(text) if not 'response' in obj: - logerr('%s: unknown format in response' % WU_KEY) - return [] + msg = "%s: no 'response' in json object" % WU_KEY + logerr(msg) + return [], [msg] response = obj['response'] if 'error' in response: - logerr('%s: error in response: %s: %s' % - (WU_KEY, - response['error']['type'], response['error']['description'])) - return [] + msg = '%s: error in response: %s: %s' % ( + WU_KEY,response['error']['type'],response['error']['description']) + logerr(msg) + return [], [msg] if issued_ts is None or now is None: n = int(time.time()) @@ -1676,28 +1796,36 @@ def WUParseForecast(text, issued_ts=None, now=None, location=None): if now is None: now = n + records = [] + msgs = [] if 'hourly_forecast' in obj: - records = WUCreateRecordsFromHourly(obj, issued_ts, now, - location=location) + records,msgs = WUCreateRecordsFromHourly(obj, issued_ts, now, + location=location) elif 'forecast' in obj: - records = WUCreateRecordsFromDaily(obj, issued_ts, now, - location=location) + records,msgs = WUCreateRecordsFromDaily(obj, issued_ts, now, + location=location) else: - records = [] - return records + msg = "%s: cannot find 'hourly_forecast' or 'forecast'" % WU_KEY + logerr(msg) + msgs.append(msg) + return records,msgs def sky2clouds(sky): - if 0 <= sky <= 5: + try: + v = int(sky) + except ValueError, e: + return None + if 0 <= v <= 5: return 'CL' - elif 5 < sky <= 25: + elif 5 < v <= 25: return 'FW' - elif 25 < sky <= 50: + elif 25 < v <= 50: return 'SC' - elif 50 < sky <= 69: + elif 50 < v <= 69: return 'B1' - elif 69 < sky <= 87: + elif 69 < v <= 87: return 'B2' - elif 87 < sky <= 100: + elif 87 < v <= 100: return 'OV' return None @@ -1879,9 +2007,12 @@ def str2float(n, s): def WUCreateRecordsFromHourly(fc, issued_ts, now, location=None): '''create from hourly10day''' + msgs = [] records = [] + cnt = 0 for period in fc['hourly_forecast']: try: + cnt += 1 r = {} r['method'] = WU_KEY r['usUnits'] = weewx.US @@ -1890,7 +2021,7 @@ def WUCreateRecordsFromHourly(fc, issued_ts, now, location=None): r['event_ts'] = str2int('epoch', period['FCTTIME']['epoch']) r['hour'] = str2int('hour', period['FCTTIME']['hour']) r['duration'] = 3600 - r['clouds'] = sky2clouds(int(period['sky'])) + r['clouds'] = sky2clouds(period['sky']) r['temp'] = str2float('temp', period['temp']['english']) r['dewpoint'] = str2float('dewpoint',period['dewpoint']['english']) r['humidity'] = str2int('humidity', period['humidity']) @@ -1907,14 +2038,20 @@ def WUCreateRecordsFromHourly(fc, issued_ts, now, location=None): r['location'] = location records.append(r) except Exception, e: - logerr('%s: failure in hourly forecast: %s' % (WU_KEY, e)) - return records + msg = '%s: failure in hourly forecast period %d: %s' % ( + WU_KEY, cnt, e) + msgs.append(msg) + logerr(msg) + return records, msgs def WUCreateRecordsFromDaily(fc, issued_ts, now, location=None): '''create from forecast10day data''' + msgs = [] records = [] + cnt = 0 for period in fc['forecast']['simpleforecast']['forecastday']: try: + cnt += 1 r = {} r['method'] = WU_KEY r['usUnits'] = weewx.US @@ -1939,8 +2076,11 @@ def WUCreateRecordsFromDaily(fc, issued_ts, now, location=None): r['location'] = location records.append(r) except Exception, e: - logerr('%s: failure in daily forecast: %s' % (WU_KEY, e)) - return records + msg = '%s: failure in daily forecast period %d: %s' % ( + WU_KEY, cnt, e) + msgs.append(msg) + logerr(msg) + return records, msgs # ----------------------------------------------------------------------------- @@ -1966,7 +2106,7 @@ class XTideForecast(Forecast): def __init__(self, engine, config_dict): super(XTideForecast, self).__init__(engine, config_dict, XT_KEY, interval=1209600, max_age=2419200) - d = config_dict['Forecast'].get(XT_KEY, {}) + d = config_dict.get('Forecast', {}).get(XT_KEY, {}) self.tideprog = d.get('prog', XT_PROG) self.tideargs = d.get('args', XT_ARGS) self.location = d['location'] @@ -2282,7 +2422,9 @@ class ForecastVariables(SearchList): ''' fd = generator.config_dict.get('Forecast', {}) sd = generator.skin_dict.get('Forecast', {}) - db = generator._getArchive(fd['database']) + + self.database = generator._getArchive(fd['database']) + self.table = fd.get('table','archive') self.latitude = generator.stn_info.latitude_f self.longitude = generator.stn_info.longitude_f @@ -2298,8 +2440,8 @@ class ForecastVariables(SearchList): self.labels['Weather'] = dict(weather_label_dict.items() + label_dict.get('Weather', {}).items()) self.labels['Zambretti'] = dict(zambretti_label_dict.items() + label_dict.get('Zambretti', {}).items()) - self.database = db - self.table = fd.get('table','archive') + self.db_max_tries = 3 + self.db_retry_wait = 5 # seconds def get_extension(self, timespan, archivedb, statsdb): return {'forecast': self} @@ -2308,18 +2450,31 @@ class ForecastVariables(SearchList): sql = "select dateTime,issued_ts,event_ts,hilo,offset,usUnits,location from %s where method = 'XTide' and dateTime = (select dateTime from %s where method = 'XTide' order by dateTime desc limit 1) and event_ts >= %d order by dateTime asc" % (self.table, self.table, from_ts) if max_events is not None: sql += ' limit %d' % max_events - records = [] - for rec in self.database.genSql(sql): - r = {} - r['dateTime'] = self._create_value(context, rec[0], 'group_time') - r['issued_ts'] = self._create_value(context, rec[1], 'group_time') - r['event_ts'] = self._create_value(context, rec[2], 'group_time') - r['hilo'] = rec[3] - r['offset'] = self._create_value(context, rec[4], 'group_altitude', - unit_system=rec[5]) - r['location'] = rec[6] - records.append(r) - return records + for count in range(self.db_max_tries): + try: + records = [] + for rec in self.database.genSql(sql): + r = {} + r['dateTime'] = self._create_value(context, + rec[0], 'group_time') + r['issued_ts'] = self._create_value(context, + rec[1], 'group_time') + r['event_ts'] = self._create_value(context, + rec[2], 'group_time') + r['hilo'] = rec[3] + r['offset'] = self._create_value(context, + rec[4], 'group_altitude', + unit_system=rec[5]) + r['location'] = rec[6] + records.append(r) + return records + except Exception, e: + logerr('get tides failed (attempt %d of %d): %s' % + ((count+1), self.db_max_tries, e)) + logdbg('waiting %d seconds before retry' % + self.db_retry_wait) + time.sleep(self.db_retry_wait) + return [] def _getRecords(self, fid, from_ts, to_ts, max_events=1): '''get the latest requested forecast of indicated type for the @@ -2329,14 +2484,23 @@ class ForecastVariables(SearchList): sql = "select * from %s where method = '%s' and event_ts >= %d and event_ts <= %d and dateTime = (select dateTime from %s where method = '%s' order by dateTime desc limit 1) order by event_ts asc" % (self.table, fid, from_ts, to_ts, self.table, fid) if max_events is not None: sql += ' limit %d' % max_events - records = [] - columns = self.database.connection.columnsOf(self.table) - for rec in self.database.genSql(sql): - r = {} - for i,f in enumerate(columns): - r[f] = rec[i] - records.append(r) - return records + for count in range(self.db_max_tries): + try: + records = [] + columns = self.database.connection.columnsOf(self.table) + for rec in self.database.genSql(sql): + r = {} + for i,f in enumerate(columns): + r[f] = rec[i] + records.append(r) + return records + except Exception, e: + logerr('get %s failed (attempt %d of %d): %s' % + (fid, (count+1), self.db_max_tries, e)) + logdbg('waiting %d seconds before retry' % + self.db_retry_wait) + time.sleep(self.db_retry_wait) + return [] def _create_value(self, context, value_str, group, units=None, unit_system=weewx.US): @@ -2390,15 +2554,24 @@ class ForecastVariables(SearchList): and is good for about 6 hours. So there is no difference between the created timestamp and event timestamp.''' sql = "select dateTime,zcode from %s where method = 'Zambretti' order by dateTime desc limit 1" % self.table - record = self.database.getSql(sql) - if record is None: - return { 'dateTime' : '', 'issued_ts' : '', 'event_ts' : '', - 'code' : '', 'text' : '' } - th = self._create_value('zambretti', record[0], 'group_time') - code = record[1] - text = self.labels['Zambretti'].get(code, code) - return { 'dateTime' : th, 'issued_ts' : th, 'event_ts' : th, - 'code' : code, 'text' : text, } + for count in range(self.db_max_tries): + try: + record = self.database.getSql(sql) + if record is not None: + th = self._create_value('zambretti', record[0], 'group_time') + code = record[1] + text = self.labels['Zambretti'].get(code, code) + return { 'dateTime' : th, 'issued_ts' : th, + 'event_ts' : th, + 'code' : code, 'text' : text, } + except Exception, e: + logerr('get zambretti failed (attempt %d of %d): %s' % + ((count+1), self.db_max_tries, e)) + logdbg('waiting %d seconds before retry' % + self.db_retry_wait) + time.sleep(self.db_retry_wait) + return { 'dateTime' : '', 'issued_ts' : '', 'event_ts' : '', + 'code' : '', 'text' : '' } def weather_periods(self, fid, from_ts=None, to_ts=None, max_events=240): '''Returns forecast records for the indicated source from the diff --git a/bin/user/test/test_forecast.py b/bin/user/test/test_forecast.py index 04681cef..b8eb7fed 100644 --- a/bin/user/test/test_forecast.py +++ b/bin/user/test/test_forecast.py @@ -299,6 +299,7 @@ class FakeData(object): def gen_fake_zambretti_data(): ts = int(time.mktime((2013,8,22,12,0,0,0,0,-1))) codes = ['A', 'B', 'C', 'D', 'E', 'F', 'A', 'A', 'A'] + records = [] for code in codes: record = {} record['method'] = 'Zambretti' @@ -308,7 +309,8 @@ class FakeData(object): record['event_ts'] = ts record['zcode'] = code ts += 300 - yield record + records.append(record) + return records @staticmethod def gen_fake_nws_data(): @@ -15162,17 +15164,17 @@ class ForecastTest(unittest.TestCase): # next record gives us a trend event.record = {'barometer': 29.834685721179159, 'usUnits': 1, 'dateTime': 1378143900, 'windDir': 90.0} record = zf.get_forecast(event) - self.assertEqual(record, {'event_ts': 1378143900, 'dateTime': 1378143900, 'zcode': 'C', 'issued_ts': 1378143900, 'method': 'Zambretti', 'usUnits': 1}) + self.assertEqual(record, [{'event_ts': 1378143900, 'dateTime': 1378143900, 'zcode': 'C', 'issued_ts': 1378143900, 'method': 'Zambretti', 'usUnits': 1}]) # now the pressure goes up slightly event.record = {'barometer': 29.835649151484603, 'usUnits': 1, 'dateTime': 1378144200, 'windDir': 90.0} record = zf.get_forecast(event) - self.assertEqual(record, {'event_ts': 1378144200, 'dateTime': 1378144200, 'zcode': 'K', 'issued_ts': 1378144200, 'method': 'Zambretti', 'usUnits': 1}) + self.assertEqual(record, [{'event_ts': 1378144200, 'dateTime': 1378144200, 'zcode': 'K', 'issued_ts': 1378144200, 'method': 'Zambretti', 'usUnits': 1}]) # now the pressure drops event.record = {'barometer': 29.0, 'usUnits': 1, 'dateTime': 1378144500, 'windDir': 90.0} record = zf.get_forecast(event) - self.assertEqual(record, {'event_ts': 1378144500, 'dateTime': 1378144500, 'zcode': 'L', 'issued_ts': 1378144500, 'method': 'Zambretti', 'usUnits': 1}) + self.assertEqual(record, [{'event_ts': 1378144500, 'dateTime': 1378144500, 'zcode': 'L', 'issued_ts': 1378144500, 'method': 'Zambretti', 'usUnits': 1}]) def test_zambretti_units(self): '''ensure that zambretti works with both US and METRIC''' @@ -15193,12 +15195,12 @@ class ForecastTest(unittest.TestCase): # next record gives us a trend event.record = {'barometer': 1010.20245852, 'usUnits': weewx.METRIC, 'dateTime': 1378143900, 'windDir': 90.0} record = zf.get_forecast(event) - self.assertEqual(record, {'event_ts': 1378143900, 'dateTime': 1378143900, 'zcode': 'C', 'issued_ts': 1378143900, 'method': 'Zambretti', 'usUnits': 1}) + self.assertEqual(record, [{'event_ts': 1378143900, 'dateTime': 1378143900, 'zcode': 'C', 'issued_ts': 1378143900, 'method': 'Zambretti', 'usUnits': 1}]) # now the pressure goes up slightly event.record = {'barometer': 1010.23508027, 'usUnits': weewx.METRIC, 'dateTime': 1378144200, 'windDir': 90.0} record = zf.get_forecast(event) - self.assertEqual(record, {'event_ts': 1378144200, 'dateTime': 1378144200, 'zcode': 'K', 'issued_ts': 1378144200, 'method': 'Zambretti', 'usUnits': 1}) + self.assertEqual(record, [{'event_ts': 1378144200, 'dateTime': 1378144200, 'zcode': 'K', 'issued_ts': 1378144200, 'method': 'Zambretti', 'usUnits': 1}]) def test_zambretti_bogus_values(self): '''confirm behavior when we get bogus values''' @@ -15230,7 +15232,7 @@ class ForecastTest(unittest.TestCase): event.record['windDir'] = 180 event.record['dateTime'] = 1368303780 c = f.get_forecast(event) - self.assertEqual(c, {'event_ts': 1368303780, 'dateTime': 1368303780, 'zcode': 'C', 'issued_ts': 1368303780, 'method': 'Zambretti', 'usUnits': 1}) + self.assertEqual(c, [{'event_ts': 1368303780, 'dateTime': 1368303780, 'zcode': 'C', 'issued_ts': 1368303780, 'method': 'Zambretti', 'usUnits': 1}]) event.record['barometer'] = 1030 event.record['windDir'] = None event.record['dateTime'] = 1368304080 @@ -15492,7 +15494,7 @@ SW (WU_API_KEY, WU_LOCATION)) fcast = forecast.WUDownloadForecast(WU_API_KEY, WU_LOCATION, url=url) print fcast - records = forecast.WUParseForecast(fcast) + records,msgs = forecast.WUParseForecast(fcast) print records def xtest_wu_forecast_from_file(self): @@ -15503,7 +15505,7 @@ SW for line in f: lines.append(line) f.close() - records = forecast.WUParseForecast(''.join(lines)) + records,msgs = forecast.WUParseForecast(''.join(lines)) print records def xtest_wu_download_daily(self): @@ -15522,7 +15524,8 @@ SW def test_wu_parse_forecast_daily(self): ts = 1377298279 - records = forecast.WUParseForecast(WU_BOS_DAILY, issued_ts=ts, now=ts) + records,msgs = forecast.WUParseForecast(WU_BOS_DAILY, + issued_ts=ts, now=ts) self.assertEqual(records[0:2], [ {'clouds': 'B2', 'temp': 61.5, 'hour': 23, 'event_ts': 1368673200, 'qpf': 0.10000000000000001, 'windSpeed': 15.0, 'pop': 50, 'dateTime': 1377298279, 'windDir': u'SSW', 'tempMin': 55.0, 'qsf': 0.0, 'windGust': 19.0, 'duration': 86400, 'humidity': 69, 'issued_ts': 1377298279, 'method': 'WU', 'usUnits': 1, 'tempMax': 68.0}, {'clouds': 'FW', 'temp': 65.5, 'hour': 23, 'event_ts': 1368759600, 'qpf': 0.0, 'windSpeed': 19.0, 'pop': 10, 'dateTime': 1377298279, 'windDir': 'W', 'tempMin': 54.0, 'qsf': 0.0, 'windGust': 23.0, 'duration': 86400, 'humidity': 42, 'issued_ts': 1377298279, 'method': 'WU', 'usUnits': 1, 'tempMax': 77.0} @@ -15530,7 +15533,8 @@ SW def test_wu_parse_forecast_hourly(self): ts = 1378215570 - records = forecast.WUParseForecast(WU_BOS_HOURLY, issued_ts=ts, now=ts) + records,msgs = forecast.WUParseForecast(WU_BOS_HOURLY, + issued_ts=ts, now=ts) self.assertEqual(records[0:2], [ {'windDir': u'S', 'clouds': 'OV', 'temp': 72.0, 'hour': 22, 'event_ts': 1378173600, 'uvIndex': 0, 'qpf': None, 'pop': 100, 'dateTime': 1378215570, 'dewpoint': 69.0, 'windSpeed': 3.0, 'obvis': None, 'rainshwrs': 'C', 'duration': 3600, 'tstms': 'S', 'humidity': 90, 'issued_ts': 1378215570, 'method': 'WU', 'usUnits': 1, 'qsf': None}, {'windDir': u'S', 'clouds': 'OV', 'temp': 72.0, 'hour': 23, 'event_ts': 1378177200, 'uvIndex': 0, 'qpf': 0.040000000000000001, 'pop': 80, 'dateTime': 1378215570, 'dewpoint': 68.0, 'windSpeed': 1.0, 'obvis': 'PF', 'rainshwrs': 'C', 'duration': 3600, 'tstms': 'S', 'humidity': 87, 'issued_ts': 1378215570, 'method': 'WU', 'usUnits': 1, 'qsf': None} @@ -15545,14 +15549,14 @@ SW def test_wu_detect_download_errors(self): '''ensure proper behavior when server replies with error''' - records = forecast.WUParseForecast(WU_ERROR_NOKEY) + records,msgs = forecast.WUParseForecast(WU_ERROR_NOKEY) self.assertEqual(records, []) def test_wu_template_periods_daily(self): '''verify the period behavior''' - records = forecast.WUParseForecast(WU_TENANTS_HARBOR_DAILY, - issued_ts=1378090800, - now=1378090800) + records,msgs = forecast.WUParseForecast(WU_TENANTS_HARBOR_DAILY, + issued_ts=1378090800, + now=1378090800) template = create_template(PERIODS_TEMPLATE, 'WU', '1378090800') self.runTemplateTest('test_wu_template_periods_daily', 'user.forecast.WUForecast', records, template, @@ -15574,9 +15578,9 @@ SW def test_wu_template_summary_daily(self): '''verify the summary behavior''' - records = forecast.WUParseForecast(WU_TENANTS_HARBOR_DAILY, - issued_ts=1378090800, - now=1378090800) + records,msgs = forecast.WUParseForecast(WU_TENANTS_HARBOR_DAILY, + issued_ts=1378090800, + now=1378090800) template = create_template(SUMMARY_TEMPLATE, 'WU', '1378090800') self.runTemplateTest('test_wu_template_summary_daily', 'user.forecast.WUForecast', records, template, @@ -15613,9 +15617,9 @@ SSW def test_wu_template_summary_daily_metric(self): '''verify the summary behavior''' - records = forecast.WUParseForecast(WU_TENANTS_HARBOR_DAILY, - issued_ts=1378090800, - now=1378090800) + records,msgs = forecast.WUParseForecast(WU_TENANTS_HARBOR_DAILY, + issued_ts=1378090800, + now=1378090800) template = create_template(SUMMARY_TEMPLATE, 'WU', '1378090800') self.runTemplateTest('test_wu_template_summary_daily_metric', 'user.forecast.WUForecast', records, template, @@ -15652,9 +15656,9 @@ SSW def test_wu_template_summary_periods_daily(self): '''verify the summary behavior using periods''' - records = forecast.WUParseForecast(WU_TENANTS_HARBOR_DAILY, - issued_ts=1378090800, - now=1378090800) + records,msgs = forecast.WUParseForecast(WU_TENANTS_HARBOR_DAILY, + issued_ts=1378090800, + now=1378090800) template = create_template(SUMMARY_PERIODS_TEMPLATE,'WU','1378090800') self.runTemplateTest('test_wu_template_summary_periods_daily', 'user.forecast.WUForecast', records, template, @@ -15676,9 +15680,9 @@ forecast for for the day 01-Sep-2013 00:00 as of 01-Sep-2013 23:00 def test_wu_template_summary_periods_daily_metric(self): '''verify the summary behavior using periods''' - records = forecast.WUParseForecast(WU_TENANTS_HARBOR_DAILY, - issued_ts=1378090800, - now=1378090800) + records,msgs = forecast.WUParseForecast(WU_TENANTS_HARBOR_DAILY, + issued_ts=1378090800, + now=1378090800) template = create_template(SUMMARY_PERIODS_TEMPLATE,'WU','1378090800') self.runTemplateTest('test_wu_template_summary_periods_daily_metric', 'user.forecast.WUForecast', records, template, @@ -15700,9 +15704,9 @@ forecast for for the day 01-Sep-2013 00:00 as of 01-Sep-2013 23:00 def test_wu_template_table_daily(self): '''exercise the period and summary template elements''' - records = forecast.WUParseForecast(WU_TENANTS_HARBOR_DAILY, - issued_ts=1378090800, - now=1378090800) + records,msgs = forecast.WUParseForecast(WU_TENANTS_HARBOR_DAILY, + issued_ts=1378090800, + now=1378090800) template = create_template(TABLE_TEMPLATE, 'WU', '1378090800') self.runTemplateLineTest('test_wu_template_table_daily', 'user.forecast.WUForecast', records, template, @@ -15710,9 +15714,9 @@ forecast for for the day 01-Sep-2013 00:00 as of 01-Sep-2013 23:00 def test_wu_template_periods_hourly(self): '''verify the period behavior for hourly''' - records = forecast.WUParseForecast(WU_BOS_HOURLY, - issued_ts=1378173600, - now=1378173600) + records,msgs = forecast.WUParseForecast(WU_BOS_HOURLY, + issued_ts=1378173600, + now=1378173600) template = create_template(PERIODS_TEMPLATE, 'WU', '1378173600') self.runTemplateTest('test_wu_template_periods_hourly', 'user.forecast.WUForecast', records, template, @@ -15744,9 +15748,9 @@ forecast for for the day 01-Sep-2013 00:00 as of 01-Sep-2013 23:00 def test_wu_template_summary_hourly(self): '''verify the summary behavior for hourly''' - records = forecast.WUParseForecast(WU_BOS_HOURLY, - issued_ts=1378173600, - now=1378173600) + records,msgs = forecast.WUParseForecast(WU_BOS_HOURLY, + issued_ts=1378173600, + now=1378173600) template = create_template(SUMMARY_TEMPLATE, 'WU', '1378173600') self.runTemplateTest('test_wu_template_summary_hourly', 'user.forecast.WUForecast', records, template, @@ -15786,9 +15790,9 @@ S def test_wu_template_table_hourly(self): '''exercise the period and summary template elements for hourly''' - records = forecast.WUParseForecast(WU_BOS_HOURLY, - issued_ts=1378173600, - now=1378173600) + records,msgs = forecast.WUParseForecast(WU_BOS_HOURLY, + issued_ts=1378173600, + now=1378173600) template = create_template(TABLE_TEMPLATE, 'WU', '1378173600') self.runTemplateLineTest('test_wu_template_table_hourly', 'user.forecast.WUForecast', records, template, @@ -15796,22 +15800,24 @@ S def test_wu_template_table(self): '''exercise the period and summary template elements''' - records = forecast.WUParseForecast(WU_BOS_HOURLY, - issued_ts=1378173600, - now=1378173600) + records,msgs = forecast.WUParseForecast(WU_BOS_HOURLY, + issued_ts=1378173600, + now=1378173600) template = create_template(TABLE_TEMPLATE, 'WU', '1378173600') - template = template.replace('period.event_ts.raw', 'period.event_ts.raw, periods=$periods') + template = template.replace('period.event_ts.raw', + 'period.event_ts.raw, periods=$periods') self.runTemplateLineTest('test_wu_template_table', 'user.forecast.WUForecast', records, template, 514) def test_wu_inorther26(self): '''test forecast for inorther26''' - records = forecast.WUParseForecast(WU_INORTHER26_HOURLY, - issued_ts=1384053615, - now=1384053615) + records,msgs = forecast.WUParseForecast(WU_INORTHER26_HOURLY, + issued_ts=1384053615, + now=1384053615) template = create_template(TABLE_TEMPLATE, 'WU', '1384053615') - template = template.replace('period.event_ts.raw', 'period.event_ts.raw, periods=$periods') + template = template.replace('period.event_ts.raw', + 'period.event_ts.raw, periods=$periods') self.runTemplateLineTest('test_wu_inorther26', 'user.forecast.WUForecast', records, template, 493) @@ -16106,9 +16112,9 @@ $ts $a.sunrise $a.sunset $gmstr config_dict = create_config(tdir, 'user.forecast.ZambrettiForecast') method_id = 'Zambretti' table = 'archive' - dbspec = config_dict['Forecast']['database'] - archive = forecast.Forecast.setup_database(dbspec, table, method_id, - config_dict, + archive = forecast.Forecast.setup_database('forecast_sqlite', + config_dict['Databases']['forecast_sqlite'], + table, forecast.defaultForecastSchema) # create a zambretti forecaster and simulator with which to test @@ -16170,7 +16176,8 @@ $ts $a.sunrise $a.sunset $gmstr size1 = os.path.getsize(dbfile) # there should be one remaining after a prune - forecast.Forecast.prune_forecasts(archive, table, method_id, ts, True) + forecast.Forecast.prune_forecasts(archive, table, method_id, ts) + forecast.Forecast.vacuum_database(archive, method_id) records = forecast.Forecast.get_saved_forecasts(archive, table, method_id) self.assertEqual(len(records), 1) diff --git a/bin/wee_config_ws23xx b/bin/wee_config_ws23xx index c4689e46..ae7d9a41 100644 --- a/bin/wee_config_ws23xx +++ b/bin/wee_config_ws23xx @@ -82,6 +82,7 @@ def main(): print 'Driver version %s' % weewx.drivers.ws23xx.DRIVER_VERSION altitude_m = weewx.drivers.ws23xx.getaltitudeM(config_dict) station = weewx.drivers.ws23xx.WS23xx(altitude=altitude_m, + config_dict=config_dict, **config_dict['WS23xx']) # Do what we need to do... @@ -94,7 +95,7 @@ def main(): history(station, ts=ts) elif options.settime: setclock(station, prompt) - elif options.interval: + elif options.interval is not None: setinterval(station, options.interval, prompt) elif options.clear: clearhistory(station, prompt) @@ -116,7 +117,7 @@ def current(station): print packet break -def history(station, ts=0, count=0): +def history(station, ts=None, count=0): """Display the indicated number of records or records since timestamp""" print "Querying the station for historical records..." for i,r in enumerate(station.genArchiveRecords(since_ts=ts, count=count)): @@ -147,6 +148,7 @@ def setclock(station, prompt): print "Set clock cancelled." def setinterval(station, interval, prompt): + print "Changing the interval will clear the station memory." v = station.getArchiveInterval() ans = None while ans not in ['y', 'n']: diff --git a/bin/weewx/abstractstation.py b/bin/weewx/abstractstation.py index b9b3a13b..a54e6555 100644 --- a/bin/weewx/abstractstation.py +++ b/bin/weewx/abstractstation.py @@ -19,6 +19,9 @@ class AbstractStation(object): @property def archive_interval(self): raise NotImplementedError("Property 'archive_interval' not implemented") + + def genStartupRecords(self, last_ts): + return self.genArchiveRecords(last_ts) def genLoopPackets(self): raise NotImplementedError("Method 'genLoopPackets' not implemented") diff --git a/bin/weewx/cheetahgenerator.py b/bin/weewx/cheetahgenerator.py index 2d0d8b43..e61884b8 100644 --- a/bin/weewx/cheetahgenerator.py +++ b/bin/weewx/cheetahgenerator.py @@ -504,8 +504,10 @@ class Stats(SearchList): as $day.outTemp.max""" def get_extension(self, timespan, archivedb, statsdb): - heatbase = self.generator.skin_dict['Units']['DegreeDays'].get('heating_base') - coolbase = self.generator.skin_dict['Units']['DegreeDays'].get('heating_base') + units_dict = self.generator.skin_dict.get('Units', {}) + dd_dict = units_dict.get('DegreeDays', {}) + heatbase = dd_dict.get('heating_base', None) + coolbase = dd_dict.get('cooling_base', None) heatbase_t = (float(heatbase[0]), heatbase[1], "group_temperature") if heatbase else default_heatbase coolbase_t = (float(coolbase[0]), coolbase[1], "group_temperature") if coolbase else default_coolbase diff --git a/bin/weewx/drivers/fousb.py b/bin/weewx/drivers/fousb.py index e8d97d78..7dced278 100644 --- a/bin/weewx/drivers/fousb.py +++ b/bin/weewx/drivers/fousb.py @@ -220,6 +220,8 @@ import weeutil.weeutil import weewx.abstractstation import weewx.wxformulas +DRIVER_VERSION = '1.4' + def loader(config_dict, engine): altitude_m = getaltitudeM(config_dict) station = FineOffsetUSB(altitude=altitude_m,**config_dict['FineOffsetUSB']) @@ -328,10 +330,8 @@ def pywws2weewx(p, ts, pressure_offset, altitude, packet[k] = None # track the pointer used to obtain the data - if p.has_key('ptr'): - packet['ptr'] = int(p['ptr']) - if p.has_key('delay'): - packet['delay'] = int(p['delay']) + packet['ptr'] = int(p['ptr']) if p.has_key('ptr') else None + packet['delay'] = int(p['delay']) if p.has_key('delay') else None # station status is an integer if packet['status'] is not None: @@ -359,18 +359,21 @@ def pywws2weewx(p, ts, pressure_offset, altitude, packet['altimeter'] = sp2ap(adjp, altitude) # calculate the rain increment from the rain total - # watch for spurious rain counter decrement. if small decrement then it - # is a sensor glitch. if decrement is significant, then it is a counter - # wraparound. + # watch for spurious rain counter decrement. if decrement is significant + # then it is a counter wraparound. a small decrement is either a sensor + # glitch or a read from a previous record. total = packet['rain'] packet['rainTotal'] = packet['rain'] if packet['rain'] is not None and last_rain is not None: if packet['rain'] < last_rain: + pstr = '0x%04x' % packet['ptr'] if packet['ptr'] is not None else 'None' if last_rain - packet['rain'] < rain_max * 0.3 * 0.5: - loginf('ignoring spurious rain counter decrement: new: %s old: %s' % (packet['rain'], last_rain)) + loginf('ignoring spurious rain counter decrement (%s): ' + 'new: %s old: %s' % (pstr, packet['rain'], last_rain)) packet['rainTotal'] = last_rain else: - loginf('rain counter wraparound detected: new: %s old: %s' % (packet['rain'], last_rain)) + loginf('rain counter wraparound detected (%s): ' + 'new: %s old: %s' % (pstr, packet['rain'], last_rain)) total += rain_max * 0.3 packet['rain'] = calculate_rain(total, last_rain) @@ -381,13 +384,19 @@ def pywws2weewx(p, ts, pressure_offset, altitude, # report rainfall in log to diagnose rain counter issues if weewx.debug: if packet['rain'] is not None and packet['rain'] > 0: - logdbg('got rainfall of %.2f cm (new: %.2f old: %.2f)' % (packet['rain'], packet['rainTotal'], last_rain)) + logdbg('got rainfall of %.2f cm (new: %.2f old: %.2f)' % + (packet['rain'], packet['rainTotal'], last_rain)) if packet['rainRate'] is not None and packet['rainRate'] > 0: - logdbg('calculated rainrate of %.2f cm/hr (%.2f cm in %d seconds)' % (packet['rainRate'], packet['rain'], int(ts - last_rain_ts))) + logdbg('calculated rainrate of %.2f cm/hr ' + '(%.2f cm in %d seconds)' % (packet['rainRate'], + packet['rain'], + int(ts - last_rain_ts))) # if the rain rate is bogus, ignore the rain and rainRate values if packet['rainRate'] is not None and packet['rainRate'] > max_rain_rate: - logerr('maximum rain rate exceeded: max: %.2f rate: %.2f cm/hr (%.2f cm in %d s)' % (max_rain_rate, packet['rainRate'], packet['rain'], int(ts - last_rain_ts))) + logerr('maximum rain rate exceeded: max: %.2f rate: %.2f cm/hr ' + '(%.2f cm in %d s)' % (max_rain_rate, packet['rainRate'], + packet['rain'], int(ts - last_rain_ts))) packet['rain'] = None packet['rainRate'] = None @@ -667,7 +676,7 @@ class FineOffsetUSB(weewx.abstractstation.AbstractStation): [Optional. Default is 15 seconds] wait_before_retry: How long to wait after a failure before retrying. - [Optional. Default is 5 seconds] + [Optional. Default is 30 seconds] max_tries: How many times to try before giving up. [Optional. Default is 3] @@ -701,7 +710,7 @@ class FineOffsetUSB(weewx.abstractstation.AbstractStation): self.polling_interval = int(stn_dict.get('polling_interval', 60)) self.max_rain_rate = int(stn_dict.get('max_rain_rate', 24)) self.timeout = float(stn_dict.get('timeout', 15.0)) - self.wait_before_retry = float(stn_dict.get('wait_before_retry', 60.0)) + self.wait_before_retry = float(stn_dict.get('wait_before_retry', 30.0)) self.max_tries = int(stn_dict.get('max_tries', 3)) self.interface = int(stn_dict.get('interface', 0)) self.vendor_id = int(stn_dict.get('vendor_id', '0x1941'), 0) @@ -738,8 +747,16 @@ class FineOffsetUSB(weewx.abstractstation.AbstractStation): self._magic_numbers = ['55aa'] self._last_magic = None + # FIXME: get last_rain_arc and last_rain_ts_arc from database + + loginf('driver version is %s' % DRIVER_VERSION) + loginf('polling mode is %s' % self.polling_mode) + if self.polling_mode.lower() == PERIODIC_POLLING.lower(): + loginf('polling interval is %s' % self.polling_interval) + loginf('altitude is %s meters' % str(self.altitude)) + loginf('pressure offset is %s' % str(self.pressure_offset)) + self.openPort() - self._setup() # Unfortunately there is no provision to obtain the model from the station # itself, so use what is specified from the configuration file. @@ -796,14 +813,6 @@ class FineOffsetUSB(weewx.abstractstation.AbstractStation): return dev return None - # FIXME: get last_rain_arc and last_rain_ts_arc from database on startup - def _setup(self): - loginf('polling mode is %s' % self.polling_mode) - if self.polling_mode.lower() == PERIODIC_POLLING.lower(): - loginf('polling interval is %s' % self.polling_interval) - loginf('altitude is %s meters' % str(self.altitude)) - loginf('pressure offset is %s' % str(self.pressure_offset)) - # There is no point in using the station clock since it cannot be trusted and # since we cannot synchronize it with the computer clock. @@ -852,6 +861,7 @@ class FineOffsetUSB(weewx.abstractstation.AbstractStation): self._last_rain_arc, self._last_rain_ts_arc, self.max_rain_rate) data['interval'] = r['interval'] + data['ptr'] = r['ptr'] self._last_rain_arc = data['rainTotal'] self._last_rain_ts_arc = ts logdbg('returning archive record %s' % ts) @@ -872,6 +882,8 @@ class FineOffsetUSB(weewx.abstractstation.AbstractStation): If we get USB read failures, retry until we get something valid. """ nerr = 0 + old_ptr = None + interval = self._archive_interval_minutes() while True: try: if self.polling_mode.lower() == ADAPTIVE_POLLING.lower(): @@ -881,12 +893,20 @@ class FineOffsetUSB(weewx.abstractstation.AbstractStation): yield data elif self.polling_mode.lower() == PERIODIC_POLLING.lower(): new_ptr = self.current_pos() + if new_ptr < data_start: + raise ObservationError('bad pointer: 0x%04x' % new_ptr) block = self.get_raw_data(new_ptr, unbuffered=True) if len(block) != reading_len[self.data_format]: raise ObservationError('wrong block length: expected: %d actual: %d' % (reading_len[self.data_format], len(block))) - nerr = 0 data = _decode(block, reading_format[self.data_format]) + delay = data.get('delay', None) + if delay is None: + raise ObservationError('no delay found in observation') + if new_ptr != old_ptr and delay >= interval: + raise ObservationError('ignoring suspected bogus data from 0x%04x (delay=%s interval=%s)' % (new_ptr, delay, interval)) + old_ptr = new_ptr data['ptr'] = new_ptr + nerr = 0 yield data time.sleep(self.polling_interval) else: diff --git a/bin/weewx/drivers/te923.py b/bin/weewx/drivers/te923.py index 8b06e6c5..c1966418 100644 --- a/bin/weewx/drivers/te923.py +++ b/bin/weewx/drivers/te923.py @@ -138,6 +138,9 @@ claimInterface. # TODO: consider open/close on each read instead of keeping open # TODO: try doing bulkRead instead of interruptRead +# TODO: sensor data use 32 bytes, but each historical record is 38 bytes. what +# do the other 4 bytes represent? + # FIXME: speed up transfers: # date;PYTHONPATH=bin python bin/weewx/drivers/te923.py --records 0 > b; date # Tue Nov 26 10:37:36 EST 2013 @@ -158,7 +161,7 @@ import weeutil import weewx.abstractstation import weewx.wxformulas -DRIVER_VERSION = '0.6' +DRIVER_VERSION = '0.7' DEBUG_READ = 0 DEBUG_DECODE = 0 DEBUG_PRESSURE = 0 @@ -362,19 +365,20 @@ class TE923(weewx.abstractstation.AbstractStation): DEBUG_PRESSURE = int(stn_dict.get('debug_pressure', 0)) self.altitude = stn_dict['altitude'] - self.polling_interval = stn_dict.get('polling_interval', 10) self.model = stn_dict.get('model', 'TE923') - self.calc_windchill = stn_dict.get('calculate_windchill', False) - self.sensor_map = stn_dict.get('sensor_map', DEFAULT_SENSOR_MAP) - self.battery_map = stn_dict.get('battery_map', DEFAULT_BATTERY_MAP) self.max_tries = int(stn_dict.get('max_tries', 5)) self.retry_wait = int(stn_dict.get('retry_wait', 30)) + self.polling_interval = int(stn_dict.get('polling_interval', 10)) + self.calc_windchill = weeutil.weeutil.tobool(stn_dict.get('calculate_windchill', False)) + self.sensor_map = stn_dict.get('sensor_map', DEFAULT_SENSOR_MAP) + self.battery_map = stn_dict.get('battery_map', DEFAULT_BATTERY_MAP) self.memory_size = stn_dict.get('memory_size', 'small') vendor_id = int(stn_dict.get('vendor_id', '0x1130'), 0) product_id = int(stn_dict.get('product_id', '0x6801'), 0) device_id = stn_dict.get('device_id', None) + loginf('driver version is %s' % DRIVER_VERSION) loginf('polling interval is %s' % str(self.polling_interval)) loginf('windchill will be %s' % ('calculated' if self.calc_windchill else 'read from station')) @@ -582,34 +586,22 @@ def decode_th(buf, i): (i, 0+offset, buf[0+offset], 1+offset, buf[1+offset], 2+offset, buf[2+offset])) if bcd2int(buf[0+offset] & 0x0f) > 9: - if DEBUG_DECODE > 1: - logdbg("TMP%d buffer 0 & 0x0f > 9" % i) - # FIXME: wview uses 0x0c || 0x0b instead of 0x06 || 0x0b + # FIXME: wview uses 0x0c || 0x0b instead of 0x06 || 0x0b for invalid if buf[0+offset] & 0x0f == 0x06 or buf[0+offset] & 0x0f == 0x0b: - if DEBUG_DECODE > 1: - logdbg("TMP%d buffer 0 & 0x0f = 0x0c or 0x0b" % i) data[tstate] = STATE_OUT_OF_RANGE else: - if DEBUG_DECODE > 1: - logdbg("TMP%d other error in buffer 0" % i) data[tstate] = STATE_INVALID if buf[1+offset] & 0x40 != 0x40 and i > 0: - if DEBUG_DECODE > 1: - logdbg("TMP%d buffer 1 bit 6 set" % i) data[tstate] = STATE_OUT_OF_RANGE # FIXME: what about missing link for temperature? if data[tstate] == STATE_OK: data[tlabel] = bcd2int(buf[0+offset]) / 10.0 \ + bcd2int(buf[1+offset] & 0x0f) * 10.0 - if DEBUG_DECODE > 1: - logdbg("TMP%d before is %0.2f" % (i, data[tlabel])) if buf[1+offset] & 0x20 == 0x20: data[tlabel] += 0.05 if buf[1+offset] & 0x80 != 0x80: data[tlabel] *= -1 - if DEBUG_DECODE > 1: - logdbg("TMP%d after is %0.2f" % (i, data[tlabel])) else: data[tlabel] = None @@ -770,7 +762,7 @@ def decode_windchill(buf): def decode_status(buf): data = {} if DEBUG_DECODE: - logdbg("STT BUFF[22]=%02x" % buf[22]) + logdbg("STT BUF[22]=%02x" % buf[22]) if buf[22] & 0x0f == 0x0f: data['storm'] = None data['forecast'] = None @@ -864,7 +856,7 @@ class Station(object): # # FIXME: must be a better way to know when there is no more data def _raw_read(self, addr, timeout=50): - reqbuf = [0x05, 0x0AF, 0x00, 0x00, 0x00, 0x00, 0xAF, 0xFE] + reqbuf = [0x05, 0xAF, 0x00, 0x00, 0x00, 0x00, 0xAF, 0xFE] reqbuf[4] = addr / 0x10000 reqbuf[3] = (addr - (reqbuf[4] * 0x10000)) / 0x100 reqbuf[2] = addr - (reqbuf[4] * 0x10000) - (reqbuf[3] * 0x100) @@ -878,25 +870,23 @@ class Station(object): if ret != 8: raise BadRead('Unexpected response to data request: %s != 8' % ret) + time.sleep(0.1) # te923tool is 0.3 + start_ts = time.time() rbuf = [] - time.sleep(0.3) try: - buf = self.handle.interruptRead(self.ENDPOINT_IN, - self.READ_LENGTH, timeout) - while buf: - nbytes = buf[0] - if DEBUG_READ: - msg = 'raw: ' - msg += ' '.join(["%02x" % buf[x] for x in range(8)]) - logdbg(msg) - if nbytes > 7 or nbytes > len(buf)-1: - raise BadRead("Bogus length during read: %d" % nbytes) - rbuf.extend(buf[1:1+nbytes]) - time.sleep(0.15) + while time.time() - start_ts < 5: buf = self.handle.interruptRead(self.ENDPOINT_IN, self.READ_LENGTH, timeout) + if buf: + nbytes = buf[0] + if nbytes > 7 or nbytes > len(buf)-1: + raise BadRead("Bogus length during read: %d" % nbytes) + rbuf.extend(buf[1:1+nbytes]) + else: + break + time.sleep(0.009) # te923tool is 0.15 except usb.USBError, e: - logdbg(e) + logdbg('usb error while reading: %s' % e) if len(rbuf) < 34: raise BadRead("Not enough bytes: %d < 34" % len(rbuf)) @@ -913,18 +903,16 @@ class Station(object): def _read(self, addr, max_tries=100): if DEBUG_READ: logdbg("reading station at address 0x%06x" % addr) - cnt = 0 - while cnt < max_tries: - cnt += 1 + for cnt in range(max_tries): try: buf = self._raw_read(addr) if DEBUG_READ: logdbg("BUF " + ' '.join(["%02x" % x for x in buf])) return buf except BadRead, e: - logdbg(e) + logdbg("Bad read (attempt %d of %d): %s" % (cnt+1,max_tries,e)) else: - raise BadRead("No data after %d attempts to read" % cnt) + raise BadRead("No data after %d read attempts" % max_tries) def gen_blocks(self, count=None): """generator that returns consecutive blocks of station memory""" @@ -976,7 +964,14 @@ class Station(object): yield addr,record def get_record(self, addr=None, now_year=None, now_month=None): - """return a single record from station""" + """return a single record from station and address of the next + + Each historical record is 38 bytes (0x26) long. Records start at + memory address 0x101 (257). The index of the latest record is at + address 0xff (255), indicating the offset from the starting address. + + On small memory stations, the last 32 bytes of memory are never used. + """ if now_year is None or now_month is None: now = int(time.time()) tt = time.localtime(now) @@ -985,6 +980,10 @@ class Station(object): if addr is None: buf = self._read(0xfb) +# start = buf[5] +# if start == 0: +# start = 0xd0 # FIXME: only for small memory model +# addr = 0x101 + (buf[5]-1) * 0x26 addr = (buf[3] * 0x100 + buf[5]) * 0x26 + 0x101 buf = self._read(addr) diff --git a/bin/weewx/drivers/wmr200.py b/bin/weewx/drivers/wmr200.py index 593ccf59..0bb4c5e2 100644 --- a/bin/weewx/drivers/wmr200.py +++ b/bin/weewx/drivers/wmr200.py @@ -21,6 +21,10 @@ # pylint: disable-msg=W0603 # suppress weewx driver methods not implemented # pylint: disable-msg=W0223 +# suppress weewx driver methods non-conforming name +# pylint: disable-msg=C0103 +# suppress too many lines in module +# pylint: disable-msg=C0302 """Classes and functions to interfacing with an Oregon Scientific WMR200 station Oregon Scientific @@ -147,6 +151,13 @@ def loader(config_dict, engine): return station +class WMR200ProtocolError(weewx.WeeWxIOError): + """Used to signal a protocol error condition""" + def __init__(self, msg): + super(WMR200ProtocolError, self).__init__() + self._msg = msg + + class UsbDevice(object): """General class to handles all access to device via USB bus.""" def __init__(self): @@ -181,10 +192,10 @@ class UsbDevice(object): except usb.USBError, exception: logcrt(('open_device() Unable to open USB interface.' ' Reason: %s' % exception)) - raise weewx.WeeWxIOError(exception) + raise weewx.WakeupError(exception) except AttributeError, exception: logcrt('open_device() Device not specified.') - raise weewx.WeeWxIOError(exception) + raise weewx.WakeupError(exception) # Detach any old claimed interfaces try: @@ -195,9 +206,9 @@ class UsbDevice(object): try: self.handle.claimInterface(self.interface) except usb.USBError, exception: - logcrt('open_device() Unable to' - ' claim USB interface. Reason: %s' % exception) - raise weewx.WeeWxIOError(exception) + logcrt(('open_device() Unable to' + ' claim USB interface. Reason: %s' % exception)) + raise weewx.WakeupError(exception) def close_device(self): """Closes a device for access.""" @@ -216,9 +227,9 @@ class UsbDevice(object): the first byte that are valid protocol bytes. Only the valid protocol bytes are returned. """ if not self.handle: - logerr(('read_device() No USB handle' - ' for usb_device Read')) - raise weewx.WeeWxIOError('No USB handle for usb_device Read') + msg = 'read_device() No USB handle for usb_device Read' + logerr(msg) + raise weewx.WeeWxIOError(msg) try: report = self.handle.interruptRead(self.in_endpoint, @@ -227,9 +238,9 @@ class UsbDevice(object): # I think this value indicates that the buffer has overflowed. if report[0] == 8: - log_msg = 'USB read_device overflow error' - logerr(log_msg) - raise WMR200AccessError(log_msg) + msg = 'USB read_device overflow error' + logerr(msg) + raise weewx.WeeWxIOError(msg) self.byte_cnt_rd += len(report) # The first byte is the size of valid data following. @@ -251,14 +262,14 @@ class UsbDevice(object): # have been exhausted. We have to send a heartbeat command # to tell the weather console to start streaming live data # again. - log_msg = 'read_device() USB Error Reason:%s' % ex if ex.args[0].find('No data available') == -1: - logerr(log_msg) - return None + msg = 'read_device() USB Error Reason:%s' % ex + logerr(msg) + raise weewx.WeeWxIOError(msg) else: # No data avail...not an error but probably ok. - logdbg('No data received in' - ' %d seconds' % int(self.timeout_read)) + logdbg(('No data received in' + ' %d seconds' % int(self.timeout_read))) return [] def write_device(self, buf): @@ -268,9 +279,9 @@ class UsbDevice(object): value = 0x00000220 if not self.handle: - log_msg = 'No USB handle for usb_device Write' - logerr(log_msg) - raise weewx.WeeWxIOError(log_msg) + msg = 'No USB handle for usb_device Write' + logerr(msg) + raise weewx.WeeWxIOError(msg) try: if DEBUG_WRITES: @@ -284,32 +295,11 @@ class UsbDevice(object): 0x0000000, # index _WMR200_USB_RESET_TIMEOUT) # timeout except usb.USBError, exception: - logerr(('write_device() Unable to' - ' send USB control message')) - logerr('**** %s' % exception) + msg = ('write_device() Unable to' + ' send USB control message %d' % exception) + logerr(msg) # Convert to a Weewx error: - raise weewx.WakeupError(exception) - - -class WMR200ProtocolError(weewx.WeeWxIOError): - """Used to signal a protocol error condition""" - def __init__(self, msg): - super(WMR200ProtocolError, self).__init__() - self._msg = msg - - -class WMR200CheckSumError(weewx.WeeWxIOError): - """Used to signal a protocol error condition""" - def __init__(self, msg): - super(WMR200CheckSumError, self).__init__() - self._msg = msg - - -class WMR200AccessError(weewx.WeeWxIOError): - """Used to signal a USB or device access error condition""" - def __init__(self, msg): - super(WMR200AccessError, self).__init__() - self._msg = msg + raise weewx.WeeWxIOError(exception) class Packet(object): @@ -322,6 +312,7 @@ class Packet(object): pkt_cmd = 0 pkt_name = 'AbstractPacket' pkt_len = 0 + pkt_id = 0 def __init__(self, wmr200): """Initialize base elements of the packet parser.""" # Keep reference to the wmr200 for any special considerations @@ -335,6 +326,9 @@ class Packet(object): self._bogus_packet = False # Add the command byte as the first field self.append_data(self.pkt_cmd) + # Packet identifier + Packet.pkt_id += 1 + self.pkt_id = Packet.pkt_id @staticmethod def host_timestamp(): @@ -408,9 +402,9 @@ class Packet(object): return cksum except IndexError: - str_val = 'Packet too small to compute 16 bit checksum' - logerr(str_val) - raise WMR200CheckSumError(str_val) + msg = 'Packet too small to compute 16 bit checksum' + logerr(msg) + raise WMR200ProtocolError(msg) def _checksum_field(self): """Returns the checksum field of the current packet. @@ -420,21 +414,35 @@ class Packet(object): try: return (self._pkt_data[-1] << 8) | self._pkt_data[-2] except IndexError: - str_val = 'Packet too small to contain 16 bit checksum' - logerr(str_val) - raise WMR200CheckSumError(str_val) + msg = 'Packet too small to contain 16 bit checksum' + logerr(msg) + raise WMR200ProtocolError(msg) def verify_checksum(self): """Verifies packet for checksum correctness. - Raises exception upon checksum failure as this is a catastrophic - event.""" - if self._checksum_calculate() != self._checksum_field(): - str_val = ('Checksum error act:%x exp:%x' + Raises exception upon checksum failure unless configured to drop.""" + if not self._bogus_packet \ + and self._checksum_calculate() != self._checksum_field(): + msg = ('Checksum error act:%x exp:%x' % (self._checksum_calculate(), self._checksum_field())) - logerr(str_val) + logerr(msg) logerr(self.to_string_raw(' packet:')) - raise WMR200CheckSumError(str_val) + if self.wmr200.ignore_checksum: + logerr('Dropping packet') + self._bogus_packet = True + return + + raise weewx.CRCError(msg) + + def record_timestamp(self): + """Returns the epoch timestamp in the record.""" + try: + return self._record['dateTime'] + except (KeyError, NameError): + msg = 'record_timestamp() Timestamp not set in record' + logerr(msg) + raise weewx.ViolatedPrecondition(msg) def packet_timestamp(self): """Pulls the timestamp from the packet. @@ -459,19 +467,23 @@ class Packet(object): logerr(log_msg) raise WMR200ProtocolError(log_msg) + def time_drift(self): + """Returns the difference between PC time and the packet timestamp. + This value is approximate as all timestamps from a given archive + interval will be the same while PC time marches onwards.""" + time_drift = self.host_timestamp() - self.packet_timestamp() + loginf('Time drift in seconds between host and console:%d' % + time_drift) + return time_drift + def timestamp(self): """Returns either that timestamp or the PC time based upon configuration. Caches the last timestamp to add to packets that do not provide timestamps.""" # Calculate the drift between pc time and the console time. # Only done first time through. - if self.wmr200.time_delta is None: - # This value is approximate as all timestamps from a given archive - # interval will be the same while host time marches onwards. - self.wmr200.time_delta = self.host_timestamp() - \ - self.packet_timestamp() - loginf('Time drift in seconds between host and console:%d' % - self.wmr200.time_delta) + if self.wmr200.time_drift is None: + self.wmr200.time_drift = self.time_drift() if self.wmr200.use_pc_time: self.wmr200.last_time_epoch = self.host_timestamp() @@ -490,14 +502,20 @@ class Packet(object): """Debug method method to print the processed packet. Must be called after the Process() method.""" - out = ' Packet cooked: ' - out += '%s ' % self.pkt_name - out += '%s ' % weeutil.weeutil.timestamp_to_string\ - (self.timestamp()) - out += 'len:%d ' % self.size_actual() - out += 'fields:%d ' % len(self._record) - out += str(self._record) - logdbg(out) + try: + out = ' Packet cooked: ' + out += 'id:%d ' % self.pkt_id + out += '%s ' % self.pkt_name + out += '%s ' % weeutil.weeutil.timestamp_to_string\ + (self.record_timestamp()) + out += 'len:%d ' % self.size_actual() + out += 'fields:%d ' % len(self._record) + out += str(self._record) + logdbg(out) + except (KeyError, NameError): + msg = 'print_cooked() called before proper setup' + logerr(msg) + raise weewx.ViolatedPrecondition(msg) class PacketLive(Packet): """Packets with live sensor data from console.""" @@ -543,7 +561,12 @@ class PacketArchive(Packet): """Returns a records field to be processed by the weewx engine.""" super(PacketArchive, self).packet_process() self._record.update({'dateTime' : self.packet_timestamp(), }) + self._record.update({'interval' : \ + int(self.wmr200.archive_interval / 60), }) + def timestamp_adjust(self, delta): + """Archive records may need time adjustment when using PC time.""" + self._record['dateTime'] += int(delta) class PacketControl(Packet): """Packets with protocol control info from console.""" @@ -593,9 +616,9 @@ class PacketControl(Packet): class PacketHistoryReady(PacketControl): - """Packet parser for archived data is ready to receive.""" + """Packet parser for control command acknowledge.""" pkt_cmd = 0xd1 - pkt_name = 'Archive Avail' + pkt_name = 'CmdAck' pkt_len = 1 def __init__(self, wmr200): super(PacketHistoryReady, self).__init__(wmr200) @@ -625,24 +648,31 @@ class PacketHistoryData(PacketArchive): def packet_process(self): """Returns a records field to be processed by the weewx engine.""" super(PacketHistoryData, self).packet_process() + try: + self._record.update(decode_rain(self, self._pkt_data[ 7:20])) + self._record.update(decode_wind(self, self._pkt_data[20:27])) + self._record.update(decode_uvi(self, self._pkt_data[27:28])) + self._record.update(decode_pressure(self, self._pkt_data[28:32])) + # Number of sensors starting at zero inclusive. + num_sensors = self._pkt_data[32] + if DEBUG_PACKETS_ARCHIVE: + loginf('Detected temp sensors:%d' % num_sensors) - self._record.update(decode_rain(self, self._pkt_data[ 7:20])) - self._record.update(decode_wind(self, self._pkt_data[20:27])) - self._record.update(decode_uvi(self, self._pkt_data[27:28])) - self._record.update(decode_pressure(self, self._pkt_data[28:32])) - num_sensors = self._pkt_data[32] - if DEBUG_PACKETS_ARCHIVE: - loginf('Detected temp sensors:%d' % num_sensors) + for i in xrange(0, num_sensors+1): + base = 33 + i*7 + self._record.update(decode_temp(self, + self._pkt_data[base:base+7])) - for i in xrange(0, num_sensors): - base = 33 + i*7 - self._record.update(decode_temp(self, self._pkt_data[base:base+7])) + # Tell wmr200 console we have processed it and can handle more. + self.wmr200.request_archive_data() - # Tell wmr200 console we have processed it and can handle more. - self.wmr200.request_archive_data() + if DEBUG_PACKETS_ARCHIVE: + loginf(' Archive packet') - if DEBUG_PACKETS_ARCHIVE: - loginf(' Archive packet') + except IndexError: + msg = ('%s decode index failure' % self.pkt_name) + logerr(msg) + raise WMR200ProtocolError(msg) def decode_wind(pkt, pkt_data): @@ -660,25 +690,28 @@ def decode_wind(pkt, pkt_data): avg_speed = ((pkt_data[3] >> 4) | ((pkt_data[4] << 4))) / 10.0 # Windchill temperature. The value is in degrees F. - if pkt_data[5] != 0: - windchill = (pkt_data[5] - 32.0) * (5.0 / 9.0) + # Set default to no windchill as it may not exist. + # Convert to metric for weewx presentation. + windchill = None + if pkt_data[6] != 0x20: + if pkt_data[6] != 0x80: + windchill = (((pkt_data[6] << 8) | pkt_data[5]) - 320) \ + * (5.0 / 90.0) + elif pkt_data[6] & 0x80: + windchill = ((((pkt_data[5])*-1) -320) * (5.0/90.0)) - if pkt_data[5] != 0 or pkt_data[6] != 0x20: - windchill = (((pkt_data[6] << 8) | pkt_data[5]) - 320) \ - * (5.0 / 90.0) - else: - windchill = None - # The console returns wind speeds in m/s. Our metric system requires - # kph, so the result needs to be multiplied by 3.6. + # The console returns wind speeds in m/s. weewx requires + # kph, so the speeds needs to be converted. record = {'windSpeed' : avg_speed * 3.60, + 'windGust' : gust_speed * 3.60, 'windDir' : dir_deg, - 'usUnits' : weewx.METRIC, 'windchill' : windchill, } # Sometimes the station emits a wind gust that is less than the - # average wind. Ignore it if this is the case. - if gust_speed >= record['windSpeed']: - record['windGust'] = gust_speed * 3.60 + # average wind. weewx requires kph, so the result needs to be + # converted. + if gust_speed < avg_speed: + record['windGust'] = avg_speed * 3.60 if DEBUG_PACKETS_WIND: loginf(' Wind Dir: %s' % (WIND_DIR_MAP[pkt_data[0] & 0x0f])) @@ -688,9 +721,9 @@ def decode_wind(pkt, pkt_data): return record except IndexError: - str_val = ('%s decode index failure' % pkt.pkt_name()) - logerr(str_val) - raise WMR200ProtocolError(str_val) + msg = ('%s decode index failure' % pkt.pkt_name()) + logerr(msg) + raise WMR200ProtocolError(msg) class PacketWind(PacketLive): """Packet parser for wind.""" @@ -713,23 +746,21 @@ def decode_rain(pkt, pkt_data): """Decode the rain portion of a wmr200 packet.""" try: # Bytes 0 and 1: high and low byte of the current rainfall rate - # in 0.1 in/h - rain_rate = ((pkt_data[1] << 8) | pkt_data[0]) / 100.0 - # Bytes 2 and 3: high and low byte of the last hour rainfall in 0.1in - rain_hour = ((pkt_data[3] << 8) | pkt_data[2]) / 100.0 - # Bytes 4 and 5: high and low byte of the last day rainfall in 0.1in - rain_day = ((pkt_data[5] << 8) | pkt_data[4]) / 100.0 - # Bytes 6 and 7: high and low byte of the total rainfall in 0.1in - rain_total = ((pkt_data[7] << 8) | pkt_data[6]) / 100.0 - # NB: in my experiments with the WMR100, it registers in increments of - # 0.04 inches. Per Ejeklint's notes have you divide the packet values by - # 10, but this would result in an 0.4 inch bucket --- too big. So, I'm - # dividing by 100. + # in 0.01 in/h. Convert into metric. + rain_rate = (((pkt_data[1] & 0x0f) << 8) | pkt_data[0]) / 100.0 * 2.54 + # Bytes 2 and 3: high and low byte of the last hour rainfall in 0.01in + # Convert into metric. + rain_hour = ((pkt_data[3] << 8) | pkt_data[2]) / 100.0 * 2.54 + # Bytes 4 and 5: high and low byte of the last day rainfall in 0.01in + # Convert into metric. + rain_day = ((pkt_data[5] << 8) | pkt_data[4]) / 100.0 * 2.54 + # Bytes 6 and 7: high and low byte of the total rainfall in 0.01in + # Convert into metric. + rain_total = ((pkt_data[7] << 8) | pkt_data[6]) / 100.0 * 2.54 record = {'rainRate' : rain_rate, 'hourRain' : rain_hour, 'dayRain' : rain_day, - 'totalRain' : rain_total, - 'usUnits' : weewx.US} + 'totalRain' : rain_total} if DEBUG_PACKETS_RAIN: loginf(" Rain rate:%.02f hour_rain:%.02f day_rain:%.02f" % (rain_rate, rain_hour, rain_day)) @@ -737,9 +768,9 @@ def decode_rain(pkt, pkt_data): return record except IndexError: - str_val = ('%s decode index failure' % pkt.pkt_name()) - logerr(str_val) - raise WMR200ProtocolError(str_val) + msg = ('%s decode index failure' % pkt.pkt_name()) + logerr(msg) + raise WMR200ProtocolError(msg) class PacketRain(PacketLive): @@ -783,9 +814,9 @@ def decode_uvi(pkt, pkt_data): return record except IndexError: - str_val = ('%s index decode index failure' % pkt.pkt_name()) - logerr(str_val) - raise WMR200ProtocolError(str_val) + msg = ('%s index decode index failure' % pkt.pkt_name()) + logerr(msg) + raise WMR200ProtocolError(msg) class PacketUvi(PacketLive): @@ -840,9 +871,9 @@ def decode_pressure(pkt, pkt_data): return record except IndexError: - str_val = ('%s index decode index failure' % pkt.pkt_name()) - logerr(str_val) - raise WMR200ProtocolError(str_val) + msg = ('%s index decode index failure' % pkt.pkt_name()) + logerr(msg) + raise WMR200ProtocolError(msg) class PacketPressure(PacketLive): @@ -869,7 +900,13 @@ def decode_temp(pkt, pkt_data): # sensors. # Byte 0: low nibble contains sensor ID. 0 for base station. sensor_id = pkt_data[0] & 0x0f + # '00 Temp steady + # '01 Temp rising + # '10 Temp falling temp_trend = (pkt_data[0] >> 6) & 0x3 + # '00 Humidity steady + # '01 Humidity rising + # '10 Humidity falling hum_trend = (pkt_data[0] >> 4) & 0x3 # The high nible contains the sign indicator. @@ -926,9 +963,9 @@ def decode_temp(pkt, pkt_data): return record except IndexError: - str_val = ('%s index decode index failure' % pkt.pkt_name()) - logerr(str_val) - raise WMR200ProtocolError(str_val) + msg = ('%s index decode index failure' % pkt.pkt_name()) + logerr(msg) + raise WMR200ProtocolError(msg) class PacketTemperature(PacketLive): @@ -975,21 +1012,20 @@ class PacketStatus(PacketLive): 'rxCheckPercent' : 1.0 }) # This information is sent to syslog - if self.wmr200.sensor_stat: - if self._pkt_data[2] & 0x2: - logwar('Sensor 1 fault (temp/hum outdoor)') + if self._pkt_data[2] & 0x2 and self.wmr200.sensor_stat: + logwar('Sensor 1 fault (temp/hum outdoor)') - if self._pkt_data[2] & 0x1: - logwar('Wind sensor fault') + if self._pkt_data[2] & 0x1 and self.wmr200.sensor_stat: + logwar('Wind sensor fault') - if self._pkt_data[3] & 0x20: - logwar('UV Sensor fault') + if self._pkt_data[3] & 0x20 and self.wmr200.sensor_stat: + logwar('UV Sensor fault') - if self._pkt_data[3] & 0x10: - logwar('Rain sensor fault') + if self._pkt_data[3] & 0x10 and self.wmr200.sensor_stat: + logwar('Rain sensor fault') - if self._pkt_data[5] & 0x20: - logwar('UV sensor: Battery low') + if self._pkt_data[5] & 0x20 and self.wmr200.sensor_stat: + logwar('UV sensor: Battery low') # This information can be passed up to weewx. if self._pkt_data[4] & 0x02: @@ -1027,8 +1063,12 @@ class PacketFactory(object): self.subclass = dict((s.pkt_cmd, s) for s in subclass_list) self.skipped_bytes = 0 + def num_packets(self): + """Returns the number of packets handled by the factory.""" + return len(self.subclass) + def get_packet(self, pkt_cmd, wmr200): - """Returns an instance of packet parser indexed from packet command. + """Returns a protocol packet instance from initial packet command byte. Returns None if there was no mapping for the protocol command. @@ -1168,10 +1208,14 @@ class PollUsbDevice(threading.Thread): # If we have sent several resets with no data, # give up and abort. if read_reset_cnt == 2: - raise weewx.WeeWxIOError(('Device unresponsive after' - 'multiple resets')) - except WMR200ProtocolError: - logerr('USB overflow') + msg = ('Device unresponsive after multiple resets') + logerr(msg) + raise weewx.RetriesExceeded(msg) + + except: + logerr('USB device read error') + raise + loginf('USB device polling thread exiting') def _append_usb_device(self, buf): @@ -1213,11 +1257,11 @@ class PollUsbDevice(threading.Thread): time.sleep(1) except usb.USBError, exception: - logerr(('reset_console() Unable to send USB control' - 'message')) - logerr('**** %s' % exception) + msg = ('reset_console() Unable to send USB control' + 'message %s' % exception) + logerr(msg) # Convert to a Weewx error: - raise weewx.WakeupError(exception) + raise weewx.WeeWxIOError(exception) def notify(self): """Gates thread to read of the device. @@ -1237,17 +1281,15 @@ class WMR200(weewx.abstractstation.AbstractStation): sensor_status: Print sensor faults or failures to syslog. [Optional] use_pc_time: Use the console timestamp or the Pc. [Optional] erase_archive: Erasae archive upon startup. [Optional] + archive_interval: Time in seconds between intervals [Optional] + ignore_checksum: Ignore checksum failures and drop packet. --- User should not typically change anything below here --- vendor_id: The USB vendor ID for the WMR [Optional] - Default is 0xfde. product_id: The USB product ID for the WM [Optional] - Default is 0xca01. interface: The USB interface [Optional] - Default is 0] in_endpoint: The IN USB endpoint used by the WMR [Optional] - Default is usb.ENDPOINT_IN + 1] """ super(WMR200, self).__init__() @@ -1264,7 +1306,17 @@ class WMR200(weewx.abstractstation.AbstractStation): self._erase_archive = \ weeutil.weeutil.tobool(stn_dict.get('erase_archive', False)) - # User configurable options but not recommended + # Archive interval in seconds. + self._archive_interval = int(stn_dict.get('archive_interval', 60)) + if self._archive_interval not in [60, 300]: + logwar('Unverified archive interval:%d sec' + % self._archive_interval) + + # Ignore checksum errors. + self._ignore_checksum = \ + weeutil.weeutil.tobool(stn_dict.get('ignore_checksum', False)) + + # Device specific hardware options. vendor_id = int(stn_dict.get('vendor_id', '0x0fde'), 0) product_id = int(stn_dict.get('product_id', '0xca01'), 0) interface = int(stn_dict.get('interface', 0)) @@ -1279,10 +1331,10 @@ class WMR200(weewx.abstractstation.AbstractStation): self.pkt = None # Setup the generator to get a byte stream from the console. - self.genByte = self._generate_bytestream + self.gen_byte = self._generate_bytestream # Calculate time delta in seconds between host and console. - self.time_delta = None + self.time_drift = None # Create USB accessor to communiate with weather console device. self.usb_device = UsbDevice() @@ -1302,9 +1354,17 @@ class WMR200(weewx.abstractstation.AbstractStation): # data stream. self._rdy_to_poke = True - # Archived packets. + # Archived packet queue. self._pkt_archive = [] + # Interval records. An aggregation of all live packets + # for a given interval. + self._interval_record = {} + self._interval_record_list = [] + + # Count of times through generate archive records. + self._gen_archive_cnt = 0 + # Create the lock to sync between main thread and watchdog thread. self._poke_lock = threading.Lock() @@ -1374,7 +1434,12 @@ class WMR200(weewx.abstractstation.AbstractStation): loginf(' Log sensor faults: %s' % self._sensor_stat) loginf(' Using PC Time: %s' % self._use_pc_time) loginf(' Erase archive data: %s' % self._erase_archive) + loginf(' Archive interval: %d' % self._archive_interval) + @property + def hardware_name(self): + """weewx api.""" + return _WMR200_DRIVER_NAME @property def altitude(self): @@ -1391,6 +1456,16 @@ class WMR200(weewx.abstractstation.AbstractStation): """Flag to use pc time rather than weather console time.""" return self._use_pc_time + @property + def archive_interval(self): + """weewx api. Time in seconds between archive intervals.""" + return self._archive_interval + + @property + def ignore_checksum(self): + """Flag to drop rather than fail on checksum errors.""" + return self._ignore_checksum + def ready_to_poke(self, val): """Set info that device is ready to be poked.""" self._poke_lock.acquire() @@ -1414,11 +1489,11 @@ class WMR200(weewx.abstractstation.AbstractStation): try: self.usb_device.write_device(buf) except usb.USBError, exception: - msg = ('Write_cmd() Unable to send USB cmd:0x%02x control message' % - cmd) + msg = (('_write_cmd() Unable to send USB cmd:0x%02x control' + ' message' % cmd)) logerr(msg) # Convert to a Weewx error: - raise weewx.WakeupError(exception) + raise weewx.WeeWxIOError(exception) def _poke_console(self): """Send a heartbeat command to the weather console. @@ -1471,7 +1546,12 @@ class WMR200(weewx.abstractstation.AbstractStation): Otherwise add the byte to the current packet. Each USB packet may stradle a protocol packet so make sure we assign the data appropriately.""" - for byte in self.genByte(): + if not self._thread_usb_poll.is_alive(): + msg = 'USB polling thread unexpectedly terminated' + logerr(msg) + raise weewx.WeeWxIOError(msg) + + for byte in self.gen_byte(): if self.pkt: self.pkt.append_data(byte) else: @@ -1515,30 +1595,33 @@ class WMR200(weewx.abstractstation.AbstractStation): # not generated from this driver. e.g. weewx.service. if self.pkt is not None and self.pkt.packet_complete(): if DEBUG_PACKETS_RAW: - loginf(self.pkt.to_string_raw(' Packet Raw:')) + loginf(self.pkt.to_string_raw(' genLoop() Packet Raw:')) + + # This will raise exception if checksum fails. + self.pkt.verify_checksum() # Drop any bogus packets. if self.pkt.is_bogus: - logerr(self.pkt.to_string_raw('Discarding bogus packet:')) + logerr(self.pkt.to_string_raw('genLoop() Discarding' + ' bogus packet:')) else: - # This will raise exception if checksum fails. - self.pkt.verify_checksum() self.pkt.packet_process() if DEBUG_PACKETS_COOKED: self.pkt.print_cooked() if self.pkt.packet_live_data(): - logdbg('Presenting weewx live packet %d' % + logdbg('genLoop() Presenting weewx live packet cnt:%d' % PacketLive.pkt_cnt) + # Add this loop packet to our interval record. + self._interval_record.update(self.pkt.packet_record()) yield self.pkt.packet_record() elif self.pkt.packet_archive_data(): # Append to archive list for next time weewx engine # requests archived packets. self._pkt_archive.append(self.pkt) - logdbg(('Retrieved archive packet rx:%d cnt:%d' % - (PacketArchive.pkt_cnt, - len(self._pkt_archive)))) + logdbg(('genLoop() Buffering archive packet len:%d' + % len(self._pkt_archive))) else: - logdbg('Acknowledged control packet cnt:%d' % + logdbg('genLoop() Acknowledged control packet cnt:%d' % PacketControl.pkt_cnt) # Reset this packet to get ready for next one @@ -1552,45 +1635,75 @@ class WMR200(weewx.abstractstation.AbstractStation): # Pull data from the weather console. self._poll_for_data() - def hardware_name(self): - """weewx api.""" - return _WMR200_DRIVER_NAME - - def XXXgenArchiveRecords(self, since_ts): - ##def genArchiveRecords(self, since_ts): + def genArchiveRecords(self, since_ts): """A generator function to return archive packets from the wmr200. weewx api to return archive records. - since_ts: A timestamp. All data since (but not including) this time - will be returned. + since_ts: A timestamp in database time. All data since but not + including this time will be returned. Pass in None for all data - yields: a sequence of dictionaries containing the data - """ + yields: a sequence of dictionary records containing the console + data.""" if since_ts: - loginf('genArchiveRecords() Getting archive packets since %s' + logdbg('genArchive() Getting archive packets since %s' % weeutil.weeutil.timestamp_to_string(since_ts)) else : - loginf('genArchiveRecords() Getting all archive packets') + logdbg('genArchive() Getting all archive packets') + since_ts = 0 - cnt = 0 - loginf(('genArchiveRecords() archive packets:%d' % - len(self._pkt_archive))) - while len(self._pkt_archive): - pkt = self._pkt_archive.pop(0) - pkt.print_cooked() - cnt += 1 - if since_ts and pkt.packet_timestamp() > since_ts: - loginf(('genArchiveRecords() yielding archive record:%s' % - weeutil.weeutil.timestamp_to_string( - pkt.packet_timestamp()))) - yield pkt.packet_record() - else: - loginf(('genArchiveRecords() dropping archive records:%s' % - weeutil.weeutil.timestamp_to_string( - pkt.packet_timestamp()))) - loginf(('genArchiveRecords() Handled archive record cnt:%d' % - cnt)) + # Create the current archive record + if self._interval_record: + self._interval_record.update( + {'interval' : int(self.archive_interval / 60), }) + logdbg(('genArchive() Creating current interval record:%s' + % self._interval_record)) + # The timestamp is already either pc time or console time + # based upon console configuration. + self._interval_record_list.append(self._interval_record) + self._interval_record = {} + + if len(self._pkt_archive) and self._gen_archive_cnt: + # Present the archive packets received from the genLoop() + # processing. If PC time is set, we must have at least one + # live packet to calculate timestamps in PC time. + loginf(('genArchive() Still receiving archive packets len:%d' % + len(self._pkt_archive))) + + while self._pkt_archive: + pkt = self._pkt_archive.pop(0) + # If we are using PC time we need to adjust the record timestamp + # with the PC drift. + if self.use_pc_time: + logdbg(('genArchive() Using pc time so adjusting archive' + ' record time by %d' % self.time_drift)) + pkt.timestamp_adjust(self.time_drift) + # The archive packets have all been processed already. + if DEBUG_PACKETS_COOKED: + pkt.print_cooked() + if pkt.record_timestamp() > since_ts: + logdbg(('genArchive() Yielding received archive record' + ' after requested timestamp')) + yield pkt.packet_record() + else: + loginf(('genArchive() Ignoring received archive record' + ' before requested timestamp')) + + # We have no archived packets since last time though this callback. + # We have a count to ensure we've been here at least once before. + # Its safe to assume that the archived packets have been drained. Now + # we can push all our current-archive packets we've accumulated. + else: + while self._interval_record_list: + interval_record = self._interval_record_list.pop(0) + # Now yield the current archive records + logdbg(('genArchive() Yielding current interval record:%s' + % interval_record)) + yield interval_record + + # Number of times through this callback. This is done to allow time + # after startup for archive packets to come through if any are avail. + self._gen_archive_cnt += 1 def closePort(self): """Closes the USB port to the device. @@ -1603,7 +1716,7 @@ class WMR200(weewx.abstractstation.AbstractStation): self._poll_device_enable = False # Join with the polling thread. self._thread_usb_poll.join() - if self._thread_usb_poll.isAlive(): + if self._thread_usb_poll.is_alive(): logerr('USB polling thread still alive') else: loginf('USB polling thread expired') @@ -1612,7 +1725,7 @@ class WMR200(weewx.abstractstation.AbstractStation): self.sock_wr.send('shutdown') # Join with the watchdog thread. self._thread_watchdog.join() - if self._thread_watchdog.isAlive(): + if self._thread_watchdog.is_alive(): logerr('Watchdog thread still alive') else: loginf('Watchdog thread expired') diff --git a/bin/weewx/drivers/ws23xx.py b/bin/weewx/drivers/ws23xx.py index c994982c..c125d895 100644 --- a/bin/weewx/drivers/ws23xx.py +++ b/bin/weewx/drivers/ws23xx.py @@ -27,9 +27,6 @@ # # This immplementation copies directly from Russell Stuart's implementation, # but only the parts required to read from and write to the weather station. -# -# The wview implementation copies from the Open2300 implementation. It reads -# each sensor multiple times to avoid spikes in data. """Classes and functions for interfacing with WS-23xx weather stations. @@ -37,7 +34,7 @@ LaCrosse made a number of stations in the 23xx series, including: WS-2300, WS-2308, WS-2310, WS-2315, WS-2317, WS-2357 -The stations were also sold as the TFA Dostman and TechnoLine 2350. +The stations were also sold as the TFA Matrix and TechnoLine 2350. The WWVB receiver is located in the console. @@ -49,13 +46,15 @@ To do a factory reset, press and hold PRESSURE and WIND for 5 seconds. A single bucket tip is 0.0204 in (0.518 mm). The station has 175 history records. That is just over 7 days of data with -the default history recording interval of 60 minutes (59 in the console). +the default history recording interval of 60 minutes. The station supports both wireless and wired communication between the sensors and a station console. Wired connection updates data every 8 seconds. Wireless connection updates data in 16 to 128 second intervals, depending on wind speed and rain activity. +The connection type can be one of 0=cable, 3=lost, 15=wireless + sensor update frequency: 32 seconds when wind speed > 22.36 mph (wireless) @@ -70,36 +69,174 @@ console update frequency: It is possible to increase the rate of wireless updates: - http://www.wikihow.com/Modify-a-Lacrosse-Ws2300-for-Frequent-Wireless-Updates + http://www.wxforum.net/index.php?topic=2196.0 -This implementation polls the station. Use the polling_interval parameter -to specify how often to poll for data. If not specified, the polling interval -will adapt based on connection type and status. +Sensors are connected by unshielded phone cables. RF interference can cause +random spikes in data, with one symptom being values of 25.5 m/s or 91.8 km/h +for the wind speed. To reduce the number of spikes in data, replace with +shielded cables: -Instruments are connected by unshielded phone cables. To reduce the number of -spikes in data, replace with shielded cables. + http://www.lavrsen.dk/sources/weather/windmod.htm -The connection type can be one of 0=cable, 3=lost, f=wireless +The station records wind speed and direction, but has no notion of gust. + +The station calculates windchill and dewpoint. The station has a serial connection to the computer. This driver does not keep the serial port open for long periods. Instead, the driver opens the serial port, reads data, then closes the port. +This driver polls the station. Use the polling_interval parameter to specify +how often to poll for data. If not specified, the polling interval will adapt +based on connection type and status. + USB-Serial Converters With a USB-serial converter one can connect the station to a computer with only USB ports, but not every converter will work properly. Perhaps the two -most common converters are based on the Prolific and LTDI chipsets. Many -people report better luck with the LTDI-based converters. Some converters +most common converters are based on the Prolific and FTDI chipsets. Many +people report better luck with the FTDI-based converters. Some converters that use the Prolific chipset (PL2303) will work, but not all of them. Known to work: ATEN UC-232A +Discrepancies Between Implementations + +As of December 2013, there are significant differences between the open2300, +wview, and ws2300 implementations. Current version numbers are as follows: + + open2300 1.11 + ws2300 1.8 + wview 5.20.2 + +History Interval + +The factory default is 60 minutes. The value stored in the console is one +less than the actual value (in minutes). So for the factory default of 60, +the console stores 59. The minimum interval is 1. + +ws2300.py reports the actual value from the console, e.g., 59 when the +interval is 60. open2300 reports the interval, e.g., 60 when the interval +is 60. wview ignores the interval. + +Detecting Bogus Sensor Values + +wview queries the station 3 times for each sensor then accepts the value only +if the three values were close to each other. + +open2300 sleeps 10 seconds if a wind measurement indicates invalid or overflow. + +The ws2300.py implementation includes overflow and validity flags for values +from the wind sensors. It does not retry based on invalid or overflow. + +Wind Speed + +There is disagreement about how to calculate wind speed and how to determine +whether the wind speed is valid. + +This driver introduces a WindConversion object that uses open2300/wview +decoding so that wind speeds match that of open2300/wview. ws2300 1.8 +incorrectly uses bcd2num instead of bin2num. This bug is fixed in this driver. + +The memory map indicates the following: + +addr smpl description +0x527 0 Wind overflow flag: 0 = normal +0x528 0 Wind minimum code: 0=min, 1=--.-, 2=OFL +0x529 0 Windspeed: binary nibble 0 [m/s * 10] +0x52A 0 Windspeed: binary nibble 1 [m/s * 10] +0x52B 0 Windspeed: binary nibble 2 [m/s * 10] +0x52C 8 Wind Direction = nibble * 22.5 degrees +0x52D 8 Wind Direction 1 measurement ago +0x52E 9 Wind Direction 2 measurement ago +0x52F 8 Wind Direction 3 measurement ago +0x530 7 Wind Direction 4 measurement ago +0x531 7 Wind Direction 5 measurement ago +0x532 0 + +wview 5.20.2 implementation (wview apparently copied from open2300): + +read 3 bytes starting at 0x527 + +0x527 x[0] +0x528 x[1] +0x529 x[2] + +if ((x[0] != 0x00) || + ((x[1] == 0xff) && (((x[2] & 0xf) == 0) || ((x[2] & 0xf) == 1)))) { + fail +} else { + dir = (x[2] >> 4) * 22.5 + speed = ((((x[2] & 0xf) << 8) + (x[1])) / 10.0 * 2.23693629) + maxdir = dir + maxspeed = speed +} + +open2300 1.10 implementation: + +read 6 bytes starting at 0x527 + +0x527 x[0] +0x528 x[1] +0x529 x[2] +0x52a x[3] +0x52b x[4] +0x52c x[5] + +if ((x[0] != 0x00) || + ((x[1] == 0xff) && (((x[2] & 0xf) == 0) || ((x[2] & 0xf) == 1)))) { + sleep 10 +} else { + dir = x[2] >> 4 + speed = ((((x[2] & 0xf) << 8) + (x[1])) / 10.0) + dir0 = (x[2] >> 4) * 22.5 + dir1 = (x[3] & 0xf) * 22.5 + dir2 = (x[3] >> 4) * 22.5 + dir3 = (x[4] & 0xf) * 22.5 + dir4 = (x[4] >> 4) * 22.5 + dir5 = (x[5] & 0xf) * 22.5 +} + +ws2300.py 1.8 implementation: + +read 1 nibble starting at 0x527 +read 1 nibble starting at 0x528 +read 4 nibble starting at 0x529 +read 3 nibble starting at 0x529 +read 1 nibble starting at 0x52c +read 1 nibble starting at 0x52d +read 1 nibble starting at 0x52e +read 1 nibble starting at 0x52f +read 1 nibble starting at 0x530 +read 1 nibble starting at 0x531 + +0x527 overflow +0x528 validity +0x529 speed[0] +0x52a speed[1] +0x52b speed[2] +0x52c dir[0] + +speed: ((x[2] * 100 + x[1] * 10 + x[0]) % 1000) / 10 +velocity: (x[2] * 100 + x[1] * 10 + x[0]) / 10 + +dir = data[0] * 22.5 +speed = (bcd2num(data) % 10**3 + 0) / 10**1 +velocity = (bcd2num(data[:3])/10.0, bin2num(data[3:4]) * 22.5) + +bcd2num([a,b,c]) -> c*100+b*10+a + """ # TODO: use pyserial instead of LinuxSerialPort +# TODO: put the __enter__ and __exit__ scaffolding on serial port, not Station # FIXME: unless we can get setTime to work, just ignore the console clock +# FIXME: detect bogus wind speed/direction +# i see these when the wind instrument is disconnected: +# ws 26.399999 +# wsh 21 +# w0 135 import optparse import syslog @@ -116,7 +253,7 @@ import weeutil.weeutil import weewx.abstractstation import weewx.wxformulas -DRIVER_VERSION = '0.6' +DRIVER_VERSION = '0.18' DEFAULT_PORT = '/dev/ttyUSB0' def logmsg(dst, msg): @@ -136,7 +273,8 @@ def logerr(msg): def loader(config_dict, engine): altitude_m = getaltitudeM(config_dict) - station = WS23xx(altitude=altitude_m, **config_dict['WS23xx']) + station = WS23xx(altitude=altitude_m, config_dict=config_dict, + **config_dict['WS23xx']) return station # FIXME: the pressure calculations belong in wxformulas @@ -212,6 +350,37 @@ def calculate_rain(newtotal, oldtotal): delta = None return delta +# FIXME: this goes in weeutil.weeutil +def calculate_rain_rate(delta, curr_ts, last_ts): + """Calculate the rain rate based on the time between two rain readings. + + delta: rainfall since last reading, in units of x + + curr_ts: timestamp of current reading, in seconds + + last_ts: timestamp of last reading, in seconds + + return: rain rate in x per hour + + If the period between readings is zero, ignore the rainfall since there + is no way to calculate a rate with no period.""" + + if curr_ts is None: + return None + if last_ts is None: + last_ts = curr_ts + if delta is not None: + period = curr_ts - last_ts + if period != 0: + rate = 3600 * delta / period + else: + rate = None + if delta != 0: + loginf('rain rate period is zero, ignoring rainfall of %f' % delta) + else: + rate = None + return rate + class WS23xx(weewx.abstractstation.AbstractStation): """Driver for LaCrosse WS23xx stations.""" @@ -245,26 +414,42 @@ class WS23xx(weewx.abstractstation.AbstractStation): """ self._last_rain = None self._last_cn = None + self._poll_wait = 60 self.altitude = stn_dict['altitude'] - self.port = stn_dict.get('port', DEFAULT_PORT) - self.polling_interval = stn_dict.get('polling_interval', None) self.model = stn_dict.get('model', 'LaCrosse WS23xx') - self.calc_windchill = stn_dict.get('calculate_windchill', False) - self.calc_dewpoint = stn_dict.get('calculate_dewpoint', False) + self.port = stn_dict.get('port', DEFAULT_PORT) + self.max_tries = int(stn_dict.get('max_tries', 5)) + self.retry_wait = int(stn_dict.get('retry_wait', 30)) + self.polling_interval = stn_dict.get('polling_interval', None) + if self.polling_interval is not None: + self.polling_interval = int(self.polling_interval) + self.calc_windchill = weeutil.weeutil.tobool(stn_dict.get('calculate_windchill', False)) + self.calc_dewpoint = weeutil.weeutil.tobool(stn_dict.get('calculate_dewpoint', False)) self.pressure_offset = stn_dict.get('pressure_offset', None) if self.pressure_offset is not None: self.pressure_offset = float(self.pressure_offset) - self.max_tries = int(stn_dict.get('max_tries', 5)) - self.retry_wait = int(stn_dict.get('retry_wait', 30)) + self.disable_catchup = weeutil.weeutil.tobool(stn_dict.get('disable_catchup', False)) - loginf('serial port is %s' % str(self.port)) - loginf('pressure offset is %s' % str(self.pressure_offset)) + loginf('driver version is %s' % DRIVER_VERSION) + loginf('serial port is %s' % self.port) + loginf('pressure offset is %s' % self.pressure_offset) + loginf('polling interval is %s' % self.polling_interval) loginf('windchill will be %s' % ('calculated' if self.calc_windchill else 'read from station')) loginf('dewpoint will be %s' % ('calculated' if self.calc_dewpoint else 'read from station')) + # FIXME: this is a hack until we modify the driver api to have an + # explicit genCatchupRecords method + self.force_recgen = weeutil.weeutil.tobool(stn_dict.get('force_software_record_generation', True)) + if self.force_recgen: + config_dict = stn_dict['config_dict'] + recgen = config_dict['StdArchive']['record_generation'] + if recgen.lower() != 'software': + loginf("forcing record_generation to 'software'") + config_dict['StdArchive']['record_generation'] = 'software' + @property def hardware_name(self): return self.model @@ -279,14 +464,11 @@ class WS23xx(weewx.abstractstation.AbstractStation): def genLoopPackets(self): ntries = 0 - wait = 60 while ntries < self.max_tries: ntries += 1 - serial_port = None try: - serial_port = LinuxSerialPort(self.port) - ws = Ws2300(serial_port) - data = get_raw_data(ws, SENSOR_IDS) + s = Station(self.port) + data = s.get_raw_data(SENSOR_IDS) packet = data_to_packet(data, int(time.time() + 0.5), altitude=self.altitude, pressure_offset=self.pressure_offset, @@ -296,266 +478,89 @@ class WS23xx(weewx.abstractstation.AbstractStation): self._last_rain = packet['rainTotal'] ntries = 0 yield packet - wait = self.get_wait(wait, data['cn']) - time.sleep(wait) - except Exception, e: + + if self.polling_interval is not None: + self._poll_wait = self.polling_interval + if data['cn'] != self._last_cn: + conn_info = get_conn_info(data['cn']) + loginf("connection changed from %s to %s" % + (get_conn_info(self._last_cn)[0], conn_info[0])) + self._last_cn = data['cn'] + if self.polling_interval is None: + loginf("using %s second polling interval" + " for %s connection" % + (conn_info[1], conn_info[0])) + self._poll_wait = conn_info[1] + time.sleep(self._poll_wait) + except Ws2300.Ws2300Exception, e: logerr("Failed attempt %d of %d to get LOOP data: %s" % (ntries, self.max_tries, e)) logdbg("Waiting %d seconds before retry" % self.retry_wait) time.sleep(self.retry_wait) finally: - if serial_port is not None: - serial_port.close() - serial_port = None + s.close() else: msg = "Max retries (%d) exceeded for LOOP data" % self.max_tries logerr(msg) raise weewx.RetriesExceeded(msg) def genArchiveRecords(self, since_ts, count=0): - serial_port = None - try: + if self.disable_catchup: + raise NotImplementedError + with Station(self.port) as s: last_rain = None - serial_port = LinuxSerialPort(self.port) - ws = Ws2300(serial_port) - interval = get_archive_interval(ws) - for ts,data in gen_records(ws, since_ts=since_ts, count=count): + for ts,data in s.gen_records(since_ts=since_ts, count=count): record = data_to_packet(data, ts, altitude=self.altitude, pressure_offset=self.pressure_offset, last_rain=last_rain, calc_dewpoint=True, calc_windchill=True) - record['interval'] = interval + record['interval'] = data['interval'] last_rain = record['rainTotal'] yield record - finally: - if serial_port is not None: - serial_port.close() - serial_port = None - -# FIXME: do not use station time until we can set it # def getTime(self) : -# serial_port = None -# try: -# serial_port = LinuxSerialPort(self.port) -# ws = Ws2300(serial_port) -# return get_time(ws) -# finally: -# if serial_port is not None: -# serial_port.close() -# serial_port = None +# with Station(self.port) as s: +# return s.get_time() # def setTime(self, ts): -# serial_port = None -# try: -# serial_port = LinuxSerialPort(self.port) -# ws = Ws2300(serial_port) -# set_time(ws, ts) -# finally: -# if serial_port is not None: -# serial_port.close() -# serial_port = None +# with Station(self.port) as s: +# s.set_time(ts) def getArchiveInterval(self): - serial_port = None - try: - serial_port = LinuxSerialPort(self.port) - ws = Ws2300(serial_port) - return get_archive_interval(ws) - finally: - if serial_port is not None: - serial_port.close() - serial_port = None + with Station(self.port) as s: + return s.get_archive_interval() def setArchiveInterval(self, interval): - serial_port = None - try: - serial_port = LinuxSerialPort(self.port) - ws = Ws2300(serial_port) - set_archive_interval(ws, interval) - finally: - if serial_port is not None: - serial_port.close() - serial_port = None + with Station(self.port) as s: + s.set_archive_interval(interval) def getConfig(self): - serial_port = None - try: - serial_port = LinuxSerialPort(self.port) - ws = Ws2300(serial_port) - data = get_raw_data(ws, Measure.IDS.keys()) + with Station(self.port) as s: + data = s.get_raw_data(Measure.IDS.keys()) fdata = {} for key in data: fdata[Measure.IDS[key].name] = data[key] return fdata - finally: - if serial_port is not None: - serial_port.close() - serial_port = None def getRecordCount(self): - serial_port = None - try: - serial_port = LinuxSerialPort(self.port) - ws = Ws2300(serial_port) - return get_record_count(ws) - finally: - if serial_port is not None: - serial_port.close() - serial_port = None + with Station(self.port) as s: + return s.get_record_count() - def get_wait(self, wait, conn): - if self.polling_interval is not None: - wait = self.polling_interval - if conn != self._last_cn: - loginf("connection changed from %s to %s" % - (get_conn_info(self._last_cn)[0], get_conn_info(conn)[0])) - if self.polling_interval is None: - conn_info = get_conn_info(conn) - loginf("using %s second polling interval for %s connection" % - (conn_info[1], conn_info[0])) - wait = conn_info[1] - self._last_cn = conn - return wait + def clearHistory(self): + with Station(self.port) as s: + s.clear_memory() # ids for current weather conditions and connection type -SENSOR_IDS = [ - 'it','ih','ot','oh','pa', 'ws','wsh','w0','rh','rt','dp','wc','cn' ] +SENSOR_IDS = [ 'it','ih','ot','oh','pa','wind','rh','rt','dp','wc','cn' ] # polling interval, in seconds, for various connection types POLLING_INTERVAL = { 0:("cable",8), 3:("lost",60), 15:("wireless",30) } def get_conn_info(conn_type): return POLLING_INTERVAL.get(conn_type, ("unknown",60)) -def set_time(ws, ts): - """Set station time to indicated unix epoch.""" - logdbg('setting station clock to %s' % - weeutil.weeutil.timestamp_to_string(ts)) - for m in [Measure.IDS['sd'], Measure.IDS['st']]: - data = m.conv.value2binary(ts) - cmd = m.conv.write(data, None) - ws.write_safe(m.address, *cmd[1:]) - -def get_time(ws): - """Return station time as unix epoch.""" - data = get_raw_data(ws, ['sw']) - ts = int(data['sw']) - logdbg('station clock is %s' % weeutil.weeutil.timestamp_to_string(ts)) - return ts - -def set_archive_interval(ws, interval): - """Set the archive interval in minutes.""" - logdbg('setting hardware archive interval to %s minutes' % interval) - for m,v in [(Measure.IDS['hi'],interval), # archive interval - (Measure.IDS['hc'],1), # time till next sample - (Measure.IDS['hn'],0)]: # number of valid records - data = m.conv.value2binary(v) - cmd = m.conv.write(data, None) - ws.write_safe(m.address, *cmd[1:]) - -def get_archive_interval(ws): - """Return archive interval in minutes.""" - data = get_raw_data(ws, ['hi']) - x = int(data['hi']) - logdbg('station archive interval is %s minutes' % x) - return x - -def clear_memory(ws): - """Clear station memory.""" - logdbg('clearing console memory') - for m,v in [(Measure.IDS['hn'],0)]: # number of valid records - data = m.conv.value2binary(v) - cmd = m.conv.write(data, None) - ws.write_safe(m.address, *cmd[1:]) - -def get_record_count(ws): - data = get_raw_data(ws, ['hn']) - x = int(data['hn']) - logdbg('record count is %s' % x) - return x - -def gen_records(ws, since_ts=None, count=None, use_computer_clock=True): - """Get latest count records from the station from oldest to newest. If - count is 0 or None, return all records. - - The station has a history interval, and it records when the last history - sample was saved. So as long as the interval does not change between the - first and last records, we are safe to infer timestamps for each record. - This assumes that if the station loses power then the memory will be - cleared. - - There is no timestamp associated with each record - we have to guess. - The station tells us the time until the next record and the epoch of the - latest record, based on the station's clock. So we can use that or use - the computer clock to guess the timestamp for each record.""" - - # FIXME: this is not atomic - if we overlap an interval, data are bogus - - measures = [ Measure.IDS['hi'], Measure.IDS['hw'], - Measure.IDS['hc'], Measure.IDS['hn'] ] - raw_data = read_measurements(ws, measures) - interval = 1 + int(measures[0].conv.binary2value(raw_data[0])) # minutes - latest_ts = int(measures[1].conv.binary2value(raw_data[1])) # epoch - time_to_next = int(measures[2].conv.binary2value(raw_data[2])) # minutes - numrec = int(measures[3].conv.binary2value(raw_data[3])) - - now = int(time.time()) - cstr = 'station' - if use_computer_clock: - latest_ts = now - (interval - time_to_next) * 60 - cstr = 'computer' - logdbg("using %s clock with latest_ts of %s" % - (cstr, weeutil.weeutil.timestamp_to_string(latest_ts))) - - if since_ts is not None: - count = int((now - since_ts) / (interval * 60)) - logdbg("count is %d to satisfy timestamp of %s" % - (count, weeutil.weeutil.timestamp_to_string(since_ts))) - if count == 0: - return - - if count and count > numrec: - count = numrec - if count and count > HistoryMeasure.MAX_HISTORY_RECORDS: - count = HistoryMeasure.MAX_HISTORY_RECORDS - - HistoryMeasure.set_constants(ws) - recno_s = 0 - recno_e = count if count else HistoryMeasure.MAX_HISTORY_RECORDS - last_ts = latest_ts - (recno_e-1) * interval * 60 - logdbg("downloading %d records from station" % recno_e) - - measures = [HistoryMeasure(n) for n in range(recno_e-1, recno_s-1, -1)] - raw_data = read_measurements(ws, measures) - for measure, nybbles in zip(measures, raw_data): - value = measure.conv.binary2value(nybbles) - data_dict = { - 'it': value.temp_indoor, - 'ih': value.humidity_indoor, - 'ot': value.temp_outdoor, - 'oh': value.humidity_outdoor, - 'pa': value.pressure_absolute, - 'rt': value.rain, - 'ws': value.wind_speed, - 'w0': value.wind_direction, - 'wsh': None, # no gust in history - 'rh': None, # no rain rate in history - 'dp': None, # no dewpoint in history - 'wc': None, # no windchill in history - } - yield last_ts, data_dict - last_ts += interval * 60 - -def get_raw_data(ws, labels): - """Get raw data from the station, return as dictionary.""" - measures = [ Measure.IDS[m] for m in labels ] - raw_data = read_measurements(ws, measures) - data_dict = dict(zip(labels, [ m.conv.binary2value(d) for m, d in zip(measures, raw_data) ])) - return data_dict - def data_to_packet(data, ts, altitude=0, pressure_offset=None, last_rain=None, calc_dewpoint=False, calc_windchill=False): """Convert raw data to format and units required by weewx. @@ -566,8 +571,9 @@ def data_to_packet(data, ts, altitude=0, pressure_offset=None, last_rain=None, uv index unitless unitless pressure mbar mbar wind speed m/s km/h - wind gust m/s km/h wind dir degree degree + wind gust None + wind gust dir None rain mm cm rain rate cm/h """ @@ -581,15 +587,20 @@ def data_to_packet(data, ts, altitude=0, pressure_offset=None, last_rain=None, packet['outHumidity'] = data['oh'] packet['pressure'] = data['pa'] - packet['windSpeed'] = data['ws'] - if packet['windSpeed'] is not None: - packet['windSpeed'] *= 3.6 # weewx wants km/h - packet['windDir'] = data['w0'] if packet['windSpeed'] else None + ws,wd,wso,wsv = data['wind'] + if wso == 0 and wsv == 0: + packet['windSpeed'] = ws + if packet['windSpeed'] is not None: + packet['windSpeed'] *= 3.6 # weewx wants km/h + packet['windDir'] = wd if packet['windSpeed'] else None + else: + loginf('invalid wind reading: speed=%s dir=%s overflow=%s invalid=%s' % + (ws,wd,wso,wsv)) + packet['windSpeed'] = None + packet['windDir'] = None - packet['windGust'] = data['wsh'] - if packet['windGust'] is not None: - packet['windGust'] *= 3.6 # weewx wants km/h - packet['windGustDir'] = data['w0'] if packet['windGust'] else None + packet['windGust'] = None + packet['windGustDir'] = None packet['rainTotal'] = data['rt'] if packet['rainTotal'] is not None: @@ -629,9 +640,173 @@ def data_to_packet(data, ts, altitude=0, pressure_offset=None, last_rain=None, return packet -#============================================================================== +class Station(object): + """Wrap the Ws2300 object so we can easily open serial port, read/write, + close serial port without all of the try/except/finally scaffolding.""" + + def __init__(self, port): + logdbg('create LinuxSerialPort') + self.serial_port = LinuxSerialPort(port) + logdbg('create Ws2300') + self.ws = Ws2300(self.serial_port) + + def __enter__(self): + logdbg('station enter') + return self + + def __exit__(self, type, value, traceback): + logdbg('station exit') + self.ws = None + self.close() + + def close(self): + logdbg('close LinuxSerialPort') + self.serial_port.close() + self.serial_port = None + + def set_time(self, ts): + """Set station time to indicated unix epoch.""" + logdbg('setting station clock to %s' % + weeutil.weeutil.timestamp_to_string(ts)) + for m in [Measure.IDS['sd'], Measure.IDS['st']]: + data = m.conv.value2binary(ts) + cmd = m.conv.write(data, None) + self.ws.write_safe(m.address, *cmd[1:]) + + def get_time(self): + """Return station time as unix epoch.""" + data = self.get_raw_data(['sw']) + ts = int(data['sw']) + logdbg('station clock is %s' % weeutil.weeutil.timestamp_to_string(ts)) + return ts + + def set_archive_interval(self, interval): + """Set the archive interval in minutes.""" + if int(interval) < 1: + raise ValueError, 'archive interval must be greater than zero' + logdbg('setting hardware archive interval to %s minutes' % interval) + interval -= 1 + for m,v in [(Measure.IDS['hi'],interval), # archive interval in minutes + (Measure.IDS['hc'],1), # time till next sample in minutes + (Measure.IDS['hn'],0)]: # number of valid records + data = m.conv.value2binary(v) + cmd = m.conv.write(data, None) + self.ws.write_safe(m.address, *cmd[1:]) + + def get_archive_interval(self): + """Return archive interval in minutes.""" + data = self.get_raw_data(['hi']) + x = 1 + int(data['hi']) + logdbg('station archive interval is %s minutes' % x) + return x + + def clear_memory(self): + """Clear station memory.""" + logdbg('clearing console memory') + for m,v in [(Measure.IDS['hn'],0)]: # number of valid records + data = m.conv.value2binary(v) + cmd = m.conv.write(data, None) + self.ws.write_safe(m.address, *cmd[1:]) + + def get_record_count(self): + data = self.get_raw_data(['hn']) + x = int(data['hn']) + logdbg('record count is %s' % x) + return x + + def gen_records(self, since_ts=None, count=None, use_computer_clock=True): + """Get latest count records from the station from oldest to newest. If + count is 0 or None, return all records. + + The station has a history interval, and it records when the last + history sample was saved. So as long as the interval does not change + between the first and last records, we are safe to infer timestamps + for each record. This assumes that if the station loses power then + the memory will be cleared. + + There is no timestamp associated with each record - we have to guess. + The station tells us the time until the next record and the epoch of + the latest record, based on the station's clock. So we can use that + or use the computer clock to guess the timestamp for each record. + + To ensure accurate data, the first record must be read within one + minute of the initial read and the remaining records must be read + within numrec * interval minutes. + """ + + logdbg("gen_records: since_ts=%s count=%s clock=%s" % + (since_ts, count, use_computer_clock)) + measures = [ Measure.IDS['hi'], Measure.IDS['hw'], + Measure.IDS['hc'], Measure.IDS['hn'] ] + raw_data = read_measurements(self.ws, measures) + interval = 1+int(measures[0].conv.binary2value(raw_data[0])) # minute + latest_ts = int(measures[1].conv.binary2value(raw_data[1])) # epoch + time_to_next = int(measures[2].conv.binary2value(raw_data[2])) # minute + numrec = int(measures[3].conv.binary2value(raw_data[3])) + + now = int(time.time()) + cstr = 'station' + if use_computer_clock: + latest_ts = now - (interval - time_to_next) * 60 + cstr = 'computer' + logdbg("using %s clock with latest_ts of %s" % + (cstr, weeutil.weeutil.timestamp_to_string(latest_ts))) + + if not count: + count = HistoryMeasure.MAX_HISTORY_RECORDS + if since_ts is not None: + count = int((now - since_ts) / (interval * 60)) + logdbg("count is %d to satisfy timestamp of %s" % + (count, weeutil.weeutil.timestamp_to_string(since_ts))) + if count == 0: + return + if count > numrec: + count = numrec + if count > HistoryMeasure.MAX_HISTORY_RECORDS: + count = HistoryMeasure.MAX_HISTORY_RECORDS + + # station is about to overwrite first record, so skip it + if time_to_next <= 1 and count == HistoryMeasure.MAX_HISTORY_RECORDS: + count -= 1 + + logdbg("downloading %d records from station" % count) + HistoryMeasure.set_constants(self.ws) + measures = [HistoryMeasure(n) for n in range(count-1, -1, -1)] + raw_data = read_measurements(self.ws, measures) + last_ts = latest_ts - (count-1) * interval * 60 + last_rain = None + for measure, nybbles in zip(measures, raw_data): + value = measure.conv.binary2value(nybbles) + delta = calculate_rain(value.rain, last_rain) + rainrate = calculate_rain_rate(delta, last_ts, last_ts-interval*60) + data_dict = { + 'interval': interval, + 'it': value.temp_indoor, + 'ih': value.humidity_indoor, + 'ot': value.temp_outdoor, + 'oh': value.humidity_outdoor, + 'pa': value.pressure_absolute, + 'rt': value.rain, + 'wind': (value.wind_speed, value.wind_direction, 0, 0), + 'rh': rainrate, + 'dp': None, # no dewpoint in history + 'wc': None, # no windchill in history + } + yield last_ts, data_dict + last_ts += interval * 60 + last_rain = value.rain + + def get_raw_data(self, labels): + """Get raw data from the station, return as dictionary.""" + measures = [ Measure.IDS[m] for m in labels ] + raw_data = read_measurements(self.ws, measures) + data_dict = dict(zip(labels, [ m.conv.binary2value(d) for m, d in zip(measures, raw_data) ])) + return data_dict + + +# ============================================================================= # The following code was adapted from ws2300.py by Russell Stuart -#============================================================================== +# ============================================================================= VERSION = "1.8 2013-08-26" @@ -1344,14 +1519,32 @@ class WindVelocityConversion(Conversion): def __init__(self): Conversion.__init__(self, "ms,d", 4, "wind speed and direction") def binary2value(self, data): - return (bcd2num(data[:3])/10.0, bin2num(data[3:4]) * 22.5) + return (bin2num(data[:3])/10.0, bin2num(data[3:4]) * 22.5) def value2binary(self, value): - return num2bcd(value[0]*10, 3) + num2bin((value[1] + 11.5) / 22.5, 1) + return num2bin(value[0]*10, 3) + num2bin((value[1] + 11.5) / 22.5, 1) def str(self, value): return "%.1f,%g" % value def parse(self, str): return tuple([float(x) for x in str.split(",")]) +# The ws2300 1.8 implementation does not calculate wind speed correctly - +# it uses bcd2num instead of bin2num. This conversion object uses bin2num +# decoding and it reads all wind data in a single transcation so that we do +# not suffer coherency problems. +class WindConversion(Conversion): + def __init__(self): + Conversion.__init__(self, "ms,d,o,v", 12, "wind speed, dir, validity") + def binary2value(self, data): + overflow = data[0] + validity = data[1] + speed = bin2num(data[2:5]) / 10.0 + direction = data[5] * 22.5 + return (speed, direction, overflow, validity) + def str(self, value): + return "%.1f,%g,%s,%s" % value + def parse(self, str): + return tuple([float(x) for x in str.split(",")]) + # # For non-numerical values. # @@ -1547,7 +1740,8 @@ conv_rain = BcdConversion("mm", 6, 2, "rain") conv_temp = BcdConversion("C", 4, 2, "temperature", -3000) conv_per2 = BinConversion("s", 2, 1, "time interval", 5) conv_per3 = BinConversion("min", 3, 0, "time interval") -conv_wspd = BcdConversion("m/s", 3, 1, "speed") +conv_wspd = BinConversion("m/s", 3, 1, "speed") +conv_wind = WindConversion() # # Define a measurement on the Ws2300. This encapsulates: @@ -1815,6 +2009,8 @@ Measure(0x6b5, "hc", conv_per3, "history time till sample") Measure(0x6b8, "hw", conv_stmp, "history last sample when") Measure(0x6c2, "hp", conv_rec2, "history last record pointer",reset=0) Measure(0x6c4, "hn", conv_rec2, "history number of records", reset=0) +# get all of the wind info in a single invocation +Measure(0x527, "wind", conv_wind, "wind") # # Read the requests. @@ -1894,25 +2090,19 @@ def main(): if options.port: port = options.port - serial_port = None - try: - serial_port = LinuxSerialPort(port) - ws = Ws2300(serial_port) + with Station(port) as s: if options.readings: - data = get_raw_data(ws, SENSOR_IDS) + data = s.get_raw_data(SENSOR_IDS) print data if options.records is not None: - for ts,record in gen_records(ws, count=options.records): + for ts,record in s.gen_records(count=options.records): print ts,record if options.measure: - data = get_raw_data(ws, [options.measure]) + data = s.get_raw_data([options.measure]) print data if options.hm: for m in Measure.IDS: print "%s\t%s" % (m, Measure.IDS[m].name) - finally: - if serial_port is not None: - serial_port.close() if __name__ == '__main__': main() diff --git a/bin/weewx/imagegenerator.py b/bin/weewx/imagegenerator.py index eb518454..1ccc420d 100644 --- a/bin/weewx/imagegenerator.py +++ b/bin/weewx/imagegenerator.py @@ -40,7 +40,7 @@ class ImageGenerator(weewx.reportengine.CachedReportGenerator): def setup(self): self.image_dict = self.skin_dict['ImageGenerator'] - self.title_dict = self.skin_dict['Labels']['Generic'] + self.title_dict = self.skin_dict.get('Labels', {}).get('Generic', {}) self.converter = weewx.units.Converter.fromSkinDict(self.skin_dict) self.formatter = weewx.units.Formatter.fromSkinDict(self.skin_dict) self.unit_helper= weewx.units.UnitInfoHelper(self.formatter, self.converter) diff --git a/bin/weewx/station.py b/bin/weewx/station.py index 8b3a4a36..1f3aa6c0 100644 --- a/bin/weewx/station.py +++ b/bin/weewx/station.py @@ -64,16 +64,19 @@ class Station(object): self.stn_info = stn_info # Add a bunch of formatted attributes: - hemispheres = skin_dict['Labels'].get('hemispheres', ('N','S','E','W')) - latlon_formats = skin_dict['Labels'].get('latlon_formats') + label_dict = skin_dict.get('Labels', {}) + hemispheres = label_dict.get('hemispheres', ('N','S','E','W')) + latlon_formats = label_dict.get('latlon_formats') self.latitude = weeutil.weeutil.latlon_string(stn_info.latitude_f, - hemispheres[0:2], 'lat', latlon_formats) + hemispheres[0:2], + 'lat', latlon_formats) self.longitude = weeutil.weeutil.latlon_string(stn_info.longitude_f, - hemispheres[2:4], 'lon', latlon_formats) - self.altitude = weewx.units.ValueHelper(value_t=stn_info.altitude_vt, - formatter=formatter, - converter=converter) - self.rain_year_str = time.strftime("%b", (0, self.rain_year_start, 1, 0,0,0,0,0,-1)) + hemispheres[2:4], + 'lon', latlon_formats) + self.altitude = weewx.units.ValueHelper(value_t=stn_info.altitude_vt, + formatter=formatter, + converter=converter) + self.rain_year_str = time.strftime("%b", (0, self.rain_year_start, 1, 0,0,0,0,0,-1)) self.uptime = weeutil.weeutil.secs_to_string(time.time() - weewx.launchtime_ts) if weewx.launchtime_ts else '' self.version = weewx.__version__ # The following works on Linux only: diff --git a/bin/weewx/wxengine.py b/bin/weewx/wxengine.py index 12896d3c..cdc3ef58 100644 --- a/bin/weewx/wxengine.py +++ b/bin/weewx/wxengine.py @@ -413,7 +413,7 @@ class StdArchive(StdService): super(StdArchive, self).__init__(engine, config_dict) # Get the archive interval from the configuration file - software_archive_interval = config_dict['StdArchive'].as_int('archive_interval') + self.archive_interval = config_dict['StdArchive'].as_int('archive_interval') # If the station supports a hardware archive interval use that instead, but # warn if they mismatch: @@ -455,7 +455,7 @@ class StdArchive(StdService): # data still on the station, but not yet put in the database. Not # all consoles can do this, so be prepared to catch the exception: try: - self._catchup() + self._catchup(self.engine.console.genStartupRecords) except NotImplementedError: pass @@ -517,7 +517,7 @@ class StdArchive(StdService): # be raised if the console does not support it. In that case, fall # back to software generation. try: - self._catchup() + self._catchup(self.engine.console.genArchiveRecords) except NotImplementedError: self._software_catchup() else: @@ -560,24 +560,26 @@ class StdArchive(StdService): def shutDown(self): self.archive.close() self.statsDb.close() - - def _catchup(self): + + def _catchup(self, generator): """Pull any unarchived records off the console and archive them. - If the hardware does not support hardware archives, an exception of type - NotImplementedError will be thrown.""" + If the hardware does not support hardware archives, an exception of + type NotImplementedError will be thrown.""" # Find out when the archive was last updated. lastgood_ts = self.archive.lastGoodStamp() try: - # Now ask the console for any new records since then. (Not all consoles - # support this feature). - for record in self.engine.console.genArchiveRecords(lastgood_ts): - self.engine.dispatchEvent(weewx.Event(weewx.NEW_ARCHIVE_RECORD, record=record, origin='hardware')) + # Now ask the console for any new records since then. + # (Not all consoles support this feature). + for record in generator(lastgood_ts): + self.engine.dispatchEvent(weewx.Event(weewx.NEW_ARCHIVE_RECORD, + record=record, + origin='hardware')) except weewx.HardwareError, e: syslog.syslog(syslog.LOG_ERR, "wxengine: Internal error detected. Catchup abandoned") - syslog.syslog(syslog.LOG_ERR, "**** %s" % e) + syslog.syslog(syslog.LOG_ERR, "**** %s" % e) def _software_catchup(self): # Extract a record out of the old accumulator. diff --git a/docs/changes.txt b/docs/changes.txt index 34c17479..58cde9d2 100644 --- a/docs/changes.txt +++ b/docs/changes.txt @@ -21,13 +21,28 @@ have any UV data. If software is specified for record_generation, do not try to use the stations archive interval. +Fixed bug when reading cooling_base option. + +Default to sane behavior if skin does not define Labels. + Fixed bug in setting of CheetahGenerator options. Fixed qsf and qpf summary values in forecast module. +Fixed handling of empty sky cover fields in WU forecasts. + Forecast module now considers the fctcode, condition, and wx fields for precipitation and obstructions to visibility. +Added options to forecast module to help diagnose parsing failures and new +forecast formats. + +Added retries when saving forecast to database and when reading from database. + +Fixes to the Fine Offset driver to eliminate spikes caused by reading from +memory before the pointer had been updated (not the same thing as an unstable +read). + Added driver for LaCrosse 2300 series of weather stations. Added driver for Hideki TE923 series of weather stations. diff --git a/util/init.d/weewx.debian b/util/init.d/weewx.debian index 3dbc65dd..d74eef7b 100755 --- a/util/init.d/weewx.debian +++ b/util/init.d/weewx.debian @@ -57,7 +57,7 @@ SCRIPTNAME=/etc/init.d/$NAME # check using ps not the pid file. pid file could be leftover. do_start() { - NPROC=`ps ax | grep $WEEWX_BIN | grep weewx.pid | wc -l` + NPROC=`ps ax | grep $WEEWX_BIN | grep $NAME.pid | wc -l` if [ $NPROC != 0 ]; then return 1 fi @@ -105,7 +105,7 @@ case "$1" in esac ;; status) - NPROC=`ps ax | grep $WEEWX_BIN | grep weewx.pid | wc -l` + NPROC=`ps ax | grep $WEEWX_BIN | grep $NAME.pid | wc -l` if [ $NPROC -gt 1 ]; then MSG="running multiple times" elif [ $NPROC = 1 ]; then diff --git a/util/logrotate.d/weewx b/util/logrotate.d/weewx index 86129e23..a92d32c5 100644 --- a/util/logrotate.d/weewx +++ b/util/logrotate.d/weewx @@ -5,10 +5,12 @@ compress delaycompress notifempty - create 644 root adm +# create 644 root adm + create 644 syslog adm sharedscripts postrotate - /etc/init.d/rsyslog stop - /etc/init.d/rsyslog start + reload rsyslog > /dev/null 2>&1 +# /etc/init.d/rsyslog stop +# /etc/init.d/rsyslog start endscript } diff --git a/util/logwatch/scripts/services/weewx b/util/logwatch/scripts/services/weewx index 8b94cb26..e8b7bd7d 100755 --- a/util/logwatch/scripts/services/weewx +++ b/util/logwatch/scripts/services/weewx @@ -2,93 +2,91 @@ # $Id$ # logwatch script to process weewx log files # Copyright 2013 Matthew Wall -# -# Revision History -# 0.6 11nov13 -# * record unstable reads on fousb -# 0.5 01nov13 -# * recognize more restful log output -# 0.4 12oct13 -# * recognize more fousb log output -# * recognize more ws28xx log output -# * track forecasting counts -# 0.3 09oct13 -# * match cheetahgenerator -# * match failed restful uploads -# * recognize forecast events -# * match weewx HUPs -# * recognize new driver startup diagnostics -# * recognize new weewx wxengine startup diagnostics -# * recognize ws28xx driver entries -# 0.2 03jan13 -# * better labels for counts -# 0.1 01jan13 -# * initial release # FIXME: break this into modules instead of a single, monolithic blob use strict; +my %counts; +my %errors; + +# keys for individual counts my $STARTUPS = 'wxengine: startups'; my $HUP_RESTARTS = 'wxengine: restart from HUP'; +my $KBD_INTERRUPTS = 'wxengine: keyboard interrupts'; my $ARCHIVE_RECORDS_ADDED = 'archive: records added'; my $IMAGES_GENERATED = 'genimages: images generated'; my $FILES_GENERATED = 'filegenerator: files generated'; my $FILES_COPIED = 'reportengine: files copied'; my $RECORDS_PUBLISHED = 'restful: records published'; my $RECORDS_SKIPPED = 'restful: records skipped'; +my $RECORDS_FAILED = 'restful: publish failed'; my $FOUSB_UNSTABLE_READS = 'fousb: unstable reads'; my $FOUSB_MAGIC_NUMBERS = 'fousb: unrecognised magic number'; +my $FOUSB_RAIN_COUNTER = 'fousb: rain counter decrement'; my $FOUSB_LOST_LOG_SYNC = 'fousb: lost log sync'; my $FOUSB_LOST_SYNC = 'fousb: lost sync'; my $FOUSB_MISSED_DATA = 'fousb: missed data'; my $FOUSB_STATION_SYNC = 'fousb: station sync'; +my $WS23XX_CONNECTION_CHANGE = 'ws23xx: connection change'; my $FORECAST_RECORDS = 'forecast: records generated'; my $FORECAST_PRUNINGS = 'forecast: prunings'; my $FORECAST_DOWNLOADS = 'forecast: downloads'; -my %counts = ( - $STARTUPS, 0, - $HUP_RESTARTS, 0, - $ARCHIVE_RECORDS_ADDED, 0, - $IMAGES_GENERATED, 0, - $FILES_GENERATED, 0, - $FILES_COPIED, 0, - $RECORDS_PUBLISHED, 0, - $RECORDS_SKIPPED, 0, - $FOUSB_UNSTABLE_READS, 0, - $FOUSB_MAGIC_NUMBERS, 0, - $FOUSB_LOST_LOG_SYNC, 0, - $FOUSB_LOST_SYNC, 0, - $FOUSB_MISSED_DATA, 0, - $FOUSB_STATION_SYNC, 0, - $FORECAST_RECORDS, 0, - $FORECAST_PRUNINGS, 0, - $FORECAST_DOWNLOADS, 0, -); -my $RECORDS_FAILED = 'restful: publish failed'; -my %errors; + +# any lines that do not match the patterns we define my @unmatched = (); -# keep details of fine offset behavior to help debug the consoles +# track upload errors to help diagnose network/server issues +my @upload_errors = (); + +# keep details of ws23xx behavior +my @conn_change = (); + +# keep details of fine offset behavior my @station_status = (); my @unstable_reads = (); my @magic_numbers = (); +my @rain_counter = (); + +my %itemized = ( + 'upload failures', \@upload_errors, + 'fousb station status', \@station_status, + 'fousb unstable reads', \@unstable_reads, + 'fousb magic numbers', \@magic_numbers, + 'fousb rain counter', \@rain_counter, + 'ws23xx connection changes', \@conn_change, + ); while(defined($_ = )) { chomp; - if (/Archive: added archive record/) { - $counts{$ARCHIVE_RECORDS_ADDED} += 1; - } elsif (/genimages: Generated (\d+) images/) { - $counts{$IMAGES_GENERATED} += $1; - } elsif (/filegenerator: generated (\d+)/ || - /cheetahgenerator: generated (\d+)/) { - $counts{$FILES_GENERATED} += $1; - } elsif (/wxengine: Starting up weewx version/) { + if (/wxengine: Starting up weewx version/) { $counts{$STARTUPS} += 1; } elsif (/wxengine: Received signal HUP/) { $counts{$HUP_RESTARTS} += 1; + } elsif (/wxengine: Keyboard interrupt/) { + $counts{$KBD_INTERRUPTS} += 1; + } elsif (/Archive: added archive record/) { + $counts{$ARCHIVE_RECORDS_ADDED} += 1; + } elsif (/genimages: Generated (\d+) images/) { + $counts{$IMAGES_GENERATED} += $1; + } elsif (/genimages: aggregate interval required for aggregate type/ || + /genimages: line type \S+ skipped/) { + $errors{$_} = $errors{$_} ? $errors{$_} + 1 : 1; + } elsif (/filegenerator: generated (\d+)/ || + /cheetahgenerator: generated (\d+)/) { + $counts{$FILES_GENERATED} += $1; } elsif (/reportengine: copied (\d+) files/) { $counts{$FILES_COPIED} += $1; + } elsif (/restful: Skipped record/) { + $counts{$RECORDS_SKIPPED} += 1; + } elsif (/restful: Published record/) { + $counts{$RECORDS_PUBLISHED} += 1; + } elsif (/restful: Unable to publish record/) { + my $key = $RECORDS_FAILED; + if (/restful: Unable to publish record \d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d \S\S\S \(\d+\) to (\S+)/) { + $key .= ' to site ' . $1; + } + $errors{$key} = $errors{$key} ? $errors{$key} + 1 : 1; } elsif (/fousb: station status/) { push @station_status, $_; } elsif (/fousb: unstable read: blocks differ/) { @@ -97,6 +95,9 @@ while(defined($_ = )) { } elsif (/fousb: unrecognised magic number/) { push @magic_numbers, $_; $counts{$FOUSB_MAGIC_NUMBERS} += 1; + } elsif (/fousb: rain counter decrement/) { + push @rain_counter, $_; + $counts{$FOUSB_RAIN_COUNTER} += 1; } elsif (/fousb: lost log sync/) { $counts{$FOUSB_LOST_LOG_SYNC} += 1; } elsif (/fousb: lost sync/) { @@ -105,6 +106,9 @@ while(defined($_ = )) { $counts{$FOUSB_MISSED_DATA} += 1; } elsif (/fousb: synchronising to the weather station/) { $counts{$FOUSB_STATION_SYNC} += 1; + } elsif (/ws23xx: connection changed from/) { + push @conn_change, $_; + $counts{$WS23XX_CONNECTION_CHANGE} += 1; } elsif (/forecast: .* generated 1 forecast record/) { $counts{$FORECAST_RECORDS} += 1; } elsif (/forecast: .* got (\d+) forecast records/) { @@ -113,23 +117,19 @@ while(defined($_ = )) { $counts{$FORECAST_PRUNINGS} += 1; } elsif (/forecast: .* downloading forecast/) { $counts{$FORECAST_DOWNLOADS} += 1; - } elsif (/restful: Skipped record/) { - $counts{$RECORDS_SKIPPED} += 1; - } elsif (/restful: Published record/) { - $counts{$RECORDS_PUBLISHED} += 1; - } elsif (/genimages: aggregate interval required for aggregate type/ || - /genimages: line type \S+ skipped/) { - $errors{$_} = $errors{$_} ? $errors{$_} + 1 : 1; - } elsif (/restful: Unable to publish record/) { - my $key = $RECORDS_FAILED; - if (/restful: Unable to publish record \d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d \S\S\S \(\d+\) to (\S+)/) { - $key .= ' to site ' . $1; - } - $errors{$key} = $errors{$key} ? $errors{$key} + 1 : 1; - } elsif (/reportengine: Running reports for latest time/ || + } elsif (/awekas: Failed upload to AWEKAS/ || + /cosm: Failed upload to COSM/ || + /emoncms: Failed upload to EmonCMS/ || + /owm: Failed upload to OpenWeatherMap/ || + /seg: Failed upload to SmartEnergyGroups/ || + /wbug: Failed upload to WeatherBug/) { + push @upload_errors, $_; + } elsif (/last message repeated/ || + /reportengine: Running reports for latest time/ || /reportengine: Found configuration file/ || /reportengine: FTP upload not requested/ || /reportengine: Running report / || # only when debug=1 + /reportengine: rsync upload not requested/ || /restful: station will register with/ || /restful: Registration interval/ || /\*\*\*\* Registration interval/ || @@ -143,10 +143,10 @@ while(defined($_ = )) { /stats: Created schema for statistical database/ || /stats: Schema exists with/ || /\*\*\*\* \'station\'/ || + /\*\*\*\* required parameter \'\'station\'\'/ || /\*\*\*\* Waiting 60 seconds then retrying/ || /wxengine: Station does not support reading the time/ || /wxengine: Starting main packet loop/ || - /wxengine: Keyboard interrupt/ || /wxengine: Shut down StdReport thread/ || /wxengine: Shut down StdRESTful thread/ || /wxengine: Loading service/ || @@ -171,7 +171,9 @@ while(defined($_ = )) { /wxengine: pid file is / || /wxengine: Use LOOP data in/ || /wxengine: Received signal/ || + /cheetahgenerator: Running / || /cheetahgenerator: skip/ || + /fousb: driver version is/ || /fousb: found station on USB/ || /fousb: altitude is/ || /fousb: archive interval is/ || @@ -189,6 +191,7 @@ while(defined($_ = )) { /fousb: avoid/ || /fousb: setting sensor clock/ || /fousb: setting station clock/ || + /fousb: estimated log time/ || /fousb: returning archive record/ || /fousb: packet timestamp/ || /fousb: log timestamp/ || @@ -196,6 +199,7 @@ while(defined($_ = )) { /fousb: get \d+ records since/ || /fousb: synchronised to/ || /fousb: pressures:/ || + /fousb: status / || /ws28xx: MainThread: driver version is/ || /ws28xx: MainThread: frequency is/ || /ws28xx: MainThread: altitude is/ || @@ -260,6 +264,31 @@ while(defined($_ = )) { /ws28xx: RFComm: CHistoryDataSet.read/ || /ws28xx: RFComm: testConfigChanged/ || /ws28xx: RFComm: SetTime/ || + /ws23xx: driver version is / || + /ws23xx: polling interval is / || + /ws23xx: station archive interval is / || + /ws23xx: using computer clock with / || + /ws23xx: using \d+ sec\S* polling interval/ || + /ws23xx: windchill will be / || + /ws23xx: dewpoint will be / || + /ws23xx: pressure offset is / || + /ws23xx: serial port is / || + /ws23xx: downloading \d+ records from station/ || + /ws23xx: count is \d+ to satisfy timestamp/ || + /ws23xx: windchill: / || + /ws23xx: dewpoint: / || + /ws23xx: station clock is / || + /te923: driver version is / || + /te923: polling interval is / || + /te923: windchill will be / || + /te923: sensor map is / || + /te923: battery map is / || + /te923: Found device on USB/ || + /owfs: driver version is / || + /owfs: interface is / || + /owfs: polling interval is / || + /owfs: sensor map is / || + /cmon: cpuinfo: / || /forecast: .* starting thread/ || /forecast: .* terminating thread/ || /forecast: .* not yet time to do the forecast/ || @@ -268,18 +297,43 @@ while(defined($_ = )) { /forecast: .* tstr=/ || /forecast: .* interval=\d+ max_age=/ || /forecast: .* deleted forecasts/ || + /forecast: .* saved \d+ forecast records/ || + /forecast: ZambrettiThread: Zambretti: generating/ || + /forecast: ZambrettiThread: Zambretti: pressure/ || + /forecast: ZambrettiThread: Zambretti: code is/ || /forecast: NWSThread: NWS: forecast matrix/ || /forecast: XTideThread: XTide: tide matrix/ || /forecast: XTideThread: XTide: generating tides/ || - /emoncms: Failed upload attempt/ || - /emoncms: Failed upload to EmonCMS/ || - /\*\*\*\* Failed upload to EmonCMS/ || - /seg: Failed upload attempt/ || - /seg: Failed upload to SmartEnergyGroups/ || - /\*\*\*\* Failed upload to SmartEnergyGroups/ || + /awekas: Failed upload attempt/ || + /awekas: code/ || + /awekas: read/ || + /awekas: url/ || + /awekas: data/ || /cosm: Failed upload attempt/ || - /cosm: Failed upload to COSM/ || - /\*\*\*\* Failed upload to COSM/) { + /cosm: code/ || + /cosm: read/ || + /cosm: url/ || + /cosm: data/ || + /emoncms: Failed upload attempt/ || + /emoncms: code/ || + /emoncms: read/ || + /emoncms: data/ || + /emoncms: url/ || + /owm: Failed upload attempt/ || + /owm: code/ || + /owm: read/ || + /owm: url/ || + /owm: data/ || + /seg: Failed upload attempt/ || + /seg: code/ || + /seg: read/ || + /seg: url/ || + /seg: data/ || + /wbug: Failed upload attempt/ || + /wbug: code/ || + /wbug: read/ || + /wbug: url/ || + /wbug: data/) { # ignore } elsif (! /weewx/) { # ignore @@ -294,13 +348,13 @@ foreach my $k (sort keys %counts) { printf(" %-40s %6d\n", $k, $counts{$k}); } -report("fousb station status", \@station_status) if $#station_status >= 0; -report("fousb unstable reads", \@unstable_reads) if $#unstable_reads >= 0; -report("fousb magic numbers", \@magic_numbers) if $#magic_numbers >= 0; - print "\nerrors:\n"; foreach my $k (keys %errors) { - printf(" %3d %s\n", $errors{$k}, $k); + printf(" %-40s %6d\n", $k, $errors{$k}); +} + +foreach my $k (sort keys %itemized) { + report($k, $itemized{$k}) if scalar @{$itemized{$k}} > 0; } report("unmatched lines", \@unmatched) if $#unmatched >= 0;