From 07044ae2b019ceae1f17ff9ffda940aa700ad80e Mon Sep 17 00:00:00 2001 From: Tom Keffer Date: Sun, 9 Nov 2014 17:46:25 +0000 Subject: [PATCH] Added aggregation types .last and .lasttime. Documented them. Included in test suites. Still have to support them in aggregation vectors. Added new appendix to Customizing Guide that documents the various aggregation types. --- TODO.txt | 3 + bin/weewx/manager.py | 89 ++++++----- bin/weewx/test/expected/index.html | 8 + bin/weewx/test/expected/metric/index.html | 8 + .../test_skins/StandardTest/index.html.tmpl | 8 + bin/weewx/units.py | 1 + docs/changes.txt | 9 +- docs/customizing.htm | 139 +++++++++++++++++- 8 files changed, 220 insertions(+), 45 deletions(-) diff --git a/TODO.txt b/TODO.txt index b8ebce7f..4646eebb 100644 --- a/TODO.txt +++ b/TODO.txt @@ -24,6 +24,9 @@ May need to update the guide on packaging an extension. On startup, the log should indicate not only which binding is in use, but what it is bound to (sqlite or mysql). +Make .last and .lasttime work for aggregation vectors. + + diff --git a/bin/weewx/manager.py b/bin/weewx/manager.py index fcf472b9..dc07e19d 100644 --- a/bin/weewx/manager.py +++ b/bin/weewx/manager.py @@ -836,35 +836,38 @@ meta_replace_str = """REPLACE INTO %s_day__metadata VALUES(?, ?)""" select_update_str = """SELECT value FROM %s_day__metadata WHERE name = 'lastUpdate';""" # Set of SQL statements to be used for calculating aggregate statistics. Key is the aggregation type. -sqlDict = {'min' : "SELECT MIN(min) FROM %(table_name)s_day_%(day_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s", - 'minmax' : "SELECT MIN(max) FROM %(table_name)s_day_%(day_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s", - 'max' : "SELECT MAX(max) FROM %(table_name)s_day_%(day_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s", - 'maxmin' : "SELECT MAX(min) FROM %(table_name)s_day_%(day_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s", - 'meanmin' : "SELECT AVG(min) FROM %(table_name)s_day_%(day_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s", - 'meanmax' : "SELECT AVG(max) FROM %(table_name)s_day_%(day_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s", - 'maxsum' : "SELECT MAX(sum) FROM %(table_name)s_day_%(day_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s", - 'mintime' : "SELECT mintime FROM %(table_name)s_day_%(day_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s AND " \ - "min = (SELECT MIN(min) FROM %(table_name)s_day_%(day_key)s WHERE dateTime >= %(start)s AND dateTime <%(stop)s)", - 'maxmintime' : "SELECT mintime FROM %(table_name)s_day_%(day_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s AND " \ - "min = (SELECT MAX(min) FROM %(table_name)s_day_%(day_key)s WHERE dateTime >= %(start)s AND dateTime <%(stop)s)", - 'maxtime' : "SELECT maxtime FROM %(table_name)s_day_%(day_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s AND " \ - "max = (SELECT MAX(max) FROM %(table_name)s_day_%(day_key)s WHERE dateTime >= %(start)s AND dateTime <%(stop)s)", - 'minmaxtime' : "SELECT maxtime FROM %(table_name)s_day_%(day_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s AND " \ - "max = (SELECT MIN(max) FROM %(table_name)s_day_%(day_key)s WHERE dateTime >= %(start)s AND dateTime <%(stop)s)", - 'maxsumtime' : "SELECT maxtime FROM %(table_name)s_day_%(day_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s AND " \ - "sum = (SELECT MAX(sum) FROM %(table_name)s_day_%(day_key)s WHERE dateTime >= %(start)s AND dateTime <%(stop)s)", - 'gustdir' : "SELECT max_dir FROM %(table_name)s_day_%(day_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s AND " \ - "max = (SELECT MAX(max) FROM %(table_name)s_day_%(day_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s)", - 'sum' : "SELECT SUM(sum) FROM %(table_name)s_day_%(day_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s", - 'count' : "SELECT SUM(count) FROM %(table_name)s_day_%(day_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s", - 'avg' : "SELECT SUM(wsum),SUM(sumtime) FROM %(table_name)s_day_%(day_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s", - 'rms' : "SELECT SUM(wsquaresum),SUM(sumtime) FROM %(table_name)s_day_%(day_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s", - 'vecavg' : "SELECT SUM(xsum),SUM(ysum),SUM(dirsumtime) FROM %(table_name)s_day_%(day_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s", - 'vecdir' : "SELECT SUM(xsum),SUM(ysum) FROM %(table_name)s_day_%(day_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s", - 'max_ge' : "SELECT SUM(max >= %(val)s) FROM %(table_name)s_day_%(day_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s", - 'max_le' : "SELECT SUM(max <= %(val)s) FROM %(table_name)s_day_%(day_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s", - 'min_le' : "SELECT SUM(min <= %(val)s) FROM %(table_name)s_day_%(day_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s", - 'sum_ge' : "SELECT SUM(sum >= %(val)s) FROM %(table_name)s_day_%(day_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s"} +sqlDict = {'min' : "SELECT MIN(min) FROM %(table_name)s_day_%(obs_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s", + 'minmax' : "SELECT MIN(max) FROM %(table_name)s_day_%(obs_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s", + 'max' : "SELECT MAX(max) FROM %(table_name)s_day_%(obs_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s", + 'maxmin' : "SELECT MAX(min) FROM %(table_name)s_day_%(obs_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s", + 'meanmin' : "SELECT AVG(min) FROM %(table_name)s_day_%(obs_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s", + 'meanmax' : "SELECT AVG(max) FROM %(table_name)s_day_%(obs_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s", + 'maxsum' : "SELECT MAX(sum) FROM %(table_name)s_day_%(obs_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s", + 'mintime' : "SELECT mintime FROM %(table_name)s_day_%(obs_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s AND " \ + "min = (SELECT MIN(min) FROM %(table_name)s_day_%(obs_key)s WHERE dateTime >= %(start)s AND dateTime <%(stop)s)", + 'maxmintime' : "SELECT mintime FROM %(table_name)s_day_%(obs_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s AND " \ + "min = (SELECT MAX(min) FROM %(table_name)s_day_%(obs_key)s WHERE dateTime >= %(start)s AND dateTime <%(stop)s)", + 'maxtime' : "SELECT maxtime FROM %(table_name)s_day_%(obs_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s AND " \ + "max = (SELECT MAX(max) FROM %(table_name)s_day_%(obs_key)s WHERE dateTime >= %(start)s AND dateTime <%(stop)s)", + 'minmaxtime' : "SELECT maxtime FROM %(table_name)s_day_%(obs_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s AND " \ + "max = (SELECT MIN(max) FROM %(table_name)s_day_%(obs_key)s WHERE dateTime >= %(start)s AND dateTime <%(stop)s)", + 'maxsumtime' : "SELECT maxtime FROM %(table_name)s_day_%(obs_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s AND " \ + "sum = (SELECT MAX(sum) FROM %(table_name)s_day_%(obs_key)s WHERE dateTime >= %(start)s AND dateTime <%(stop)s)", + 'gustdir' : "SELECT max_dir FROM %(table_name)s_day_%(obs_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s AND " \ + "max = (SELECT MAX(max) FROM %(table_name)s_day_%(obs_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s)", + 'sum' : "SELECT SUM(sum) FROM %(table_name)s_day_%(obs_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s", + 'count' : "SELECT SUM(count) FROM %(table_name)s_day_%(obs_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s", + 'last' : "SELECT %(obs_key)s FROM %(table_name)s WHERE dateTime > %(start)s AND dateTime <= %(stop)s AND %(obs_key)s IS NOT NULL " \ + "ORDER BY dateTime DESC LIMIT 1", + 'lasttime' : "SELECT MAX(dateTime) FROM %(table_name)s WHERE dateTime > %(start)s AND dateTime <= %(stop)s AND %(obs_key)s IS NOT NULL", + 'avg' : "SELECT SUM(wsum),SUM(sumtime) FROM %(table_name)s_day_%(obs_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s", + 'rms' : "SELECT SUM(wsquaresum),SUM(sumtime) FROM %(table_name)s_day_%(obs_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s", + 'vecavg' : "SELECT SUM(xsum),SUM(ysum),SUM(dirsumtime) FROM %(table_name)s_day_%(obs_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s", + 'vecdir' : "SELECT SUM(xsum),SUM(ysum) FROM %(table_name)s_day_%(obs_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s", + 'max_ge' : "SELECT SUM(max >= %(val)s) FROM %(table_name)s_day_%(obs_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s", + 'max_le' : "SELECT SUM(max <= %(val)s) FROM %(table_name)s_day_%(obs_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s", + 'min_le' : "SELECT SUM(min <= %(val)s) FROM %(table_name)s_day_%(obs_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s", + 'sum_ge' : "SELECT SUM(sum >= %(val)s) FROM %(table_name)s_day_%(obs_key)s WHERE dateTime >= %(start)s AND dateTime < %(stop)s"} #=============================================================================== # Class DaySummaryManager @@ -1012,14 +1015,23 @@ class DaySummaryManager(Manager): val += ("group_rain",) target_val = weewx.units.convertStd(val, self.std_unit_system)[0] - # This dictionary is used for interpolating the SQL statement. - interDict = {'start' : weeutil.weeutil.startOfDay(timespan.start), - 'stop' : timespan.stop, - 'day_key' : obs_type, - 'aggregateType' : aggregateType, - 'val' : target_val, - 'table_name' : self.table_name} + # convert to lower-case: + aggregateType = aggregateType.lower() + # Get the dictionary to be used for interpolating the SQL statement. + if aggregateType == 'last' or aggregateType == 'lasttime': + interDict = {'start' : timespan.start, + 'stop' : timespan.stop, + 'obs_key' : obs_type, + 'table_name' : self.table_name} + else: + interDict = {'start' : weeutil.weeutil.startOfDay(timespan.start), + 'stop' : timespan.stop, + 'obs_key' : obs_type, + 'aggregateType' : aggregateType, + 'val' : target_val, + 'table_name' : self.table_name} + # Run the query against the database: _row = self.xeqSql(sqlDict[aggregateType], interDict) @@ -1034,11 +1046,12 @@ class DaySummaryManager(Manager): _result = None # Do the required calculation for this aggregat type - elif aggregateType in ['min', 'maxmin', 'max', 'minmax', 'meanmin', 'meanmax', 'maxsum', 'sum', 'gustdir']: + elif aggregateType in ['min', 'maxmin', 'max', 'minmax', 'meanmin', 'meanmax', + 'maxsum', 'sum', 'gustdir', 'last']: # These aggregates are passed through 'as is'. _result = _row[0] - elif aggregateType in ['mintime', 'maxmintime', 'maxtime', 'minmaxtime', 'maxsumtime', + elif aggregateType in ['mintime', 'maxmintime', 'maxtime', 'minmaxtime', 'maxsumtime', 'lasttime', 'count', 'max_ge', 'max_le', 'min_le', 'sum_ge']: # These aggregates are always integers: _result = int(_row[0]) diff --git a/bin/weewx/test/expected/index.html b/bin/weewx/test/expected/index.html index 84bc3c9b..ef1d6eb8 100644 --- a/bin/weewx/test/expected/index.html +++ b/bin/weewx/test/expected/index.html @@ -209,6 +209,14 @@ Time of min temperature: 07:00 + + Last temperature of the day + 63.6°F + + + Time of the last temperature of the day + 09/04/10 00:00:00 + Max Temperature in alt_binding 59.3°F diff --git a/bin/weewx/test/expected/metric/index.html b/bin/weewx/test/expected/metric/index.html index 7b253733..e12efcf9 100644 --- a/bin/weewx/test/expected/metric/index.html +++ b/bin/weewx/test/expected/metric/index.html @@ -209,6 +209,14 @@ Time of min temperature: 07:00 + + Last temperature of the day + 17.6°C + + + Time of the last temperature of the day + 09/04/10 00:00:00 + Max Temperature in alt_binding 15.2°C diff --git a/bin/weewx/test/test_skins/StandardTest/index.html.tmpl b/bin/weewx/test/test_skins/StandardTest/index.html.tmpl index cf15764c..90ac412c 100644 --- a/bin/weewx/test/test_skins/StandardTest/index.html.tmpl +++ b/bin/weewx/test/test_skins/StandardTest/index.html.tmpl @@ -237,6 +237,14 @@ Time of min temperature: $day.outTemp.mintime + + Last temperature of the day + $day.outTemp.last + + + Time of the last temperature of the day + $day.outTemp.lasttime.format("%x %X") + Max Temperature in alt_binding $day($data_binding='alt_binding').outTemp.max diff --git a/bin/weewx/units.py b/bin/weewx/units.py index c1847332..df06ab70 100644 --- a/bin/weewx/units.py +++ b/bin/weewx/units.py @@ -100,6 +100,7 @@ agg_group = {'mintime' : "group_time", 'maxtime' : "group_time", 'minmaxtime' : "group_time", "maxsumtime" : "group_time", + "lasttime" : "group_time", 'count' : "group_count", 'max_ge' : "group_count", 'max_le' : "group_count", diff --git a/docs/changes.txt b/docs/changes.txt index 7309a4c3..5624c4e8 100644 --- a/docs/changes.txt +++ b/docs/changes.txt @@ -17,9 +17,16 @@ Allow other databases to be used in images. The archive interval can now change within a database without consequences. -Introduced a new tag $last, which uses the last available timestamp +Introduced a new tag $latest, which uses the last available timestamp in a data binding (which may not be the same as the "current" timestamp). +Introduced a new aggregation type ".last", which returns the last +value in an aggregation interval. E.g., $week.outTemp.last would +return the last temperature seen in the week. + +Introduced a new aggregation type ".lasttime" which returns +the time of the above. + Total rewrite of how devices are configured. A single utility wee_config_device replaces each of the device-specific configuration utilities. diff --git a/docs/customizing.htm b/docs/customizing.htm index c436963d..7f984dbe 100644 --- a/docs/customizing.htm +++ b/docs/customizing.htm @@ -766,12 +766,10 @@ $month.outTemp.max

aggregation is an aggregation type. - This is something like 'min', 'sum', 'mintime'. If you + If you ask for $month.outTemp.avg you are asking for - the average outside temperature for the month. The table - Statistical Types - shows what aggregation types are available for which types. + the average outside temperature for the month. Possible + aggregation types are given in Appendix: Aggregation types.

optional_unit_conversion is an optional unit @@ -787,6 +785,8 @@ $month.outTemp.max href="#formatting_options">Formatting Options below.

+ +

Unit conversion options

The tag optional_unit_conversion can be used with either current observations or aggregations. If @@ -4285,7 +4285,134 @@ Options: -

Appendix: Units

+

Appendix: Aggregation types

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Aggregation types +
Aggregation typeMeaning
'avg'The average value in the aggregation period.
'sum'The sum of values in the aggregation period.
'count'The number of non-null values in the aggregation period.
'min'The minimum value in the aggregation period.
'mintime'The time of the minimum value.
'max'The maximum value in the aggregation period.
'maxtime'The time of the maximum value.
'maxmin'The maximum daily minimum in the aggregation period. Aggregation period + must be one day or longer.
'maxmintime'The time of the maximum daily minimum.
'minmax'The minimum daily maximum in the aggregation period. Aggregation period + must be one day or longer.
'minmaxtime'The time of the minimum daily maximum.
'maxsum'The maximum daily sum in the aggregation period. Aggregation + period must be one day or longer.
'maxsumtime'The time of the maximum daily sum.
'meanmin'The average daily minimum in the aggregation period. Aggregation + period must be one day or longer.
'meanmax'The average daily maximum in the aggregation period. Aggregation + period must be one day or longer.
'gustdir'The direction of the max gust in the aggregation period.
'last'The last value in the aggregation period.
'lasttime'The time of the last value in the aggregation period.
'max_ge(val)'The number of days where the maximum value is greater than or + equal to val. Aggregation period must be one day or longer. +
'max_le(val)'The number of days where the maximum value is less than or + equal to val. Aggregation period must be one day or longer. +
'min_le(val)'The number of days where the minimum value is less than or + equal to val. Aggregation period must be one day or longer. +
'sum_ge(val)'The number of days where the sum of value is greater than or + equal to val. Aggregation period must be one day or longer. +
'rms'The root mean square value in the aggregation period.
'vecavg'The vector average magnitude in the aggregation period.
'vecdir'The direction of the average vector.
+ +

Appendix: Units

Weewx offers three different unit systems: