From 8cbcd96c629adff5cbe0aaaed722b034ac0c16fe Mon Sep 17 00:00:00 2001 From: Tom Keffer Date: Sun, 24 Nov 2019 05:47:56 -0800 Subject: [PATCH] More realistic synthetic database. However, see TODO.md --- TODO.md | 10 ++++- bin/weewx/tests/gen_fake_data.py | 74 +++++++++++++++++++++++++------ bin/weewx/tests/test_aggregate.py | 11 +++++ bin/weewx/wxservices.py | 15 ++++--- 4 files changed, 91 insertions(+), 19 deletions(-) diff --git a/TODO.md b/TODO.md index f96cdd4f..c90c57b3 100644 --- a/TODO.md +++ b/TODO.md @@ -1,5 +1,13 @@ +### Tests + +Synthetic database used for tests: some of the types require the database, but it's not available until +after the transaction. Need to either go back later and patch in the derived types, or make them +available outside the database. Or, give up the idea of calculating them. + +`gen_fake_data.py` is now more realistic, but the expected results haven't caught back up to it yet. + ### Xtypes -Sort out how staticmethods work. +Allow expressions in aggregates. ### Install Note in the docs that doing: diff --git a/bin/weewx/tests/gen_fake_data.py b/bin/weewx/tests/gen_fake_data.py index 79b977cd..473d32a4 100644 --- a/bin/weewx/tests/gen_fake_data.py +++ b/bin/weewx/tests/gen_fake_data.py @@ -1,4 +1,5 @@ # +# # Copyright (c) 2009-2019 Tom Keffer # # See the file LICENSE.txt for your full rights. @@ -14,20 +15,35 @@ from __future__ import with_statement import logging import math +import os +import sys import time import schemas.wview import weedb import weewx.manager +import weewx.wxservices +import weewx.xtypes log = logging.getLogger(__name__) +os.environ['TZ'] = 'America/Los_Angeles' +time.tzset() + +# The start of the 'solar year' for 2009-2010 +year_start_tt = (2009, 12, 21, 9, 47, 0, 0, 0, 0) +year_start = int(time.mktime(year_start_tt)) + # Roughly nine months of data: -start_tt = (2010, 1, 1, 0, 0, 0, 0, 0, -1) -stop_tt = (2010, 9, 3, 11, 20, 0, 0, 0, -1) +start_tt = (2010, 1, 1, 0, 0, 0, 0, 0, -1) # 2010-01-01 00:00 +stop_tt = (2010, 9, 3, 11, 0, 0, 0, 0, -1) # 2010-09-03 11:00 start_ts = int(time.mktime(start_tt)) stop_ts = int(time.mktime(stop_tt)) +altitude_vt = (700, 'foot', 'group_altitude') +latitude = 45 +longitude = -125 + daily_temp_range = 40.0 annual_temp_range = 80.0 avg_temp = 40.0 @@ -40,7 +56,7 @@ weather_rain_total = 0.5 # This is inches per weather cycle avg_baro = 30.0 # Archive interval in seconds: -interval = 600 +interval = 3600 schema = schemas.wview.schema @@ -53,8 +69,8 @@ def configDatabases(config_dict, database_type): def configDatabase(config_dict, binding, start_ts=start_ts, stop_ts=stop_ts, interval=interval, amplitude=1.0, - day_phase_offset=0.0, annual_phase_offset=0.0, - weather_phase_offset=0.0): + day_phase_offset=math.pi / 4.0, annual_phase_offset=0.0, + weather_phase_offset=0.0, year_start=start_ts): """Configures the archive databases.""" global schema @@ -105,7 +121,9 @@ def configDatabase(config_dict, binding, start_ts=start_ts, stop_ts=stop_ts, int amplitude=amplitude, day_phase_offset=day_phase_offset, annual_phase_offset=annual_phase_offset, - weather_phase_offset=weather_phase_offset)) + weather_phase_offset=weather_phase_offset, + year_start=start_ts, + db_manager=archive)) t2 = time.time() delta = t2 - t1 print("\nTime to create synthetic database '%s' = %6.2fs" @@ -130,28 +148,37 @@ def configDatabase(config_dict, binding, start_ts=start_ts, stop_ts=stop_ts, int def genFakeRecords(start_ts=start_ts, stop_ts=stop_ts, interval=interval, amplitude=1.0, day_phase_offset=0.0, annual_phase_offset=0.0, - weather_phase_offset=0.0): + weather_phase_offset=0.0, year_start=start_ts, db_manager=None): + pressure_cooker = weewx.wxservices.PressureCooker(altitude_vt) + wx_types = weewx.wxservices.WXXTypes({}, altitude_vt, latitude, longitude) + weewx.xtypes.xtypes.append(pressure_cooker) + weewx.xtypes.xtypes.append(wx_types) + count = 0 for ts in range(start_ts, stop_ts + interval, interval): - daily_phase = ((ts - start_ts) * 2.0 * math.pi + day_phase_offset) / (3600 * 24.0) - annual_phase = ((ts - start_ts) * 2.0 * math.pi + annual_phase_offset) / (3600 * 24.0 * 365.0) - weather_phase = ((ts - start_ts) * 2.0 * math.pi + weather_phase_offset) / weather_cycle + daily_phase = ((ts - year_start) * 2.0 * math.pi) / (3600 * 24.0) + day_phase_offset + annual_phase = ((ts - year_start) * 2.0 * math.pi) / (3600 * 24.0 * 365.0) + annual_phase_offset + weather_phase = ((ts - year_start) * 2.0 * math.pi) / weather_cycle + weather_phase_offset record = { 'dateTime': ts, 'usUnits': weewx.US, 'interval': interval / 60, 'outTemp': 0.5 * amplitude * (-daily_temp_range * math.sin(daily_phase) - annual_temp_range * math.cos(annual_phase)) + avg_temp, - 'barometer': 0.5 * amplitude * weather_baro_range * math.sin(weather_phase) + avg_baro, + 'barometer': -0.5 * amplitude * weather_baro_range * math.sin(weather_phase) + avg_baro, 'windSpeed': abs(amplitude * weather_wind_range * (1.0 + math.sin(weather_phase))), - 'windDir': math.degrees(weather_phase) % 360.0} + 'windDir': math.degrees(weather_phase) % 360.0, + 'outHumidity': 40 * math.sin(weather_phase) + 50, + } record['windGust'] = 1.2 * record['windSpeed'] record['windGustDir'] = record['windDir'] if math.sin(weather_phase) > .95: - record['rain'] = 0.02 * amplitude if math.sin(weather_phase) > 0.98 else 0.01 * amplitude + record['rain'] = 0.08 * amplitude if math.sin(weather_phase) > 0.98 else 0.04 * amplitude else: record['rain'] = 0.0 + record['radiation'] = max(amplitude * 800 * math.sin(daily_phase - math.pi / 2.0), 0) + record['radiation'] *= 0.5 * (math.cos(annual_phase + math.pi) + 1.5) # Make every 71st observation (a prime number) a null. This is a deterministic algorithm, so it # will produce the same results every time. @@ -160,4 +187,25 @@ def genFakeRecords(start_ts=start_ts, stop_ts=stop_ts, interval=interval, if count % 71 == 0: record[obs_type] = None + if db_manager: + record['dewpoint'] = weewx.xtypes.get_scalar('dewpoint', record, db_manager)[0] + record['windchill'] = weewx.xtypes.get_scalar('windchill', record, db_manager)[0] + record['pressure'] = weewx.xtypes.get_scalar('pressure', record, db_manager)[0] + record['altimeter'] = weewx.xtypes.get_scalar('altimeter', record, db_manager)[0] + record['ET'] = weewx.xtypes.get_scalar('ET', record, db_manager)[0] + yield record + + +if __name__ == '__main__': + count = 0 + for rec in genFakeRecords(): + if count % 30 == 0: + print("Time outTemp windSpeed barometer rain radiation") + count += 1 + outTemp = "%10.1f" % rec['outTemp'] if rec['outTemp'] is not None else " N/A" + windSpeed = "%10.1f" % rec['windSpeed'] if rec['windSpeed'] is not None else " N/A" + barometer = "%10.1f" % rec['barometer'] if rec['barometer'] is not None else " N/A" + rain = "%10.2f" % rec['rain'] if rec['rain'] is not None else " N/A" + radiation = "%10.0f" % rec['radiation'] if rec['radiation'] is not None else " N/A" + print(6 * "%s" % (time.ctime(rec['dateTime']), outTemp, windSpeed, barometer, rain, radiation)) diff --git a/bin/weewx/tests/test_aggregate.py b/bin/weewx/tests/test_aggregate.py index 6593fa11..d3d39ad0 100644 --- a/bin/weewx/tests/test_aggregate.py +++ b/bin/weewx/tests/test_aggregate.py @@ -202,6 +202,17 @@ class TestAggregate(unittest.TestCase): self.assertAlmostEqual(windvec[0].imag, 3.211, 3) self.assertEqual(windvec[1:3], ('mile_per_hour', 'group_speed')) + def test_get_aggregate_expression(self): + """Test using an expression in an aggregate""" + with weewx.manager.open_manager_with_config(self.config_dict, 'wx_binding') as db_manager: + month_start_tt = (2010, 3, 1, 0, 0, 0, 0, 0, -1) + month_stop_tt = (2010, 3, 2, 0, 0, 0, 0, 0, -1) + start_ts = time.mktime(month_start_tt) + stop_ts = time.mktime(month_stop_tt) + + value = weewx.xtypes.get_aggregate('rain-ET', TimeSpan(start_ts, stop_ts), 'sum', db_manager) + print(value) + if __name__ == '__main__': unittest.main() diff --git a/bin/weewx/wxservices.py b/bin/weewx/wxservices.py index 481ead04..0e2e8e8f 100644 --- a/bin/weewx/wxservices.py +++ b/bin/weewx/wxservices.py @@ -509,7 +509,13 @@ class PressureCooker(weewx.xtypes.XType): # Get the temperature in Fahrenheit from 12 hours ago temp_12h_vt = self._get_temperature_12h(record['dateTime'], dbmanager) - if temp_12h_vt is not None: + if temp_12h_vt is None \ + or temp_12h_vt[0] is None \ + or record['outTemp'] is None \ + or record['barometer'] is None \ + or record['outHumidity'] is None: + pressure = None + else: # The following requires everything to be in US Customary units. # Rather than convert the whole record, just convert what we need: record_US = weewx.units.to_US({'usUnits': record['usUnits'], @@ -527,10 +533,9 @@ class PressureCooker(weewx.xtypes.XType): temp_12h_F[0], record_US['outHumidity'] ) - # Convert to target unit system and return - return weewx.units.convertStd((pressure, 'inHg', 'group_pressure'), record['usUnits']) - else: - return ValueTuple(None, None, None) + + # Convert to target unit system and return + return weewx.units.convertStd((pressure, 'inHg', 'group_pressure'), record['usUnits']) def altimeter(self, record): """Calculate the observation type 'altimeter'."""