mirror/weewx
1
0
Fork 0
mirror of https://github.com/weewx/weewx.git synced 2026-04-18 16:46:56 -04:00
Code Issues Packages Projects Releases Wiki Activity

More realistic synthetic database. However, see TODO.md

Browse Source
This commit is contained in:
Tom Keffer
2019-11-24 05:47:56 -08:00
parent 6ffb2f23b5
commit 8cbcd96c62
4 changed files with 91 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
TODO.md
View File

@@ -1,5 +1,13 @@
### Tests
Synthetic database used for tests: some of the types require the database, but it's not available until
after the transaction. Need to either go back later and patch in the derived types, or make them
available outside the database. Or, give up the idea of calculating them.
`gen_fake_data.py` is now more realistic, but the expected results haven't caught back up to it yet.
### Xtypes
Sort out how staticmethods work.
Allow expressions in aggregates.
### Install
Note in the docs that doing:

74
bin/weewx/tests/gen_fake_data.py
View File

@@ -1,4 +1,5 @@
#
#
# Copyright (c) 2009-2019 Tom Keffer <tkeffer@gmail.com>
#
# See the file LICENSE.txt for your full rights.
@@ -14,20 +15,35 @@ from __future__ import with_statement
import logging
import math
import os
import sys
import time
import schemas.wview
import weedb
import weewx.manager
import weewx.wxservices
import weewx.xtypes
log = logging.getLogger(__name__)
os.environ['TZ'] = 'America/Los_Angeles'
time.tzset()
# The start of the 'solar year' for 2009-2010
year_start_tt = (2009, 12, 21, 9, 47, 0, 0, 0, 0)
year_start = int(time.mktime(year_start_tt))
# Roughly nine months of data:
start_tt = (2010, 1, 1, 0, 0, 0, 0, 0, -1)
stop_tt = (2010, 9, 3, 11, 20, 0, 0, 0, -1)
start_tt = (2010, 1, 1, 0, 0, 0, 0, 0, -1) # 2010-01-01 00:00
stop_tt = (2010, 9, 3, 11, 0, 0, 0, 0, -1) # 2010-09-03 11:00
start_ts = int(time.mktime(start_tt))
stop_ts = int(time.mktime(stop_tt))
altitude_vt = (700, 'foot', 'group_altitude')
latitude = 45
longitude = -125
daily_temp_range = 40.0
annual_temp_range = 80.0
avg_temp = 40.0
@@ -40,7 +56,7 @@ weather_rain_total = 0.5 # This is inches per weather cycle
avg_baro = 30.0
# Archive interval in seconds:
interval = 600
interval = 3600
schema = schemas.wview.schema
@@ -53,8 +69,8 @@ def configDatabases(config_dict, database_type):
def configDatabase(config_dict, binding, start_ts=start_ts, stop_ts=stop_ts, interval=interval, amplitude=1.0,
day_phase_offset=0.0, annual_phase_offset=0.0,
weather_phase_offset=0.0):
day_phase_offset=math.pi / 4.0, annual_phase_offset=0.0,
weather_phase_offset=0.0, year_start=start_ts):
"""Configures the archive databases."""
global schema
@@ -105,7 +121,9 @@ def configDatabase(config_dict, binding, start_ts=start_ts, stop_ts=stop_ts, int
amplitude=amplitude,
day_phase_offset=day_phase_offset,
annual_phase_offset=annual_phase_offset,
weather_phase_offset=weather_phase_offset))
weather_phase_offset=weather_phase_offset,
year_start=start_ts,
db_manager=archive))
t2 = time.time()
delta = t2 - t1
print("\nTime to create synthetic database '%s' = %6.2fs"
@@ -130,28 +148,37 @@ def configDatabase(config_dict, binding, start_ts=start_ts, stop_ts=stop_ts, int
def genFakeRecords(start_ts=start_ts, stop_ts=stop_ts, interval=interval,
amplitude=1.0, day_phase_offset=0.0, annual_phase_offset=0.0,
weather_phase_offset=0.0):
weather_phase_offset=0.0, year_start=start_ts, db_manager=None):
pressure_cooker = weewx.wxservices.PressureCooker(altitude_vt)
wx_types = weewx.wxservices.WXXTypes({}, altitude_vt, latitude, longitude)
weewx.xtypes.xtypes.append(pressure_cooker)
weewx.xtypes.xtypes.append(wx_types)
count = 0
for ts in range(start_ts, stop_ts + interval, interval):
daily_phase = ((ts - start_ts) * 2.0 * math.pi + day_phase_offset) / (3600 * 24.0)
annual_phase = ((ts - start_ts) * 2.0 * math.pi + annual_phase_offset) / (3600 * 24.0 * 365.0)
weather_phase = ((ts - start_ts) * 2.0 * math.pi + weather_phase_offset) / weather_cycle
daily_phase = ((ts - year_start) * 2.0 * math.pi) / (3600 * 24.0) + day_phase_offset
annual_phase = ((ts - year_start) * 2.0 * math.pi) / (3600 * 24.0 * 365.0) + annual_phase_offset
weather_phase = ((ts - year_start) * 2.0 * math.pi) / weather_cycle + weather_phase_offset
record = {
'dateTime': ts,
'usUnits': weewx.US,
'interval': interval / 60,
'outTemp': 0.5 * amplitude * (-daily_temp_range * math.sin(daily_phase)
- annual_temp_range * math.cos(annual_phase)) + avg_temp,
'barometer': 0.5 * amplitude * weather_baro_range * math.sin(weather_phase) + avg_baro,
'barometer': -0.5 * amplitude * weather_baro_range * math.sin(weather_phase) + avg_baro,
'windSpeed': abs(amplitude * weather_wind_range * (1.0 + math.sin(weather_phase))),
'windDir': math.degrees(weather_phase) % 360.0}
'windDir': math.degrees(weather_phase) % 360.0,
'outHumidity': 40 * math.sin(weather_phase) + 50,
}
record['windGust'] = 1.2 * record['windSpeed']
record['windGustDir'] = record['windDir']
if math.sin(weather_phase) > .95:
record['rain'] = 0.02 * amplitude if math.sin(weather_phase) > 0.98 else 0.01 * amplitude
record['rain'] = 0.08 * amplitude if math.sin(weather_phase) > 0.98 else 0.04 * amplitude
else:
record['rain'] = 0.0
record['radiation'] = max(amplitude * 800 * math.sin(daily_phase - math.pi / 2.0), 0)
record['radiation'] *= 0.5 * (math.cos(annual_phase + math.pi) + 1.5)
# Make every 71st observation (a prime number) a null. This is a deterministic algorithm, so it
# will produce the same results every time.
@@ -160,4 +187,25 @@ def genFakeRecords(start_ts=start_ts, stop_ts=stop_ts, interval=interval,
if count % 71 == 0:
record[obs_type] = None
if db_manager:
record['dewpoint'] = weewx.xtypes.get_scalar('dewpoint', record, db_manager)[0]
record['windchill'] = weewx.xtypes.get_scalar('windchill', record, db_manager)[0]
record['pressure'] = weewx.xtypes.get_scalar('pressure', record, db_manager)[0]
record['altimeter'] = weewx.xtypes.get_scalar('altimeter', record, db_manager)[0]
record['ET'] = weewx.xtypes.get_scalar('ET', record, db_manager)[0]
yield record
if __name__ == '__main__':
count = 0
for rec in genFakeRecords():
if count % 30 == 0:
print("Time outTemp windSpeed barometer rain radiation")
count += 1
outTemp = "%10.1f" % rec['outTemp'] if rec['outTemp'] is not None else " N/A"
windSpeed = "%10.1f" % rec['windSpeed'] if rec['windSpeed'] is not None else " N/A"
barometer = "%10.1f" % rec['barometer'] if rec['barometer'] is not None else " N/A"
rain = "%10.2f" % rec['rain'] if rec['rain'] is not None else " N/A"
radiation = "%10.0f" % rec['radiation'] if rec['radiation'] is not None else " N/A"
print(6 * "%s" % (time.ctime(rec['dateTime']), outTemp, windSpeed, barometer, rain, radiation))

11
bin/weewx/tests/test_aggregate.py
View File

@@ -202,6 +202,17 @@ class TestAggregate(unittest.TestCase):
self.assertAlmostEqual(windvec[0].imag, 3.211, 3)
self.assertEqual(windvec[1:3], ('mile_per_hour', 'group_speed'))
def test_get_aggregate_expression(self):
"""Test using an expression in an aggregate"""
with weewx.manager.open_manager_with_config(self.config_dict, 'wx_binding') as db_manager:
month_start_tt = (2010, 3, 1, 0, 0, 0, 0, 0, -1)
month_stop_tt = (2010, 3, 2, 0, 0, 0, 0, 0, -1)
start_ts = time.mktime(month_start_tt)
stop_ts = time.mktime(month_stop_tt)
value = weewx.xtypes.get_aggregate('rain-ET', TimeSpan(start_ts, stop_ts), 'sum', db_manager)
print(value)
if __name__ == '__main__':
unittest.main()

15
bin/weewx/wxservices.py
View File

@@ -509,7 +509,13 @@ class PressureCooker(weewx.xtypes.XType):
# Get the temperature in Fahrenheit from 12 hours ago
temp_12h_vt = self._get_temperature_12h(record['dateTime'], dbmanager)
if temp_12h_vt is not None:
if temp_12h_vt is None \
or temp_12h_vt[0] is None \
or record['outTemp'] is None \
or record['barometer'] is None \
or record['outHumidity'] is None:
pressure = None
else:
# The following requires everything to be in US Customary units.
# Rather than convert the whole record, just convert what we need:
record_US = weewx.units.to_US({'usUnits': record['usUnits'],
@@ -527,10 +533,9 @@ class PressureCooker(weewx.xtypes.XType):
temp_12h_F[0],
record_US['outHumidity']
)
# Convert to target unit system and return
return weewx.units.convertStd((pressure, 'inHg', 'group_pressure'), record['usUnits'])
else:
return ValueTuple(None, None, None)
# Convert to target unit system and return
return weewx.units.convertStd((pressure, 'inHg', 'group_pressure'), record['usUnits'])
def altimeter(self, record):
"""Calculate the observation type 'altimeter'."""