From e235198614df6dfd72db9165ded799caa36fcc66 Mon Sep 17 00:00:00 2001 From: Tom Keffer Date: Mon, 12 Apr 2010 19:28:52 +0000 Subject: [PATCH] V1.7.0rc2 Improved error checking and logging in FTP and RESTful uploads. --- bin/weeutil/ftpupload.py | 30 ++++++++++++++++++++---------- bin/weewx/__init__.py | 2 +- bin/weewx/restful.py | 19 ++++++++++++------- docs/customizing.htm | 3 ++- docs/readme.htm | 5 ++++- setup.py | 4 ++-- skins/Standard/index.html.tmpl | 24 ++++++++++++------------ skins/Standard/month.html.tmpl | 24 ++++++++++++------------ skins/Standard/week.html.tmpl | 24 ++++++++++++------------ skins/Standard/year.html.tmpl | 24 ++++++++++++------------ weewx.conf | 4 ++-- 11 files changed, 91 insertions(+), 72 deletions(-) diff --git a/bin/weeutil/ftpupload.py b/bin/weeutil/ftpupload.py index 9ee92df2..e0fcd08a 100644 --- a/bin/weeutil/ftpupload.py +++ b/bin/weeutil/ftpupload.py @@ -15,7 +15,6 @@ import ftplib import cPickle import time import syslog -import socket class FtpUpload: """Uploads a directory and all its descendants to a remote server. @@ -96,26 +95,37 @@ class FtpUpload: full_remote_path = os.path.join(remote_dir_path, filename) STOR_cmd = "STOR %s" % full_remote_path - fd = open(full_local_path, "r") + # Retry up to max_tries times: for count in range(self.max_tries): try: + # If we have to retry, we should probably reopen the file as well. + # Hence, the open is in the inner loop: + fd = open(full_local_path, "r") ftp_server.storbinary(STOR_cmd, fd) - except ftplib.all_errors, e: + except (ftplib.all_errors, IOError), e: + # Unsuccessful. Log it and go around again. syslog.syslog(syslog.LOG_ERR, "ftpupload: attempt #%d. Failed uploading %s. Reason: %s" % (count+1, full_remote_path, e)) - if count >= self.max_tries -1 : - syslog.syslog(syslog.LOG_ERR, "ftpupload: Failed to upload file %s" % full_remote_path) - raise ftp_server.set_pasv(self.passive) else: - fd.close() + # Success. Log it, break out of the loop n_uploaded += 1 + fileset.add(full_local_path) syslog.syslog(syslog.LOG_DEBUG, "ftpupload: Uploaded file %s" % full_remote_path) break - fileset.add(full_local_path) + finally: + # This is always executed on every loop. Close the file. + try: + fd.close() + except: + pass + else: + # This is executed only if the loop terminates naturally (without a break statement), + # meaning the upload failed max_tries times. Log it, move on to the next file. + syslog.syslog(syslog.LOG_ERR, "ftpupload: Failed to upload file %s" % full_remote_path) finally: try: ftp_server.quit() - except socket.error: + except: pass timestamp = time.time() @@ -194,7 +204,7 @@ if __name__ == '__main__': import configobj weewx.debug = 1 - syslog.openlog('reportengine', syslog.LOG_PID|syslog.LOG_CONS) + syslog.openlog('ftpupload', syslog.LOG_PID|syslog.LOG_CONS) syslog.setlogmask(syslog.LOG_UPTO(syslog.LOG_DEBUG)) if len(sys.argv) < 2 : diff --git a/bin/weewx/__init__.py b/bin/weewx/__init__.py index 937d077f..d26995e6 100644 --- a/bin/weewx/__init__.py +++ b/bin/weewx/__init__.py @@ -12,7 +12,7 @@ """ import time -__version__="1.7.0rc1" +__version__="1.7.0rc2" # Holds the program launch time in unix epoch seconds: # Useful for calculating 'uptime.' diff --git a/bin/weewx/restful.py b/bin/weewx/restful.py index a8c98c87..6a171196 100644 --- a/bin/weewx/restful.py +++ b/bin/weewx/restful.py @@ -64,23 +64,27 @@ class RESTful(object): try: _response = urllib2.urlopen(_url) except (urllib2.URLError, socket.error), e: + # Unsuccessful. Log it and go around again for another try syslog.syslog(syslog.LOG_ERR, "restful: Failed attempt #%d to upload to %s" % (_count+1, self.site)) syslog.syslog(syslog.LOG_ERR, " **** Reason: %s" % (e,)) - if _count >= self.max_tries -1 : - syslog.syslog(syslog.LOG_ERR, "restful: Failed to upload to %s" % self.site) - raise else: - # A bad station ID or password will not throw an exception, but - # it will have the error encoded in the return message: + # No exception thrown, but we're still not done. + # We have to also check for a bad station ID or password. + # It will have the error encoded in the return message: for line in _response: # PWSweather signals with 'ERROR', WU with 'INVALID': if line.startswith('ERROR') or line.startswith('INVALID'): # Bad login. No reason to retry. Log it and raise an exception. syslog.syslog(syslog.LOG_ERR, "restful: %s returns %s. Aborting." % (self.site, line)) raise FailedPost, line - # If we get here, we have been successful. Just return. + # Does not seem to be an error. We're done. return - + else: + # This is executed only if the loop terminates normally, meaning + # the upload failed max_tries times. Log it. + syslog.syslog(syslog.LOG_ERR, "restful: Failed to upload to %s" % self.site) + raise IOError, "Failed ftp upload to site %s after %d tries" % (self.site, self.max_tries) + @staticmethod def extractRecordFrom(archive, time_ts): """Get a record from the archive database. @@ -230,6 +234,7 @@ class RESTThread(threading.Thread): # but we keep them separate to support V2.5: except (IOError, socket.error), e: syslog.syslog(syslog.LOG_ERR, "restful: Unable to publish record %s to %s station %s" % (time_str, station.site, station.station)) + syslog.syslog(syslog.LOG_ERR, " **** %s" % e) if hasattr(e, 'reason'): syslog.syslog(syslog.LOG_ERR, " **** Failed to reach server. Reason: %s" % e.reason) if hasattr(e, 'code'): diff --git a/docs/customizing.htm b/docs/customizing.htm index 502f8525..2cc0d945 100644 --- a/docs/customizing.htm +++ b/docs/customizing.htm @@ -1038,7 +1038,8 @@ them).

If set, the NOAA radar image will be displayed. If commented out, no image will be displayed.

googleAnalyticsId

-

If you have a Google Analytics ID, you can set it here. The Google Javascript +

If you have a Google Analytics ID, you can set it here. The Google +Analytics Javascript code will then be included, enabling analytics of your website usage. If commented out, the code will not be included.

Extending [Extras]

diff --git a/docs/readme.htm b/docs/readme.htm index 59c343dd..ba2faa7b 100644 --- a/docs/readme.htm +++ b/docs/readme.htm @@ -566,7 +566,10 @@ FTP'ing data to a web server or sending data to the Weather Underground. Twe

This section covers options relating to the entire weather station setup.

WEEWX_ROOT

Set to the root directory of the weewx data file hierarchy -for this station, nominally '/home/weewx'. This value +for this station, nominally '/home/weewx'. The +weewx data subdirectories skins, +archive, and public_html are expected to be +found here. This value will be set automatically by the setup script setup.py to reflect the choice you made in the configuration file setup.cfg. Required. No default.

diff --git a/setup.py b/setup.py index a135ed1a..b3b8a2cb 100755 --- a/setup.py +++ b/setup.py @@ -60,10 +60,10 @@ from distutils.command.install_data import install_data from distutils.command.install_lib import install_lib from distutils.command.sdist import sdist +# Make sure we can find the bin subdirectory: bin_dir = os.path.join(os.getcwd(), 'bin') -print bin_dir sys.path.insert(0, bin_dir) -print sys.path + from weewx import __version__ as VERSION class My_install_lib(install_lib): diff --git a/skins/Standard/index.html.tmpl b/skins/Standard/index.html.tmpl index f53e3339..e1b71fa4 100644 --- a/skins/Standard/index.html.tmpl +++ b/skins/Standard/index.html.tmpl @@ -360,19 +360,19 @@ - ## Include the Google Analytics code if the user has supplied an ID: + ## Include the Google Analytics code if the user has supplied an ID: #if $Extras.has_key('googleAnalyticsId') - - - #end if + + + #end if diff --git a/skins/Standard/month.html.tmpl b/skins/Standard/month.html.tmpl index 40d4f2c9..1c3beed9 100644 --- a/skins/Standard/month.html.tmpl +++ b/skins/Standard/month.html.tmpl @@ -414,19 +414,19 @@ - ## Include the Google Analytics code if the user has supplied an ID: + ## Include the Google Analytics code if the user has supplied an ID: #if $Extras.has_key('googleAnalyticsId') - - - #end if + + + #end if diff --git a/skins/Standard/week.html.tmpl b/skins/Standard/week.html.tmpl index b4a7eee1..65b10de2 100644 --- a/skins/Standard/week.html.tmpl +++ b/skins/Standard/week.html.tmpl @@ -412,19 +412,19 @@ - ## Include the Google Analytics code if the user has supplied an ID: + ## Include the Google Analytics code if the user has supplied an ID: #if $Extras.has_key('googleAnalyticsId') - - - #end if + + + #end if diff --git a/skins/Standard/year.html.tmpl b/skins/Standard/year.html.tmpl index 30bb0b72..d6936722 100644 --- a/skins/Standard/year.html.tmpl +++ b/skins/Standard/year.html.tmpl @@ -263,19 +263,19 @@ - ## Include the Google Analytics code if the user has supplied an ID: + ## Include the Google Analytics code if the user has supplied an ID: #if $Extras.has_key('googleAnalyticsId') - - - #end if + + + #end if diff --git a/weewx.conf b/weewx.conf index 11f42320..78127d47 100644 --- a/weewx.conf +++ b/weewx.conf @@ -29,7 +29,7 @@ debug = 0 socket_timeout = 20 # Current version -version = 1.7.0rc1 +version = 1.7.0rc2 ############################################################################################ @@ -202,7 +202,7 @@ version = 1.7.0rc1 # What skin this report should be based on: skin = Standard # You can override values in the skin configuration file. - # Here, as an example, we override what unit to use for pressure measurements. + # Here, as an example, we override what unit to use for pressure measurements: [[[Units]]] [[[[Groups]]]] group_pressure=mbar