V1.7.0rc2

Improved error checking and logging in FTP and RESTful uploads.
2026-04-18 00:26:57 -04:00 · 2010-04-12 19:28:52 +00:00
parent 2ad1dc98bb
commit e235198614
11 changed files with 91 additions and 72 deletions
--- a/bin/weeutil/ftpupload.py
+++ b/bin/weeutil/ftpupload.py
@@ -15,7 +15,6 @@ import ftplib
 import cPickle
 import time
 import syslog
-import socket

 class FtpUpload:
    """Uploads a directory and all its descendants to a remote server.
@@ -96,26 +95,37 @@ class FtpUpload:
    
                    full_remote_path = os.path.join(remote_dir_path, filename)
                    STOR_cmd = "STOR %s" % full_remote_path
-                    fd = open(full_local_path, "r")
+                    # Retry up to max_tries times:
                    for count in range(self.max_tries):
                        try:
+                            # If we have to retry, we should probably reopen the file as well.
+                            # Hence, the open is in the inner loop:
+                            fd = open(full_local_path, "r")
                            ftp_server.storbinary(STOR_cmd, fd)
-                        except ftplib.all_errors, e:
+                        except (ftplib.all_errors, IOError), e:
+                            # Unsuccessful. Log it and go around again.
                            syslog.syslog(syslog.LOG_ERR, "ftpupload: attempt #%d. Failed uploading %s. Reason: %s" % (count+1, full_remote_path, e))
-                            if count >= self.max_tries -1 :
-                                syslog.syslog(syslog.LOG_ERR, "ftpupload: Failed to upload file %s" % full_remote_path)
-                                raise
                            ftp_server.set_pasv(self.passive)
                        else:
-                            fd.close()
+                            # Success. Log it, break out of the loop
                            n_uploaded += 1
+                            fileset.add(full_local_path)
                            syslog.syslog(syslog.LOG_DEBUG, "ftpupload: Uploaded file %s" % full_remote_path)
                            break
-                    fileset.add(full_local_path)
+                        finally:
+                            # This is always executed on every loop. Close the file.
+                            try:
+                                fd.close()
+                            except:
+                                pass
+                    else:
+                        # This is executed only if the loop terminates naturally (without a break statement),
+                        # meaning the upload failed max_tries times. Log it, move on to the next file.
+                        syslog.syslog(syslog.LOG_ERR, "ftpupload: Failed to upload file %s" % full_remote_path)
        finally:
            try:
                ftp_server.quit()
-            except socket.error:
+            except:
                pass
        
        timestamp = time.time()
@@ -194,7 +204,7 @@ if __name__ == '__main__':
    import configobj
    
    weewx.debug = 1
-    syslog.openlog('reportengine', syslog.LOG_PID|syslog.LOG_CONS)
+    syslog.openlog('ftpupload', syslog.LOG_PID|syslog.LOG_CONS)
    syslog.setlogmask(syslog.LOG_UPTO(syslog.LOG_DEBUG))

    if len(sys.argv) < 2 :
--- a/bin/weewx/init.py
+++ b/bin/weewx/init.py
@@ -12,7 +12,7 @@
 """
 import time

-__version__="1.7.0rc1"
+__version__="1.7.0rc2"

 # Holds the program launch time in unix epoch seconds:
 # Useful for calculating 'uptime.'
--- a/bin/weewx/restful.py
+++ b/bin/weewx/restful.py
@@ -64,23 +64,27 @@ class RESTful(object):
            try:
                _response = urllib2.urlopen(_url)
            except (urllib2.URLError, socket.error), e:
+                # Unsuccessful. Log it and go around again for another try
                syslog.syslog(syslog.LOG_ERR, "restful: Failed attempt #%d to upload to %s" % (_count+1, self.site))
                syslog.syslog(syslog.LOG_ERR, "   ****  Reason: %s" % (e,))
-                if _count >= self.max_tries -1 :
-                    syslog.syslog(syslog.LOG_ERR, "restful: Failed to upload to %s" % self.site)
-                    raise
            else:
-                # A bad station ID or password will not throw an exception, but
-                # it will have the error encoded in the return message:
+                # No exception thrown, but we're still not done.
+                # We have to also check for a bad station ID or password.
+                # It will have the error encoded in the return message:
                for line in _response:
                    # PWSweather signals with 'ERROR', WU with 'INVALID':
                    if line.startswith('ERROR') or line.startswith('INVALID'):
                        # Bad login. No reason to retry. Log it and raise an exception.
                        syslog.syslog(syslog.LOG_ERR, "restful: %s returns %s. Aborting." % (self.site, line))
                        raise FailedPost, line
-                # If we get here, we have been successful.  Just return.
+                # Does not seem to be an error. We're done.
                return
-    
+        else:
+            # This is executed only if the loop terminates normally, meaning
+            # the upload failed max_tries times. Log it.
+            syslog.syslog(syslog.LOG_ERR, "restful: Failed to upload to %s" % self.site)
+            raise IOError, "Failed ftp upload to site %s after %d tries" % (self.site, self.max_tries)
+
    @staticmethod
    def extractRecordFrom(archive, time_ts):
        """Get a record from the archive database. 
@@ -230,6 +234,7 @@ class RESTThread(threading.Thread):
                # but we keep them separate to support V2.5:
                except (IOError, socket.error), e:
                    syslog.syslog(syslog.LOG_ERR, "restful: Unable to publish record %s to %s station %s" % (time_str, station.site, station.station))
+                    syslog.syslog(syslog.LOG_ERR, "   ****  %s" % e)
                    if hasattr(e, 'reason'):
                        syslog.syslog(syslog.LOG_ERR, "   ****  Failed to reach server. Reason: %s" % e.reason)
                    if hasattr(e, 'code'):
--- a/docs/customizing.htm
+++ b/docs/customizing.htm
@@ -1038,7 +1038,8 @@ them).</p>
 <p>If set, the NOAA radar image will be displayed. If commented out, no image will 
 be displayed.</p>
 <h4 class="config_option">googleAnalyticsId</h4>
-<p>If you have a Google Analytics ID, you can set it here. The Google Javascript 
+<p>If you have a <a href="http://www.google.com/analytics/">Google Analytics ID</a>, you can set it here. The Google 
+Analytics Javascript 
 code will then be included, enabling analytics of your website usage. If 
 commented out, the code will not be included.</p>
 <h4>Extending <span class="code">[Extras]</span></h4>
--- a/docs/readme.htm
+++ b/docs/readme.htm
@@ -566,7 +566,10 @@ FTP&#39;ing data to a web server or sending data to the Weather Underground. Twe
 <p>This section covers options relating to the entire weather station setup. </p>
 <h4 class="config_option">WEEWX_ROOT</h4>
 <p>Set to the root directory of the <span class="code">weewx</span> data file hierarchy 
-for this station, nominally &#39;<span class="code">/home/weewx</span>&#39;. This value 
+for this station, nominally &#39;<span class="code">/home/weewx</span>&#39;. The 
+weewx data subdirectories <span class="code">skins</span>, <span class="code">
+archive</span>, and <span class="code">public_html</span> are expected to be 
+found here. This value 
 will be set automatically by the setup script <span class="code">setup.py</span> 
 to reflect the choice you made in the configuration file <span class="code">setup.cfg</span>. 
 Required. No default.</p>
--- a/setup.py
+++ b/setup.py
@@ -60,10 +60,10 @@ from distutils.command.install_data import install_data
 from distutils.command.install_lib  import install_lib
 from distutils.command.sdist import sdist

+# Make sure we can find the bin subdirectory:
 bin_dir = os.path.join(os.getcwd(), 'bin')
-print bin_dir
 sys.path.insert(0, bin_dir)
-print sys.path  
+
 from weewx import __version__ as VERSION

 class My_install_lib(install_lib):
--- a/skins/Standard/index.html.tmpl
+++ b/skins/Standard/index.html.tmpl
@@ -360,19 +360,19 @@
      </div>
    </div>

-	## Include the Google Analytics code if the user has supplied an ID:
+    ## Include the Google Analytics code if the user has supplied an ID:
    #if $Extras.has_key('googleAnalyticsId')
-	<script type="text/javascript">
-		var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
-		document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
-	</script>
-	<script type="text/javascript">
-		try{
-			var pageTracker = _gat._getTracker("$Extras.googleAnalyticsId");
-			pageTracker._trackPageview();
-		} catch(err) {}
-	</script>
-	#end if
+    <script type="text/javascript">
+        var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
+        document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
+    </script>
+    <script type="text/javascript">
+        try{
+            var pageTracker = _gat._getTracker("$Extras.googleAnalyticsId");
+            pageTracker._trackPageview();
+        } catch(err) {}
+    </script>
+    #end if

 </body>

--- a/skins/Standard/month.html.tmpl
+++ b/skins/Standard/month.html.tmpl
@@ -414,19 +414,19 @@
      </div>
    </div> <!-- End id "container" -->

-	## Include the Google Analytics code if the user has supplied an ID:
+    ## Include the Google Analytics code if the user has supplied an ID:
    #if $Extras.has_key('googleAnalyticsId')
-	<script type="text/javascript">
-		var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
-		document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
-	</script>
-	<script type="text/javascript">
-		try{
-			var pageTracker = _gat._getTracker("$Extras.googleAnalyticsId");
-			pageTracker._trackPageview();
-		} catch(err) {}
-	</script>
-	#end if
+    <script type="text/javascript">
+        var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
+        document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
+    </script>
+    <script type="text/javascript">
+        try{
+            var pageTracker = _gat._getTracker("$Extras.googleAnalyticsId");
+            pageTracker._trackPageview();
+        } catch(err) {}
+    </script>
+    #end if

  </body>
 </html>
--- a/skins/Standard/week.html.tmpl
+++ b/skins/Standard/week.html.tmpl
@@ -412,19 +412,19 @@
      </div>
    </div> <!-- End id "container" -->

-	## Include the Google Analytics code if the user has supplied an ID:
+    ## Include the Google Analytics code if the user has supplied an ID:
    #if $Extras.has_key('googleAnalyticsId')
-	<script type="text/javascript">
-		var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
-		document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
-	</script>
-	<script type="text/javascript">
-		try{
-			var pageTracker = _gat._getTracker("$Extras.googleAnalyticsId");
-			pageTracker._trackPageview();
-		} catch(err) {}
-	</script>
-	#end if
+    <script type="text/javascript">
+        var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
+        document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
+    </script>
+    <script type="text/javascript">
+        try{
+            var pageTracker = _gat._getTracker("$Extras.googleAnalyticsId");
+            pageTracker._trackPageview();
+        } catch(err) {}
+    </script>
+    #end if

  </body>
 </html>
--- a/skins/Standard/year.html.tmpl
+++ b/skins/Standard/year.html.tmpl
@@ -263,19 +263,19 @@
      </div>
    </div> <!-- End id "container" -->

-	## Include the Google Analytics code if the user has supplied an ID:
+    ## Include the Google Analytics code if the user has supplied an ID:
    #if $Extras.has_key('googleAnalyticsId')
-	<script type="text/javascript">
-		var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
-		document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
-	</script>
-	<script type="text/javascript">
-		try{
-			var pageTracker = _gat._getTracker("$Extras.googleAnalyticsId");
-			pageTracker._trackPageview();
-		} catch(err) {}
-	</script>
-	#end if
+    <script type="text/javascript">
+        var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
+        document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
+    </script>
+    <script type="text/javascript">
+        try{
+            var pageTracker = _gat._getTracker("$Extras.googleAnalyticsId");
+            pageTracker._trackPageview();
+        } catch(err) {}
+    </script>
+    #end if

  </body>
 </html>
--- a/weewx.conf
+++ b/weewx.conf
@@ -29,7 +29,7 @@ debug = 0
 socket_timeout = 20

 # Current version
-version = 1.7.0rc1
+version = 1.7.0rc2

 ############################################################################################

@@ -202,7 +202,7 @@ version = 1.7.0rc1
        # What skin this report should be based on:
        skin = Standard
        # You can override values in the skin configuration file.
-        # Here, as an example, we override what unit to use for pressure measurements.
+        # Here, as an example, we override what unit to use for pressure measurements:
        [[[Units]]]
            [[[[Groups]]]]
                group_pressure=mbar