mirror of
https://github.com/weewx/weewx.git
synced 2026-04-18 16:46:56 -04:00
305 lines
12 KiB
Python
305 lines
12 KiB
Python
#
|
|
# Copyright (c) 2009-2015 Tom Keffer <tkeffer@gmail.com>
|
|
#
|
|
# See the file LICENSE.txt for your full rights.
|
|
#
|
|
"""For uploading files to a remove server via FTP"""
|
|
|
|
from __future__ import absolute_import
|
|
from __future__ import print_function
|
|
from __future__ import with_statement
|
|
|
|
import ftplib
|
|
import logging
|
|
import os
|
|
import sys
|
|
import time
|
|
|
|
from six.moves import cPickle
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
class FtpUpload(object):
|
|
"""Uploads a directory and all its descendants to a remote server.
|
|
|
|
Keeps track of when a file was last uploaded, so it is uploaded only
|
|
if its modification time is newer."""
|
|
|
|
def __init__(self, server,
|
|
user, password,
|
|
local_root, remote_root,
|
|
port=21,
|
|
name="FTP",
|
|
passive=True,
|
|
max_tries=3,
|
|
secure=False,
|
|
debug=0,
|
|
secure_data=True):
|
|
"""Initialize an instance of FtpUpload.
|
|
|
|
After initializing, call method run() to perform the upload.
|
|
|
|
server: The remote server to which the files are to be uploaded.
|
|
|
|
user,
|
|
password : The user name and password that are to be used.
|
|
|
|
name: A unique name to be given for this FTP session. This allows more
|
|
than one session to be uploading from the same local directory. [Optional.
|
|
Default is 'FTP'.]
|
|
|
|
passive: True to use passive mode; False to use active mode. [Optional.
|
|
Default is True (passive mode)]
|
|
|
|
max_tries: How many times to try creating a directory or uploading
|
|
a file before giving up [Optional. Default is 3]
|
|
|
|
secure: Set to True to attempt an FTP over TLS (FTPS) session.
|
|
|
|
debug: Set to 1 for extra debug information, 0 otherwise.
|
|
|
|
secure_data: If a secure session is requested (option secure=True),
|
|
should we attempt a secure data connection as well? This option is useful
|
|
due to a bug in the Python FTP client library. See Issue #284.
|
|
[Optional. Default is True]
|
|
"""
|
|
self.server = server
|
|
self.user = user
|
|
self.password = password
|
|
self.local_root = os.path.normpath(local_root)
|
|
self.remote_root = os.path.normpath(remote_root)
|
|
self.port = port
|
|
self.name = name
|
|
self.passive = passive
|
|
self.max_tries = max_tries
|
|
self.secure = secure
|
|
self.debug = debug
|
|
self.secure_data = secure_data
|
|
|
|
def run(self):
|
|
"""Perform the actual upload.
|
|
|
|
returns: the number of files uploaded."""
|
|
|
|
if self.secure:
|
|
try:
|
|
FTPClass = ftplib.FTP_TLS
|
|
except AttributeError:
|
|
FTPClass = ftplib.FTP
|
|
log.debug("Your version of Python does not support FTPS. Using insecure connection.")
|
|
self.secure = False
|
|
else:
|
|
FTPClass = ftplib.FTP
|
|
|
|
# Get the timestamp and members of the last upload:
|
|
(timestamp, fileset) = self.getLastUpload()
|
|
|
|
n_uploaded = 0
|
|
# Try to connect to the ftp server up to max_tries times:
|
|
|
|
try:
|
|
if self.secure:
|
|
log.debug("Attempting secure connection to %s" % self.server)
|
|
else:
|
|
log.debug("Attempting connection to %s" % self.server)
|
|
for count in range(self.max_tries):
|
|
try:
|
|
ftp_server = FTPClass()
|
|
ftp_server.connect(self.server, self.port)
|
|
|
|
if self.debug:
|
|
ftp_server.set_debuglevel(self.debug)
|
|
|
|
ftp_server.login(self.user, self.password)
|
|
ftp_server.set_pasv(self.passive)
|
|
if self.secure and self.secure_data:
|
|
ftp_server.prot_p()
|
|
log.debug("Secure data connection to %s" % self.server)
|
|
else:
|
|
log.debug("Connected to %s" % self.server)
|
|
break
|
|
except ftplib.all_errors as e:
|
|
log.error("Unable to connect or log into server : %s" % e)
|
|
else:
|
|
# This is executed only if the loop terminates naturally (without a break statement),
|
|
# meaning the ftp connection failed max_tries times. Abandon ftp upload
|
|
log.error("Attempted %d times to connect to server %s. Giving up."
|
|
% (self.max_tries, self.server))
|
|
return n_uploaded
|
|
|
|
# Walk the local directory structure
|
|
for (dirpath, unused_dirnames, filenames) in os.walk(self.local_root):
|
|
|
|
# Strip out the common local root directory. What is left
|
|
# will be the relative directory both locally and remotely.
|
|
local_rel_dir_path = dirpath.replace(self.local_root, '.')
|
|
if self._skipThisDir(local_rel_dir_path):
|
|
continue
|
|
# This is the absolute path to the remote directory:
|
|
remote_dir_path = os.path.normpath(os.path.join(self.remote_root, local_rel_dir_path))
|
|
|
|
# Make the remote directory if necessary:
|
|
self._make_remote_dir(ftp_server, remote_dir_path)
|
|
|
|
# Now iterate over all members of the local directory:
|
|
for filename in filenames:
|
|
|
|
full_local_path = os.path.join(dirpath, filename)
|
|
# See if this file can be skipped:
|
|
if self._skipThisFile(timestamp, fileset, full_local_path):
|
|
continue
|
|
|
|
full_remote_path = os.path.join(remote_dir_path, filename)
|
|
STOR_cmd = "STOR %s" % full_remote_path
|
|
# Retry up to max_tries times:
|
|
for count in range(self.max_tries):
|
|
# If we have to retry, we should probably reopen the file as well.
|
|
# Hence, the open is in the inner loop:
|
|
with open(full_local_path, 'rb') as fd:
|
|
try:
|
|
ftp_server.storbinary(STOR_cmd, fd)
|
|
except ftplib.all_errors as e:
|
|
# Unsuccessful. Log it and go around again.
|
|
log.error("Attempt #%d. Failed uploading %s to %s. Reason: %s"
|
|
% (count + 1, full_remote_path, self.server, e))
|
|
ftp_server.set_pasv(self.passive)
|
|
else:
|
|
# Success. Log it, break out of the loop
|
|
n_uploaded += 1
|
|
fileset.add(full_local_path)
|
|
log.debug("Uploaded file %s" % full_remote_path)
|
|
break
|
|
else:
|
|
# This is executed only if the loop terminates naturally (without a break statement),
|
|
# meaning the upload failed max_tries times. Log it, move on to the next file.
|
|
log.error("Failed to upload file %s" % full_remote_path)
|
|
finally:
|
|
try:
|
|
ftp_server.quit()
|
|
except:
|
|
pass
|
|
|
|
timestamp = time.time()
|
|
self.saveLastUpload(timestamp, fileset)
|
|
return n_uploaded
|
|
|
|
def getLastUpload(self):
|
|
"""Reads the time and members of the last upload from the local root"""
|
|
|
|
timeStampFile = os.path.join(self.local_root, "#%s.last" % self.name)
|
|
|
|
# If the file does not exist, an IOError exception will be raised.
|
|
# If the file exists, but is truncated, an EOFError will be raised.
|
|
# Either way, be prepared to catch it.
|
|
try:
|
|
with open(timeStampFile, "rb") as f:
|
|
timestamp = cPickle.load(f)
|
|
fileset = cPickle.load(f)
|
|
except (IOError, EOFError, cPickle.PickleError, AttributeError):
|
|
timestamp = 0
|
|
fileset = set()
|
|
# Either the file does not exist, or it is garbled.
|
|
# Either way, it's safe to remove it.
|
|
try:
|
|
os.remove(timeStampFile)
|
|
except OSError:
|
|
pass
|
|
|
|
return (timestamp, fileset)
|
|
|
|
def saveLastUpload(self, timestamp, fileset):
|
|
"""Saves the time and members of the last upload in the local root."""
|
|
timeStampFile = os.path.join(self.local_root, "#%s.last" % self.name)
|
|
with open(timeStampFile, "wb") as f:
|
|
cPickle.dump(timestamp, f)
|
|
cPickle.dump(fileset, f)
|
|
|
|
def _make_remote_dir(self, ftp_server, remote_dir_path):
|
|
"""Make a remote directory if necessary."""
|
|
# Try to make the remote directory up max_tries times, then give up.
|
|
for unused_count in range(self.max_tries):
|
|
try:
|
|
ftp_server.mkd(remote_dir_path)
|
|
except ftplib.all_errors as e:
|
|
# Got an exception. It might be because the remote directory already exists:
|
|
if sys.exc_info()[0] is ftplib.error_perm:
|
|
msg = str(e).strip()
|
|
# If a directory already exists, some servers respond with a '550' ("Requested action not taken") code,
|
|
# others with a '521' ("Access denied" or "Pathname already exists") code.
|
|
if msg.startswith('550') or msg.startswith('521'):
|
|
# Directory already exists
|
|
return
|
|
log.error("Got error while attempting to make remote directory %s" % remote_dir_path)
|
|
log.error(" **** Error: %s" % e)
|
|
else:
|
|
log.debug("Made directory %s" % remote_dir_path)
|
|
return
|
|
else:
|
|
log.error("Unable to create remote directory %s" % remote_dir_path)
|
|
raise IOError("Unable to create remote directory %s" % remote_dir_path)
|
|
|
|
def _skipThisDir(self, local_dir):
|
|
|
|
return os.path.basename(local_dir) in ('.svn', 'CVS')
|
|
|
|
def _skipThisFile(self, timestamp, fileset, full_local_path):
|
|
|
|
filename = os.path.basename(full_local_path)
|
|
if filename[-1] == '~' or filename[0] == '#':
|
|
return True
|
|
|
|
if full_local_path not in fileset:
|
|
return False
|
|
|
|
if os.stat(full_local_path).st_mtime > timestamp:
|
|
return False
|
|
|
|
# Filename is in the set, and is up to date.
|
|
return True
|
|
|
|
|
|
if __name__ == '__main__':
|
|
import socket
|
|
|
|
import configobj
|
|
|
|
import weewx
|
|
import weeutil.logger
|
|
|
|
weewx.debug = 1
|
|
|
|
weeutil.logger.setup('ftpupload', {})
|
|
|
|
if len(sys.argv) < 2:
|
|
print("""Usage: ftpupload.py path-to-configuration-file [path-to-be-ftp'd]""")
|
|
sys.exit(weewx.CMD_ERROR)
|
|
|
|
try:
|
|
config_dict = configobj.ConfigObj(sys.argv[1], file_error=True, encoding='utf-8')
|
|
except IOError:
|
|
print("Unable to open configuration file %s" % sys.argv[1])
|
|
raise
|
|
|
|
if len(sys.argv) == 2:
|
|
try:
|
|
ftp_dir = os.path.join(config_dict['WEEWX_ROOT'],
|
|
config_dict['StdReport']['HTML_ROOT'])
|
|
except KeyError:
|
|
print("No HTML_ROOT in configuration dictionary.")
|
|
sys.exit(1)
|
|
else:
|
|
ftp_dir = sys.argv[2]
|
|
|
|
socket.setdefaulttimeout(10)
|
|
|
|
ftp_upload = FtpUpload(config_dict['StdReport']['FTP']['server'],
|
|
config_dict['StdReport']['FTP']['user'],
|
|
config_dict['StdReport']['FTP']['password'],
|
|
ftp_dir,
|
|
config_dict['StdReport']['FTP']['path'],
|
|
'FTP',
|
|
config_dict['StdReport']['FTP'].as_bool('passive'),
|
|
config_dict['StdReport']['FTP'].as_int('max_tries'))
|
|
ftp_upload.run()
|