From da9ca8a1f40bfc864d77d0dfdfeaa5280804074e Mon Sep 17 00:00:00 2001 From: Jokob-sk Date: Sun, 5 Feb 2023 15:24:46 +1100 Subject: [PATCH] Plugins 0.1 - Website monitoring cleanup --- .gitignore | 1 + docker-compose.yml | 1 + front/plugins/README.md | 4 +- front/plugins/website_monitoring/config.json | 17 ++ .../website_monitoring/last_result.log | 4 +- front/plugins/website_monitoring/script.log | 13 + front/plugins/website_monitoring/script.py | 255 +++--------------- 7 files changed, 79 insertions(+), 216 deletions(-) diff --git a/.gitignore b/.gitignore index 8d580837..ea9c9e15 100755 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,6 @@ config/pialert.conf db/* front/log/* +front/plugins/**/*.log **/%40eaDir/ **/@eaDir/ diff --git a/docker-compose.yml b/docker-compose.yml index b5750732..53d149bc 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,6 +31,7 @@ services: - ${DEV_LOCATION}/front/network.php:/home/pi/pialert/front/network.php - ${DEV_LOCATION}/front/presence.php:/home/pi/pialert/front/presence.php - ${DEV_LOCATION}/front/settings.php:/home/pi/pialert/front/settings.php + - ${DEV_LOCATION}/front/plugins:/home/pi/pialert/front/plugins # DELETE END anyone trying to use this file: comment out / delete ABOVE lines, they are only for development purposes environment: - TZ=${TZ} diff --git a/front/plugins/README.md b/front/plugins/README.md index 4a83c1a0..046e58ad 100755 --- a/front/plugins/README.md +++ b/front/plugins/README.md @@ -10,7 +10,7 @@ If you wish to develop a plugin, please check the existing plugin structure. |----------------------|----------------------|----------------------| | `config.json` | yes | Contains the plugin configuration including the settings available to the user. | | `script.py` | yes | The Python script itself | - | `last_result.log` | yes | The file used to interface between PiAlert and the plugin (script). Should contain a set of testing data. | + | `last_result.log` | yes | The file used to interface between PiAlert and the plugin (script). | | `script.log` | no | Logging output (recommended) | | `README.md` | no | Amy setup considerations or overview | @@ -30,7 +30,7 @@ Used to interface between PiAlert and the plugin (script). After every scan it s | Order | Represented Column | Required | Description | |----------------------|----------------------|----------------------|----------------------| - | 0 | `Object_PrimaryID` | yes | The primary ID used to group Events under. Should be UNIQUE in the context of the last result (so in `last_result.log`) | + | 0 | `Object_PrimaryID` | yes | The primary ID used to group Events under. | | 1 | `Object_SecondaryID` | no | Optionalsecondary ID to create a relationship beween other entities, such as a MAC address | | 2 | `DateTime` | yes | When the event occured in the format `2023-01-02 15:56:30` | | 3 | `Watched_Value1` | yes | A value that is watched and users can receive notifications if it changed compared to the previously saved entry. For example IP address | diff --git a/front/plugins/website_monitoring/config.json b/front/plugins/website_monitoring/config.json index 42131112..94edafa0 100755 --- a/front/plugins/website_monitoring/config.json +++ b/front/plugins/website_monitoring/config.json @@ -38,10 +38,24 @@ "en_us" : "Enable a regular scan of your services. If you select schedule the scheduling settings from below are applied. If you select once the scan is run only once on start of the application (container) for the time specified in WEBMON_TIMEOUT setting." } + }, + { + "type": "FORCE_REPORT", + "default_value": false, + "options": [], + "name" : { + "en_us" : "Schedule" + }, + "description": + { + "en_us" : "Enable a regular scan of your services. If you select schedule the scheduling settings from below are applied. If you select once the scan is run only once on start of the application (container) for the time specified in WEBMON_TIMEOUT setting." + } + }, { "type": "RUN_SCHD", "default_value":"0 2 * * *", + "options": [], "name" : { "en_us" : "Schedule" }, @@ -54,6 +68,7 @@ { "type": "API_SQL", "default_value":"SELECT * FROM plugin_website_monitor", + "options": [], "name" : { "en_us" : "API endpoint" }, @@ -66,6 +81,7 @@ { "type": "TIMEOUT", "default_value":5, + "options": [], "name" : { "en_us" : "Run timeout" }, @@ -90,6 +106,7 @@ { "type": "ARGS", "default_value":"", + "options": [], "name" : { "en_us" : "Run timeout" }, diff --git a/front/plugins/website_monitoring/last_result.log b/front/plugins/website_monitoring/last_result.log index 9bcdbff9..d3e80925 100755 --- a/front/plugins/website_monitoring/last_result.log +++ b/front/plugins/website_monitoring/last_result.log @@ -1,2 +1,2 @@ -https://www.google.com|null|2023-01-02 15:56:30|404|0.7898|null|null|null -https://www.duckduckgo.com|192.168.0.1|2023-01-02 15:56:30|200|0.9898|null|null|Best search engine \ No newline at end of file +http://google.com|null|2023-02-05 15:23:04|200|0.407871|null|null|null +http://bing.com|null|2023-02-05 15:23:04|200|0.196052|null|null|null diff --git a/front/plugins/website_monitoring/script.log b/front/plugins/website_monitoring/script.log index e69de29b..2f3baa62 100755 --- a/front/plugins/website_monitoring/script.log +++ b/front/plugins/website_monitoring/script.log @@ -0,0 +1,13 @@ +Pi.Alert [Prototype]: +--------------------------------------------------------- +Current User: root + +Monitor Web-Services +Timestamp: 2023-02-05 15:23:03 + +Start Services Monitoring + +| Timestamp | URL | StatusCode | ResponseTime | +----------------------------------------------- +2023-02-05 15:23:04 | http://google.com | 200 | 0.407871 +2023-02-05 15:23:04 | http://bing.com | 200 | 0.196052 diff --git a/front/plugins/website_monitoring/script.py b/front/plugins/website_monitoring/script.py index 59b904de..d5f42de3 100755 --- a/front/plugins/website_monitoring/script.py +++ b/front/plugins/website_monitoring/script.py @@ -1,174 +1,71 @@ #!/usr/bin/env python +# Based on the work of https://github.com/leiweibau/Pi.Alert + +# /home/pi/pialert/front/plugins/website_monitoring/script.py urls=http://google.com,http://bing.com from __future__ import unicode_literals from time import sleep, time, strftime import requests +import pathlib + +import argparse import io #import smtplib import sys #from smtp_config import sender, password, receivers, host, port from requests.packages.urllib3.exceptions import InsecureRequestWarning -import sqlite3 + import pwd import os -con = sqlite3.connect("monitoring.db") -cur = con.cursor() +curPath = str(pathlib.Path(__file__).parent.resolve()) +log_file = curPath + '/script.log' +last_run = curPath + '/last_result.log' -#DELAY = 60 # Delay between site queries -EMAIL_INTERVAL = 1800 # Delay between alert emails - -last_email_time = {} # Monitored sites and timestamp of last alert sent - -# Message template for alert -MESSAGE = """From: {sender} -To: {receivers} -Subject: Monitor Service Notification - -You are being notified that {site} is experiencing a {status} status! -""" +print(last_run) # Workflow -def main(): - global cur - global con +def main(): + parser = argparse.ArgumentParser(description='Simple URL monitoring tool') + parser.add_argument('urls', action="store", help="urls to check separated by ','") + values = parser.parse_args() - prepare_service_monitoring_env() - service_monitoring() - print_service_monitoring_changes() - # prepare_service_monitoring_notification() - -# ----------------------------------------------------------------------------- -def prepare_service_monitoring_env (): - global con - global cur - - sql_create_table = """ CREATE TABLE IF NOT EXISTS Services_Events( - moneve_URL TEXT NOT NULL, - moneve_DateTime TEXT NOT NULL, - moneve_StatusCode NUMERIC NOT NULL, - moneve_Latency TEXT NOT NULL - ); """ - cur.execute(sql_create_table) - - sql_create_table = """ CREATE TABLE IF NOT EXISTS Services_CurrentScan( - cur_URL TEXT NOT NULL, - cur_DateTime TEXT NOT NULL, - cur_StatusCode NUMERIC NOT NULL, - cur_Latency TEXT NOT NULL, - cur_AlertEvents INTEGER DEFAULT 0, - cur_AlertDown INTEGER DEFAULT 0, - cur_StatusChanged INTEGER DEFAULT 0, - cur_LatencyChanged INTEGER DEFAULT 0 - ); """ - cur.execute(sql_create_table) - - sql_create_table = """ CREATE TABLE IF NOT EXISTS Services( - mon_URL TEXT NOT NULL, - mon_MAC TEXT, - mon_LastStatus NUMERIC NOT NULL, - mon_LastLatency TEXT NOT NULL, - mon_LastScan TEXT NOT NULL, - mon_Tags TEXT, - mon_AlertEvents INTEGER DEFAULT 0, - mon_AlertDown INTEGER DEFAULT 0, - PRIMARY KEY(mon_URL) - ); """ - cur.execute(sql_create_table) - -# Update Service with lastLatence, lastScan and lastStatus -# ----------------------------------------------------------------------------- -def set_service_update(_mon_URL, _mon_lastScan, _mon_lastStatus, _mon_lastLatence,): - global con - global cur - - sqlite_insert = """UPDATE Services SET mon_LastScan=?, mon_LastStatus=?, mon_LastLatency=? WHERE mon_URL=?;""" - - table_data = (_mon_lastScan, _mon_lastStatus, _mon_lastLatence, _mon_URL) - cur.execute(sqlite_insert, table_data) - con.commit() - -# Insert Services_Events with moneve_URL, moneve_DateTime, moneve_StatusCode and moneve_Latency -# ----------------------------------------------------------------------------- -def set_services_events(_moneve_URL, _moneve_DateTime, _moneve_StatusCode, _moneve_Latency): - global con - global cur - - sqlite_insert = """INSERT INTO Services_Events - (moneve_URL, moneve_DateTime, moneve_StatusCode, moneve_Latency) - VALUES (?, ?, ?, ?);""" - - table_data = (_moneve_URL, _moneve_DateTime, _moneve_StatusCode, _moneve_Latency) - cur.execute(sqlite_insert, table_data) - con.commit() - -# Insert Services_Events with moneve_URL, moneve_DateTime, moneve_StatusCode and moneve_Latency -# ----------------------------------------------------------------------------- -def set_services_current_scan(_cur_URL, _cur_DateTime, _cur_StatusCode, _cur_Latency): - global con - global cur - - cur.execute("SELECT * FROM Services WHERE mon_URL = ?", [_cur_URL]) - rows = cur.fetchall() - for row in rows: - _mon_AlertEvents = row[6] - _mon_AlertDown = row[7] - _mon_StatusCode = row[2] - _mon_Latency = row[3] - - if _mon_StatusCode != _cur_StatusCode: - _cur_StatusChanged = 1 + if values.urls: + with open(last_run, 'w') as last_run_logfile: + # empty file + last_run_logfile.write("") + service_monitoring(values.urls.split('=')[1].split(',')) else: - _cur_StatusChanged = 0 - - if _mon_Latency == "99999" and _mon_Latency != _cur_Latency: - _cur_LatencyChanged = 1 - elif _cur_Latency == "99999" and _mon_Latency != _cur_Latency: - _cur_LatencyChanged = 1 - else: - _cur_LatencyChanged = 0 - - sqlite_insert = """INSERT INTO Services_CurrentScan - (cur_URL, cur_DateTime, cur_StatusCode, cur_Latency, cur_AlertEvents, cur_AlertDown, cur_StatusChanged, cur_LatencyChanged) - VALUES (?, ?, ?, ?, ?, ?, ?, ?);""" - - table_data = (_cur_URL, _cur_DateTime, _cur_StatusCode, _cur_Latency, _mon_AlertEvents, _mon_AlertDown, _cur_StatusChanged, _cur_LatencyChanged) - cur.execute(sqlite_insert, table_data) - con.commit() + return # ----------------------------------------------------------------------------- def service_monitoring_log(site, status, latency): # global monitor_logfile # Log status message to log file - with open('monitor.log', 'a') as monitor_logfile: + with open(log_file, 'a') as monitor_logfile: monitor_logfile.write("{} | {} | {} | {}\n".format(strftime("%Y-%m-%d %H:%M:%S"), site, status, latency, ) ) - -# ----------------------------------------------------------------------------- -def send_alert(site, status): - """If more than EMAIL_INTERVAL seconds since last email, resend email""" - if (time() - last_email_time[site]) > EMAIL_INTERVAL: - try: - smtpObj = smtplib.SMTP(host, port) # Set up SMTP object - smtpObj.starttls() - smtpObj.login(sender, password) - smtpObj.sendmail(sender, - receivers, - MESSAGE.format(sender=sender, - receivers=", ".join(receivers), - site=site, - status=status - ) + with open(last_run, 'a') as last_run_logfile: + # https://www.duckduckgo.com|192.168.0.1|2023-01-02 15:56:30|200|0.9898|null|null|Best search engine + last_run_logfile.write("{}|{}|{}|{}|{}|{}|{}|{}\n".format( + site, + 'null', + strftime("%Y-%m-%d %H:%M:%S"), + status, + latency, + 'null', + 'null', + 'null', + ) ) - last_email_time[site] = time() # Update time of last email - print("Successfully sent email") - except smtplib.SMTPException: - print("Error sending email ({}:{})".format(host, port)) + + + # ----------------------------------------------------------------------------- def check_services_health(site): @@ -190,73 +87,19 @@ def check_services_health(site): latency = "99999" return 503, latency -# ----------------------------------------------------------------------------- Duplicat +# ----------------------------------------------------------------------------- def get_username(): return pwd.getpwuid(os.getuid())[0] -# ----------------------------------------------------------------------------- -def get_services_list(): - global cur - global con - - with open('monitor.log', 'a') as monitor_logfile: - monitor_logfile.write("... Get Services List\n") - monitor_logfile.close() - - cur.execute("SELECT mon_URL FROM Services") - rows = cur.fetchall() - - sites = [] - for row in rows: - sites.append(row[0]) - - return sites # ----------------------------------------------------------------------------- -def flush_services_current_scan(): - global cur - global con - - with open('monitor.log', 'a') as monitor_logfile: - monitor_logfile.write("... Flush previous scan results\n") - monitor_logfile.close() - - cur.execute("DELETE FROM Services_CurrentScan") - con.commit() - -# ----------------------------------------------------------------------------- -def print_service_monitoring_changes(): - global cur - global con - - print("Services Monitoring Changes") - changedStatusCode = cur.execute("SELECT COUNT() FROM Services_CurrentScan WHERE cur_StatusChanged = 1").fetchone()[0] - print("... Changed StatusCodes: ", str(changedStatusCode)) - changedLatency = cur.execute("SELECT COUNT() FROM Services_CurrentScan WHERE cur_LatencyChanged = 1").fetchone()[0] - print("... Changed Reachability: ", str(changedLatency)) - with open('monitor.log', 'a') as monitor_logfile: - monitor_logfile.write("\nServices Monitoring Changes:\n") - monitor_logfile.write("... Changed StatusCodes: " + str(changedStatusCode)) - monitor_logfile.write("\n... Changed Reachability: " + str(changedLatency)) - monitor_logfile.write("\n") - monitor_logfile.close() - -# ----------------------------------------------------------------------------- -# def prepare_service_monitoring_notification(): -# global cur -# global con - - -# ----------------------------------------------------------------------------- -def service_monitoring(): - global cur - global con +def service_monitoring(urls): # Empty Log and write new header print("Prepare Services Monitoring") print("... Prepare Logfile") - with open('monitor.log', 'w') as monitor_logfile: + with open(log_file, 'w') as monitor_logfile: monitor_logfile.write("Pi.Alert [Prototype]:\n---------------------------------------------------------\n") monitor_logfile.write("Current User: %s \n\n" % get_username()) monitor_logfile.write("Monitor Web-Services\n") @@ -264,19 +107,14 @@ def service_monitoring(): monitor_logfile.close() print("... Get Services List") - sites = get_services_list() + sites = urls - print("... Flush previous scan results") - flush_services_current_scan() print("Start Services Monitoring") - with open('monitor.log', 'a') as monitor_logfile: + with open(log_file, 'a') as monitor_logfile: monitor_logfile.write("\nStart Services Monitoring\n\n| Timestamp | URL | StatusCode | ResponseTime |\n-----------------------------------------------\n") monitor_logfile.close() - for site in sites: - last_email_time[site] = 0 # Initialize timestamp as 0 - while sites: for site in sites: status,latency = check_services_health(site) @@ -291,20 +129,13 @@ def service_monitoring(): # Write Logfile service_monitoring_log(site, status, latency) - # Insert Services_Events with moneve_URL, moneve_DateTime, moneve_StatusCode and moneve_Latency - set_services_events(site, scantime, status, latency) - # Insert Services_CurrentScan with moneve_URL, moneve_DateTime, moneve_StatusCode and moneve_Latency - set_services_current_scan(site, scantime, status, latency) sys.stdout.flush() - # Update Service with lastLatence, lastScan and lastStatus after compare with services_current_scan - set_service_update(site, scantime, status, latency) - break else: - with open('monitor.log', 'a') as monitor_logfile: + with open(log_file, 'a') as monitor_logfile: monitor_logfile.write("\n\nNo site(s) to monitor!") monitor_logfile.close()