From 7822b11d51c82ba6f832511cb723928e6eaedd22 Mon Sep 17 00:00:00 2001 From: jokob-sk Date: Sat, 8 Nov 2025 14:15:45 +1100 Subject: [PATCH] BE: plugins changed data detection Signed-off-by: jokob-sk --- server/plugin.py | 8 +-- server/scan/device_handling.py | 126 +++++++++++++++++++-------------- server/utils/datetime_utils.py | 40 +++++++++++ 3 files changed, 118 insertions(+), 56 deletions(-) diff --git a/server/plugin.py b/server/plugin.py index a62ba584..81075a3f 100755 --- a/server/plugin.py +++ b/server/plugin.py @@ -28,7 +28,7 @@ class plugin_manager: self.db = db self.all_plugins = all_plugins self.plugin_states = {} - self.name_plugins_checked = None + self.plugin_checks = {} # object cache of settings and schedules for faster lookups self._cache = {} @@ -213,7 +213,7 @@ class plugin_manager: If plugin_name is provided, only calculates stats for that plugin. Structure per plugin: { - "lastChanged": str, + "lastDataChange": str, "totalObjects": int, "newObjects": int, "changedObjects": int, @@ -238,7 +238,7 @@ class plugin_manager: changed_objects = total_objects - new_objects plugin_states[plugin_name] = { - "lastChanged": last_changed or "", + "lastDataChange": last_changed or "", "totalObjects": total_objects or 0, "newObjects": new_objects or 0, "changedObjects": changed_objects or 0, @@ -261,7 +261,7 @@ class plugin_manager: new_objects = new_objects or 0 # ensure it's int changed_objects = total_objects - new_objects plugin_states[plugin] = { - "lastChanged": last_changed or "", + "lastDataChange": last_changed or "", "totalObjects": total_objects or 0, "newObjects": new_objects or 0, "changedObjects": changed_objects or 0, diff --git a/server/scan/device_handling.py b/server/scan/device_handling.py index 76aca0c5..612e888a 100755 --- a/server/scan/device_handling.py +++ b/server/scan/device_handling.py @@ -11,7 +11,7 @@ INSTALL_PATH="/app" sys.path.extend([f"{INSTALL_PATH}/server"]) from helper import get_setting_value, check_IP_format -from utils.datetime_utils import timeNowDB +from utils.datetime_utils import timeNowDB, normalizeTimeStamp from logger import mylog, Logger from const import vendorsPath, vendorsPathNewest, sql_generateGuid from models.device_instance import DeviceInstance @@ -519,64 +519,86 @@ def create_new_devices (db): mylog('debug','[New Devices] New Devices end') db.commitDB() +#------------------------------------------------------------------------------- +# Check if plugins data changed +def check_plugin_data_changed(pm, plugins_to_check): + """ + Checks whether any of the specified plugins have updated data since their + last recorded check time. + This function compares each plugin's `lastDataChange` timestamp from + `pm.plugin_states` with its corresponding `lastDataCheck` timestamp from + `pm.plugin_checks`. If a plugin's data has changed more recently than it + was last checked, it is flagged as changed. + + Args: + pm (object): Plugin manager or state object containing: + - plugin_states (dict): Per-plugin metadata with "lastDataChange". + - plugin_checks (dict): Per-plugin last check timestamps. + plugins_to_check (list[str]): List of plugin names to validate. + + Returns: + bool: True if any plugin data has changed since last check, + otherwise False. + + Logging: + - Logs unexpected or invalid timestamps at level 'none'. + - Logs when no changes are detected at level 'debug'. + - Logs each changed plugin at level 'debug'. + """ + + plugins_changed = [] + + for plugin_name in plugins_to_check: + + last_data_change = pm.plugin_states.get(plugin_name, {}).get("lastDataChange") + last_data_check = pm.plugin_checks.get(plugin_name, "") + + if not last_data_change: + continue + + # Normalize and validate last_changed timestamp + last_changed_ts = normalizeTimeStamp(str(last_data_change)) + + if last_changed_ts == None: + mylog('none', f'[check_plugin_data_changed] Unexpected last_data_change timestamp for {plugin_name}: {last_data_change}') + + # Normalize and validate last_data_check timestamp + last_data_check_ts = normalizeTimeStamp(str(last_data_check)) + + if last_data_check_ts == None: + mylog('none', f'[check_plugin_data_changed] Unexpected last_data_check timestamp for {plugin_name}: {last_data_check}') + + # Track which plugins have newer state than last_checked + if last_data_check_ts is None or last_changed_ts is None or last_changed_ts > last_data_check_ts: + mylog('debug', f'[check_plugin_data_changed] plugin_name changed last_changed_ts | last_data_check_ts: {last_changed_ts} | {last_data_check_ts}') + plugins_changed.append(plugin_name) + + # Skip if no plugin state changed since last check + if len(plugins_changed) == 0: + mylog('debug', f'[check_plugin_data_changed] No relevant plugin changes since last check for {plugins_to_check}') + return False + + # Continue if changes detected + for p in plugins_changed: + mylog('debug', f'[check_plugin_data_changed] {p} changed (last_data_change|last_data_check): ({pm.plugin_states.get(p, {}).get("lastDataChange")}|{pm.plugin_checks.get(p)})') + + return True #------------------------------------------------------------------------------- def update_devices_names(pm): - sql = pm.db.sql - resolver = NameResolver(pm.db) - device_handler = DeviceInstance(pm.db) # --- Short-circuit if no name-resolution plugin has changed --- - name_plugins = ["DIGSCAN", "NSLOOKUP", "NBTSCAN", "AVAHISCAN"] - - # Retrieve last time name resolution was checked - last_checked = pm.name_plugins_checked - - # Normalize last_checked to datetime if it's a string - if isinstance(last_checked, str): - try: - last_checked = parser.parse(last_checked) - except (ValueError, TypeError) as e: - mylog('none', f'[Update Device Name] Could not parse last_checked timestamp: {last_checked!r} ({e})') - last_checked = None - elif not isinstance(last_checked, datetime.datetime): - last_checked = None - - # Collect and normalize valid state update timestamps for name-related plugins - state_times = [] - - for p in name_plugins: - state_updated = pm.plugin_states.get(p, {}).get("stateUpdated") - if not state_updated: - continue - - # Normalize and validate timestamp - if isinstance(state_updated, datetime.datetime): - state_times.append(state_updated) - elif isinstance(state_updated, str): - try: - state_times.append(parser.parse(state_updated)) - except Exception as e: - mylog('none', f'[Update Device Name] Failed to parse timestamp for {p}: {state_updated!r} ({e})') - else: - mylog('none', f'[Update Device Name] Unexpected timestamp type for {p}: {type(state_updated)}') - - # Determine the latest valid timestamp safely (after collecting all timestamps) - latest_state = None - try: - if state_times: - latest_state = max(state_times) - except (ValueError, TypeError) as e: - mylog('none', f'[Update Device Name] Failed to determine latest timestamp, using fallback ({e})') - latest_state = state_times[-1] if state_times else None - - - # Skip if no plugin state changed since last check - if last_checked and latest_state and latest_state <= last_checked: - mylog('debug', '[Update Device Name] No relevant name plugin changes since last check — skipping update.') + if check_plugin_data_changed(pm, ["DIGSCAN", "NSLOOKUP", "NBTSCAN", "AVAHISCAN"]) == False: + mylog('debug', '[Update Device Name] No relevant plugin changes since last check.') return + mylog('debug', '[Update Device Name] Check if unknown devices present to resolve names for or if REFRESH_FQDN enabled.') + + sql = pm.db.sql + resolver = NameResolver(pm.db) + device_handler = DeviceInstance(pm.db) + nameNotFound = "(name not found)" # Define resolution strategies in priority order @@ -674,7 +696,7 @@ def update_devices_names(pm): # --- Step 3: Log last checked time --- # After resolving names, update last checked - pm.name_plugins_checked = timeNowDB() + pm.plugin_checks = {"DIGSCAN": timeNowDB(), "AVAHISCAN": timeNowDB(), "NSLOOKUP": timeNowDB(), "NBTSCAN": timeNowDB() } #------------------------------------------------------------------------------- # Updates devPresentLastScan for parent devices based on the presence of their NICs diff --git a/server/utils/datetime_utils.py b/server/utils/datetime_utils.py index a74234b4..16bba58c 100644 --- a/server/utils/datetime_utils.py +++ b/server/utils/datetime_utils.py @@ -65,6 +65,46 @@ def timeNowDB(local=True): # Date and time methods #------------------------------------------------------------------------------- +def normalizeTimeStamp(inputTimeStamp): + """ + Normalize various timestamp formats into a datetime.datetime object. + + Supports: + - SQLite-style 'YYYY-MM-DD HH:MM:SS' + - ISO 8601 'YYYY-MM-DDTHH:MM:SSZ' + - Epoch timestamps (int or float) + - datetime.datetime objects (returned as-is) + - Empty or invalid values (returns None) + """ + if inputTimeStamp is None: + return None + + # Already a datetime + if isinstance(inputTimeStamp, datetime.datetime): + return inputTimeStamp + + # Epoch timestamp (integer or float) + if isinstance(inputTimeStamp, (int, float)): + try: + return datetime.datetime.fromtimestamp(inputTimeStamp) + except (OSError, OverflowError, ValueError): + return None + + # String formats (SQLite / ISO8601) + if isinstance(inputTimeStamp, str): + inputTimeStamp = inputTimeStamp.strip() + if not inputTimeStamp: + return None + try: + # Handles SQLite and ISO8601 automatically + return parser.parse(inputTimeStamp) + except Exception: + return None + + # Unrecognized type + return None + + # ------------------------------------------------------------------------------------------- def format_date_iso(date1: str) -> str: """Return ISO 8601 string for a date or None if empty"""