Refactor authoritative field handling and enhance device update logic

- Updated `get_source_for_field_update_with_value` to determine source values based on new field values, including handling for empty and unknown values.
- Introduced `get_overwrite_sql_clause` to build SQL conditions for authoritative overwrite checks based on plugin settings.
- Enhanced `update_devices_data_from_scan` to utilize new authoritative settings and conditions for updating device fields.
- Added new tests for source value determination and device creation to ensure proper handling of source fields.
- Created in-memory SQLite database fixtures for testing device creation and updates.
This commit is contained in:
Jokob @NetAlertX
2026-01-22 04:33:49 +00:00
parent 422a048806
commit 49e689f022
5 changed files with 931 additions and 143 deletions

View File

@@ -601,7 +601,7 @@ function toggleFieldLock(mac, fieldName) {
const sourceIndicator = lockBtn.next();
if (sourceIndicator.hasClass("input-group-addon")) {
const sourceValue = shouldLock ? "LOCKED" : "UNKNOWN";
const sourceClass = shouldLock ? "input-group-addon text-danger" : "input-group-addon text-muted";
const sourceClass = shouldLock ? "input-group-addon text-danger" : "input-group-addon pointer text-muted";
sourceIndicator.text(sourceValue);
sourceIndicator.attr("class", sourceClass);
sourceIndicator.attr("title", getString("FieldLock_Source_Label") + sourceValue);

View File

@@ -118,21 +118,64 @@ def can_overwrite_field(field_name, current_source, plugin_prefix, plugin_settin
return not current_source or current_source == "NEWDEV"
def get_source_for_field_update(field_name, plugin_prefix, is_user_override=False):
def get_overwrite_sql_clause(field_name, source_column, plugin_settings):
"""
Determine what source value should be set when a field is updated.
Build a SQL condition for authoritative overwrite checks.
Returns a SQL snippet that permits overwrite for the given field
based on SET_ALWAYS/SET_EMPTY and USER/LOCKED protection.
Args:
field_name: The field being updated (e.g., "devName").
source_column: The *Source column name (e.g., "devNameSource").
plugin_settings: dict with "set_always" and "set_empty" lists.
Returns:
str: SQL condition snippet (no leading WHERE).
"""
set_always = plugin_settings.get("set_always", [])
set_empty = plugin_settings.get("set_empty", [])
if field_name in set_always:
return f"COALESCE({source_column}, '') NOT IN ('USER', 'LOCKED')"
if field_name in set_empty or field_name not in set_always:
return f"COALESCE({source_column}, '') IN ('', 'NEWDEV')"
return f"COALESCE({source_column}, '') IN ('', 'NEWDEV')"
def get_source_for_field_update_with_value(
field_name, plugin_prefix, field_value, is_user_override=False
):
"""
Determine the source value for a field update based on the new value.
If the new value is empty or an "unknown" placeholder, return NEWDEV.
Otherwise, fall back to standard source selection rules.
Args:
field_name: The field being updated.
plugin_prefix: The unique prefix of the plugin writing (e.g., "UNIFIAPI").
Ignored if is_user_override is True.
is_user_override: If True, return "USER"; if False, return plugin_prefix.
field_value: The new value being written.
is_user_override: If True, return "USER".
Returns:
str: The source value to set for the *Source field.
"""
if is_user_override:
return "USER"
if field_value is None:
return "NEWDEV"
if isinstance(field_value, str):
stripped = field_value.strip()
if stripped in ("", "null"):
return "NEWDEV"
if stripped.lower() in ("(unknown)", "(name not found)"):
return "NEWDEV"
return plugin_prefix

View File

@@ -9,6 +9,11 @@ from models.device_instance import DeviceInstance
from scan.name_resolution import NameResolver
from scan.device_heuristics import guess_icon, guess_type
from db.db_helper import sanitize_SQL_input, list_to_where, safe_int
from db.authoritative_handler import (
get_overwrite_sql_clause,
get_plugin_authoritative_settings,
get_source_for_field_update_with_value,
)
from helper import format_ip_long
# Make sure log level is initialized correctly
@@ -56,6 +61,15 @@ def update_devices_data_from_scan(db):
sql = db.sql # TO-DO
startTime = timeNowDB()
device_columns = set()
try:
device_columns = {row["name"] for row in sql.execute("PRAGMA table_info(Devices)").fetchall()}
except Exception:
device_columns = set()
def has_column(column_name):
return column_name in device_columns if device_columns else False
# Update Last Connection
mylog("debug", "[Update Devices] 1 Last Connection")
sql.execute(f"""UPDATE Devices SET devLastConnection = '{startTime}',
@@ -69,87 +83,464 @@ def update_devices_data_from_scan(db):
WHERE NOT EXISTS (SELECT 1 FROM CurrentScan
WHERE devMac = cur_MAC) """)
# Update IP (devLastIP always updated, primary IPv4/IPv6 set based on family)
mylog("debug", "[Update Devices] - cur_IP -> devLastIP / devPrimaryIPv4 / devPrimaryIPv6")
sql.execute("""
WITH LatestIP AS (
SELECT c.cur_MAC AS mac, c.cur_IP AS ip
FROM CurrentScan c
WHERE c.cur_IP IS NOT NULL
AND c.cur_IP NOT IN ('', 'null', '(unknown)', '(Unknown)')
AND c.cur_DateTime = (
SELECT MAX(c2.cur_DateTime)
FROM CurrentScan c2
WHERE c2.cur_MAC = c.cur_MAC
AND c2.cur_IP IS NOT NULL
AND c2.cur_IP NOT IN ('', 'null', '(unknown)', '(Unknown)')
)
)
UPDATE Devices
SET devLastIP = (SELECT ip FROM LatestIP WHERE mac = devMac),
devPrimaryIPv4 = CASE
WHEN (SELECT ip FROM LatestIP WHERE mac = devMac) LIKE '%:%' THEN devPrimaryIPv4
ELSE (SELECT ip FROM LatestIP WHERE mac = devMac)
END,
devPrimaryIPv6 = CASE
WHEN (SELECT ip FROM LatestIP WHERE mac = devMac) LIKE '%:%' THEN (SELECT ip FROM LatestIP WHERE mac = devMac)
ELSE devPrimaryIPv6
END
WHERE EXISTS (SELECT 1 FROM LatestIP WHERE mac = devMac);
""")
plugin_rows = sql.execute(
"SELECT DISTINCT cur_ScanMethod FROM CurrentScan"
).fetchall()
plugin_prefixes = [row[0] for row in plugin_rows if row[0]]
if not plugin_prefixes:
plugin_prefixes = [None]
plugin_settings_cache = {}
# Update only devices with empty, NULL or (u(U)nknown) vendors
mylog("debug", "[Update Devices] - cur_Vendor -> (if empty) devVendor")
sql.execute("""UPDATE Devices
def get_plugin_settings_cached(plugin_prefix):
if plugin_prefix not in plugin_settings_cache:
plugin_settings_cache[plugin_prefix] = get_plugin_authoritative_settings(
plugin_prefix
)
return plugin_settings_cache[plugin_prefix]
for plugin_prefix in plugin_prefixes:
filter_by_scan_method = plugin_prefix is not None and plugin_prefix != ""
source_prefix = plugin_prefix if filter_by_scan_method else "NEWDEV"
plugin_settings = get_plugin_settings_cached(source_prefix)
has_last_ip_source = has_column("devLastIpSource")
has_vendor_source = has_column("devVendorSource")
has_parent_port_source = has_column("devParentPortSource")
has_parent_mac_source = has_column("devParentMacSource")
has_ssid_source = has_column("devSsidSource")
has_name_source = has_column("devNameSource")
dev_last_ip_clause = (
get_overwrite_sql_clause("devLastIP", "devLastIpSource", plugin_settings)
if has_last_ip_source
else "1=1"
)
dev_vendor_clause = (
get_overwrite_sql_clause("devVendor", "devVendorSource", plugin_settings)
if has_vendor_source
else "1=1"
)
dev_parent_port_clause = (
get_overwrite_sql_clause("devParentPort", "devParentPortSource", plugin_settings)
if has_parent_port_source
else "1=1"
)
dev_parent_mac_clause = (
get_overwrite_sql_clause("devParentMAC", "devParentMacSource", plugin_settings)
if has_parent_mac_source
else "1=1"
)
dev_ssid_clause = (
get_overwrite_sql_clause("devSSID", "devSsidSource", plugin_settings)
if has_ssid_source
else "1=1"
)
dev_name_clause = (
get_overwrite_sql_clause("devName", "devNameSource", plugin_settings)
if has_name_source
else "1=1"
)
name_is_set_always = "devName" in plugin_settings.get("set_always", [])
vendor_is_set_always = "devVendor" in plugin_settings.get("set_always", [])
parent_port_is_set_always = "devParentPort" in plugin_settings.get("set_always", [])
parent_mac_is_set_always = "devParentMAC" in plugin_settings.get("set_always", [])
ssid_is_set_always = "devSSID" in plugin_settings.get("set_always", [])
name_empty_condition = "1=1" if name_is_set_always else (
"(devName IN ('(unknown)', '(name not found)', '') OR devName IS NULL)"
)
vendor_empty_condition = "1=1" if vendor_is_set_always else (
"(devVendor IS NULL OR devVendor IN ('', 'null', '(unknown)', '(Unknown)'))"
)
parent_port_empty_condition = "1=1" if parent_port_is_set_always else (
"(devParentPort IS NULL OR devParentPort IN ('', 'null', '(unknown)', '(Unknown)'))"
)
parent_mac_empty_condition = "1=1" if parent_mac_is_set_always else (
"(devParentMAC IS NULL OR devParentMAC IN ('', 'null', '(unknown)', '(Unknown)'))"
)
ssid_empty_condition = "1=1" if ssid_is_set_always else (
"(devSSID IS NULL OR devSSID IN ('', 'null'))"
)
# Update IP (devLastIP always updated, primary IPv4/IPv6 set based on family)
mylog(
"debug",
f"[Update Devices] - ({source_prefix}) cur_IP -> devLastIP / devPrimaryIPv4 / devPrimaryIPv6",
)
last_ip_source_fragment = ", devLastIpSource = ?" if has_last_ip_source else ""
last_ip_params = (source_prefix,) if has_last_ip_source else ()
if filter_by_scan_method:
sql.execute(
f"""
WITH LatestIP AS (
SELECT c.cur_MAC AS mac, c.cur_IP AS ip
FROM CurrentScan c
WHERE c.cur_IP IS NOT NULL
AND c.cur_IP NOT IN ('', 'null', '(unknown)', '(Unknown)')
AND c.cur_ScanMethod = ?
AND c.cur_DateTime = (
SELECT MAX(c2.cur_DateTime)
FROM CurrentScan c2
WHERE c2.cur_MAC = c.cur_MAC
AND c2.cur_IP IS NOT NULL
AND c2.cur_IP NOT IN ('', 'null', '(unknown)', '(Unknown)')
AND c2.cur_ScanMethod = ?
)
)
UPDATE Devices
SET devLastIP = (SELECT ip FROM LatestIP WHERE mac = devMac),
devPrimaryIPv4 = CASE
WHEN (SELECT ip FROM LatestIP WHERE mac = devMac) LIKE '%:%' THEN devPrimaryIPv4
ELSE (SELECT ip FROM LatestIP WHERE mac = devMac)
END,
devPrimaryIPv6 = CASE
WHEN (SELECT ip FROM LatestIP WHERE mac = devMac) LIKE '%:%' THEN (SELECT ip FROM LatestIP WHERE mac = devMac)
ELSE devPrimaryIPv6
END
{last_ip_source_fragment}
WHERE EXISTS (SELECT 1 FROM LatestIP WHERE mac = devMac)
AND {dev_last_ip_clause};
""",
(plugin_prefix, plugin_prefix, *last_ip_params),
)
else:
sql.execute(
f"""
WITH LatestIP AS (
SELECT c.cur_MAC AS mac, c.cur_IP AS ip
FROM CurrentScan c
WHERE c.cur_IP IS NOT NULL
AND c.cur_IP NOT IN ('', 'null', '(unknown)', '(Unknown)')
AND c.cur_DateTime = (
SELECT MAX(c2.cur_DateTime)
FROM CurrentScan c2
WHERE c2.cur_MAC = c.cur_MAC
AND c2.cur_IP IS NOT NULL
AND c2.cur_IP NOT IN ('', 'null', '(unknown)', '(Unknown)')
)
)
UPDATE Devices
SET devLastIP = (SELECT ip FROM LatestIP WHERE mac = devMac),
devPrimaryIPv4 = CASE
WHEN (SELECT ip FROM LatestIP WHERE mac = devMac) LIKE '%:%' THEN devPrimaryIPv4
ELSE (SELECT ip FROM LatestIP WHERE mac = devMac)
END,
devPrimaryIPv6 = CASE
WHEN (SELECT ip FROM LatestIP WHERE mac = devMac) LIKE '%:%' THEN (SELECT ip FROM LatestIP WHERE mac = devMac)
ELSE devPrimaryIPv6
END
{last_ip_source_fragment}
WHERE EXISTS (SELECT 1 FROM LatestIP WHERE mac = devMac)
AND {dev_last_ip_clause};
""",
last_ip_params,
)
# Update vendor
mylog("debug", f"[Update Devices] - ({source_prefix}) cur_Vendor -> devVendor")
vendor_source_fragment = ", devVendorSource = ?" if has_vendor_source else ""
vendor_params = (source_prefix,) if has_vendor_source else ()
if filter_by_scan_method:
sql.execute(
f"""
UPDATE Devices
SET devVendor = (
SELECT cur_Vendor
FROM CurrentScan
WHERE Devices.devMac = CurrentScan.cur_MAC
AND CurrentScan.cur_ScanMethod = ?
AND CurrentScan.cur_Vendor IS NOT NULL
AND CurrentScan.cur_Vendor NOT IN ('', 'null', '(unknown)', '(Unknown)')
ORDER BY CurrentScan.cur_DateTime DESC
LIMIT 1
)
WHERE
(devVendor IS NULL OR devVendor IN ("", "null", "(unknown)", "(Unknown)"))
AND EXISTS (
SELECT 1
FROM CurrentScan
WHERE Devices.devMac = CurrentScan.cur_MAC
)""")
{vendor_source_fragment}
WHERE {vendor_empty_condition}
AND EXISTS (
SELECT 1
FROM CurrentScan
WHERE Devices.devMac = CurrentScan.cur_MAC
AND CurrentScan.cur_ScanMethod = ?
AND CurrentScan.cur_Vendor IS NOT NULL
AND CurrentScan.cur_Vendor NOT IN ('', 'null', '(unknown)', '(Unknown)')
)
AND {dev_vendor_clause}
""",
(plugin_prefix, plugin_prefix, *vendor_params),
)
else:
sql.execute(
f"""
UPDATE Devices
SET devVendor = (
SELECT cur_Vendor
FROM CurrentScan
WHERE Devices.devMac = CurrentScan.cur_MAC
AND CurrentScan.cur_Vendor IS NOT NULL
AND CurrentScan.cur_Vendor NOT IN ('', 'null', '(unknown)', '(Unknown)')
ORDER BY CurrentScan.cur_DateTime DESC
LIMIT 1
)
{vendor_source_fragment}
WHERE {vendor_empty_condition}
AND EXISTS (
SELECT 1
FROM CurrentScan
WHERE Devices.devMac = CurrentScan.cur_MAC
AND CurrentScan.cur_Vendor IS NOT NULL
AND CurrentScan.cur_Vendor NOT IN ('', 'null', '(unknown)', '(Unknown)')
)
AND {dev_vendor_clause}
""",
vendor_params,
)
# Update only devices with empty or NULL devParentPort
mylog("debug", "[Update Devices] - (if not empty) cur_Port -> devParentPort")
sql.execute("""UPDATE Devices
# Update parent port
mylog("debug", f"[Update Devices] - ({source_prefix}) cur_Port -> devParentPort")
parent_port_source_fragment = ", devParentPortSource = ?" if has_parent_port_source else ""
parent_port_params = (source_prefix,) if has_parent_port_source else ()
if filter_by_scan_method:
sql.execute(
f"""
UPDATE Devices
SET devParentPort = (
SELECT cur_Port
FROM CurrentScan
WHERE Devices.devMac = CurrentScan.cur_MAC
)
WHERE
(devParentPort IS NULL OR devParentPort IN ("", "null", "(unknown)", "(Unknown)"))
AND
EXISTS (
SELECT 1
FROM CurrentScan
WHERE Devices.devMac = CurrentScan.cur_MAC
AND CurrentScan.cur_Port IS NOT NULL AND CurrentScan.cur_Port NOT IN ("", "null")
)""")
SELECT cur_Port
FROM CurrentScan
WHERE Devices.devMac = CurrentScan.cur_MAC
AND CurrentScan.cur_ScanMethod = ?
AND CurrentScan.cur_Port IS NOT NULL
AND CurrentScan.cur_Port NOT IN ('', 'null')
ORDER BY CurrentScan.cur_DateTime DESC
LIMIT 1
)
{parent_port_source_fragment}
WHERE {parent_port_empty_condition}
AND EXISTS (
SELECT 1
FROM CurrentScan
WHERE Devices.devMac = CurrentScan.cur_MAC
AND CurrentScan.cur_ScanMethod = ?
AND CurrentScan.cur_Port IS NOT NULL
AND CurrentScan.cur_Port NOT IN ('', 'null')
)
AND {dev_parent_port_clause}
""",
(plugin_prefix, plugin_prefix, *parent_port_params),
)
else:
sql.execute(
f"""
UPDATE Devices
SET devParentPort = (
SELECT cur_Port
FROM CurrentScan
WHERE Devices.devMac = CurrentScan.cur_MAC
AND CurrentScan.cur_Port IS NOT NULL
AND CurrentScan.cur_Port NOT IN ('', 'null')
ORDER BY CurrentScan.cur_DateTime DESC
LIMIT 1
)
{parent_port_source_fragment}
WHERE {parent_port_empty_condition}
AND EXISTS (
SELECT 1
FROM CurrentScan
WHERE Devices.devMac = CurrentScan.cur_MAC
AND CurrentScan.cur_Port IS NOT NULL
AND CurrentScan.cur_Port NOT IN ('', 'null')
)
AND {dev_parent_port_clause}
""",
parent_port_params,
)
# Update only devices with empty or NULL devParentMAC
mylog("debug", "[Update Devices] - (if not empty) cur_NetworkNodeMAC -> devParentMAC")
sql.execute("""UPDATE Devices
# Update parent MAC
mylog("debug", f"[Update Devices] - ({source_prefix}) cur_NetworkNodeMAC -> devParentMAC")
parent_mac_source_fragment = ", devParentMacSource = ?" if has_parent_mac_source else ""
parent_mac_params = (source_prefix,) if has_parent_mac_source else ()
if filter_by_scan_method:
sql.execute(
f"""
UPDATE Devices
SET devParentMAC = (
SELECT cur_NetworkNodeMAC
FROM CurrentScan
WHERE Devices.devMac = CurrentScan.cur_MAC
AND CurrentScan.cur_ScanMethod = ?
AND CurrentScan.cur_NetworkNodeMAC IS NOT NULL
AND CurrentScan.cur_NetworkNodeMAC NOT IN ('', 'null')
ORDER BY CurrentScan.cur_DateTime DESC
LIMIT 1
)
WHERE
(devParentMAC IS NULL OR devParentMAC IN ("", "null", "(unknown)", "(Unknown)"))
AND
EXISTS (
SELECT 1
FROM CurrentScan
WHERE Devices.devMac = CurrentScan.cur_MAC
AND CurrentScan.cur_NetworkNodeMAC IS NOT NULL AND CurrentScan.cur_NetworkNodeMAC NOT IN ("", "null")
)
""")
{parent_mac_source_fragment}
WHERE {parent_mac_empty_condition}
AND EXISTS (
SELECT 1
FROM CurrentScan
WHERE Devices.devMac = CurrentScan.cur_MAC
AND CurrentScan.cur_ScanMethod = ?
AND CurrentScan.cur_NetworkNodeMAC IS NOT NULL
AND CurrentScan.cur_NetworkNodeMAC NOT IN ('', 'null')
)
AND {dev_parent_mac_clause}
""",
(plugin_prefix, plugin_prefix, *parent_mac_params),
)
else:
sql.execute(
f"""
UPDATE Devices
SET devParentMAC = (
SELECT cur_NetworkNodeMAC
FROM CurrentScan
WHERE Devices.devMac = CurrentScan.cur_MAC
AND CurrentScan.cur_NetworkNodeMAC IS NOT NULL
AND CurrentScan.cur_NetworkNodeMAC NOT IN ('', 'null')
ORDER BY CurrentScan.cur_DateTime DESC
LIMIT 1
)
{parent_mac_source_fragment}
WHERE {parent_mac_empty_condition}
AND EXISTS (
SELECT 1
FROM CurrentScan
WHERE Devices.devMac = CurrentScan.cur_MAC
AND CurrentScan.cur_NetworkNodeMAC IS NOT NULL
AND CurrentScan.cur_NetworkNodeMAC NOT IN ('', 'null')
)
AND {dev_parent_mac_clause}
""",
parent_mac_params,
)
# Update SSID
mylog("debug", f"[Update Devices] - ({source_prefix}) cur_SSID -> devSSID")
ssid_source_fragment = ", devSsidSource = ?" if has_ssid_source else ""
ssid_params = (source_prefix,) if has_ssid_source else ()
if filter_by_scan_method:
sql.execute(
f"""
UPDATE Devices
SET devSSID = (
SELECT cur_SSID
FROM CurrentScan
WHERE Devices.devMac = CurrentScan.cur_MAC
AND CurrentScan.cur_ScanMethod = ?
AND CurrentScan.cur_SSID IS NOT NULL
AND CurrentScan.cur_SSID NOT IN ('', 'null')
ORDER BY CurrentScan.cur_DateTime DESC
LIMIT 1
)
{ssid_source_fragment}
WHERE {ssid_empty_condition}
AND EXISTS (
SELECT 1
FROM CurrentScan
WHERE Devices.devMac = CurrentScan.cur_MAC
AND CurrentScan.cur_ScanMethod = ?
AND CurrentScan.cur_SSID IS NOT NULL
AND CurrentScan.cur_SSID NOT IN ('', 'null')
)
AND {dev_ssid_clause}
""",
(plugin_prefix, plugin_prefix, *ssid_params),
)
else:
sql.execute(
f"""
UPDATE Devices
SET devSSID = (
SELECT cur_SSID
FROM CurrentScan
WHERE Devices.devMac = CurrentScan.cur_MAC
AND CurrentScan.cur_SSID IS NOT NULL
AND CurrentScan.cur_SSID NOT IN ('', 'null')
ORDER BY CurrentScan.cur_DateTime DESC
LIMIT 1
)
{ssid_source_fragment}
WHERE {ssid_empty_condition}
AND EXISTS (
SELECT 1
FROM CurrentScan
WHERE Devices.devMac = CurrentScan.cur_MAC
AND CurrentScan.cur_SSID IS NOT NULL
AND CurrentScan.cur_SSID NOT IN ('', 'null')
)
AND {dev_ssid_clause}
""",
ssid_params,
)
# Update Name
mylog("debug", f"[Update Devices] - ({source_prefix}) cur_Name -> devName")
name_source_fragment = ", devNameSource = ?" if has_name_source else ""
name_params = (source_prefix,) if has_name_source else ()
if filter_by_scan_method:
sql.execute(
f"""
UPDATE Devices
SET devName = (
SELECT cur_Name
FROM CurrentScan
WHERE cur_MAC = devMac
AND cur_ScanMethod = ?
AND cur_Name IS NOT NULL
AND cur_Name <> 'null'
AND cur_Name <> ''
ORDER BY cur_DateTime DESC
LIMIT 1
)
{name_source_fragment}
WHERE {name_empty_condition}
AND EXISTS (
SELECT 1
FROM CurrentScan
WHERE cur_MAC = devMac
AND cur_ScanMethod = ?
AND cur_Name IS NOT NULL
AND cur_Name <> 'null'
AND cur_Name <> ''
)
AND {dev_name_clause}
""",
(plugin_prefix, plugin_prefix, *name_params),
)
else:
sql.execute(
f"""
UPDATE Devices
SET devName = (
SELECT cur_Name
FROM CurrentScan
WHERE cur_MAC = devMac
AND cur_Name IS NOT NULL
AND cur_Name <> 'null'
AND cur_Name <> ''
ORDER BY cur_DateTime DESC
LIMIT 1
)
{name_source_fragment}
WHERE {name_empty_condition}
AND EXISTS (
SELECT 1
FROM CurrentScan
WHERE cur_MAC = devMac
AND cur_Name IS NOT NULL
AND cur_Name <> 'null'
AND cur_Name <> ''
)
AND {dev_name_clause}
""",
name_params,
)
# Update only devices with empty or NULL devSite
mylog("debug", "[Update Devices] - (if not empty) cur_NetworkSite -> (if empty) devSite",)
@@ -168,23 +559,6 @@ def update_devices_data_from_scan(db):
AND CurrentScan.cur_NetworkSite IS NOT NULL AND CurrentScan.cur_NetworkSite NOT IN ("", "null")
)""")
# Update only devices with empty or NULL devSSID
mylog("debug", "[Update Devices] - (if not empty) cur_SSID -> (if empty) devSSID")
sql.execute("""UPDATE Devices
SET devSSID = (
SELECT cur_SSID
FROM CurrentScan
WHERE Devices.devMac = CurrentScan.cur_MAC
)
WHERE
(devSSID IS NULL OR devSSID IN ("", "null"))
AND EXISTS (
SELECT 1
FROM CurrentScan
WHERE Devices.devMac = CurrentScan.cur_MAC
AND CurrentScan.cur_SSID IS NOT NULL AND CurrentScan.cur_SSID NOT IN ("", "null")
)""")
# Update only devices with empty or NULL devType
mylog("debug", "[Update Devices] - (if not empty) cur_Type -> (if empty) devType")
sql.execute("""UPDATE Devices
@@ -202,43 +576,48 @@ def update_devices_data_from_scan(db):
AND CurrentScan.cur_Type IS NOT NULL AND CurrentScan.cur_Type NOT IN ("", "null")
)""")
# Update (unknown) or (name not found) Names if available
mylog("debug", "[Update Devices] - (if not empty) cur_Name -> (if empty) devName")
sql.execute(""" UPDATE Devices
SET devName = COALESCE((
SELECT cur_Name
FROM CurrentScan
WHERE cur_MAC = devMac
AND cur_Name IS NOT NULL
AND cur_Name <> 'null'
AND cur_Name <> ''
), devName)
WHERE (devName IN ('(unknown)', '(name not found)', '')
OR devName IS NULL)
AND EXISTS (
SELECT 1
FROM CurrentScan
WHERE cur_MAC = devMac
AND cur_Name IS NOT NULL
AND cur_Name <> 'null'
AND cur_Name <> ''
) """)
# Update VENDORS
recordsToUpdate = []
query = """SELECT * FROM Devices
WHERE devVendor IS NULL OR devVendor IN ("", "null", "(unknown)", "(Unknown)")
"""
vendor_settings = get_plugin_authoritative_settings("VNDRPDT")
vendor_clause = (
get_overwrite_sql_clause("devVendor", "devVendorSource", vendor_settings)
if has_column("devVendorSource")
else "1=1"
)
vendor_is_set_always = "devVendor" in vendor_settings.get("set_always", [])
if vendor_is_set_always:
query = f"""SELECT * FROM Devices
WHERE {vendor_clause}
"""
else:
query = f"""SELECT * FROM Devices
WHERE (devVendor IS NULL OR devVendor IN ("", "null", "(unknown)", "(Unknown)"))
AND {vendor_clause}
"""
for device in sql.execute(query):
vendor = query_MAC_vendor(device["devMac"])
if vendor != -1 and vendor != -2:
recordsToUpdate.append([vendor, device["devMac"]])
recordsToUpdate.append([vendor, "VNDRPDT", device["devMac"]])
if len(recordsToUpdate) > 0:
sql.executemany(
"UPDATE Devices SET devVendor = ? WHERE devMac = ? ", recordsToUpdate
)
if has_column("devVendorSource"):
sql.executemany(
f"""UPDATE Devices
SET devVendor = ?,
devVendorSource = ?
WHERE devMac = ?
AND {vendor_clause}""",
recordsToUpdate,
)
else:
sql.executemany(
"""UPDATE Devices
SET devVendor = ?
WHERE devMac = ?""",
[(row[0], row[2]) for row in recordsToUpdate],
)
# Update devPresentLastScan based on NICs presence
update_devPresentLastScan_based_on_nics(db)
@@ -524,12 +903,20 @@ def create_new_devices(db):
cur_Type,
) = row
# Preserve raw values to determine source attribution
raw_name = str(cur_Name).strip() if cur_Name else ""
raw_vendor = str(cur_Vendor).strip() if cur_Vendor else ""
raw_ip = str(cur_IP).strip() if cur_IP else ""
raw_ssid = str(cur_SSID).strip() if cur_SSID else ""
raw_parent_mac = cur_NetworkNodeMAC.strip() if cur_NetworkNodeMAC else ""
raw_parent_port = str(cur_PORT).strip() if cur_PORT else ""
# Handle NoneType
cur_Name = str(cur_Name).strip() if cur_Name else "(unknown)"
cur_Name = raw_name if raw_name else "(unknown)"
cur_Type = (
str(cur_Type).strip() if cur_Type else get_setting_value("NEWDEV_devType")
)
cur_NetworkNodeMAC = cur_NetworkNodeMAC.strip() if cur_NetworkNodeMAC else ""
cur_NetworkNodeMAC = raw_parent_mac
cur_NetworkNodeMAC = (
cur_NetworkNodeMAC
if cur_NetworkNodeMAC and cur_MAC != "Internet"
@@ -546,7 +933,7 @@ def create_new_devices(db):
)
# Derive primary IP family values
cur_IP = str(cur_IP).strip() if cur_IP else ""
cur_IP = raw_ip
cur_IP_normalized = check_IP_format(cur_IP) if ":" not in cur_IP else cur_IP
# Validate IPv6 addresses using format_ip_long for consistency (do not store integer result)
@@ -558,6 +945,33 @@ def create_new_devices(db):
primary_ipv4 = cur_IP_normalized if cur_IP_normalized and ":" not in cur_IP_normalized else ""
primary_ipv6 = cur_IP_normalized if cur_IP_normalized and ":" in cur_IP_normalized else ""
plugin_prefix = str(cur_ScanMethod).strip() if cur_ScanMethod else "NEWDEV"
dev_mac_source = get_source_for_field_update_with_value(
"devMac", plugin_prefix, cur_MAC, is_user_override=False
)
dev_name_source = get_source_for_field_update_with_value(
"devName", plugin_prefix, raw_name, is_user_override=False
)
dev_vendor_source = get_source_for_field_update_with_value(
"devVendor", plugin_prefix, raw_vendor, is_user_override=False
)
dev_last_ip_source = get_source_for_field_update_with_value(
"devLastIP", plugin_prefix, cur_IP_normalized, is_user_override=False
)
dev_ssid_source = get_source_for_field_update_with_value(
"devSSID", plugin_prefix, raw_ssid, is_user_override=False
)
dev_parent_mac_source = get_source_for_field_update_with_value(
"devParentMAC", plugin_prefix, raw_parent_mac, is_user_override=False
)
dev_parent_port_source = get_source_for_field_update_with_value(
"devParentPort", plugin_prefix, raw_parent_port, is_user_override=False
)
dev_parent_rel_type_source = "NEWDEV"
dev_fqdn_source = "NEWDEV"
dev_vlan_source = "NEWDEV"
# Preparing the individual insert statement
sqlQuery = f"""INSERT OR IGNORE INTO Devices
(
@@ -577,6 +991,16 @@ def create_new_devices(db):
devSSID,
devType,
devSourcePlugin,
devMacSource,
devNameSource,
devFqdnSource,
devLastIpSource,
devVendorSource,
devSsidSource,
devParentMacSource,
devParentPortSource,
devParentRelTypeSource,
devVlanSource,
{newDevColumns}
)
VALUES
@@ -597,6 +1021,16 @@ def create_new_devices(db):
'{sanitize_SQL_input(cur_SSID)}',
'{sanitize_SQL_input(cur_Type)}',
'{sanitize_SQL_input(cur_ScanMethod)}',
'{sanitize_SQL_input(dev_mac_source)}',
'{sanitize_SQL_input(dev_name_source)}',
'{sanitize_SQL_input(dev_fqdn_source)}',
'{sanitize_SQL_input(dev_last_ip_source)}',
'{sanitize_SQL_input(dev_vendor_source)}',
'{sanitize_SQL_input(dev_ssid_source)}',
'{sanitize_SQL_input(dev_parent_mac_source)}',
'{sanitize_SQL_input(dev_parent_port_source)}',
'{sanitize_SQL_input(dev_parent_rel_type_source)}',
'{sanitize_SQL_input(dev_vlan_source)}',
{newDevDefaults}
)"""
@@ -709,7 +1143,8 @@ def update_devices_names(pm):
If False, resolves only FQDN.
Returns:
recordsToUpdate (list): List of [newName, newFQDN, devMac] or [newFQDN, devMac] for DB update.
recordsToUpdate (list): List of
[newName, nameSource, newFQDN, fqdnSource, devMac] or [newFQDN, fqdnSource, devMac].
recordsNotFound (list): List of [nameNotFound, devMac] for DB update.
foundStats (dict): Number of successes per strategy.
notFound (int): Number of devices not resolved.
@@ -738,9 +1173,9 @@ def update_devices_names(pm):
foundStats[label] += 1
if resolve_both_name_and_fqdn:
recordsToUpdate.append([newName, newFQDN, device["devMac"]])
recordsToUpdate.append([newName, label, newFQDN, label, device["devMac"]])
else:
recordsToUpdate.append([newFQDN, device["devMac"]])
recordsToUpdate.append([newFQDN, label, device["devMac"]])
break
# If no name was resolved, queue device for "(name not found)" update
@@ -768,13 +1203,51 @@ def update_devices_names(pm):
# Apply updates to database
sql.executemany(
"UPDATE Devices SET devName = ? WHERE devMac = ?", recordsNotFound
)
sql.executemany(
"UPDATE Devices SET devName = ?, devFQDN = ? WHERE devMac = ?",
recordsToUpdate,
"""UPDATE Devices
SET devName = CASE
WHEN COALESCE(devNameSource, '') IN ('USER', 'LOCKED') THEN devName
ELSE ?
END
WHERE devMac = ?
AND COALESCE(devNameSource, '') IN ('', 'NEWDEV')""",
recordsNotFound,
)
records_by_plugin = {}
for entry in recordsToUpdate:
records_by_plugin.setdefault(entry[1], []).append(entry)
for plugin_label, plugin_records in records_by_plugin.items():
plugin_settings = get_plugin_authoritative_settings(plugin_label)
name_clause = get_overwrite_sql_clause(
"devName", "devNameSource", plugin_settings
)
fqdn_clause = get_overwrite_sql_clause(
"devFQDN", "devFqdnSource", plugin_settings
)
sql.executemany(
f"""UPDATE Devices
SET devName = CASE
WHEN {name_clause} THEN ?
ELSE devName
END,
devNameSource = CASE
WHEN {name_clause} THEN ?
ELSE devNameSource
END,
devFQDN = CASE
WHEN {fqdn_clause} THEN ?
ELSE devFQDN
END,
devFqdnSource = CASE
WHEN {fqdn_clause} THEN ?
ELSE devFqdnSource
END
WHERE devMac = ?""",
plugin_records,
)
# --- Step 2: Optionally refresh FQDN for all devices ---
if get_setting_value("REFRESH_FQDN"):
allDevices = device_handler.getAll()
@@ -791,10 +1264,30 @@ def update_devices_names(pm):
mylog("verbose", f"[Update FQDN] Names Found (DIGSCAN/AVAHISCAN/NSLOOKUP/NBTSCAN): {len(recordsToUpdate)}({res_string})",)
mylog("verbose", f"[Update FQDN] Names Not Found : {notFound}")
# Apply FQDN-only updates
sql.executemany(
"UPDATE Devices SET devFQDN = ? WHERE devMac = ?", recordsToUpdate
)
records_by_plugin = {}
for entry in recordsToUpdate:
records_by_plugin.setdefault(entry[1], []).append(entry)
for plugin_label, plugin_records in records_by_plugin.items():
plugin_settings = get_plugin_authoritative_settings(plugin_label)
fqdn_clause = get_overwrite_sql_clause(
"devFQDN", "devFqdnSource", plugin_settings
)
# Apply FQDN-only updates
sql.executemany(
f"""UPDATE Devices
SET devFQDN = CASE
WHEN {fqdn_clause} THEN ?
ELSE devFQDN
END,
devFqdnSource = CASE
WHEN {fqdn_clause} THEN ?
ELSE devFqdnSource
END
WHERE devMac = ?""",
plugin_records,
)
# Commit all database changes
pm.db.commitDB()

View File

@@ -2,11 +2,9 @@
Unit tests for authoritative field update handler.
"""
import pytest
from server.db.authoritative_handler import (
can_overwrite_field,
get_source_for_field_update,
get_source_for_field_update_with_value,
FIELD_SOURCE_MAP,
)
@@ -75,17 +73,52 @@ class TestCanOverwriteField:
)
class TestGetSourceForFieldUpdate:
"""Test source value determination for field updates."""
class TestGetSourceForFieldUpdateWithValue:
"""Test source value determination with value-based normalization."""
def test_user_override_sets_user_source(self):
"""User override should set USER source."""
assert get_source_for_field_update("devName", "UNIFIAPI", is_user_override=True) == "USER"
assert (
get_source_for_field_update_with_value(
"devName", "UNIFIAPI", "Device", is_user_override=True
)
== "USER"
)
def test_plugin_update_sets_plugin_prefix(self):
"""Plugin update should set plugin prefix as source."""
assert get_source_for_field_update("devName", "UNIFIAPI", is_user_override=False) == "UNIFIAPI"
assert get_source_for_field_update("devLastIP", "ARPSCAN", is_user_override=False) == "ARPSCAN"
assert (
get_source_for_field_update_with_value(
"devName", "UNIFIAPI", "Device", is_user_override=False
)
== "UNIFIAPI"
)
assert (
get_source_for_field_update_with_value(
"devLastIP", "ARPSCAN", "192.168.1.1", is_user_override=False
)
== "ARPSCAN"
)
def test_empty_or_unknown_values_return_newdev(self):
assert (
get_source_for_field_update_with_value(
"devName", "ARPSCAN", "", is_user_override=False
)
== "NEWDEV"
)
assert (
get_source_for_field_update_with_value(
"devName", "ARPSCAN", "(unknown)", is_user_override=False
)
== "NEWDEV"
)
def test_non_empty_value_sets_plugin_prefix(self):
assert (
get_source_for_field_update_with_value(
"devVendor", "ARPSCAN", "Acme", is_user_override=False
)
== "ARPSCAN"
)
class TestFieldSourceMapping:

View File

@@ -83,6 +83,34 @@ def scan_db():
"""
)
cur.execute(
"""
CREATE TABLE Events (
eve_MAC TEXT,
eve_IP TEXT,
eve_DateTime TEXT,
eve_EventType TEXT,
eve_AdditionalInfo TEXT,
eve_PendingAlertEmail INTEGER
)
"""
)
cur.execute(
"""
CREATE TABLE Sessions (
ses_MAC TEXT,
ses_IP TEXT,
ses_EventTypeConnection TEXT,
ses_DateTimeConnection TEXT,
ses_EventTypeDisconnection TEXT,
ses_DateTimeDisconnection TEXT,
ses_StillConnected INTEGER,
ses_AdditionalInfo TEXT
)
"""
)
conn.commit()
yield conn
conn.close()
@@ -109,6 +137,197 @@ def mock_device_handlers():
yield
@pytest.fixture
def scan_db_for_new_devices():
"""Create an in-memory SQLite database for create_new_devices tests."""
conn = sqlite3.connect(":memory:")
conn.row_factory = sqlite3.Row
cur = conn.cursor()
cur.execute(
"""
CREATE TABLE Devices (
devMac TEXT PRIMARY KEY,
devName TEXT,
devVendor TEXT,
devLastIP TEXT,
devPrimaryIPv4 TEXT,
devPrimaryIPv6 TEXT,
devFirstConnection TEXT,
devLastConnection TEXT,
devSyncHubNode TEXT,
devGUID TEXT,
devParentMAC TEXT,
devParentPort TEXT,
devSite TEXT,
devSSID TEXT,
devType TEXT,
devSourcePlugin TEXT,
devMacSource TEXT,
devNameSource TEXT,
devFqdnSource TEXT,
devLastIpSource TEXT,
devVendorSource TEXT,
devSsidSource TEXT,
devParentMacSource TEXT,
devParentPortSource TEXT,
devParentRelTypeSource TEXT,
devVlanSource TEXT,
devAlertEvents INTEGER,
devAlertDown INTEGER,
devPresentLastScan INTEGER,
devIsArchived INTEGER,
devIsNew INTEGER,
devSkipRepeated INTEGER,
devScan INTEGER,
devOwner TEXT,
devFavorite INTEGER,
devGroup TEXT,
devComments TEXT,
devLogEvents INTEGER,
devLocation TEXT,
devCustomProps TEXT,
devParentRelType TEXT,
devReqNicsOnline INTEGER
)
"""
)
cur.execute(
"""
CREATE TABLE CurrentScan (
cur_MAC TEXT,
cur_Name TEXT,
cur_Vendor TEXT,
cur_ScanMethod TEXT,
cur_IP TEXT,
cur_SyncHubNodeName TEXT,
cur_NetworkNodeMAC TEXT,
cur_PORT TEXT,
cur_NetworkSite TEXT,
cur_SSID TEXT,
cur_Type TEXT
)
"""
)
cur.execute(
"""
CREATE TABLE Events (
eve_MAC TEXT,
eve_IP TEXT,
eve_DateTime TEXT,
eve_EventType TEXT,
eve_AdditionalInfo TEXT,
eve_PendingAlertEmail INTEGER
)
"""
)
cur.execute(
"""
CREATE TABLE Sessions (
ses_MAC TEXT,
ses_IP TEXT,
ses_EventTypeConnection TEXT,
ses_DateTimeConnection TEXT,
ses_EventTypeDisconnection TEXT,
ses_DateTimeDisconnection TEXT,
ses_StillConnected INTEGER,
ses_AdditionalInfo TEXT
)
"""
)
conn.commit()
yield conn
conn.close()
def test_create_new_devices_sets_sources(scan_db_for_new_devices):
"""New device insert initializes source fields from scan method."""
cur = scan_db_for_new_devices.cursor()
cur.execute(
"""
INSERT INTO CurrentScan (
cur_MAC, cur_Name, cur_Vendor, cur_ScanMethod, cur_IP,
cur_SyncHubNodeName, cur_NetworkNodeMAC, cur_PORT,
cur_NetworkSite, cur_SSID, cur_Type
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
"AA:BB:CC:DD:EE:10",
"DeviceOne",
"AcmeVendor",
"ARPSCAN",
"192.168.1.10",
"",
"11:22:33:44:55:66",
"1",
"",
"MyWifi",
"",
),
)
scan_db_for_new_devices.commit()
settings = {
"NEWDEV_devType": "default-type",
"NEWDEV_devParentMAC": "FF:FF:FF:FF:FF:FF",
"NEWDEV_devOwner": "owner",
"NEWDEV_devGroup": "group",
"NEWDEV_devComments": "",
"NEWDEV_devLocation": "",
"NEWDEV_devCustomProps": "",
"NEWDEV_devParentRelType": "uplink",
"SYNC_node_name": "SYNCNODE",
}
def get_setting_value_side_effect(key):
return settings.get(key, "")
db = Mock()
db.sql_connection = scan_db_for_new_devices
db.sql = cur
db.commitDB = scan_db_for_new_devices.commit
with patch.multiple(
device_handling,
get_setting_value=Mock(side_effect=get_setting_value_side_effect),
safe_int=Mock(return_value=0),
):
device_handling.create_new_devices(db)
row = cur.execute(
"""
SELECT
devMacSource,
devNameSource,
devVendorSource,
devLastIpSource,
devSsidSource,
devParentMacSource,
devParentPortSource,
devParentRelTypeSource,
devFqdnSource,
devVlanSource
FROM Devices WHERE devMac = ?
""",
("AA:BB:CC:DD:EE:10",),
).fetchone()
assert row["devMacSource"] == "ARPSCAN"
assert row["devNameSource"] == "ARPSCAN"
assert row["devVendorSource"] == "ARPSCAN"
assert row["devLastIpSource"] == "ARPSCAN"
assert row["devSsidSource"] == "ARPSCAN"
assert row["devParentMacSource"] == "ARPSCAN"
assert row["devParentPortSource"] == "ARPSCAN"
assert row["devParentRelTypeSource"] == "NEWDEV"
assert row["devFqdnSource"] == "NEWDEV"
assert row["devVlanSource"] == "NEWDEV"
def test_scan_updates_newdev_device_name(scan_db, mock_device_handlers):
"""Scanner discovers name for device with NEWDEV source."""
cur = scan_db.cursor()