From 48454f6f2f10213af9cff89002847e38d27b4137 Mon Sep 17 00:00:00 2001 From: "Jokob @NetAlertX" <96159884+jokob-sk@users.noreply.github.com> Date: Fri, 27 Mar 2026 07:30:13 +0000 Subject: [PATCH 1/2] feat(plugins): Optimize badge fetching by using lightweight JSON instead of GraphQL --- front/pluginsCore.php | 75 ++++++++++++++++++------------------------- server/api.py | 2 ++ server/const.py | 5 +++ 3 files changed, 38 insertions(+), 44 deletions(-) diff --git a/front/pluginsCore.php b/front/pluginsCore.php index c76535b0..1065df12 100755 --- a/front/pluginsCore.php +++ b/front/pluginsCore.php @@ -348,51 +348,38 @@ function postPluginGraphQL(gqlField, prefix, foreignKey, dtRequest, callback) { }); } -// Fire a single batched GraphQL request to fetch the Objects dbCount for -// every plugin and populate the sidebar badges immediately on page load. -function prefetchPluginBadges() { - const apiToken = getSetting("API_TOKEN"); - const apiBase = getApiBase(); - const mac = $("#txtMacFilter").val(); - const foreignKey = (mac && mac !== "--") ? mac : null; - - // Build one aliased sub-query per visible plugin - const prefixes = pluginDefinitions - .filter(p => p.show_ui) - .map(p => p.unique_prefix); - - if (prefixes.length === 0) return; - - // GraphQL aliases must be valid identifiers — prefixes already are (A-Z0-9_) - const fkOpt = foreignKey ? `, foreignKey: "${foreignKey}"` : ''; - const fragments = prefixes.map(p => [ - `${p}_obj: pluginsObjects(options: {plugin: "${p}", page: 1, limit: 1${fkOpt}}) { dbCount }`, - `${p}_evt: pluginsEvents(options: {plugin: "${p}", page: 1, limit: 1${fkOpt}}) { dbCount }`, - `${p}_hist: pluginsHistory(options: {plugin: "${p}", page: 1, limit: 1${fkOpt}}) { dbCount }`, - ].join('\n ')).join('\n '); - - const query = `query BadgeCounts {\n ${fragments}\n }`; - - $.ajax({ - method: "POST", - url: `${apiBase}/graphql`, - headers: { "Authorization": `Bearer ${apiToken}`, "Content-Type": "application/json" }, - data: JSON.stringify({ query }), - success: function(response) { - if (response.errors) { - console.error("[plugins] badge prefetch errors:", response.errors); - return; - } - prefixes.forEach(p => { - const obj = response.data[`${p}_obj`]; - const evt = response.data[`${p}_evt`]; - const hist = response.data[`${p}_hist`]; - if (obj) { $(`#badge_${p}`).text(obj.dbCount); $(`#objCount_${p}`).text(obj.dbCount); } - if (evt) { $(`#evtCount_${p}`).text(evt.dbCount); } - if (hist) { $(`#histCount_${p}`).text(hist.dbCount); } - }); +// Fetch the lightweight plugins_stats.json (~1KB) and populate all badge +// and sub-tab counts instantly — no GraphQL, no 250MB file loads. +async function prefetchPluginBadges() { + try { + const stats = await fetchJson('table_plugins_stats.json'); + // Build lookup: { ARPSCAN: { objects: 42, events: 3, history: 100 }, ... } + const counts = {}; + for (const row of stats.data) { + const p = row.tableName; // 'objects' | 'events' | 'history' + const plugin = row.plugin; + if (!counts[plugin]) counts[plugin] = { objects: 0, events: 0, history: 0 }; + counts[plugin][p] = row.cnt; } - }); + for (const [prefix, c] of Object.entries(counts)) { + $(`#badge_${prefix}`).text(c.objects); + $(`#objCount_${prefix}`).text(c.objects); + $(`#evtCount_${prefix}`).text(c.events); + $(`#histCount_${prefix}`).text(c.history); + } + // Set 0 for plugins with no rows in any table + pluginDefinitions.filter(p => p.show_ui).forEach(p => { + const prefix = p.unique_prefix; + if (!counts[prefix]) { + $(`#badge_${prefix}`).text(0); + $(`#objCount_${prefix}`).text(0); + $(`#evtCount_${prefix}`).text(0); + $(`#histCount_${prefix}`).text(0); + } + }); + } catch (err) { + console.error('[plugins] badge prefetch failed:', err); + } } function generateTabs() { diff --git a/server/api.py b/server/api.py index fd775f47..cf4d2ad4 100755 --- a/server/api.py +++ b/server/api.py @@ -16,6 +16,7 @@ from const import ( sql_plugins_events, sql_plugins_history, sql_plugins_objects, + sql_plugins_stats, sql_language_strings, sql_notifications_all, sql_online_history, @@ -66,6 +67,7 @@ def update_api( ["plugins_events", sql_plugins_events], ["plugins_history", sql_plugins_history], ["plugins_objects", sql_plugins_objects], + ["plugins_stats", sql_plugins_stats], ["plugins_language_strings", sql_language_strings], ["notifications", sql_notifications_all], ["online_history", sql_online_history], diff --git a/server/const.py b/server/const.py index 97faa00b..1dc1e83b 100755 --- a/server/const.py +++ b/server/const.py @@ -120,6 +120,11 @@ sql_events_pending_alert = "SELECT * FROM Events where evePendingAlertEmail is sql_events_all = "SELECT rowid, * FROM Events ORDER BY eveDateTime DESC" sql_settings = "SELECT * FROM Settings" sql_plugins_objects = "SELECT * FROM Plugins_Objects" +sql_plugins_stats = """SELECT 'objects' AS tableName, plugin, COUNT(*) AS cnt FROM Plugins_Objects GROUP BY plugin + UNION ALL + SELECT 'events', plugin, COUNT(*) FROM Plugins_Events GROUP BY plugin + UNION ALL + SELECT 'history', plugin, COUNT(*) FROM Plugins_History GROUP BY plugin""" sql_language_strings = "SELECT * FROM Plugins_Language_Strings" sql_notifications_all = "SELECT * FROM Notifications" sql_online_history = "SELECT * FROM Online_History" From 4daead1f8fddc81b2a357a6696bb1f889aba783e Mon Sep 17 00:00:00 2001 From: "Jokob @NetAlertX" <96159884+jokob-sk@users.noreply.github.com> Date: Fri, 27 Mar 2026 08:08:44 +0000 Subject: [PATCH 2/2] feat(plugins): Enhance badge fetching with conditional JSON and GraphQL support --- front/pluginsCore.php | 66 ++++++++++++++++++++++++++++++++++--------- server/api.py | 7 +++++ 2 files changed, 60 insertions(+), 13 deletions(-) diff --git a/front/pluginsCore.php b/front/pluginsCore.php index 1065df12..1fa8ab13 100755 --- a/front/pluginsCore.php +++ b/front/pluginsCore.php @@ -348,28 +348,68 @@ function postPluginGraphQL(gqlField, prefix, foreignKey, dtRequest, callback) { }); } -// Fetch the lightweight plugins_stats.json (~1KB) and populate all badge -// and sub-tab counts instantly — no GraphQL, no 250MB file loads. +// Fetch badge counts for every plugin and populate sidebar + sub-tab counters. +// Fast path: static JSON (~1KB) when no MAC filter is active. +// Filtered path: batched GraphQL aliases when a foreignKey (MAC) is set. async function prefetchPluginBadges() { + const mac = $("#txtMacFilter").val(); + const foreignKey = (mac && mac !== "--") ? mac : null; + + const prefixes = pluginDefinitions + .filter(p => p.show_ui) + .map(p => p.unique_prefix); + + if (prefixes.length === 0) return; + try { - const stats = await fetchJson('table_plugins_stats.json'); - // Build lookup: { ARPSCAN: { objects: 42, events: 3, history: 100 }, ... } - const counts = {}; - for (const row of stats.data) { - const p = row.tableName; // 'objects' | 'events' | 'history' - const plugin = row.plugin; - if (!counts[plugin]) counts[plugin] = { objects: 0, events: 0, history: 0 }; - counts[plugin][p] = row.cnt; + let counts = {}; // { PREFIX: { objects: N, events: N, history: N } } + + if (!foreignKey) { + // ---- FAST PATH: lightweight pre-computed JSON ---- + const stats = await fetchJson('table_plugins_stats.json'); + for (const row of stats.data) { + const p = row.tableName; // 'objects' | 'events' | 'history' + const plugin = row.plugin; + if (!counts[plugin]) counts[plugin] = { objects: 0, events: 0, history: 0 }; + counts[plugin][p] = row.cnt; + } + } else { + // ---- FILTERED PATH: GraphQL with foreignKey ---- + const apiToken = getSetting("API_TOKEN"); + const apiBase = getApiBase(); + const fkOpt = `, foreignKey: "${foreignKey}"`; + const fragments = prefixes.map(p => [ + `${p}_obj: pluginsObjects(options: {plugin: "${p}", page: 1, limit: 1${fkOpt}}) { dbCount }`, + `${p}_evt: pluginsEvents(options: {plugin: "${p}", page: 1, limit: 1${fkOpt}}) { dbCount }`, + `${p}_hist: pluginsHistory(options: {plugin: "${p}", page: 1, limit: 1${fkOpt}}) { dbCount }`, + ].join('\n ')).join('\n '); + + const query = `query BadgeCounts {\n ${fragments}\n }`; + const response = await $.ajax({ + method: "POST", + url: `${apiBase}/graphql`, + headers: { "Authorization": `Bearer ${apiToken}`, "Content-Type": "application/json" }, + data: JSON.stringify({ query }), + }); + if (response.errors) { console.error("[plugins] badge GQL errors:", response.errors); return; } + for (const p of prefixes) { + counts[p] = { + objects: response.data[`${p}_obj`]?.dbCount ?? 0, + events: response.data[`${p}_evt`]?.dbCount ?? 0, + history: response.data[`${p}_hist`]?.dbCount ?? 0, + }; + } } + + // Update DOM for (const [prefix, c] of Object.entries(counts)) { $(`#badge_${prefix}`).text(c.objects); $(`#objCount_${prefix}`).text(c.objects); $(`#evtCount_${prefix}`).text(c.events); $(`#histCount_${prefix}`).text(c.history); } - // Set 0 for plugins with no rows in any table - pluginDefinitions.filter(p => p.show_ui).forEach(p => { - const prefix = p.unique_prefix; + // Zero out plugins with no rows in any table + prefixes.forEach(prefix => { if (!counts[prefix]) { $(`#badge_${prefix}`).text(0); $(`#objCount_${prefix}`).text(0); diff --git a/server/api.py b/server/api.py index cf4d2ad4..11834bc4 100755 --- a/server/api.py +++ b/server/api.py @@ -76,6 +76,13 @@ def update_api( ["custom_endpoint", conf.API_CUSTOM_SQL], ] + # plugins_stats is derived from plugins_objects/events/history — + # ensure it is refreshed when any of its sources are partially updated. + _STATS_SOURCES = {"plugins_objects", "plugins_events", "plugins_history"} + if updateOnlyDataSources and _STATS_SOURCES & set(updateOnlyDataSources): + if "plugins_stats" not in updateOnlyDataSources: + updateOnlyDataSources = list(updateOnlyDataSources) + ["plugins_stats"] + # Save selected database tables for dsSQL in dataSourcesSQLs: if not updateOnlyDataSources or dsSQL[0] in updateOnlyDataSources: