From 0cb7ad633287c1d4b53035eea92610444c732f9d Mon Sep 17 00:00:00 2001 From: "Jokob @NetAlertX" <96159884+jokob-sk@users.noreply.github.com> Date: Sun, 24 May 2026 00:36:23 +0000 Subject: [PATCH] BE:Fix push mode for plugins #1652 --- .github/skills/code-standards/SKILL.md | 14 ++++++++++++-- front/plugins/maintenance/maintenance.py | 16 ++++++++++++++++ server/api_server/sync_endpoint.py | 1 + server/plugin.py | 10 ++++++---- 4 files changed, 35 insertions(+), 6 deletions(-) diff --git a/.github/skills/code-standards/SKILL.md b/.github/skills/code-standards/SKILL.md index faf583d6..af1092d3 100644 --- a/.github/skills/code-standards/SKILL.md +++ b/.github/skills/code-standards/SKILL.md @@ -8,12 +8,22 @@ description: NetAlertX coding standards and conventions. Use this when writing c - ask me to review before going to each next step (mention n step out of x) (AI only) - before starting, prepare implementation plan (AI only) - ask me to review it and ask any clarifying questions first -- add test creation as last step - follow repo architecture patterns - do not place in the root of /test +- add test creation as last step - follow repo architecture patterns - do not place in the root of `/test` - code has to be maintainable, no duplicate code - follow DRY principle - maintainability of code is more important than speed of implementation - code files should be less than 500 LOC for better maintainability - DB columns must not contain underscores, use camelCase instead (e.g., deviceInstanceId, not device_instance_id) -- treat DB as temporary storage for stats, long term configuration should be stored in the /config folder, the /config folder should allow you to restore most of your functionality (excluding historical data) +- treat DB as temporary storage for stats, long term configuration should be stored in the `/config` folder, the `/config` folder should allow you to restore most of your functionality (excluding historical data) +- never access DB directly from application layers, always use helper functions in `server/db/db_helper.py` and implement new functionality in handlers (e.g., `DeviceInstance` in `server/models/device_instance.py`) +- always validate and normalize MAC addresses before writing to DB (use `normalize_mac` from `plugin_helper.py`) +- all subprocess calls must set explicit timeouts +- use `timeNowUTC` from `utils.datetime_utils` for all time-related operations and DB timestamps (store all timestamps in UTC) +- use sanitizers from `server/helper.py` for user input before storing in DB +- reuse shared mocks and factories from `test/db_test_helpers.py` for tests, never redefine them locally +- use environment variables for runtime paths, never hardcode paths or use relative paths +- follow existing code style and structure, and ensure backward compatibility with existing installations when submitting PRs +- all code needs to be scalable to handle large networks with thousands of devices (10k+) without performance degradation + ## File Length diff --git a/front/plugins/maintenance/maintenance.py b/front/plugins/maintenance/maintenance.py index 85caf0b1..a95cf477 100755 --- a/front/plugins/maintenance/maintenance.py +++ b/front/plugins/maintenance/maintenance.py @@ -12,6 +12,7 @@ from logger import mylog, Logger # noqa: E402 [flake8 lint suppression] from helper import get_setting_value # noqa: E402 [flake8 lint suppression] from const import logPath # noqa: E402 [flake8 lint suppression] from messaging.in_app import remove_old # noqa: E402 [flake8 lint suppression] +from utils.datetime_utils import timeNowUTC # noqa: E402 [flake8 lint suppression] import conf # noqa: E402 [flake8 lint suppression] from pytz import timezone # noqa: E402 [flake8 lint suppression] @@ -69,6 +70,21 @@ def main(): mylog('verbose', [f'[{pluginName}] Cleaning in-app notification history']) remove_old(MAINT_NOTI_LENGTH) + # Delete processed sync artefact files older than 24 hours. + # These are created by the SYNC plugin (Mode 3) when it renames received + # device JSON files to processed_*.log after processing. They have no value + # once processed and are not cleaned up anywhere else. + _PROCESSED_MAX_AGE_SECS = 24 * 3600 + now = timeNowUTC(as_string=False).timestamp() + deleted = 0 + for fname in os.listdir(LOG_PATH): + if fname.startswith('processed_') and fname.endswith('.log'): + fpath = os.path.join(LOG_PATH, fname) + if os.path.isfile(fpath) and (now - os.path.getmtime(fpath)) > _PROCESSED_MAX_AGE_SECS: + os.remove(fpath) + deleted += 1 + mylog('verbose', [f'[{pluginName}] Deleted {deleted} processed sync artefact file(s) from {LOG_PATH}']) + return 0 diff --git a/server/api_server/sync_endpoint.py b/server/api_server/sync_endpoint.py index 51d1704b..7556041d 100755 --- a/server/api_server/sync_endpoint.py +++ b/server/api_server/sync_endpoint.py @@ -11,6 +11,7 @@ INSTALL_PATH = os.getenv("NETALERTX_APP", "/app") # Make sure log level is initialized correctly lggr = Logger(get_setting_value('LOG_LEVEL')) + def handle_sync_get(): """Handle GET requests for SYNC (NODE → HUB).""" diff --git a/server/plugin.py b/server/plugin.py index 492f1371..4fb7f31f 100755 --- a/server/plugin.py +++ b/server/plugin.py @@ -543,10 +543,12 @@ def execute_plugin(db, all_plugins, plugin): # Append the final parameters to sqlParams sqlParams.append(tuple(base_params)) - # keep current instance log file, delete all from other nodes - if filename != "last_result.log" and os.path.exists(full_path): - os.remove(full_path) # DEBUG:TODO uncomment 🐛 - mylog("verbose", f"[Plugins] Processed and deleted file: {full_path} ") + # Delete only files received from other nodes (identified by .encoded. or .decoded. in the name). + # Local result files (e.g. last_result.ARPSCAN.log) are overwritten each cycle and must + # survive so the SYNC plugin can read and forward them to the hub. + if (".encoded." in filename or ".decoded." in filename) and os.path.exists(full_path): + os.remove(full_path) + mylog("verbose", f"[Plugins] Processed and deleted node-sync file: {full_path}") # app-db-query if plugin["data_source"] == "app-db-query":