BE:Fix push mode for plugins #1652

.github/skills/code-standards/SKILL.md

@@ -8,12 +8,22 @@ description: NetAlertX coding standards and conventions. Use this when writing c
 - ask me to review before going to each next step (mention n step out of x)  (AI only)
 - before starting, prepare implementation plan  (AI only)
 - ask me to review it and ask any clarifying questions first
-- add test creation as last step - follow repo architecture patterns - do not place in the root of /test
+- add test creation as last step - follow repo architecture patterns - do not place in the root of `/test`
 - code has to be maintainable, no duplicate code
 - follow DRY principle - maintainability of code is more important than speed of implementation
 - code files should be less than 500 LOC for better maintainability
 - DB columns must not contain underscores, use camelCase instead (e.g., deviceInstanceId, not device_instance_id)
-- treat DB as temporary storage for stats, long term configuration should be stored in the /config folder, the /config folder should allow you to restore most of your functionality (excluding historical data)
+- treat DB as temporary storage for stats, long term configuration should be stored in the `/config` folder, the `/config` folder should allow you to restore most of your functionality (excluding historical data)
+- never access DB directly from application layers, always use helper functions in `server/db/db_helper.py` and implement new functionality in handlers (e.g., `DeviceInstance` in `server/models/device_instance.py`)
+- always validate and normalize MAC addresses before writing to DB (use `normalize_mac` from `plugin_helper.py`)
+- all subprocess calls must set explicit timeouts
+- use `timeNowUTC` from `utils.datetime_utils` for all time-related operations and DB timestamps (store all timestamps in UTC)
+- use sanitizers from `server/helper.py` for user input before storing in DB
+- reuse shared mocks and factories from `test/db_test_helpers.py` for tests, never redefine them locally
+- use environment variables for runtime paths, never hardcode paths or use relative paths
+- follow existing code style and structure, and ensure backward compatibility with existing installations when submitting PRs
+- all code needs to be scalable to handle large networks with thousands of devices (10k+) without performance degradation
+
 
 ## File Length
 

front/plugins/maintenance/maintenance.py

@@ -12,6 +12,7 @@ from logger import mylog, Logger  # noqa: E402 [flake8 lint suppression]
 from helper import get_setting_value  # noqa: E402 [flake8 lint suppression]
 from const import logPath  # noqa: E402 [flake8 lint suppression]
 from messaging.in_app import remove_old  # noqa: E402 [flake8 lint suppression]
+from utils.datetime_utils import timeNowUTC  # noqa: E402 [flake8 lint suppression]
 import conf  # noqa: E402 [flake8 lint suppression]
 from pytz import timezone  # noqa: E402 [flake8 lint suppression]
 
@@ -69,6 +70,21 @@ def main():
         mylog('verbose', [f'[{pluginName}] Cleaning in-app notification history'])
         remove_old(MAINT_NOTI_LENGTH)
 
+    # Delete processed sync artefact files older than 24 hours.
+    # These are created by the SYNC plugin (Mode 3) when it renames received
+    # device JSON files to processed_*.log after processing. They have no value
+    # once processed and are not cleaned up anywhere else.
+    _PROCESSED_MAX_AGE_SECS = 24 * 3600
+    now = timeNowUTC(as_string=False).timestamp()
+    deleted = 0
+    for fname in os.listdir(LOG_PATH):
+        if fname.startswith('processed_') and fname.endswith('.log'):
+            fpath = os.path.join(LOG_PATH, fname)
+            if os.path.isfile(fpath) and (now - os.path.getmtime(fpath)) > _PROCESSED_MAX_AGE_SECS:
+                os.remove(fpath)
+                deleted += 1
+    mylog('verbose', [f'[{pluginName}] Deleted {deleted} processed sync artefact file(s) from {LOG_PATH}'])
+
     return 0
 
 

server/api_server/sync_endpoint.py

@@ -11,6 +11,7 @@ INSTALL_PATH = os.getenv("NETALERTX_APP", "/app")
 # Make sure log level is initialized correctly
 lggr = Logger(get_setting_value('LOG_LEVEL'))
 
+
 def handle_sync_get():
     """Handle GET requests for SYNC (NODE → HUB)."""
 

server/plugin.py

@@ -543,10 +543,12 @@ def execute_plugin(db, all_plugins, plugin):
                     # Append the final parameters to sqlParams
                     sqlParams.append(tuple(base_params))
 
-            # keep current instance log file, delete all from other nodes
-            if filename != "last_result.log" and os.path.exists(full_path):
-                os.remove(full_path)  # DEBUG:TODO uncomment 🐛
-                mylog("verbose", f"[Plugins] Processed and deleted file: {full_path} ")
+            # Delete only files received from other nodes (identified by .encoded. or .decoded. in the name).
+            # Local result files (e.g. last_result.ARPSCAN.log) are overwritten each cycle and must
+            # survive so the SYNC plugin can read and forward them to the hub.
+            if (".encoded." in filename or ".decoded." in filename) and os.path.exists(full_path):
+                os.remove(full_path)
+                mylog("verbose", f"[Plugins] Processed and deleted node-sync file: {full_path}")
 
     # app-db-query
     if plugin["data_source"] == "app-db-query":