From 76e7cc7b3dd7553bd1ca5463c50470c2a0f681da Mon Sep 17 00:00:00 2001 From: jokob-sk Date: Sun, 14 Jun 2026 10:35:08 +1000 Subject: [PATCH 1/2] BE: Removal of stdout.log --- .devcontainer/Dockerfile | 1 - Dockerfile | 1 - Dockerfile.debian | 1 - docs/API_LOGS.md | 3 +- docs/API_OLD.md | 1 - docs/DOCKER_COMPOSE.md | 1 + docs/PLUGINS_DEV.md | 2 +- docs/PLUGINS_DEV_DATASOURCES.md | 2 +- docs/PLUGINS_DEV_DATA_CONTRACT.md | 2 +- docs/PLUGINS_DEV_QUICK_START.md | 4 +- docs/PLUGINS_DEV_SETTINGS.md | 2 +- front/php/components/logs_defaults.json | 11 --- front/plugins/maintenance/maintenance.py | 77 +++++++++++++++---- install/debian12/start.debian12.sh | 2 +- .../services/start-backend.sh | 4 +- install/proxmox/proxmox-install-netalertx.sh | 4 +- install/ubuntu24/install.sh | 2 +- server/api_server/logs_endpoint.py | 2 +- server/api_server/mcp_endpoint.py | 2 +- server/api_server/openapi/schemas.py | 2 +- server/helper.py | 4 +- server/logger.py | 7 -- 22 files changed, 80 insertions(+), 57 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 966f297f..5a0eecc5 100755 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -89,7 +89,6 @@ ENV LOG_STDERR=${NETALERTX_LOG}/stderr.log ENV LOG_APP_PHP_ERRORS=${NETALERTX_LOG}/app.php_errors.log ENV LOG_EXECUTION_QUEUE=${NETALERTX_LOG}/execution_queue.log ENV LOG_REPORT_OUTPUT_JSON=${NETALERTX_LOG}/report_output.json -ENV LOG_STDOUT=${NETALERTX_LOG}/stdout.log ENV LOG_CRON=${NETALERTX_LOG}/cron.log ENV LOG_NGINX_ERROR=${NETALERTX_LOG}/nginx-error.log diff --git a/Dockerfile b/Dockerfile index d5081be7..58a72b7e 100755 --- a/Dockerfile +++ b/Dockerfile @@ -86,7 +86,6 @@ ENV LOG_STDERR=${NETALERTX_LOG}/stderr.log ENV LOG_APP_PHP_ERRORS=${NETALERTX_LOG}/app.php_errors.log ENV LOG_EXECUTION_QUEUE=${NETALERTX_LOG}/execution_queue.log ENV LOG_REPORT_OUTPUT_JSON=${NETALERTX_LOG}/report_output.json -ENV LOG_STDOUT=${NETALERTX_LOG}/stdout.log ENV LOG_CRON=${NETALERTX_LOG}/cron.log ENV LOG_NGINX_ERROR=${NETALERTX_LOG}/nginx-error.log diff --git a/Dockerfile.debian b/Dockerfile.debian index 4b73d1b3..0755f4a9 100755 --- a/Dockerfile.debian +++ b/Dockerfile.debian @@ -61,7 +61,6 @@ ENV LOG_STDERR=${NETALERTX_LOG}/stderr.log ENV LOG_APP_PHP_ERRORS=${NETALERTX_LOG}/app.php_errors.log ENV LOG_EXECUTION_QUEUE=${NETALERTX_LOG}/execution_queue.log ENV LOG_REPORT_OUTPUT_JSON=${NETALERTX_LOG}/report_output.json -ENV LOG_STDOUT=${NETALERTX_LOG}/stdout.log ENV LOG_CRON=${NETALERTX_LOG}/cron.log ENV LOG_NGINX_ERROR=${NETALERTX_LOG}/nginx-error.log diff --git a/docs/API_LOGS.md b/docs/API_LOGS.md index 81f85503..e014db8e 100644 --- a/docs/API_LOGS.md +++ b/docs/API_LOGS.md @@ -12,14 +12,13 @@ Only specific, pre-approved log files can be purged for security and stability r **Query Parameter:** -* `file` → The name of the log file to purge (e.g., `app.log`, `stdout.log`) +* `file` → The name of the log file to purge (e.g., `app.log`) **Allowed Files:** ``` app.log IP_changes.log -stdout.log stderr.log app.php_errors.log execution_queue.log diff --git a/docs/API_OLD.md b/docs/API_OLD.md index 4b6434eb..c5f229e7 100755 --- a/docs/API_OLD.md +++ b/docs/API_OLD.md @@ -352,7 +352,6 @@ This API endpoint retrieves files from the `/tmp/log` folder. | `report_output.json` | JSON format report output | | `report_output.txt` | Text format report output | | `stderr.log` | Logs of standard error output | -| `stdout.log` | Logs of standard output | ## API Endpoint: /config files diff --git a/docs/DOCKER_COMPOSE.md b/docs/DOCKER_COMPOSE.md index 92c93cc3..d5c98959 100755 --- a/docs/DOCKER_COMPOSE.md +++ b/docs/DOCKER_COMPOSE.md @@ -170,6 +170,7 @@ However, if you prefer to have direct, file-level access to your configuration f ``` **After (Using a Local Folder / Bind Mount):** + Make sure to replace `/local_data_dir` with your actual path. The format is `::`. ```yaml diff --git a/docs/PLUGINS_DEV.md b/docs/PLUGINS_DEV.md index 57193043..905426c6 100755 --- a/docs/PLUGINS_DEV.md +++ b/docs/PLUGINS_DEV.md @@ -352,7 +352,7 @@ See: [UI Components](PLUGINS_DEV_UI_COMPONENTS.md) - **Settings Helper:** `/app/server/helper.py` - Use `get_setting_value()` in scripts - **Example Plugins:** `/app/front/plugins/*/` - Study working implementations - **Logs:** `/tmp/log/plugins/` - Plugin output and execution logs -- **Backend Logs:** `/tmp/log/stdout.log` - Core system logs +- **Backend Logs:** `/tmp/log/app.log` - Core system logs --- diff --git a/docs/PLUGINS_DEV_DATASOURCES.md b/docs/PLUGINS_DEV_DATASOURCES.md index cc3b7700..6225bee2 100644 --- a/docs/PLUGINS_DEV_DATASOURCES.md +++ b/docs/PLUGINS_DEV_DATASOURCES.md @@ -364,7 +364,7 @@ sqlite> SELECT ... ; ```bash # Watch backend logs -tail -f /tmp/log/stdout.log | grep -i "data_source\|MYPREFIX" +tail -f /tmp/log/app.log | grep -i "data_source\|MYPREFIX" ``` --- diff --git a/docs/PLUGINS_DEV_DATA_CONTRACT.md b/docs/PLUGINS_DEV_DATA_CONTRACT.md index 2ca03bca..a1f2cdc7 100644 --- a/docs/PLUGINS_DEV_DATA_CONTRACT.md +++ b/docs/PLUGINS_DEV_DATA_CONTRACT.md @@ -245,7 +245,7 @@ cat /tmp/log/plugins/last_result.YOURPREFIX.log | awk -F'|' '{print NF}' | sort **Check core processing in logs:** ```bash -tail -f /tmp/log/stdout.log | grep -i "YOURPREFIX\|Plugins_Objects" +tail -f /tmp/log/app.log | grep -i "YOURPREFIX\|Plugins_Objects" ``` ## See Also diff --git a/docs/PLUGINS_DEV_QUICK_START.md b/docs/PLUGINS_DEV_QUICK_START.md index 6bac4ae5..d9a17766 100644 --- a/docs/PLUGINS_DEV_QUICK_START.md +++ b/docs/PLUGINS_DEV_QUICK_START.md @@ -143,7 +143,7 @@ ls -la /tmp/log/plugins/last_result.MYPLN.log cat /tmp/log/plugins/last_result.MYPLN.log # Check backend logs for errors -tail -f /tmp/log/stdout.log | grep "my_plugin\|MYPLN" +tail -f /tmp/log/app.log | grep "my_plugin\|MYPLN" ``` ## Next Steps @@ -162,7 +162,7 @@ Now that you have a working basic plugin: |-------|----------| | "Module not found" errors | Ensure `sys.path` includes `/app/server` and `/app/front/plugins` | | Settings not appearing | Restart backend and clear browser cache | -| Results not showing up | Check `/tmp/log/plugins/*.log` and `/tmp/log/stdout.log` for errors | +| Results not showing up | Check `/tmp/log/plugins/*.log` and `/tmp/log/app.log` for errors | | Permission denied | Plugin runs in container, use absolute paths like `/app/front/plugins/...` | ## Resources diff --git a/docs/PLUGINS_DEV_SETTINGS.md b/docs/PLUGINS_DEV_SETTINGS.md index 552bbf91..f25dc46a 100644 --- a/docs/PLUGINS_DEV_SETTINGS.md +++ b/docs/PLUGINS_DEV_SETTINGS.md @@ -507,7 +507,7 @@ mylog('none', f"Setting value: {value}") Check backend logs: ```bash -tail -f /tmp/log/stdout.log | grep -i "setting\|MYPLN" +tail -f /tmp/log/app.log | grep -i "setting\|MYPLN" ``` ## See Also diff --git a/front/php/components/logs_defaults.json b/front/php/components/logs_defaults.json index b40f2bb2..95fd6a7c 100755 --- a/front/php/components/logs_defaults.json +++ b/front/php/components/logs_defaults.json @@ -70,17 +70,6 @@ "filePath": "__NETALERTX_LOG__/db_is_locked.log", "textAreaCssClass": "logs logs-small" }, - { - "buttons": [ - { - "labelStringCode": "Maint_PurgeLog", - "event": "logManage('stdout.log', 'cleanLog')" - } - ], - "fileName": "stdout.log", - "filePath": "__NETALERTX_LOG__/stdout.log", - "textAreaCssClass": "logs logs-small" - }, { "buttons": [ { diff --git a/front/plugins/maintenance/maintenance.py b/front/plugins/maintenance/maintenance.py index a95cf477..81180cbe 100755 --- a/front/plugins/maintenance/maintenance.py +++ b/front/plugins/maintenance/maintenance.py @@ -2,7 +2,6 @@ import os import sys -from collections import deque # Register NetAlertX directories INSTALL_PATH = os.getenv('NETALERTX_APP', '/app') @@ -36,34 +35,49 @@ def main(): MAINT_LOG_LENGTH = int(get_setting_value('MAINT_LOG_LENGTH')) MAINT_NOTI_LENGTH = int(get_setting_value('MAINT_NOTI_LENGTH')) - logFiles = ["app.log", "nginx-error.log", "stdout.log"] + logFiles = ["app.log", "nginx-error.log"] # Check if set if MAINT_LOG_LENGTH != 0: + MAX_TAIL_SIZE = 0.1 * 1024 * 1024 # 0.1 MB + for fileEntry in logFiles: - mylog('verbose', [f'[{pluginName}] Cleaning file']) + logFile = os.path.join(logPath, fileEntry) - logFile = logPath + "/" + fileEntry + if not os.path.isfile(logFile): + mylog('verbose', [f'[{pluginName}] File not found: {fileEntry}']) + continue - mylog('verbose', [f'[{pluginName}] {fileEntry} size BEFORE: {os.path.getsize(logFile)}']) + size_before = os.path.getsize(logFile) - # Using a deque to efficiently keep the last N lines - lines_to_keep = deque(maxlen=MAINT_LOG_LENGTH) + mylog('verbose', [f'[{pluginName}] {fileEntry} size BEFORE: {size_before}']) - with open(logFile, 'r') as file: - # Read lines from the file and store the last N lines - for line in file: - lines_to_keep.append(line) + try: - with open(logFile, 'w') as file: - # Write the last N lines back to the file - file.writelines(lines_to_keep) + if size_before > MAX_TAIL_SIZE: - mylog('verbose', [f'[{pluginName}] {fileEntry} size AFTER: {os.path.getsize(logFile)}']) + mylog('verbose', [f'[{pluginName}] {fileEntry} exceeds {MAX_TAIL_SIZE} bytes, truncating']) - mylog('verbose', [f'[{pluginName}] Cleanup of {fileEntry} finished']) + with open(logFile, 'r+b') as f: + f.truncate(0) + + else: + + lines_to_keep = tail_file(logFile, MAINT_LOG_LENGTH) + + with open(logFile, 'r+b') as f: + f.seek(0) + f.truncate() + f.writelines(lines_to_keep) + + size_after = os.path.getsize(logFile) + + mylog('verbose', [f'[{pluginName}] {fileEntry} size AFTER: {size_after}']) + + except Exception as e: + mylog('none', [f'[{pluginName}] Failed to clean {fileEntry}: {e}']) # Check if set if MAINT_NOTI_LENGTH != 0: @@ -88,6 +102,37 @@ def main(): return 0 +def tail_file(filepath, num_lines): + """ + Return the last num_lines lines from a file without reading the entire file. + """ + if num_lines <= 0: + return [] + + block_size = 8192 + + with open(filepath, 'rb') as f: + f.seek(0, os.SEEK_END) + file_size = f.tell() + + blocks = [] + lines_found = 0 + position = file_size + + while position > 0 and lines_found <= num_lines: + read_size = min(block_size, position) + position -= read_size + + f.seek(position) + block = f.read(read_size) + + blocks.append(block) + lines_found += block.count(b'\n') + + data = b''.join(reversed(blocks)) + return data.splitlines(keepends=True)[-num_lines:] + + # =============================================================================== # BEGIN # =============================================================================== diff --git a/install/debian12/start.debian12.sh b/install/debian12/start.debian12.sh index 311ff49d..e633dfe5 100755 --- a/install/debian12/start.debian12.sh +++ b/install/debian12/start.debian12.sh @@ -112,7 +112,7 @@ fi # Create an empty log files # Create the execution_queue.log file if it doesn't exist -touch "${INSTALL_DIR}"/log/{app.log,execution_queue.log,app_front.log,app.php_errors.log,stderr.log,stdout.log,db_is_locked.log} +touch "${INSTALL_DIR}"/log/{app.log,execution_queue.log,app_front.log,app.php_errors.log,stderr.log,db_is_locked.log} touch "${INSTALL_DIR}"/api/user_notifications.json # Create plugins sub-directory if it doesn't exist in case a custom log folder is used mkdir -p "${INSTALL_DIR}"/log/plugins diff --git a/install/production-filesystem/services/start-backend.sh b/install/production-filesystem/services/start-backend.sh index 77e6dfff..e40f7b53 100755 --- a/install/production-filesystem/services/start-backend.sh +++ b/install/production-filesystem/services/start-backend.sh @@ -11,6 +11,6 @@ done # Force kill if graceful shutdown failed killall -KILL python3 &>/dev/null -echo "Starting python3 $(cat /services/config/python/backend-extra-launch-parameters 2>/dev/null) -m server > ${NETALERTX_LOG}/stdout.log 2> >(tee ${NETALERTX_LOG}/stderr.log >&2)" read -ra EXTRA_PARAMS < <(cat /services/config/python/backend-extra-launch-parameters 2>/dev/null) -exec python3 "${EXTRA_PARAMS[@]}" -m server > "${NETALERTX_LOG}/stdout.log" 2> >(tee "${NETALERTX_LOG}/stderr.log" >&2) +echo "Starting python3 ${EXTRA_PARAMS[*]} -m server (stdout+stderr merged)" +exec python3 "${EXTRA_PARAMS[@]}" -m server 2>&1 diff --git a/install/proxmox/proxmox-install-netalertx.sh b/install/proxmox/proxmox-install-netalertx.sh index 351aa19a..27fd3a5a 100755 --- a/install/proxmox/proxmox-install-netalertx.sh +++ b/install/proxmox/proxmox-install-netalertx.sh @@ -307,7 +307,7 @@ mkdir -p "${INSTALL_DIR}"/log/plugins chown -R www-data:www-data "${INSTALL_DIR}"/log/plugins # Create the execution_queue.log file if it doesn't exist -touch ${INSTALL_DIR}/log/{app.log,execution_queue.log,app_front.log,app.php_errors.log,stderr.log,stdout.log,db_is_locked.log} +touch ${INSTALL_DIR}/log/{app.log,execution_queue.log,app_front.log,app.php_errors.log,stderr.log,db_is_locked.log} touch ${INSTALL_DIR}/api/user_notifications.json chown -R www-data:www-data "${INSTALL_DIR}"/log "${INSTALL_DIR}"/api chmod -R ug+rwX "${INSTALL_DIR}"/log "${INSTALL_DIR}"/api @@ -319,7 +319,7 @@ chown www-data:www-data "${INSTALL_DIR}/log" "${INSTALL_DIR}/api" mkdir -p "${INSTALL_DIR}/log/plugins" # Create log and api files directly as the www-data user to ensure correct ownership from the start. -sudo -u www-data touch ${INSTALL_DIR}/log/{app.log,execution_queue.log,app_front.log,app.php_errors.log,stderr.log,stdout.log,db_is_locked.log} +sudo -u www-data touch ${INSTALL_DIR}/log/{app.log,execution_queue.log,app_front.log,app.php_errors.log,stderr.log,db_is_locked.log} sudo -u www-data touch ${INSTALL_DIR}/api/user_notifications.json # Set final permissions for all created files and directories. diff --git a/install/ubuntu24/install.sh b/install/ubuntu24/install.sh index 588054d6..b61b8385 100755 --- a/install/ubuntu24/install.sh +++ b/install/ubuntu24/install.sh @@ -276,7 +276,7 @@ mount -t tmpfs -o noexec,nosuid,nodev tmpfs "${INSTALL_DIR}/api" # Create log files if they don't exist echo "[INSTALL] Creating log files if they don't exist" -touch "${INSTALL_DIR}"/log/{app.log,execution_queue.log,app_front.log,app.php_errors.log,stderr.log,stdout.log,db_is_locked.log} +touch "${INSTALL_DIR}"/log/{app.log,execution_queue.log,app_front.log,app.php_errors.log,stderr.log,db_is_locked.log} touch "${INSTALL_DIR}"/api/user_notifications.json # Create plugins sub-directory if it doesn't exist in case a custom log folder is used mkdir -p "${INSTALL_DIR}"/log/plugins diff --git a/server/api_server/logs_endpoint.py b/server/api_server/logs_endpoint.py index d3a8fd50..1461708e 100644 --- a/server/api_server/logs_endpoint.py +++ b/server/api_server/logs_endpoint.py @@ -26,7 +26,7 @@ def clean_log(log_file): flask.Response: JSON response with success and message keys """ allowed_files = [ - 'app.log', 'app_front.log', 'IP_changes.log', 'stdout.log', 'stderr.log', + 'app.log', 'app_front.log', 'IP_changes.log', 'stderr.log', 'app.php_errors.log', 'execution_queue.log', 'db_is_locked.log' ] diff --git a/server/api_server/mcp_endpoint.py b/server/api_server/mcp_endpoint.py index 827338a2..eb56f65b 100644 --- a/server/api_server/mcp_endpoint.py +++ b/server/api_server/mcp_endpoint.py @@ -812,7 +812,7 @@ def _list_resources() -> List[Dict[str, Any]]: # Log files log_files = [ - ("stdout.log", "Backend stdout log"), + ("app.log", "Backend log"), ("stderr.log", "Backend stderr log"), ("app_front.log", "Frontend commands log"), ("app.php_errors.log", "PHP errors log") diff --git a/server/api_server/openapi/schemas.py b/server/api_server/openapi/schemas.py index 3b1728d7..86baa66e 100644 --- a/server/api_server/openapi/schemas.py +++ b/server/api_server/openapi/schemas.py @@ -57,7 +57,7 @@ NOTIFICATION_LEVELS = Literal["info", "warning", "error", "alert", "interrupt"] ALLOWED_TABLES = Literal["Devices", "Events", "Sessions", "Settings", "CurrentScan", "Online_History", "Plugins_Objects", "Plugins_History"] ALLOWED_LOG_FILES = Literal[ - "app.log", "app_front.log", "IP_changes.log", "stdout.log", "stderr.log", + "app.log", "app_front.log", "IP_changes.log", "stderr.log", "app.php_errors.log", "execution_queue.log", "db_is_locked.log" ] diff --git a/server/helper.py b/server/helper.py index bfbf65c1..ad1ef109 100755 --- a/server/helper.py +++ b/server/helper.py @@ -17,7 +17,7 @@ import ipaddress import conf from const import applicationPath, fullConfPath, fullDbPath, dbPath, confPath, apiPath -from logger import mylog, logResult +from logger import mylog # Register NetAlertX directories using runtime configuration INSTALL_PATH = applicationPath @@ -70,7 +70,7 @@ def initialiseFile(pathToCheck, defaultFile): mylog("none", ["[Setup] (" + defaultFile + ") copied over successfully to (" + pathToCheck + ")."],) # write stdout and stderr into .log files for debugging if needed - logResult(stdout, stderr) # TO-DO should be changed to mylog + # logResult(stdout, stderr) # TO-DO should be changed to mylog except subprocess.CalledProcessError as e: # An error occured, handle it diff --git a/server/logger.py b/server/logger.py index 6f0ef4a0..c3a8c3ec 100755 --- a/server/logger.py +++ b/server/logger.py @@ -146,13 +146,6 @@ def append_file_binary(file_path, input_data): file.write(input_data) -def logResult(stdout, stderr): - if stderr is not None: - append_file_binary(logPath + "/stderr.log", stderr) - if stdout is not None: - append_file_binary(logPath + "/stdout.log", stdout) - - def append_line_to_file(pPath, pText): if sys.version_info < (3, 0): file = io.open(pPath, mode="a", encoding="utf-8") From 83b10b4a708a97385be11db280dea6646a5b32f5 Mon Sep 17 00:00:00 2001 From: jokob-sk Date: Sun, 14 Jun 2026 10:53:52 +1000 Subject: [PATCH 2/2] BE: Removal of stdout.log --- front/plugins/maintenance/maintenance.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/front/plugins/maintenance/maintenance.py b/front/plugins/maintenance/maintenance.py index 81180cbe..f59ff5f6 100755 --- a/front/plugins/maintenance/maintenance.py +++ b/front/plugins/maintenance/maintenance.py @@ -40,7 +40,7 @@ def main(): # Check if set if MAINT_LOG_LENGTH != 0: - MAX_TAIL_SIZE = 0.1 * 1024 * 1024 # 0.1 MB + MAX_TAIL_SIZE = MAINT_LOG_LENGTH * 80 # Bytes = lines * approx 80 chars per log line for fileEntry in logFiles: @@ -56,14 +56,10 @@ def main(): try: - if size_before > MAX_TAIL_SIZE: - - mylog('verbose', [f'[{pluginName}] {fileEntry} exceeds {MAX_TAIL_SIZE} bytes, truncating']) - - with open(logFile, 'r+b') as f: - f.truncate(0) - + if size_before <= MAX_TAIL_SIZE: + mylog('verbose', [f'[{pluginName}] {fileEntry} already within limit, skipping']) else: + mylog('verbose', [f'[{pluginName}] {fileEntry} exceeds limit, trimming to last {MAINT_LOG_LENGTH} lines']) lines_to_keep = tail_file(logFile, MAINT_LOG_LENGTH)