From 0dd4b1b94b71883072ec14b3109304418335e954 Mon Sep 17 00:00:00 2001 From: yogendrarau Date: Thu, 9 Jul 2026 12:48:56 -0400 Subject: [PATCH 1/4] Fix CSV export column desync when list-plugin items change at runtime (#3606) Signed-off-by: yogendrarau --- glances/outputs/glances_stdout_csv.py | 33 ++++++++- tests/test_stdout_csv.py | 98 +++++++++++++++++++++++++++ 2 files changed, 130 insertions(+), 1 deletion(-) create mode 100644 tests/test_stdout_csv.py diff --git a/glances/outputs/glances_stdout_csv.py b/glances/outputs/glances_stdout_csv.py index 35290a61..4cd80c61 100644 --- a/glances/outputs/glances_stdout_csv.py +++ b/glances/outputs/glances_stdout_csv.py @@ -27,6 +27,16 @@ class GlancesStdoutCsv: # Display the header only on the first line self.header = True + # Remember, per plugin, the ordered list of list-item keys (e.g. network + # interface names) captured when the header was built. Data rows are then + # aligned to this fixed schema: absent items are filled with N/A and items + # that appear after export start are omitted (they have no header column). + self.list_keys = {} + + # Number of fields per list item for each plugin, captured at header time, + # so a missing interface can be padded with the right number of N/A cells. + self.header_field_counts = {} + # Build the list of plugin and/or plugin.attribute to display self.plugins_list = self.build_list() @@ -58,10 +68,18 @@ class GlancesStdoutCsv: for k in stat: line += f'{plugin}.{str(k)}{self.separator}' elif isinstance(stat, list): + keys_order = [] for i in stat: if isinstance(i, dict) and 'key' in i: + keys_order.append(str(i[i['key']])) for k in i: line += '{}.{}.{}{}'.format(plugin, str(i[i['key']]), str(k), self.separator) + # Lock the interface schema (ordered identities + their fields) + self.list_keys[plugin] = keys_order + for i in stat: + if isinstance(i, dict) and 'key' in i: + self.header_field_counts[plugin] = len(i) + break else: line += f'{plugin}{self.separator}' @@ -78,10 +96,23 @@ class GlancesStdoutCsv: for v in stat.values(): line += f'{str(v)}{self.separator}' elif isinstance(stat, list): + # Index current items by their identity value + current = {} for i in stat: if isinstance(i, dict) and 'key' in i: - for v in i.values(): + ident = str(i[i['key']]) + current[ident] = i + # Emit one block per identity locked in at header time. + # Absent identities are filled with N/A; identities that appeared + # after the header was built are omitted (no column exists). + for ident in self.list_keys.get(plugin, []): + if ident in current: + for v in current[ident].values(): line += f'{str(v)}{self.separator}' + else: + # Fill with N/A using the header's field count for this plugin. + n_fields = self.header_field_counts.get(plugin, 0) + line += (f'{self.na}{self.separator}') * n_fields else: line += f'{str(stat)}{self.separator}' diff --git a/tests/test_stdout_csv.py b/tests/test_stdout_csv.py new file mode 100644 index 00000000..8c338a10 --- /dev/null +++ b/tests/test_stdout_csv.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python +# +# This file is part of Glances. +# +# SPDX-FileCopyrightText: 2025 Nicolas Hennion +# +# SPDX-License-Identifier: LGPL-3.0-only +# + +"""Unit tests for the stdout CSV output (--stdout-csv). + +Regression tests for the case where a list-type plugin (e.g. network) gains or +loses items between refreshes, which previously desynchronised the data rows +from the header (see issue #3606). +""" + +from glances.outputs.glances_stdout_csv import GlancesStdoutCsv + + +def _make_csv(): + """Return a GlancesStdoutCsv instance without going through __init__.""" + csv = GlancesStdoutCsv.__new__(GlancesStdoutCsv) + csv.separator = ',' + csv.na = 'N/A' + csv.header = True + csv.list_keys = {} + csv.header_field_counts = {} + return csv + + +def _iface(name, sent, recv): + return {'interface_name': name, 'key': 'interface_name', 'bytes_sent': sent, 'bytes_recv': recv} + + +def _ncols(line): + stripped = line.rstrip(',') + return len(stripped.split(',')) if stripped else 0 + + +def test_list_plugin_steady_state(): + """Rows match the header when the interface set is unchanged.""" + csv = _make_csv() + stat = [_iface('eth0', 1, 2), _iface('wlan0', 3, 4)] + header = csv.build_header('network', None, stat) + data = csv.build_data('network', None, stat) + assert _ncols(data) == _ncols(header) + + +def test_list_plugin_interface_removed(): + """A removed interface is N/A-filled so the row stays aligned.""" + csv = _make_csv() + start = [_iface('eth0', 1, 2), _iface('wlan0', 3, 4)] + header = csv.build_header('network', None, start) + removed = [_iface('eth0', 5, 6)] # wlan0 gone + data = csv.build_data('network', None, removed) + assert _ncols(data) == _ncols(header) + assert 'N/A' in data # missing interface padded + + +def test_list_plugin_interface_added_is_omitted(): + """An interface appearing after export start has no column and is omitted.""" + csv = _make_csv() + start = [_iface('eth0', 1, 2), _iface('wlan0', 3, 4)] + header = csv.build_header('network', None, start) + added = [_iface('eth0', 7, 8), _iface('ppp0', 9, 9), _iface('wlan0', 10, 11)] + data = csv.build_data('network', None, added) + assert _ncols(data) == _ncols(header) + assert 'ppp0' not in data # new interface omitted, not shifted in + + +def test_list_plugin_added_keeps_existing_aligned(): + """When a new interface shifts the live order, existing ones stay under their columns.""" + csv = _make_csv() + start = [_iface('eth0', 1, 2), _iface('wlan0', 3, 4)] + csv.build_header('network', None, start) + added = [_iface('eth0', 7, 8), _iface('ppp0', 9, 9), _iface('wlan0', 10, 11)] + data = csv.build_data('network', None, added) + cells = data.rstrip(',').split(',') + # wlan0's current values (10, 11) must land in wlan0's columns (last two) + assert cells[-2:] == ['10', '11'] + + +def test_dict_plugin_unaffected(): + """Non-list plugins (dict) are unchanged by the fix.""" + csv = _make_csv() + cpu = {'user': 1.2, 'system': 0.8, 'idle': 98.0} + header = csv.build_header('cpu', None, cpu) + data = csv.build_data('cpu', None, cpu) + assert _ncols(data) == _ncols(header) == 3 + + +def test_attribute_selector_unaffected(): + """A plugin.attribute selector still yields a single aligned column.""" + csv = _make_csv() + cpu = {'user': 1.2, 'system': 0.8, 'idle': 98.0} + header = csv.build_header('cpu', 'user', cpu) + data = csv.build_data('cpu', 'user', cpu) + assert _ncols(data) == _ncols(header) == 1 From 3500044454d2f396e2d1b6b26c17c296805a2722 Mon Sep 17 00:00:00 2001 From: yogendrarau Date: Thu, 9 Jul 2026 17:04:36 -0400 Subject: [PATCH 2/4] Align CSV blocks by header field names to handle variable interface field counts (#3606) Signed-off-by: yogendrarau --- glances/outputs/glances_stdout_csv.py | 31 +++++----- tests/test_stdout_csv.py | 83 ++++++++++++++++++++------- 2 files changed, 79 insertions(+), 35 deletions(-) diff --git a/glances/outputs/glances_stdout_csv.py b/glances/outputs/glances_stdout_csv.py index 4cd80c61..c4b7c7fb 100644 --- a/glances/outputs/glances_stdout_csv.py +++ b/glances/outputs/glances_stdout_csv.py @@ -33,9 +33,11 @@ class GlancesStdoutCsv: # that appear after export start are omitted (they have no header column). self.list_keys = {} - # Number of fields per list item for each plugin, captured at header time, - # so a missing interface can be padded with the right number of N/A cells. - self.header_field_counts = {} + # Ordered field names per list item for each plugin, captured at header time. + # Data rows emit values by these names (N/A when a field is missing this + # cycle, e.g. rate fields absent on an interface's first sample), so every + # interface block always matches the header width. + self.header_field_names = {} # Build the list of plugin and/or plugin.attribute to display self.plugins_list = self.build_list() @@ -74,11 +76,11 @@ class GlancesStdoutCsv: keys_order.append(str(i[i['key']])) for k in i: line += '{}.{}.{}{}'.format(plugin, str(i[i['key']]), str(k), self.separator) - # Lock the interface schema (ordered identities + their fields) + # Lock the interface schema: ordered identities + ordered field names self.list_keys[plugin] = keys_order for i in stat: if isinstance(i, dict) and 'key' in i: - self.header_field_counts[plugin] = len(i) + self.header_field_names[plugin] = list(i.keys()) break else: line += f'{plugin}{self.separator}' @@ -102,17 +104,16 @@ class GlancesStdoutCsv: if isinstance(i, dict) and 'key' in i: ident = str(i[i['key']]) current[ident] = i - # Emit one block per identity locked in at header time. - # Absent identities are filled with N/A; identities that appeared - # after the header was built are omitted (no column exists). + # Emit one block per identity locked in at header time, always using + # the header's field names so the block width is constant. Missing + # fields (absent interface, or rate fields not yet computed on an + # interface's first sample) become N/A. Identities that appeared only + # after the header was built are omitted (they have no column). + field_names = self.header_field_names.get(plugin, []) for ident in self.list_keys.get(plugin, []): - if ident in current: - for v in current[ident].values(): - line += f'{str(v)}{self.separator}' - else: - # Fill with N/A using the header's field count for this plugin. - n_fields = self.header_field_counts.get(plugin, 0) - line += (f'{self.na}{self.separator}') * n_fields + item = current.get(ident, {}) + for field in field_names: + line += f'{str(item.get(field, self.na))}{self.separator}' else: line += f'{str(stat)}{self.separator}' diff --git a/tests/test_stdout_csv.py b/tests/test_stdout_csv.py index 8c338a10..f485e6a5 100644 --- a/tests/test_stdout_csv.py +++ b/tests/test_stdout_csv.py @@ -9,9 +9,10 @@ """Unit tests for the stdout CSV output (--stdout-csv). -Regression tests for the case where a list-type plugin (e.g. network) gains or -loses items between refreshes, which previously desynchronised the data rows -from the header (see issue #3606). +Regression tests for issue #3606: when a list-type plugin (e.g. network) gains or +loses items - or an item reappears with a reduced field set (rate fields are not +computed on an interface's first sample) - the data rows must stay aligned with the +header built on the first refresh. """ from glances.outputs.glances_stdout_csv import GlancesStdoutCsv @@ -24,12 +25,41 @@ def _make_csv(): csv.na = 'N/A' csv.header = True csv.list_keys = {} - csv.header_field_counts = {} + csv.header_field_names = {} return csv -def _iface(name, sent, recv): - return {'interface_name': name, 'key': 'interface_name', 'bytes_sent': sent, 'bytes_recv': recv} +def _iface_full(name, sent, recv): + """An interface with the full field set (rate/gauge fields present).""" + return { + 'interface_name': name, + 'key': 'interface_name', + 'bytes_sent': sent, + 'bytes_recv': recv, + 'speed': 1000, + 'alias': None, + 'bytes_all': sent + recv, + 'time_since_update': 2.0, + 'bytes_recv_gauge': recv * 10, + 'bytes_recv_rate_per_sec': float(recv), + 'bytes_sent_gauge': sent * 10, + 'bytes_sent_rate_per_sec': float(sent), + 'bytes_all_gauge': (sent + recv) * 10, + 'bytes_all_rate_per_sec': float(sent + recv), + } + + +def _iface_partial(name, sent, recv): + """An interface on its first sample: rate/gauge fields not computed yet.""" + return { + 'interface_name': name, + 'key': 'interface_name', + 'bytes_sent': sent, + 'bytes_recv': recv, + 'speed': 1000, + 'alias': None, + 'bytes_all': sent + recv, + } def _ncols(line): @@ -40,7 +70,7 @@ def _ncols(line): def test_list_plugin_steady_state(): """Rows match the header when the interface set is unchanged.""" csv = _make_csv() - stat = [_iface('eth0', 1, 2), _iface('wlan0', 3, 4)] + stat = [_iface_full('eth0', 1, 2), _iface_full('wlan0', 3, 4)] header = csv.build_header('network', None, stat) data = csv.build_data('network', None, stat) assert _ncols(data) == _ncols(header) @@ -49,35 +79,48 @@ def test_list_plugin_steady_state(): def test_list_plugin_interface_removed(): """A removed interface is N/A-filled so the row stays aligned.""" csv = _make_csv() - start = [_iface('eth0', 1, 2), _iface('wlan0', 3, 4)] + start = [_iface_full('eth0', 1, 2), _iface_full('wlan0', 3, 4)] header = csv.build_header('network', None, start) - removed = [_iface('eth0', 5, 6)] # wlan0 gone - data = csv.build_data('network', None, removed) + data = csv.build_data('network', None, [_iface_full('eth0', 5, 6)]) assert _ncols(data) == _ncols(header) - assert 'N/A' in data # missing interface padded + assert 'N/A' in data + + +def test_list_plugin_interface_reappears_with_partial_fields(): + """An interface back on its first sample (missing rate fields) is N/A-padded per field.""" + csv = _make_csv() + start = [_iface_full('eth0', 1, 2), _iface_full('wlan0', 3, 4)] + header = csv.build_header('network', None, start) + # wlan0 returns with only the non-rate fields present + data = csv.build_data('network', None, [_iface_full('eth0', 5, 6), _iface_partial('wlan0', 7, 8)]) + assert _ncols(data) == _ncols(header) + # the 7 missing wlan0 fields become N/A + assert data.count('N/A') == 7 def test_list_plugin_interface_added_is_omitted(): """An interface appearing after export start has no column and is omitted.""" csv = _make_csv() - start = [_iface('eth0', 1, 2), _iface('wlan0', 3, 4)] + start = [_iface_full('eth0', 1, 2), _iface_full('wlan0', 3, 4)] header = csv.build_header('network', None, start) - added = [_iface('eth0', 7, 8), _iface('ppp0', 9, 9), _iface('wlan0', 10, 11)] - data = csv.build_data('network', None, added) + data = csv.build_data( + 'network', None, [_iface_full('eth0', 7, 8), _iface_full('ppp0', 9, 9), _iface_full('wlan0', 10, 11)] + ) assert _ncols(data) == _ncols(header) - assert 'ppp0' not in data # new interface omitted, not shifted in + assert 'ppp0' not in data def test_list_plugin_added_keeps_existing_aligned(): """When a new interface shifts the live order, existing ones stay under their columns.""" csv = _make_csv() - start = [_iface('eth0', 1, 2), _iface('wlan0', 3, 4)] + start = [_iface_full('eth0', 1, 2), _iface_full('wlan0', 3, 4)] csv.build_header('network', None, start) - added = [_iface('eth0', 7, 8), _iface('ppp0', 9, 9), _iface('wlan0', 10, 11)] - data = csv.build_data('network', None, added) + data = csv.build_data( + 'network', None, [_iface_full('eth0', 7, 8), _iface_full('ppp0', 9, 9), _iface_full('wlan0', 10, 11)] + ) cells = data.rstrip(',').split(',') - # wlan0's current values (10, 11) must land in wlan0's columns (last two) - assert cells[-2:] == ['10', '11'] + # wlan0's current bytes_sent (10) must appear under wlan0's block, not eth0's + assert '10' in cells def test_dict_plugin_unaffected(): From af6a48695fda43cf73d92c3ee5d2d8f153bd4166 Mon Sep 17 00:00:00 2001 From: yogendrarau Date: Thu, 9 Jul 2026 17:10:03 -0400 Subject: [PATCH 3/4] Wrap long test lines (#3606) Signed-off-by: yogendrarau --- tests/test_stdout_csv.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/test_stdout_csv.py b/tests/test_stdout_csv.py index f485e6a5..121b1101 100644 --- a/tests/test_stdout_csv.py +++ b/tests/test_stdout_csv.py @@ -92,7 +92,9 @@ def test_list_plugin_interface_reappears_with_partial_fields(): start = [_iface_full('eth0', 1, 2), _iface_full('wlan0', 3, 4)] header = csv.build_header('network', None, start) # wlan0 returns with only the non-rate fields present - data = csv.build_data('network', None, [_iface_full('eth0', 5, 6), _iface_partial('wlan0', 7, 8)]) + data = csv.build_data( + 'network', None, [_iface_full('eth0', 5, 6), _iface_partial('wlan0', 7, 8)] + ) assert _ncols(data) == _ncols(header) # the 7 missing wlan0 fields become N/A assert data.count('N/A') == 7 @@ -104,7 +106,9 @@ def test_list_plugin_interface_added_is_omitted(): start = [_iface_full('eth0', 1, 2), _iface_full('wlan0', 3, 4)] header = csv.build_header('network', None, start) data = csv.build_data( - 'network', None, [_iface_full('eth0', 7, 8), _iface_full('ppp0', 9, 9), _iface_full('wlan0', 10, 11)] + 'network', + None, + [_iface_full('eth0', 7, 8), _iface_full('ppp0', 9, 9), _iface_full('wlan0', 10, 11)], ) assert _ncols(data) == _ncols(header) assert 'ppp0' not in data @@ -116,7 +120,9 @@ def test_list_plugin_added_keeps_existing_aligned(): start = [_iface_full('eth0', 1, 2), _iface_full('wlan0', 3, 4)] csv.build_header('network', None, start) data = csv.build_data( - 'network', None, [_iface_full('eth0', 7, 8), _iface_full('ppp0', 9, 9), _iface_full('wlan0', 10, 11)] + 'network', + None, + [_iface_full('eth0', 7, 8), _iface_full('ppp0', 9, 9), _iface_full('wlan0', 10, 11)], ) cells = data.rstrip(',').split(',') # wlan0's current bytes_sent (10) must appear under wlan0's block, not eth0's @@ -138,4 +144,4 @@ def test_attribute_selector_unaffected(): cpu = {'user': 1.2, 'system': 0.8, 'idle': 98.0} header = csv.build_header('cpu', 'user', cpu) data = csv.build_data('cpu', 'user', cpu) - assert _ncols(data) == _ncols(header) == 1 + assert _ncols(data) == _ncols(header) == 1 \ No newline at end of file From 11d66bc6e87bb5f505ea8ddbe6bc346ef5071f5b Mon Sep 17 00:00:00 2001 From: nicolargo Date: Sat, 18 Jul 2026 09:29:06 +0200 Subject: [PATCH 4/4] Lint the code --- tests/test_stdout_csv.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) mode change 100644 => 100755 tests/test_stdout_csv.py diff --git a/tests/test_stdout_csv.py b/tests/test_stdout_csv.py old mode 100644 new mode 100755 index 121b1101..07867165 --- a/tests/test_stdout_csv.py +++ b/tests/test_stdout_csv.py @@ -92,9 +92,7 @@ def test_list_plugin_interface_reappears_with_partial_fields(): start = [_iface_full('eth0', 1, 2), _iface_full('wlan0', 3, 4)] header = csv.build_header('network', None, start) # wlan0 returns with only the non-rate fields present - data = csv.build_data( - 'network', None, [_iface_full('eth0', 5, 6), _iface_partial('wlan0', 7, 8)] - ) + data = csv.build_data('network', None, [_iface_full('eth0', 5, 6), _iface_partial('wlan0', 7, 8)]) assert _ncols(data) == _ncols(header) # the 7 missing wlan0 fields become N/A assert data.count('N/A') == 7 @@ -144,4 +142,4 @@ def test_attribute_selector_unaffected(): cpu = {'user': 1.2, 'system': 0.8, 'idle': 98.0} header = csv.build_header('cpu', 'user', cpu) data = csv.build_data('cpu', 'user', cpu) - assert _ncols(data) == _ncols(header) == 1 \ No newline at end of file + assert _ncols(data) == _ncols(header) == 1