diff --git a/glances/plugins/plugin/model.py b/glances/plugins/plugin/model.py index ce0a764d..526bb641 100644 --- a/glances/plugins/plugin/model.py +++ b/glances/plugins/plugin/model.py @@ -53,9 +53,69 @@ fields_unit_type = { } +class LazyViews(dict): + """Views for a list-of-dicts plugin, built per key on first access. + + processlist can hold tens of thousands of items while the UI shows a few dozen rows, and + in curses mode nothing reads the views at all, so building them up front is pure waste. + Anything that reads the whole object goes through get_views(), which materialises it + first, so consumers still see a plain and complete dict. + """ + + def __init__(self, plugin, raw, key_field): + super().__init__() + self._plugin = plugin + self._items = {item[key_field]: item for item in raw} + + def _build(self, key): + item = self._items[key] # a genuinely unknown key raises KeyError, as a dict would + return {field: self._plugin._build_view_for_field(key=key, field=field) for field in item} + + def __missing__(self, key): + built = self._build(key) + super().__setitem__(key, built) + return built + + # Membership is deliberately left as dict's own: it reports what has been built, not what + # could be. _build_view_for_field() asks whether a previous view exists before indexing + # into it, and answering "yes" for an entry that is only about to be created sends it + # straight back in here. + + def materialize(self): + """Build every remaining view and return self as a fully populated dict.""" + for key in self._items: + if not super().__contains__(key): + super().__setitem__(key, self._build(key)) + return self + + def __iter__(self): + self.materialize() + return super().__iter__() + + def __len__(self): + self.materialize() + return super().__len__() + + def keys(self): + self.materialize() + return super().keys() + + def values(self): + self.materialize() + return super().values() + + def items(self): + self.materialize() + return super().items() + + class GlancesPluginModel: """Main class for Glances plugin model.""" + # Build the per-item views on demand instead of up front. Only worth it for plugins whose + # item count is large and unrelated to how many rows the UI shows. + lazy_views = False + def __init__(self, args=None, config=None, items_history_list=None, stats_init_value={}, fields_description=None): """Init the plugin of plugins model class. @@ -669,6 +729,13 @@ class GlancesPluginModel: """ ret = {} + # hide_zero makes _build_view_for_field() read the previous self.views, which a lazy + # container cannot provide: it is itself self.views by then, so the lookup would + # recurse into the entry being built. Fall back to building everything up front. + if self.lazy_views and not self.hide_zero and isinstance(self.get_raw(), list) and self.get_key() is not None: + self.views = LazyViews(self, self.get_raw(), self.get_key()) + return self.views + if self.get_raw() is not None and isinstance(self.get_raw(), list) and self.get_key() is not None: # Stats are stored in a list of dict (ex: DISKIO, NETWORK, FS...) for i in self.get_raw(): @@ -706,6 +773,9 @@ class GlancesPluginModel: item_views = self.views else: item_views = self.views[item] + if isinstance(item_views, LazyViews): + # The caller gets the object itself, so hand out a fully built one. + item_views = item_views.materialize() if key is None: return item_views if key not in item_views: diff --git a/glances/plugins/processlist/__init__.py b/glances/plugins/processlist/__init__.py index 2e084490..1fd8dd56 100644 --- a/glances/plugins/processlist/__init__.py +++ b/glances/plugins/processlist/__init__.py @@ -122,6 +122,10 @@ class ProcesslistPlugin(GlancesPluginModel): stats is a list """ + # The list holds every process on the machine while the UI shows a few dozen rows, and + # the curses output never reads the views at all. Build them on demand. + lazy_views = True + # Default list of processes stats to be grabbed / displayed # Can be altered by glances_processes.disable_stats enable_stats = [ diff --git a/tests/test_lazy_views.py b/tests/test_lazy_views.py new file mode 100644 index 00000000..ae36ac71 --- /dev/null +++ b/tests/test_lazy_views.py @@ -0,0 +1,80 @@ +"""Tests for the lazily built process views.""" + +import json +from unittest import mock + +import pytest + +from glances.plugins.plugin.model import LazyViews +from glances.plugins.processlist import ProcesslistPlugin + + +def make_process(pid): + return { + 'pid': pid, + 'key': 'pid', + 'name': f'proc{pid}', + 'cmdline': [f'proc{pid}'], + 'username': 'someone', + 'status': 'S', + 'nice': 0, + 'num_threads': 1, + 'cpu_percent': 0.0, + 'memory_percent': 0.1, + 'memory_info': {'rss': 1024, 'vms': 2048}, + 'cpu_times': {'user': 1.0, 'system': 1.0}, + 'io_counters': [0, 0, 0, 0, 0], + 'time_since_update': 1.0, + } + + +@pytest.fixture +def plugin(): + p = ProcesslistPlugin(args=mock.Mock(time=2), config=None) + p.stats = [make_process(pid) for pid in range(10)] + return p + + +def built_count(views): + """How many entries exist without asking the lazy container to build more.""" + return dict.__len__(views) + + +def test_views_are_lazy_by_default(plugin): + plugin.update_views() + assert isinstance(plugin.views, LazyViews) + assert built_count(plugin.views) == 0 + + +def test_reading_one_key_builds_only_that_key(plugin): + plugin.update_views() + view = plugin.views[4] + assert view['cpu_percent']['decoration'] is not None + assert built_count(plugin.views) == 1 + + +def test_unknown_key_raises(plugin): + plugin.update_views() + with pytest.raises(KeyError): + plugin.views[999999] + + +def test_get_views_materializes_everything(plugin): + plugin.update_views() + views = plugin.get_views() + assert len(views) == 10 + assert built_count(views) == 10 + + +def test_json_contains_every_process(plugin): + plugin.update_views() + assert len(json.loads(plugin.get_json_views())) == 10 + + +def test_hide_zero_falls_back_to_eager_views(plugin): + """hide_zero reads the previous view of a key, which a lazy container cannot provide.""" + plugin.hide_zero = True + plugin.hide_zero_fields = ['cpu_percent'] + plugin.update_views() + assert not isinstance(plugin.views, LazyViews) + assert len(plugin.views) == 10