From 2353e49655238436733c4ccc0f6bafe433058ed8 Mon Sep 17 00:00:00 2001 From: nicolargo Date: Sun, 31 May 2026 17:33:37 +0200 Subject: [PATCH 1/3] Make only one API call for multiple containers (use sparse=True option). Also correct behavor for Podman and LXD. --- glances/plugins/containers/engines/docker.py | 85 ++++++++++++++++++-- glances/plugins/containers/engines/lxd.py | 7 +- glances/plugins/containers/engines/podman.py | 26 +++++- 3 files changed, 107 insertions(+), 11 deletions(-) diff --git a/glances/plugins/containers/engines/docker.py b/glances/plugins/containers/engines/docker.py index 7c6be50d..c818def7 100644 --- a/glances/plugins/containers/engines/docker.py +++ b/glances/plugins/containers/engines/docker.py @@ -9,6 +9,7 @@ """Docker Extension unit for Glances' Containers plugin.""" import time +from concurrent.futures import ThreadPoolExecutor from typing import Any from glances.globals import nativestr, pretty_date, replace_special_chars @@ -29,6 +30,13 @@ else: disable_plugin_docker = False +# Issue #3559: number of containers inspected concurrently per update cycle. +# docker-py's default list() inspects each container sequentially (one synchronous +# API call per container). Kept under docker-py's default connection pool size to +# avoid "connection pool is full" warnings. +MAX_CONCURRENT_INSPECT = 6 + + class DockerStatsFetcher: MANDATORY_MEMORY_FIELDS = ['usage', 'limit'] @@ -235,6 +243,10 @@ class DockerExtension: self.ext_name = "containers (Docker)" self.stats_fetchers = {} + # Issue #3559: cache the (immutable) image tags per container id to avoid + # one inspect_image API call per container on every refresh. + self.image_cache = {} + self.connect() def connect(self) -> None: @@ -269,7 +281,11 @@ class DockerExtension: try: # Issue #1152: Docker module doesn't export details about stopped containers # The Containers/all key of the configuration file should be set to True - containers = self.client.containers.list(all=all_tag) + # Issue #3559: docker-py's list() inspects each container sequentially (one + # synchronous API call per container), which dominates the update time when + # there are many containers. Use sparse=True to get the list in a single API + # call, then inspect the containers concurrently (see _inspect_concurrently). + containers = self.client.containers.list(all=all_tag, sparse=True) self.display_error = True except Exception as e: if self.display_error: @@ -279,6 +295,9 @@ class DockerExtension: logger.debug(f"{self.ext_name} plugin - Can't get containers list ({e})") return version_stats, [] + # Inspect the containers concurrently to populate their attributes (issue #3559) + containers = self._inspect_concurrently(containers) + # Start new thread for new container for container in containers: if container.id not in self.stats_fetchers: @@ -288,7 +307,8 @@ class DockerExtension: self.stats_fetchers[container.id] = DockerStatsFetcher(container) # Stop threads for non-existing containers - absent_containers = set(self.stats_fetchers.keys()) - {c.id for c in containers} + current_ids = {c.id for c in containers} + absent_containers = set(self.stats_fetchers.keys()) - current_ids for container_id in absent_containers: # Stop the StatsFetcher logger.debug(f"{self.ext_name} plugin - Stop thread for old container {container_id[:12]}") @@ -296,15 +316,69 @@ class DockerExtension: # Delete the StatsFetcher from the dict del self.stats_fetchers[container_id] + # Drop image cache entries for non-existing containers (issue #3559) + for container_id in set(self.image_cache) - current_ids: + del self.image_cache[container_id] + # Get stats for all containers container_stats = [self.generate_stats(container) for container in containers] return version_stats, container_stats + def _inspect_concurrently(self, containers: list) -> list: + """Inspect the given (sparse) containers concurrently and return the live ones. + + docker-py's `list()` inspects containers one at a time, i.e. O(N) synchronous + API calls on the caller thread (issue #3559). `list(sparse=True)` returns + summary-only objects in a single call; `reload()` performs the per-container + inspect, which we parallelise here to keep the update time roughly independent + of the number of containers. Containers removed between the list and the inspect + are dropped. + """ + if not containers: + return [] + + def reload_container(container): + try: + container.reload() + return container + except Exception as e: + # Container removed between list and inspect, or inspect failed + logger.debug(f"{self.ext_name} plugin - Can't inspect container {container.id[:12]} ({e})") + return None + + max_workers = min(len(containers), MAX_CONCURRENT_INSPECT) + with ThreadPoolExecutor(max_workers=max_workers) as executor: + reloaded = executor.map(reload_container, containers) + + return [container for container in reloaded if container is not None] + @property def key(self) -> str: """Return the key of the list.""" return 'name' + def _get_image(self, container): + """Return the container image tags, cached per container id (issue #3559). + + `container.image` performs an inspect_image API call. The image of a container + is immutable for its lifetime, so the result is cached to avoid one synchronous + API call per container on every refresh. + """ + if container.id in self.image_cache: + return self.image_cache[container.id] + + try: + # API fails on Unraid - See issue #2233 + # Access container.image only once (it triggers an inspect_image API call) + tags = container.image.tags + image = (','.join(tags if tags else []),) + except (requests.exceptions.HTTPError, docker.errors.NullResource): + # Container plugin crashes with docker.errors.NullResource on Podman pod infra containers issue #3498 + image = '' + + self.image_cache[container.id] = image + return image + def generate_stats(self, container) -> dict[str, Any]: # Init the stats for the current container # Manage healthy status (see issue #3402) @@ -331,12 +405,7 @@ class DockerExtension: } # Container Image - try: - # API fails on Unraid - See issue #2233 - stats['image'] = (','.join(container.image.tags if container.image.tags else []),) - except (requests.exceptions.HTTPError, docker.errors.NullResource): - # Container plugin crashes with docker.errors.NullResource on Podman pod infra containers issue #3498 - stats['image'] = '' + stats['image'] = self._get_image(container) if container.attrs['Config'].get('Entrypoint', None): stats['command'].extend(container.attrs['Config'].get('Entrypoint', [])) diff --git a/glances/plugins/containers/engines/lxd.py b/glances/plugins/containers/engines/lxd.py index 26338220..f1e34ef7 100644 --- a/glances/plugins/containers/engines/lxd.py +++ b/glances/plugins/containers/engines/lxd.py @@ -245,7 +245,12 @@ class LxdExtension: # List instances try: - instances = self.client.instances.all() + # Issue #3559: recursion=1 fetches every instance's config (status, devices, + # created_at, ...) in a single API call. Without it, pylxd creates name-only + # instances and each attribute access below (i.status, expanded_devices, ...) + # lazily triggers one synchronous GET per instance, i.e. O(N) calls per cycle. + # State (CPU/MEM/NET/IO) still comes from the background pollers, not from here. + instances = self.client.instances.all(recursion=1) # In a cluster, only show instances running on this node if self.local_node: instances = [i for i in instances if getattr(i, 'location', None) == self.local_node] diff --git a/glances/plugins/containers/engines/podman.py b/glances/plugins/containers/engines/podman.py index 204683fa..ec34ba02 100644 --- a/glances/plugins/containers/engines/podman.py +++ b/glances/plugins/containers/engines/podman.py @@ -295,6 +295,10 @@ class PodmanExtension: self.pods_stats_fetcher = None self.container_stats_fetchers = {} + # Issue #3559: cache the (immutable) image tags per container id to avoid + # one inspect_image API call per container on every refresh. + self.image_cache = {} + self.connect() def connect(self): @@ -354,7 +358,8 @@ class PodmanExtension: self.container_stats_fetchers[container.id] = PodmanContainerStatsFetcher(container) # Stop threads for non-existing containers - absent_containers = set(self.container_stats_fetchers.keys()) - {c.id for c in containers} + current_ids = {c.id for c in containers} + absent_containers = set(self.container_stats_fetchers.keys()) - current_ids for container_id in absent_containers: # Stop the StatsFetcher logger.debug(f"{self.ext_name} plugin - Stop thread for old container {container_id[:12]}") @@ -362,6 +367,10 @@ class PodmanExtension: # Delete the StatsFetcher from the dict del self.container_stats_fetchers[container_id] + # Drop image cache entries for non-existing containers (issue #3559) + for container_id in set(self.image_cache) - current_ids: + del self.image_cache[container_id] + # Get stats for all containers container_stats = [self.generate_stats(container) for container in containers] @@ -378,13 +387,26 @@ class PodmanExtension: """Return the key of the list.""" return "name" + def _get_image(self, container): + """Return the container image tags, cached per container id (issue #3559). + + `container.image` performs an inspect_image API call. The image of a container + is immutable for its lifetime, so the result is cached to avoid one synchronous + API call per container on every refresh. + """ + if container.id not in self.image_cache: + # Access container.image only once (it triggers an inspect_image API call) + tags = container.image.tags + self.image_cache[container.id] = ",".join(tags if tags else []) + return self.image_cache[container.id] + def generate_stats(self, container) -> dict[str, Any]: # Init the stats for the current container stats = { "key": self.key, "name": nativestr(container.name), "id": container.id, - "image": ",".join(container.image.tags if container.image.tags else []), + "image": self._get_image(container), "status": container.attrs["State"], "created": container.attrs["Created"], "command": container.attrs.get("Command") or [], From e68e9f44526ecc6368d28fb39fd04b63b2f9e660 Mon Sep 17 00:00:00 2001 From: nicolargo Date: Sun, 31 May 2026 17:34:25 +0200 Subject: [PATCH 2/3] Add unit test to containers/docker plugin --- tests/test_plugin_docker.py | 119 ++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100755 tests/test_plugin_docker.py diff --git a/tests/test_plugin_docker.py b/tests/test_plugin_docker.py new file mode 100755 index 00000000..889b73be --- /dev/null +++ b/tests/test_plugin_docker.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python +# +# Glances - An eye on your system +# +# SPDX-FileCopyrightText: 2026 Nicolas Hennion +# +# SPDX-License-Identifier: LGPL-3.0-only +# + +"""Tests for the Docker container engine (issue #3559 performance fix).""" + +from unittest.mock import MagicMock, patch + +import pytest + +from glances.plugins.containers.engines import docker as docker_engine + + +class FakeImage: + def __init__(self, tags): + self.tags = tags + + +class FakeContainer: + """Minimal container double counting image lookups and inspects (reload).""" + + def __init__(self, cid, status="exited", tags=None): + self.id = cid + self.name = f"name-{cid}" + self.attrs = { + "State": {"Status": status}, + "Created": "2026-01-01T00:00:00Z", + "Config": {"Entrypoint": None, "Cmd": ["sh"]}, + "HostConfig": {}, + } + self._tags = tags if tags is not None else [f"img-{cid}:latest"] + self.ports = {} + self.image_access_count = 0 + self.reload_count = 0 + + @property + def image(self): + # container.image performs an inspect_image API call in docker-py + self.image_access_count += 1 + return FakeImage(self._tags) + + def reload(self): + # reload() performs the per-container inspect in docker-py + self.reload_count += 1 + + +def make_extension(containers): + """Build a DockerExtension without connecting to a real daemon.""" + ext = docker_engine.DockerExtension.__new__(docker_engine.DockerExtension) + ext.disable = False + ext.display_error = True + ext.ext_name = "containers (Docker)" + ext.stats_fetchers = {} + ext.image_cache = {} + ext.client = MagicMock() + ext.client.containers.list.return_value = containers + return ext + + +def test_list_is_sparse(): + """The list must be fetched sparse (single API call, no per-container inspect).""" + containers = [FakeContainer("a"), FakeContainer("b")] + ext = make_extension(containers) + with patch.object(docker_engine, "DockerStatsFetcher", MagicMock()): + ext.update(all_tag=True) + ext.client.containers.list.assert_called_once_with(all=True, sparse=True) + + +def test_containers_are_inspected_every_cycle(): + """Each container is inspected (reload) once per update cycle (fresh status/uptime).""" + containers = [FakeContainer("a"), FakeContainer("b"), FakeContainer("c")] + ext = make_extension(containers) + with patch.object(docker_engine, "DockerStatsFetcher", MagicMock()): + ext.update(all_tag=True) + ext.update(all_tag=True) + assert all(c.reload_count == 2 for c in containers) + + +def test_image_is_cached_across_cycles(): + """Issue #3559: image (inspect_image) must be looked up once, then cached.""" + containers = [FakeContainer("a"), FakeContainer("b")] + ext = make_extension(containers) + with patch.object(docker_engine, "DockerStatsFetcher", MagicMock()): + ext.update(all_tag=True) + ext.update(all_tag=True) + ext.update(all_tag=True) + # 3 cycles, but the immutable image is only fetched once per container + assert all(c.image_access_count == 1 for c in containers) + + +def test_image_field_format_preserved(): + """The exposed image field keeps its historical shape (tuple of joined tags).""" + containers = [FakeContainer("a", tags=["redis:7"])] + ext = make_extension(containers) + with patch.object(docker_engine, "DockerStatsFetcher", MagicMock()): + _, stats = ext.update(all_tag=True) + assert stats[0]["image"] == ("redis:7",) + + +def test_image_cache_evicts_removed_containers(): + """Image cache must not grow unbounded: entries for gone containers are dropped.""" + first = [FakeContainer("a"), FakeContainer("b")] + ext = make_extension(first) + with patch.object(docker_engine, "DockerStatsFetcher", MagicMock()): + ext.update(all_tag=True) + assert set(ext.image_cache) == {"a", "b"} + # Container "b" disappears + ext.client.containers.list.return_value = [FakeContainer("a")] + ext.update(all_tag=True) + assert set(ext.image_cache) == {"a"} + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) From 2e5ab02bda271b8d0558baa7ba0075229ee79cf1 Mon Sep 17 00:00:00 2001 From: nicolargo Date: Sun, 31 May 2026 17:35:44 +0200 Subject: [PATCH 3/3] Get back to default ThreadedIterableStreamer.sleep_duration to 0.1 second --- glances/stats_streamer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glances/stats_streamer.py b/glances/stats_streamer.py index 3acfa11a..ec31dab2 100644 --- a/glances/stats_streamer.py +++ b/glances/stats_streamer.py @@ -18,7 +18,7 @@ class ThreadedIterableStreamer: Use `ThreadedIterableStreamer.stats` to access the latest streamed results """ - def __init__(self, iterable, initial_stream_value=None, sleep_duration=0.01): + def __init__(self, iterable, initial_stream_value=None, sleep_duration=0.1): """ iterable: an Iterable instance that needs to be streamed """