From fc7106c012768240f41e5c8ec3de98e1a4edd87a Mon Sep 17 00:00:00 2001 From: nicolargo Date: Sat, 6 Jan 2018 11:20:50 +0100 Subject: [PATCH 1/6] First rush for the issue #1105. Bug with Process IO when Glances starts --- NEWS | 1 + glances/processes.py | 60 ++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 2 +- 3 files changed, 62 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index d566617c..9d9fc8df 100644 --- a/NEWS +++ b/NEWS @@ -17,6 +17,7 @@ Enhancements and new features: * Remove graph export from Glances #1206 * Add a code of conduct for Glances project's participants #1211 * Context switches bottleneck identification #1212 + * Take advantage of the PSUtil issue #1025 (Add process_iter(attrs, ad_value)) #1105 Bugs corrected: diff --git a/glances/processes.py b/glances/processes.py index 0412358a..6767dc09 100644 --- a/glances/processes.py +++ b/glances/processes.py @@ -149,6 +149,8 @@ class GlancesProcesses(object): return int(f.read()) except (OSError, IOError): return None + else: + return None @property def max_processes(self): @@ -414,6 +416,64 @@ class GlancesProcesses(object): # Update the maximum process ID (pid) number self.processcount['pid_max'] = self.pid_max + # Only work whith PsUtil + mandatories_attr = ['cmdline', 'cpu_percent', 'cpu_times', + 'io_counters', 'memory_info', 'memory_percent', + 'name', 'nice', 'pid', + 'ppid', 'status', 'username'] + self.processlist = [p.info for p in sorted(psutil.process_iter(attrs=mandatories_attr, + ad_value=None), + key=lambda p: p.info['cpu_percent'])] + + # Add metadata + for proc in self.processlist: + proc['time_since_update'] = time_since_update + # Status + proc['status'] = str(proc['status'])[:1].upper() + # Process IO + # procstat['io_counters'] is a list: + # [read_bytes, write_bytes, read_bytes_old, write_bytes_old, io_tag] + # If io_tag = 0 > Access denied (display "?") + # If io_tag = 1 > No access denied (display the IO rate) + # Availability: all platforms except macOS and Illumos/Solaris + if proc['io_counters'] is not None: + proc_io = proc['io_counters'] + io_new = [proc_io.read_bytes, proc_io.write_bytes] + # For IO rate computation + # Append saved IO r/w bytes + try: + proc['io_counters'] = io_new + self.io_old[proc['pid']] + except KeyError: + proc['io_counters'] = io_new + [0, 0] + # then save the IO r/w bytes + self.io_old[proc['pid']] = io_new + io_tag = 1 + else: + proc['io_counters'] = [0, 0] + [0, 0] + io_tag = 0 + + # Append the IO tag (for display) + proc['io_counters'] += [io_tag] + + def update_OLD(self): + """Update the processes stats.""" + # Reset the stats + self.processlist = [] + self.reset_processcount() + + # Do not process if disable tag is set + if self.disable_tag: + return + + # Get the time since last update + time_since_update = getTimeSinceLastUpdate('process_disk') + + # Reset the max dict + self.reset_max_values() + + # Update the maximum process ID (pid) number + self.processcount['pid_max'] = self.pid_max + # Build an internal dict with only mandatories stats (sort keys) processdict = {} excluded_processes = set() diff --git a/requirements.txt b/requirements.txt index 4cc0c6c0..00399543 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -psutil==5.2.2 +psutil==5.4.2 From 9a40395efb7d32c22c6adfb3efa9bb933cdce057 Mon Sep 17 00:00:00 2001 From: nicolargo Date: Sat, 6 Jan 2018 16:41:47 +0100 Subject: [PATCH 2/6] Correct IO counter and max cpu/mem --- glances/plugins/glances_processlist.py | 3 +- glances/processes.py | 38 +++++++++++++++----------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/glances/plugins/glances_processlist.py b/glances/plugins/glances_processlist.py index 12d8a72c..3c5fcc89 100644 --- a/glances/plugins/glances_processlist.py +++ b/glances/plugins/glances_processlist.py @@ -304,7 +304,8 @@ class Plugin(GlancesPlugin): msg = '{:>10}'.format('?') ret.append(self.curse_add_line(msg, optional=True)) # IO read/write - if 'io_counters' in p: + if 'io_counters' in p and p['io_counters'][4] == 1: + # Display rate if stats is available and io_tag ([4]) == 1 # IO read io_rs = int((p['io_counters'][0] - p['io_counters'][2]) / p['time_since_update']) if io_rs == 0: diff --git a/glances/processes.py b/glances/processes.py index 6767dc09..83f1782a 100644 --- a/glances/processes.py +++ b/glances/processes.py @@ -407,29 +407,34 @@ class GlancesProcesses(object): if self.disable_tag: return - # Get the time since last update - time_since_update = getTimeSinceLastUpdate('process_disk') - - # Reset the max dict - self.reset_max_values() - - # Update the maximum process ID (pid) number - self.processcount['pid_max'] = self.pid_max - - # Only work whith PsUtil + # Grab the stats mandatories_attr = ['cmdline', 'cpu_percent', 'cpu_times', 'io_counters', 'memory_info', 'memory_percent', 'name', 'nice', 'pid', 'ppid', 'status', 'username'] + # and build the processes stats list self.processlist = [p.info for p in sorted(psutil.process_iter(attrs=mandatories_attr, ad_value=None), key=lambda p: p.info['cpu_percent'])] - # Add metadata + # Update the maximum process ID (pid) number + self.processcount['pid_max'] = self.pid_max + + # Compute the maximum value for keys in self._max_values_list + # Reset the max dict + self.reset_max_values() + # Compute max + for k in self._max_values_list: + self.set_max_values(k, max(i[k] for i in self.processlist)) + + # Loop over processes and add metadata for proc in self.processlist: - proc['time_since_update'] = time_since_update - # Status + # Time since last update (for disk_io rate computation) + proc['time_since_update'] = getTimeSinceLastUpdate('process_disk') + + # Process status (only keep the first char) proc['status'] = str(proc['status'])[:1].upper() + # Process IO # procstat['io_counters'] is a list: # [read_bytes, write_bytes, read_bytes_old, write_bytes_old, io_tag] @@ -437,17 +442,18 @@ class GlancesProcesses(object): # If io_tag = 1 > No access denied (display the IO rate) # Availability: all platforms except macOS and Illumos/Solaris if proc['io_counters'] is not None: - proc_io = proc['io_counters'] - io_new = [proc_io.read_bytes, proc_io.write_bytes] + io_new = [proc['io_counters'].read_bytes, + proc['io_counters'].write_bytes] # For IO rate computation # Append saved IO r/w bytes try: proc['io_counters'] = io_new + self.io_old[proc['pid']] + io_tag = 1 except KeyError: proc['io_counters'] = io_new + [0, 0] + io_tag = 0 # then save the IO r/w bytes self.io_old[proc['pid']] = io_new - io_tag = 1 else: proc['io_counters'] = [0, 0] + [0, 0] io_tag = 0 From 6f62a9f2e8418fb4e7c224942843d5fcab87882f Mon Sep 17 00:00:00 2001 From: nicolargo Date: Sat, 6 Jan 2018 16:42:43 +0100 Subject: [PATCH 3/6] Correct IO counter and max cpu/mem --- glances/processes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/glances/processes.py b/glances/processes.py index 83f1782a..d43b7223 100644 --- a/glances/processes.py +++ b/glances/processes.py @@ -421,8 +421,8 @@ class GlancesProcesses(object): self.processcount['pid_max'] = self.pid_max # Compute the maximum value for keys in self._max_values_list - # Reset the max dict - self.reset_max_values() + # # Reset the max dict + # self.reset_max_values() # Compute max for k in self._max_values_list: self.set_max_values(k, max(i[k] for i in self.processlist)) From 8f7fbafa4667819745230557c3074c0d1a1d0409 Mon Sep 17 00:00:00 2001 From: nicolargo Date: Tue, 9 Jan 2018 18:45:06 +0100 Subject: [PATCH 4/6] Max value does not work --- glances/processes.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/glances/processes.py b/glances/processes.py index d43b7223..897acd62 100644 --- a/glances/processes.py +++ b/glances/processes.py @@ -420,13 +420,6 @@ class GlancesProcesses(object): # Update the maximum process ID (pid) number self.processcount['pid_max'] = self.pid_max - # Compute the maximum value for keys in self._max_values_list - # # Reset the max dict - # self.reset_max_values() - # Compute max - for k in self._max_values_list: - self.set_max_values(k, max(i[k] for i in self.processlist)) - # Loop over processes and add metadata for proc in self.processlist: # Time since last update (for disk_io rate computation) @@ -461,6 +454,11 @@ class GlancesProcesses(object): # Append the IO tag (for display) proc['io_counters'] += [io_tag] + # Compute the maximum value for keys in self._max_values_list + # Compute max + for k in self._max_values_list: + self.set_max_values(k, max(i[k] for i in self.processlist)) + def update_OLD(self): """Update the processes stats.""" # Reset the stats From 1dbec8c8f64d82d927a1cf88b45c4a9b2ced97bb Mon Sep 17 00:00:00 2001 From: nicolargo Date: Tue, 16 Jan 2018 21:44:33 +0100 Subject: [PATCH 5/6] Correct Max CPU MEM display --- glances/plugins/glances_processlist.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/glances/plugins/glances_processlist.py b/glances/plugins/glances_processlist.py index 3c5fcc89..6f56ee7a 100644 --- a/glances/plugins/glances_processlist.py +++ b/glances/plugins/glances_processlist.py @@ -21,6 +21,7 @@ import os import shlex +import copy from datetime import timedelta from glances.compat import iteritems @@ -80,7 +81,7 @@ class Plugin(GlancesPlugin): self.nb_log_core = 0 # Get the max values (dict) - self.max_values = glances_processes.max_values() + self.max_values = copy.deepcopy(glances_processes.max_values()) # Get the maximum PID number # Use to optimize space (see https://github.com/nicolargo/glances/issues/959) @@ -111,7 +112,8 @@ class Plugin(GlancesPlugin): self.stats = glances_processes.getlist() # Get the max values (dict) - self.max_values = glances_processes.max_values() + # Use Deep copy to avoid change between update and display + self.max_values = copy.deepcopy(glances_processes.max_values()) elif self.input_method == 'snmp': # No SNMP grab for processes @@ -445,6 +447,8 @@ class Plugin(GlancesPlugin): self.__msg_curse_header(ret, process_sort_key, args) # Process list + logger.info(self.max_values) + logger.info([(i['cpu_percent'], i['name']) for i in self.stats]) if glances_processes.is_tree_enabled(): ret.extend(self.get_process_tree_curses_data( self.__sort_stats(process_sort_key), args, first_level=True, From aa17412856a67803664224bc52a8f1d8a2356e2c Mon Sep 17 00:00:00 2001 From: nicolargo Date: Tue, 16 Jan 2018 21:55:54 +0100 Subject: [PATCH 6/6] Remove debug message --- glances/plugins/glances_processlist.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/glances/plugins/glances_processlist.py b/glances/plugins/glances_processlist.py index 6f56ee7a..e27ab6dd 100644 --- a/glances/plugins/glances_processlist.py +++ b/glances/plugins/glances_processlist.py @@ -447,8 +447,6 @@ class Plugin(GlancesPlugin): self.__msg_curse_header(ret, process_sort_key, args) # Process list - logger.info(self.max_values) - logger.info([(i['cpu_percent'], i['name']) for i in self.stats]) if glances_processes.is_tree_enabled(): ret.extend(self.get_process_tree_curses_data( self.__sort_stats(process_sort_key), args, first_level=True,