[PATCH 1/3] python-linux-procfs: Progate error to user if a pid is completed
by John Kacur
If a pid is completed and disappears a FileNotFoundError will occur
because /proc/pid/stat will disappear too.
It is not possible to check for the file first because it could still
disappear between the time of the check and the time of use.
Progate this error to the user.
The user should handle this with a try, except clause and ignore it if
an exception occurs.
Signed-off-by: John Kacur <jkacur(a)redhat.com>
---
procfs/procfs.py | 19 ++++++++++++++++---
1 file changed, 16 insertions(+), 3 deletions(-)
diff --git a/procfs/procfs.py b/procfs/procfs.py
index 408b2bcd0a31..a0e9977214fe 100755
--- a/procfs/procfs.py
+++ b/procfs/procfs.py
@@ -130,7 +130,12 @@ class pidstat:
def __init__(self, pid, basedir="/proc"):
self.pid = pid
- self.load(basedir)
+ try:
+ self.load(basedir)
+ except FileNotFoundError:
+ # The file representing the pid has disappeared
+ # propagate the error to the user to handle
+ raise
def __getitem__(self, fieldname):
return self.fields[fieldname]
@@ -151,7 +156,11 @@ class pidstat:
return fieldname in self.fields
def load(self, basedir="/proc"):
- f = open("%s/%d/stat" % (basedir, self.pid))
+ try:
+ f = open("%s/%d/stat" % (basedir, self.pid))
+ except FileNotFoundError:
+ # The pid has disappeared, propagate the error
+ raise
fields = f.readline().strip().split(') ')
f.close()
fields = fields[0].split(' (') + fields[1].split()
@@ -338,7 +347,11 @@ class process:
else:
sclass = pidstatus
- setattr(self, attr, sclass(self.pid, self.basedir))
+ try:
+ setattr(self, attr, sclass(self.pid, self.basedir))
+ except FileNotFoundError:
+ # The pid has disappeared, progate the error
+ raise
elif attr == "cmdline":
self.load_cmdline()
elif attr == "threads":
--
2.31.1
2 years
[PATCH 1/3] python-linux-procfs: Propagate error to user if a pid is completed
by John Kacur
If a pid is completed and disappears a FileNotFoundError will occur
because /proc/pid/stat will disappear too.
It is not possible to check for the file first because it could still
disappear between the time of the check and the time of use.
Propagate this error to the user.
The user should handle this with a try, except clause and ignore it if
an exception occurs.
Signed-off-by: John Kacur <jkacur(a)redhat.com>
---
procfs/procfs.py | 19 ++++++++++++++++---
1 file changed, 16 insertions(+), 3 deletions(-)
diff --git a/procfs/procfs.py b/procfs/procfs.py
index 408b2bcd0a31..a0e9977214fe 100755
--- a/procfs/procfs.py
+++ b/procfs/procfs.py
@@ -130,7 +130,12 @@ class pidstat:
def __init__(self, pid, basedir="/proc"):
self.pid = pid
- self.load(basedir)
+ try:
+ self.load(basedir)
+ except FileNotFoundError:
+ # The file representing the pid has disappeared
+ # propagate the error to the user to handle
+ raise
def __getitem__(self, fieldname):
return self.fields[fieldname]
@@ -151,7 +156,11 @@ class pidstat:
return fieldname in self.fields
def load(self, basedir="/proc"):
- f = open("%s/%d/stat" % (basedir, self.pid))
+ try:
+ f = open("%s/%d/stat" % (basedir, self.pid))
+ except FileNotFoundError:
+ # The pid has disappeared, propagate the error
+ raise
fields = f.readline().strip().split(') ')
f.close()
fields = fields[0].split(' (') + fields[1].split()
@@ -338,7 +347,11 @@ class process:
else:
sclass = pidstatus
- setattr(self, attr, sclass(self.pid, self.basedir))
+ try:
+ setattr(self, attr, sclass(self.pid, self.basedir))
+ except FileNotFoundError:
+ # The pid has disappeared, progate the error
+ raise
elif attr == "cmdline":
self.load_cmdline()
elif attr == "threads":
--
2.31.1
2 years
[PATCH] python-linux-procfs: Fix traceback with non-utf8 chars in the /proc/PID/cmdline
by John Kacur
Fix traceback if there are non-utf8 characters in the /proc/PID/cmdline
Signed-off-by: John Kacur <jkacur(a)redhat.com>
---
procfs/procfs.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/procfs/procfs.py b/procfs/procfs.py
index 3b7474cccb01..408b2bcd0a31 100755
--- a/procfs/procfs.py
+++ b/procfs/procfs.py
@@ -357,9 +357,9 @@ class process:
return hasattr(self, attr)
def load_cmdline(self):
- f = open("/proc/%d/cmdline" % self.pid)
- self.cmdline = f.readline().strip().split('\0')[:-1]
- f.close()
+ with open("/proc/%d/cmdline" % self.pid, mode='rb') as f:
+ cmdline = f.readline().decode(encoding='unicode_escape')
+ self.cmdline = cmdline.strip().split('\0')[:-1]
def load_threads(self):
self.threads = pidstats("/proc/%d/task/" % self.pid)
--
2.31.1
2 years
[PATCH] tuna: Make it clear that include and isolate affects IRQs in help
by John Kacur
Make it clear in the online help (-h) and in the manpage
that
-i --isolate
-I --include
affects IRQs as well as threads
Signed-off-by: John Kacur <jkacur(a)redhat.com>
---
docs/tuna.8 | 4 ++--
tuna-cmd.py | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/docs/tuna.8 b/docs/tuna.8
index b6431be5e117..bd6299b5a771 100644
--- a/docs/tuna.8
+++ b/docs/tuna.8
@@ -26,10 +26,10 @@ Apply changes described in profile
List preloaded profiles
.TP
\fB\-i\fR, \fB\-\-isolate\fR
-Move all threads away from CPU\-LIST. Requires \fB\-c\fR or \fB-S\fR.
+Move all allowed threads and IRQs away from CPU\-LIST. Requires \fB\-c\fR or \fB-S\fR.
.TP
\fB\-I\fR, \fB\-\-include\fR
-Allow all threads to run on CPU\-LIST. Requires \fB\-c\fR or \fB-S\fR.
+Allow all allowed threads and IRQs to run on CPU\-LIST. Requires \fB\-c\fR or \fB-S\fR.
.TP
\fB\-m\fR, \fB\-\-move\fR
Move selected entities to CPU\-LIST. Requires \fB\-c\fR and either \fB-t\fR or \fB-q\fR.
diff --git a/tuna-cmd.py b/tuna-cmd.py
index 46ce86efa036..d209a2e89493 100755
--- a/tuna-cmd.py
+++ b/tuna-cmd.py
@@ -63,9 +63,9 @@ def usage():
_('Explicitly disable usage of perf in GUI for process view')))
print(fmt % ('-f, --filter',
_('Display filter the selected entities')))
- print(fmt % ('-i, --isolate', _('Move all threads away from %(cpulist)s') %
+ print(fmt % ('-i, --isolate', _('Move all allowed threads and IRQs away from %(cpulist)s') %
{"cpulist": _('CPU-LIST')}))
- print(fmt % ('-I, --include', _('Allow all threads to run on %(cpulist)s') %
+ print(fmt % ('-I, --include', _('Allow all allowed threads and IRQs to run on %(cpulist)s') %
{"cpulist": _('CPU-LIST')}))
print(fmt % ('-K, --no_kthreads',
_('Operations will not affect kernel threads')))
--
2.31.1
2 years
[PATCH] tuna: RFC: Use cpusets to isolate cpus
by John Kacur
Use cpusets to isolate cpus
This is a proof of concept, some more work would have to be done
especially in tuna to deal with situations such as attempting to create a new
isolated cpuset when one already exists, and so on.
After running the following command on a machine with no cpusets
su -c './tuna-cmd.py -c "5-7" -i'
We have the base cpuset which by convention is mounted here /dev/cpuset
and we have
cpuset 'tuna0' which has all the cpus that were not to be isolated
cpuset 'tuna1' which has all the cpus that were to be isolated
and tuna1 is a cpu_exclusive set
All the tasks that were migratable from the base cpuset are now in tuna0
and no tasks are in tuna1
ls -ld /dev/cpuset
dr-xr-xr-x. 4 root root 0 Nov 8 15:20 /dev/cpuset
[root@fionn ~]# cd /dev/cpuset
[root@fionn cpuset]# cat tuna0/cpus
0-4,8-11
[root@fionn cpuset]# cat tuna1/cpus
5-7
[root@fionn cpuset]# cat tuna0/cpu_exclusive
0
[root@fionn cpuset]# cat tuna1/cpu_exclusive
1
Some tasks are not migrate-able, after the migration
There are (as an example)
138 tasks that couldn't be migrated
971 that were migrated
and 0 in the new cpuset with the isolated cpus
[jkacur@fionn tuna]$ cat /dev/cpuset/tasks | wc -l
138
[jkacur@fionn tuna]$ cat /dev/cpuset/tuna0/tasks | wc -l
961
[jkacur@fionn tuna]$ cat /dev/cpuset/tuna1/tasks | wc -l
0
The programmer could now use affinity to run new tasks on the isolated
cpus
pushed to branch devel/cpusets so folks can try it out.
Signed-off-by: John Kacur <jkacur(a)redhat.com>
---
tuna/cpuset.py | 225 +++++++++++++++++++++++++++++++++++++++++++++++++
tuna/tuna.py | 33 ++++++++
2 files changed, 258 insertions(+)
create mode 100755 tuna/cpuset.py
diff --git a/tuna/cpuset.py b/tuna/cpuset.py
new file mode 100755
index 000000000000..526ca2975d4d
--- /dev/null
+++ b/tuna/cpuset.py
@@ -0,0 +1,225 @@
+#!/usr/bin/python3
+# Copyright 2021 John Kacur <jkacur(a)redhat.com>
+""" Module for creating and using cpusets """
+
+import errno
+import os
+import subprocess
+import sys
+from glob import glob
+
+import procfs
+
+
+class Cpuset:
+ """ Class for manipulating cpusets """
+
+ mpath = '/dev/cpuset'
+
+ def __init__(self, name=None):
+ self.cpuset_name = None
+ self._cpuset_path = None
+ self.create_cpuset(name)
+ self.tasks = []
+
+ @property
+ def cpuset_path(self):
+ """ returns the cpuset_path, for example /dev/cpuset/tuna0 """
+ return self._cpuset_path
+
+ def __str__(self):
+ return (f'cpuset_name={self.cpuset_name}, cpuset_path={self.cpuset_path}')
+
+ def create_cpuset(self, name=None):
+ """ creates a cpuset below /dev/cpuset """
+ self.cpuset_name = name
+ path = os.path.join(Cpuset.mpath, self.cpuset_name)
+ if not os.path.exists(path):
+ os.mkdir(path)
+ self._cpuset_path = path
+
+ def assign_cpus(self, cpu_str):
+ """ Assign cpus in list-format to a cpuset """
+ path = os.path.join(self._cpuset_path, "cpus")
+ with open(path, 'w') as f:
+ f.write(cpu_str)
+
+ def write_pid(self, pid):
+ """ place a pid in a cpuset """
+ path = os.path.join(self._cpuset_path, "tasks")
+ pid_str = str(pid)
+ with open(path, 'wb', buffering=0) as f:
+ b = pid_str.encode('utf-8')
+ try:
+ f.write(b)
+ f.flush()
+ except OSError as err:
+ if err.args[0] == errno.EINVAL:
+ pass
+ if err.args[0] == errno.ESRCH:
+ pass
+
+ def write_memnode(self, memnode):
+ """ place memnode in a cpuset """
+ path = os.path.join(self._cpuset_path, "mems")
+ memnode_str = str(memnode)
+ with open(path, 'w') as f:
+ f.write(memnode_str)
+
+ def write_cpu_exclusive(self, cpu_exclusive):
+ """ turn cpu_explusive on or off """
+ path = os.path.join(self._cpuset_path, "cpu_exclusive")
+ cpu_exclusive_str = str(cpu_exclusive)
+ with open(path, 'w') as f:
+ f.write(cpu_exclusive_str)
+
+ def set_cpu_exclusive(self):
+ """ turn cpu_exclusive on """
+ self.write_cpu_exclusive('1')
+
+ def unset_cpu_exclusive(self):
+ """ turn cpu_exlusive off """
+ self.write_cpu_exclusive('0')
+
+ def write_memory_migrate(self, memory_migrate):
+ """ turn memory migrate on or off """
+ path = os.path.join(self._cpuset_path, "memory_migrate")
+ memory_migrate_str = str(memory_migrate)
+ with open(path, 'w') as f:
+ f.write(memory_migrate_str)
+
+ def set_memory_migrate(self):
+ """ turn memory_migrate on """
+ self.write_memory_migrate('1')
+
+ def unset_memory_migrate(self):
+ """ turn memory migrate off """
+ self.write_memory_migrate('0')
+
+ def get_tasks(self):
+ """ get the tasks currently in a cpuset """
+ path = os.path.join(self.cpuset_path, "tasks")
+ with open(path, 'r') as f:
+ task = f.readline().strip()
+ while task:
+ self.tasks.append(task)
+ task = f.readline().strip()
+ return self.tasks
+
+
+class CpusetsInit(Cpuset):
+ """
+ CpusetsInit inits cpusets if they are currently off
+ this class will create /dev/cpuset and mount it
+ It will also test whether the kernel supports cpusets
+ This class is a cpuset itself, it is the base cpuset /dev/cpuset
+ """
+
+ mpath = '/dev/cpuset'
+
+ def __init__(self):
+ self._supported = self._cpuset_supported()
+ if self._supported:
+ self.create_cpuset_dir()
+ self.mount_cpuset()
+ self._numa_nodes = self._get_numa_nodes()
+ super().__init__('/dev/cpuset')
+
+ @property
+ def supported(self):
+ """ return True if cpuset is supported """
+ return self._supported
+
+ @property
+ def numa_nodes(self):
+ """ return the number of numa nodes """
+ return self._numa_nodes
+
+ @staticmethod
+ def _cpuset_supported():
+ """
+ Return True if cpusets are supported
+ if the entry 'node cpuset' is in /proc/filesystems
+ then cpusets are supported
+ This function is called by the __init__
+ Users of this class should use the supported method
+ """
+ dpath = '/proc/filesystems'
+ with open(dpath) as fp:
+ line = fp.readline().strip()
+ while line:
+ elems = line.split()
+ if len(elems) == 2 and elems[1] == "cpuset":
+ return True
+ line = fp.readline().strip()
+ return False
+
+ @staticmethod
+ def _get_numa_nodes():
+ """
+ return the number of numa nodes
+ used by init to set _numa_nodes
+ Users of this class should use the "numa_nodes" method
+ """
+ return len(glob('/sys/devices/system/node/node*'))
+
+ @staticmethod
+ def create_cpuset_dir():
+ """ create /dev/cpuset if it doesn't already exist """
+ path = Cpuset.mpath
+ if not os.path.exists(path):
+ os.mkdir(path)
+
+ @staticmethod
+ def mount_cpuset():
+ """ mount /dev/cpuset if it exists and isn't already mounted """
+ path = Cpuset.mpath
+ if os.path.exists(path) and not os.path.ismount(path):
+ args = ['mount', '-t', 'cpuset', 'cpuset', path]
+ subprocess.run(args, check=True)
+
+
+class TaskMigrate:
+ """ a class for migrating a set of pids from one cpuset to another """
+
+ def __init__(self, cpuset_from, cpuset_to):
+ self.cpuset_from = cpuset_from
+ self.cpuset_to = cpuset_to
+ self.cpuset_to.set_memory_migrate()
+
+ def migrate(self):
+ """ do the migration, only attempt on pids where this is allowed """
+ # ps = procfs.pidstats()
+ # ps.reload_threads()
+ for task in self.cpuset_from.get_tasks():
+ self.cpuset_to.write_pid(task)
+
+
+if __name__ == '__main__':
+
+ cpusets_init = CpusetsInit()
+ # tasks = cpusets_init.get_tasks()
+ print(f'cpusets_init.supported = {cpusets_init.supported}')
+ if cpusets_init.supported:
+ print(f'cpusets_init.numa_nodes = {cpusets_init.numa_nodes}')
+ # print(f'tasks = {tasks}')
+ cpuset0 = Cpuset('tuna0')
+ cpuset1 = Cpuset('tuna1')
+
+ cpuset0.assign_cpus('0-4,8-11')
+ cpuset1.assign_cpus('5-7')
+ cpuset0.write_memnode('0')
+ cpuset1.write_memnode('0')
+ cpuset1.set_cpu_exclusive()
+ cpuset0.set_memory_migrate()
+
+ tm = TaskMigrate(cpusets_init, cpuset0)
+ tm.migrate()
+
+ sys.exit(0)
+ for dirpath, dirnames, fnames in \
+ os.walk(cpusets_init.mpath, topdown=True):
+ print(f'dirpath = {dirpath}')
+ print(f'dirnames = {dirnames}')
+ print(f'fnames = {fnames}')
+ print('\n')
diff --git a/tuna/tuna.py b/tuna/tuna.py
index 6b6edcc8ff72..d3758e0cb731 100755
--- a/tuna/tuna.py
+++ b/tuna/tuna.py
@@ -14,6 +14,7 @@ import procfs
from procfs import utilist
import help
import tuna_sched
+import cpuset
try:
fntable
@@ -332,7 +333,39 @@ def parse_irq_affinity_filename(filename, nr_cpus):
return utilist.bitmasklist(line, nr_cpus)
+def _isolate_cpus(cpus, nr_cpus):
+
+ base_cpuset = cpuset.CpusetsInit()
+
+ if not base_cpuset.supported:
+ print('Warning: cpusets are not supported')
+ return
+
+ # cpus not in the isolated list
+ cpus0 = set(range(nr_cpus)) - set(cpus)
+ cpus0 = list_to_cpustring(cpus0)
+
+ # cpuset of cpus not in the isolated list
+ tuna0 = cpuset.Cpuset('tuna0')
+ tuna0.assign_cpus(cpus0)
+ tuna0.write_memnode('0')
+
+ # cpuset to be isolated
+ tuna1 = cpuset.Cpuset('tuna1')
+ cpus1 = list_to_cpustring(cpus)
+ tuna1.assign_cpus(cpus1)
+ tuna1.write_memnode('0')
+ tuna1.set_cpu_exclusive()
+
+ # Migrate all migratable tasks from the base_cpuset
+ # to the cpusset with the cpus not in the isolated list
+ tm = cpuset.TaskMigrate(base_cpuset, tuna0)
+ tm.migrate()
+
def isolate_cpus(cpus, nr_cpus):
+ _isolate_cpus(cpus, nr_cpus)
+ return
+
fname = sys._getframe().f_code.co_name # Function name
ps = procfs.pidstats()
ps.reload_threads()
--
2.31.1
2 years