Use cpusets to isolate cpus
This is a proof of concept, some more work would have to be done
especially in tuna to deal with situations such as attempting to create a new
isolated cpuset when one already exists, and so on.
After running the following command on a machine with no cpusets
su -c './tuna-cmd.py -c "5-7" -i'
We have the base cpuset which by convention is mounted here /dev/cpuset
and we have
cpuset 'tuna0' which has all the cpus that were not to be isolated
cpuset 'tuna1' which has all the cpus that were to be isolated
and tuna1 is a cpu_exclusive set
All the tasks that were migratable from the base cpuset are now in tuna0
and no tasks are in tuna1
ls -ld /dev/cpuset
dr-xr-xr-x. 4 root root 0 Nov 8 15:20 /dev/cpuset
[root@fionn ~]# cd /dev/cpuset
[root@fionn cpuset]# cat tuna0/cpus
0-4,8-11
[root@fionn cpuset]# cat tuna1/cpus
5-7
[root@fionn cpuset]# cat tuna0/cpu_exclusive
0
[root@fionn cpuset]# cat tuna1/cpu_exclusive
1
Some tasks are not migrate-able, after the migration
There are (as an example)
138 tasks that couldn't be migrated
971 that were migrated
and 0 in the new cpuset with the isolated cpus
[jkacur@fionn tuna]$ cat /dev/cpuset/tasks | wc -l
138
[jkacur@fionn tuna]$ cat /dev/cpuset/tuna0/tasks | wc -l
961
[jkacur@fionn tuna]$ cat /dev/cpuset/tuna1/tasks | wc -l
0
The programmer could now use affinity to run new tasks on the isolated
cpus
pushed to branch devel/cpusets so folks can try it out.
Signed-off-by: John Kacur <jkacur(a)redhat.com>
---
tuna/cpuset.py | 225 +++++++++++++++++++++++++++++++++++++++++++++++++
tuna/tuna.py | 33 ++++++++
2 files changed, 258 insertions(+)
create mode 100755 tuna/cpuset.py
diff --git a/tuna/cpuset.py b/tuna/cpuset.py
new file mode 100755
index 000000000000..526ca2975d4d
--- /dev/null
+++ b/tuna/cpuset.py
@@ -0,0 +1,225 @@
+#!/usr/bin/python3
+# Copyright 2021 John Kacur <jkacur(a)redhat.com>
+""" Module for creating and using cpusets """
+
+import errno
+import os
+import subprocess
+import sys
+from glob import glob
+
+import procfs
+
+
+class Cpuset:
+ """ Class for manipulating cpusets """
+
+ mpath = '/dev/cpuset'
+
+ def __init__(self, name=None):
+ self.cpuset_name = None
+ self._cpuset_path = None
+ self.create_cpuset(name)
+ self.tasks = []
+
+ @property
+ def cpuset_path(self):
+ """ returns the cpuset_path, for example /dev/cpuset/tuna0 """
+ return self._cpuset_path
+
+ def __str__(self):
+ return (f'cpuset_name={self.cpuset_name}, cpuset_path={self.cpuset_path}')
+
+ def create_cpuset(self, name=None):
+ """ creates a cpuset below /dev/cpuset """
+ self.cpuset_name = name
+ path = os.path.join(Cpuset.mpath, self.cpuset_name)
+ if not os.path.exists(path):
+ os.mkdir(path)
+ self._cpuset_path = path
+
+ def assign_cpus(self, cpu_str):
+ """ Assign cpus in list-format to a cpuset """
+ path = os.path.join(self._cpuset_path, "cpus")
+ with open(path, 'w') as f:
+ f.write(cpu_str)
+
+ def write_pid(self, pid):
+ """ place a pid in a cpuset """
+ path = os.path.join(self._cpuset_path, "tasks")
+ pid_str = str(pid)
+ with open(path, 'wb', buffering=0) as f:
+ b = pid_str.encode('utf-8')
+ try:
+ f.write(b)
+ f.flush()
+ except OSError as err:
+ if err.args[0] == errno.EINVAL:
+ pass
+ if err.args[0] == errno.ESRCH:
+ pass
+
+ def write_memnode(self, memnode):
+ """ place memnode in a cpuset """
+ path = os.path.join(self._cpuset_path, "mems")
+ memnode_str = str(memnode)
+ with open(path, 'w') as f:
+ f.write(memnode_str)
+
+ def write_cpu_exclusive(self, cpu_exclusive):
+ """ turn cpu_explusive on or off """
+ path = os.path.join(self._cpuset_path, "cpu_exclusive")
+ cpu_exclusive_str = str(cpu_exclusive)
+ with open(path, 'w') as f:
+ f.write(cpu_exclusive_str)
+
+ def set_cpu_exclusive(self):
+ """ turn cpu_exclusive on """
+ self.write_cpu_exclusive('1')
+
+ def unset_cpu_exclusive(self):
+ """ turn cpu_exlusive off """
+ self.write_cpu_exclusive('0')
+
+ def write_memory_migrate(self, memory_migrate):
+ """ turn memory migrate on or off """
+ path = os.path.join(self._cpuset_path, "memory_migrate")
+ memory_migrate_str = str(memory_migrate)
+ with open(path, 'w') as f:
+ f.write(memory_migrate_str)
+
+ def set_memory_migrate(self):
+ """ turn memory_migrate on """
+ self.write_memory_migrate('1')
+
+ def unset_memory_migrate(self):
+ """ turn memory migrate off """
+ self.write_memory_migrate('0')
+
+ def get_tasks(self):
+ """ get the tasks currently in a cpuset """
+ path = os.path.join(self.cpuset_path, "tasks")
+ with open(path, 'r') as f:
+ task = f.readline().strip()
+ while task:
+ self.tasks.append(task)
+ task = f.readline().strip()
+ return self.tasks
+
+
+class CpusetsInit(Cpuset):
+ """
+ CpusetsInit inits cpusets if they are currently off
+ this class will create /dev/cpuset and mount it
+ It will also test whether the kernel supports cpusets
+ This class is a cpuset itself, it is the base cpuset /dev/cpuset
+ """
+
+ mpath = '/dev/cpuset'
+
+ def __init__(self):
+ self._supported = self._cpuset_supported()
+ if self._supported:
+ self.create_cpuset_dir()
+ self.mount_cpuset()
+ self._numa_nodes = self._get_numa_nodes()
+ super().__init__('/dev/cpuset')
+
+ @property
+ def supported(self):
+ """ return True if cpuset is supported """
+ return self._supported
+
+ @property
+ def numa_nodes(self):
+ """ return the number of numa nodes """
+ return self._numa_nodes
+
+ @staticmethod
+ def _cpuset_supported():
+ """
+ Return True if cpusets are supported
+ if the entry 'node cpuset' is in /proc/filesystems
+ then cpusets are supported
+ This function is called by the __init__
+ Users of this class should use the supported method
+ """
+ dpath = '/proc/filesystems'
+ with open(dpath) as fp:
+ line = fp.readline().strip()
+ while line:
+ elems = line.split()
+ if len(elems) == 2 and elems[1] == "cpuset":
+ return True
+ line = fp.readline().strip()
+ return False
+
+ @staticmethod
+ def _get_numa_nodes():
+ """
+ return the number of numa nodes
+ used by init to set _numa_nodes
+ Users of this class should use the "numa_nodes" method
+ """
+ return len(glob('/sys/devices/system/node/node*'))
+
+ @staticmethod
+ def create_cpuset_dir():
+ """ create /dev/cpuset if it doesn't already exist """
+ path = Cpuset.mpath
+ if not os.path.exists(path):
+ os.mkdir(path)
+
+ @staticmethod
+ def mount_cpuset():
+ """ mount /dev/cpuset if it exists and isn't already mounted """
+ path = Cpuset.mpath
+ if os.path.exists(path) and not os.path.ismount(path):
+ args = ['mount', '-t', 'cpuset', 'cpuset', path]
+ subprocess.run(args, check=True)
+
+
+class TaskMigrate:
+ """ a class for migrating a set of pids from one cpuset to another """
+
+ def __init__(self, cpuset_from, cpuset_to):
+ self.cpuset_from = cpuset_from
+ self.cpuset_to = cpuset_to
+ self.cpuset_to.set_memory_migrate()
+
+ def migrate(self):
+ """ do the migration, only attempt on pids where this is allowed """
+ # ps = procfs.pidstats()
+ # ps.reload_threads()
+ for task in self.cpuset_from.get_tasks():
+ self.cpuset_to.write_pid(task)
+
+
+if __name__ == '__main__':
+
+ cpusets_init = CpusetsInit()
+ # tasks = cpusets_init.get_tasks()
+ print(f'cpusets_init.supported = {cpusets_init.supported}')
+ if cpusets_init.supported:
+ print(f'cpusets_init.numa_nodes = {cpusets_init.numa_nodes}')
+ # print(f'tasks = {tasks}')
+ cpuset0 = Cpuset('tuna0')
+ cpuset1 = Cpuset('tuna1')
+
+ cpuset0.assign_cpus('0-4,8-11')
+ cpuset1.assign_cpus('5-7')
+ cpuset0.write_memnode('0')
+ cpuset1.write_memnode('0')
+ cpuset1.set_cpu_exclusive()
+ cpuset0.set_memory_migrate()
+
+ tm = TaskMigrate(cpusets_init, cpuset0)
+ tm.migrate()
+
+ sys.exit(0)
+ for dirpath, dirnames, fnames in \
+ os.walk(cpusets_init.mpath, topdown=True):
+ print(f'dirpath = {dirpath}')
+ print(f'dirnames = {dirnames}')
+ print(f'fnames = {fnames}')
+ print('\n')
diff --git a/tuna/tuna.py b/tuna/tuna.py
index 6b6edcc8ff72..d3758e0cb731 100755
--- a/tuna/tuna.py
+++ b/tuna/tuna.py
@@ -14,6 +14,7 @@ import procfs
from procfs import utilist
import help
import tuna_sched
+import cpuset
try:
fntable
@@ -332,7 +333,39 @@ def parse_irq_affinity_filename(filename, nr_cpus):
return utilist.bitmasklist(line, nr_cpus)
+def _isolate_cpus(cpus, nr_cpus):
+
+ base_cpuset = cpuset.CpusetsInit()
+
+ if not base_cpuset.supported:
+ print('Warning: cpusets are not supported')
+ return
+
+ # cpus not in the isolated list
+ cpus0 = set(range(nr_cpus)) - set(cpus)
+ cpus0 = list_to_cpustring(cpus0)
+
+ # cpuset of cpus not in the isolated list
+ tuna0 = cpuset.Cpuset('tuna0')
+ tuna0.assign_cpus(cpus0)
+ tuna0.write_memnode('0')
+
+ # cpuset to be isolated
+ tuna1 = cpuset.Cpuset('tuna1')
+ cpus1 = list_to_cpustring(cpus)
+ tuna1.assign_cpus(cpus1)
+ tuna1.write_memnode('0')
+ tuna1.set_cpu_exclusive()
+
+ # Migrate all migratable tasks from the base_cpuset
+ # to the cpusset with the cpus not in the isolated list
+ tm = cpuset.TaskMigrate(base_cpuset, tuna0)
+ tm.migrate()
+
def isolate_cpus(cpus, nr_cpus):
+ _isolate_cpus(cpus, nr_cpus)
+ return
+
fname = sys._getframe().f_code.co_name # Function name
ps = procfs.pidstats()
ps.reload_threads()
--
2.31.1