Change in vdsm[master]: numa: move stuff to numa

mpolednik at redhat.com mpolednik at redhat.com
Tue Feb 9 12:38:08 UTC 2016


Martin Polednik has uploaded a new change for review.

Change subject: numa: move stuff to numa
......................................................................

numa: move stuff to numa

wip

Change-Id: I2dffad8663ca50a67f117b1c74d4459e667d9f43
Signed-off-by: Martin Polednik <mpolednik at redhat.com>
---
M lib/vdsm/Makefile.am
A lib/vdsm/numa.py
M tests/capsTests.py
M tests/hoststatsTests.py
M tests/numaUtilsTests.py
M tests/samplingTests.py
M vdsm.spec.in
M vdsm/caps.py
M vdsm/numaUtils.py
M vdsm/virt/hoststats.py
M vdsm/virt/sampling.py
11 files changed, 113 insertions(+), 82 deletions(-)


  git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/86/53286/1

diff --git a/lib/vdsm/Makefile.am b/lib/vdsm/Makefile.am
index a1628ab..4732924 100644
--- a/lib/vdsm/Makefile.am
+++ b/lib/vdsm/Makefile.am
@@ -43,6 +43,7 @@
 	libvirtconnection.py \
 	m2cutils.py \
 	netconfpersistence.py \
+	numa.py \
 	panic.py \
 	password.py \
 	ppc64HardwareInfo.py \
diff --git a/lib/vdsm/numa.py b/lib/vdsm/numa.py
new file mode 100644
index 0000000..90bc6e5
--- /dev/null
+++ b/lib/vdsm/numa.py
@@ -0,0 +1,82 @@
+#
+# Copyright 2016 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
+#
+# Refer to the README and COPYING files for full details of the license
+#
+
+from __future__ import absolute_import
+
+import xml.etree.ElementTree as ET
+
+from . import utils
+from . import libvirtconnection
+
+
+def _get_libvirt_caps():
+    return libvirtconnection.get().getCapabilities()
+
+
+ at utils.memoized
+def topology(capabilities=None):
+    if capabilities is None:
+        capabilities = _get_libvirt_caps()
+    caps = ET.fromstring(capabilities)
+    host = caps.find('host')
+    cells = host.find('.//cells')
+    cellsInfo = {}
+    cellSets = cells.findall('cell')
+    for cell in cellSets:
+        cellInfo = {}
+        cpus = []
+        for cpu in cell.iter(tag='cpu'):
+            cpus.append(int(cpu.get('id')))
+        cellInfo['cpus'] = cpus
+        cellIndex = cell.get('id')
+        memInfo = memory_by_cell(int(cellIndex))
+        cellInfo['totalMemory'] = memInfo['total']
+        cellsInfo[cellIndex] = cellInfo
+    return cellsInfo
+
+
+def memory_by_cell(index):
+    """
+    Get the memory stats of a specified numa node, the unit is MiB.
+
+    :param cell: the index of numa node
+    :type cell: int
+    :return: dict like {'total': '49141', 'free': '46783'}
+    """
+    cellMemInfo = libvirtconnection.get().getMemoryStats(index, 0)
+    cellMemInfo['total'] = str(cellMemInfo['total'] / 1024)
+    cellMemInfo['free'] = str(cellMemInfo['free'] / 1024)
+    return cellMemInfo
+
+
+ at utils.memoized
+def distances(capabilities=None):
+    if capabilities is None:
+        capabilities = _get_libvirt_caps()
+    caps = ET.fromstring(capabilities)
+    cells = caps.find('host').find('.//cells').findall('cell')
+    distances = {}
+    for cell in cells:
+        cellIndex = cell.get('id')
+        distances[cellIndex] = []
+        for sibling in cell.find('distances').findall('sibling'):
+            distances[cellIndex].append(sibling.get('value'))
+
+    return distances
diff --git a/tests/capsTests.py b/tests/capsTests.py
index a701f24..ce3cca5 100644
--- a/tests/capsTests.py
+++ b/tests/capsTests.py
@@ -28,6 +28,7 @@
 import caps
 from vdsm import commands
 from vdsm import cpuarch
+from vdsm import numa
 from vdsm import utils
 
 
@@ -114,13 +115,13 @@
         for res, s in zip(expectedRes, sign):
             self.assertEqual(res, caps._parseKeyVal(lines, s))
 
-    @MonkeyPatch(caps, 'getMemoryStatsByNumaCell', lambda x: {
+    @MonkeyPatch(numa, 'memory_by_cell', lambda x: {
         'total': '49141', 'free': '46783'})
-    @MonkeyPatch(caps, '_getCapsXMLStr', lambda: _getTestData(
+    @MonkeyPatch(numa, '_get_libvirt_caps', lambda: _getTestData(
         "caps_libvirt_amd_6274.out"))
     def testNumaTopology(self):
         # 2 x AMD 6272 (with Modules)
-        t = caps.getNumaTopology()
+        t = numa.topology()
         expectedNumaInfo = {
             '0': {'cpus': [0, 1, 2, 3, 4, 5, 6, 7], 'totalMemory': '49141'},
             '1': {'cpus': [8, 9, 10, 11, 12, 13, 14, 15],
@@ -131,10 +132,10 @@
                   'totalMemory': '49141'}}
         self.assertEqual(t, expectedNumaInfo)
 
-    @MonkeyPatch(caps, '_getCapsXMLStr', lambda: _getTestData(
+    @MonkeyPatch(numa, '_get_libvirt_caps', lambda: _getTestData(
         'caps_libvirt_ibm_S822L_le.out'))
     def testNumaNodeDistance(self):
-        t = caps.getNumaNodeDistance()
+        t = numa.distances()
         expectedDistanceInfo = {'0': ['10', '20', '40', '40'],
                                 '1': ['20', '10', '40', '40'],
                                 '16': ['40', '40', '10', '20'],
@@ -302,11 +303,11 @@
                     'pc-i440fx-rhel7.0.0']
         self.assertEqual(expected, result)
 
-    @MonkeyPatch(caps, 'getMemoryStatsByNumaCell', lambda x: {
+    @MonkeyPatch(numa, 'memory_by_cell', lambda x: {
         'total': '1', 'free': '1'})
-    def test_getNumaTopology(self):
+    def test_topology(self):
         capsData = self._readCaps("caps_libvirt_intel_i73770_nosnap.out")
-        result = caps.getNumaTopology(capsData)
+        result = numa.topology(capsData)
         # only check cpus, memory does not come from file
         expected = [0, 1, 2, 3, 4, 5, 6, 7]
         self.assertEqual(expected, result['0']['cpus'])
diff --git a/tests/hoststatsTests.py b/tests/hoststatsTests.py
index 4841805..3a2e9f2 100644
--- a/tests/hoststatsTests.py
+++ b/tests/hoststatsTests.py
@@ -24,7 +24,7 @@
 
 from virt import hoststats
 
-import caps
+from vdsm import numa
 
 from testlib import VdsmTestCase as TestCaseBase
 from monkeypatch import MonkeyPatchScope
@@ -135,7 +135,7 @@
         first_sample = fake.HostSample(1.0, {0: cpu_sample, 1: cpu_sample})
         last_sample = fake.HostSample(2.0, {0: cpu_sample, 1: cpu_sample})
 
-        with MonkeyPatchScope([(caps, 'getNumaTopology',
+        with MonkeyPatchScope([(numa, 'topology',
                                 self._fakeNumaTopology)]):
             result = hoststats._get_cpu_core_stats(first_sample, last_sample)
             self.assertEqual(len(result), 2)
@@ -149,7 +149,7 @@
         first_sample = fake.HostSample(1.0, {0: cpu_sample})
         last_sample = fake.HostSample(2.0, {0: cpu_sample, 1: cpu_sample})
 
-        with MonkeyPatchScope([(caps, 'getNumaTopology',
+        with MonkeyPatchScope([(numa, 'topology',
                                 self._fakeNumaTopology)]):
             result = hoststats._get_cpu_core_stats(first_sample, last_sample)
             self.assertEqual(len(result), 1)
@@ -162,7 +162,7 @@
         # CPU one suddenly came online and the second sample is still missing
         last_sample = fake.HostSample(2.0, {0: cpu_sample})
 
-        with MonkeyPatchScope([(caps, 'getNumaTopology',
+        with MonkeyPatchScope([(numa, 'topology',
                                 self._fakeNumaTopology)]):
             result = hoststats._get_cpu_core_stats(first_sample, last_sample)
             self.assertEqual(len(result), 1)
diff --git a/tests/numaUtilsTests.py b/tests/numaUtilsTests.py
index 3b6ae6f..c13eea8 100644
--- a/tests/numaUtilsTests.py
+++ b/tests/numaUtilsTests.py
@@ -26,8 +26,8 @@
 from monkeypatch import MonkeyPatch
 from monkeypatch import MonkeyPatchScope
 
-import caps
 import numaUtils
+from vdsm import numa
 
 import vmfakelib as fake
 
@@ -60,8 +60,8 @@
         self.assertEqual(vcpuPids, expectedVcpuPids)
 
     @MonkeyPatch(numaUtils, 'supervdsm', fake.SuperVdsm())
-    @MonkeyPatch(caps,
-                 'getNumaTopology',
+    @MonkeyPatch(numa,
+                 'topology',
                  lambda: {'0': {'cpus': [0, 1, 2, 3],
                                 'totalMemory': '49141'},
                           '1': {'cpus': [4, 5, 6, 7],
diff --git a/tests/samplingTests.py b/tests/samplingTests.py
index 1aa3120..edb4f7a 100644
--- a/tests/samplingTests.py
+++ b/tests/samplingTests.py
@@ -24,10 +24,9 @@
 import threading
 
 from vdsm import ipwrapper
+from vdsm import numa
 from vdsm.password import ProtectedPassword
 import virt.sampling as sampling
-
-import caps
 
 from testValidation import ValidateRunningAsRoot
 from testlib import permutations, expandPermutations
@@ -202,9 +201,9 @@
                 }
             }
 
-        return MonkeyPatchScope([(caps, 'getNumaTopology',
+        return MonkeyPatchScope([(numa, 'topology',
                                   fakeNumaTopology),
-                                 (caps, 'getMemoryStatsByNumaCell',
+                                 (numa, 'memory_by_cell',
                                   fakeMemoryStats)])
 
     def testMemoryStatsWithZeroMemoryAsString(self):
diff --git a/vdsm.spec.in b/vdsm.spec.in
index 725e28e..dc5314b 100644
--- a/vdsm.spec.in
+++ b/vdsm.spec.in
@@ -1134,6 +1134,7 @@
 %{python_sitelib}/%{vdsm_name}/network/sourceroute.py*
 %{python_sitelib}/%{vdsm_name}/network/sourceroutethread.py*
 %{python_sitelib}/%{vdsm_name}/network/utils.py*
+%{python_sitelib}/%{vdsm_name}/numa.py*
 %{python_sitelib}/%{vdsm_name}/password.py*
 %{python_sitelib}/%{vdsm_name}/panic.py*
 %{python_sitelib}/%{vdsm_name}/ppc64HardwareInfo.py*
diff --git a/vdsm/caps.py b/vdsm/caps.py
index e6cc817..73ddf8b 100644
--- a/vdsm/caps.py
+++ b/vdsm/caps.py
@@ -39,6 +39,7 @@
 from vdsm import hooks
 from vdsm import libvirtconnection
 from vdsm import netinfo
+from vdsm import numa
 from vdsm import host
 from vdsm import commands
 from vdsm import utils
@@ -210,58 +211,6 @@
             logging.debug("libvirt is missing '%s': live merge disabled", flag)
             return False
     return True
-
-
- at utils.memoized
-def getNumaTopology(capabilities=None):
-    if capabilities is None:
-        capabilities = _getCapsXMLStr()
-    caps = ET.fromstring(capabilities)
-    host = caps.find('host')
-    cells = host.find('.//cells')
-    cellsInfo = {}
-    cellSets = cells.findall('cell')
-    for cell in cellSets:
-        cellInfo = {}
-        cpus = []
-        for cpu in cell.iter(tag='cpu'):
-            cpus.append(int(cpu.get('id')))
-        cellInfo['cpus'] = cpus
-        cellIndex = cell.get('id')
-        memInfo = getMemoryStatsByNumaCell(int(cellIndex))
-        cellInfo['totalMemory'] = memInfo['total']
-        cellsInfo[cellIndex] = cellInfo
-    return cellsInfo
-
-
-def getMemoryStatsByNumaCell(cell):
-    """
-    Get the memory stats of a specified numa node, the unit is MiB.
-
-    :param cell: the index of numa node
-    :type cell: int
-    :return: dict like {'total': '49141', 'free': '46783'}
-    """
-    cellMemInfo = libvirtconnection.get().getMemoryStats(cell, 0)
-    cellMemInfo['total'] = str(cellMemInfo['total'] / 1024)
-    cellMemInfo['free'] = str(cellMemInfo['free'] / 1024)
-    return cellMemInfo
-
-
- at utils.memoized
-def getNumaNodeDistance(capabilities=None):
-    if capabilities is None:
-        capabilities = _getCapsXMLStr()
-    caps = ET.fromstring(capabilities)
-    cells = caps.find('host').find('.//cells').findall('cell')
-    distances = {}
-    for cell in cells:
-        cellIndex = cell.get('id')
-        distances[cellIndex] = []
-        for sibling in cell.find('distances').findall('sibling'):
-            distances[cellIndex].append(sibling.get('value'))
-
-    return distances
 
 
 @utils.memoized
@@ -570,8 +519,8 @@
     else:
         caps['rngSources'] = _getRngSources()
 
-    caps['numaNodes'] = getNumaTopology()
-    caps['numaNodeDistance'] = getNumaNodeDistance()
+    caps['numaNodes'] = numa.topology()
+    caps['numaNodeDistance'] = numa.distances()
     caps['autoNumaBalancing'] = getAutoNumaBalancingInfo()
 
     caps['selinux'] = _getSELinux()
diff --git a/vdsm/numaUtils.py b/vdsm/numaUtils.py
index 10a9254..38f7e1d 100644
--- a/vdsm/numaUtils.py
+++ b/vdsm/numaUtils.py
@@ -23,9 +23,8 @@
 import os.path
 import xml.etree.cElementTree as ET
 
+from vdsm import numa
 from vdsm import supervdsm
-
-import caps
 
 # xml file name -> (last mtime, cached value)
 _libvirt_vcpu_pids_cache = {}
@@ -146,7 +145,7 @@
 
 def _get_mapping_pcpu_to_pnode():
     pcpu_to_pnode = {}
-    for node_index, numa_node in caps.getNumaTopology().iteritems():
+    for node_index, numa_node in numa.topology().iteritems():
         for pcpu_id in numa_node['cpus']:
             pcpu_to_pnode[pcpu_id] = int(node_index)
     return pcpu_to_pnode
diff --git a/vdsm/virt/hoststats.py b/vdsm/virt/hoststats.py
index b58d170..342b703 100644
--- a/vdsm/virt/hoststats.py
+++ b/vdsm/virt/hoststats.py
@@ -22,9 +22,9 @@
 
 import six
 
+from vdsm import numa
 from vdsm import utils
 
-import caps
 import v2v
 
 
@@ -108,7 +108,7 @@
         return ("%.2f" % (jiffies / interval))
 
     cpu_core_stats = {}
-    for node_index, numa_node in six.iteritems(caps.getNumaTopology()):
+    for node_index, numa_node in six.iteritems(numa.topology()):
         cpu_cores = numa_node['cpus']
         for cpu_core in cpu_cores:
             try:
diff --git a/vdsm/virt/sampling.py b/vdsm/virt/sampling.py
index 91b10a0..06760de 100644
--- a/vdsm/virt/sampling.py
+++ b/vdsm/virt/sampling.py
@@ -32,12 +32,11 @@
 
 from vdsm.constants import P_VDSM_RUN, P_VDSM_CLIENT_LOG
 from vdsm import ipwrapper
+from vdsm import numa
 from vdsm.netinfo import nics, bonding, vlans
 from vdsm import utils
 
 from .utils import ExpiringCache
-
-import caps
 
 _THP_STATE_PATH = '/sys/kernel/mm/transparent_hugepage/enabled'
 if not os.path.exists(_THP_STATE_PATH):
@@ -165,10 +164,10 @@
     """
     def __init__(self):
         self.nodesMemSample = {}
-        numaTopology = caps.getNumaTopology()
+        numaTopology = numa.topology()
         for nodeIndex in numaTopology:
             nodeMemSample = {}
-            memInfo = caps.getMemoryStatsByNumaCell(int(nodeIndex))
+            memInfo = numa.memory_by_cell(int(nodeIndex))
             nodeMemSample['memFree'] = memInfo['free']
             # in case the numa node has zero memory assigned, report the whole
             # memory as used


-- 
To view, visit https://gerrit.ovirt.org/53286
To unsubscribe, visit https://gerrit.ovirt.org/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I2dffad8663ca50a67f117b1c74d4459e667d9f43
Gerrit-PatchSet: 1
Gerrit-Project: vdsm
Gerrit-Branch: master
Gerrit-Owner: Martin Polednik <mpolednik at redhat.com>


More information about the vdsm-patches mailing list