Change in vdsm[master]: Move the host stats dict creation to host module

ybronhei at redhat.com ybronhei at redhat.com
Sun May 1 09:55:13 UTC 2016


Yaniv Bronhaim has uploaded a new change for review.

Change subject: Move the host stats dict creation to host module
......................................................................

Move the host stats dict creation to host module

This work is part of reporting the metric of host stats. This patch help
to centerelize the code around host capabilities.

Change-Id: I626a673c429d486ac78b309c2f31cb58af8852b2
Signed-off-by: Yaniv Bronhaim <ybronhei at redhat.com>
---
M lib/vdsm/host.py
M vdsm/API.py
2 files changed, 174 insertions(+), 157 deletions(-)


  git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/74/56874/1

diff --git a/lib/vdsm/host.py b/lib/vdsm/host.py
index 1cf633a..5318ce6 100644
--- a/lib/vdsm/host.py
+++ b/lib/vdsm/host.py
@@ -20,15 +20,23 @@
 from __future__ import absolute_import
 
 
+import errno
 import os
 import logging
-from vdsm.utils import memoized
+import time
+from vdsm import cpuarch
+from vdsm import hooks
+from vdsm import utils
+from vdsm.config import config
+from vdsm.define import Kbytes, Mbytes
+from vdsm.virt import hoststats
+from vdsm.virt import vmstatus
+from vdsm.virt import sampling
 from .commands import execCmd
 from . import constants
-from . import cpuarch
 
 
- at memoized
+ at utils.memoized
 def uuid():
     host_UUID = None
 
@@ -65,3 +73,161 @@
         logging.error("Error retrieving host UUID", exc_info=True)
 
     return host_UUID
+
+
+def _readSwapTotalFree():
+    meminfo = utils.readMemInfo()
+    return meminfo['SwapTotal'] / 1024, meminfo['SwapFree'] / 1024
+
+
+# take a rough estimate on how much free mem is available for new vm
+# memTotal = memFree + memCached + mem_used_by_non_qemu + resident  .
+# simply returning (memFree + memCached) is not good enough, as the
+# resident set size of qemu processes may grow - up to  memCommitted.
+# Thus, we deduct the growth potential of qemu processes, which is
+# (memCommitted - resident)
+
+def _memCommitted(cif):
+    """
+    Return the amount of memory (Mb) committed for VMs
+    """
+    committed = 0
+    for v in cif.vmContainer.values():
+        committed += v.memCommitted
+    return committed
+
+
+def _memAvailable(cif):
+    """
+    Return an approximation of available memory for new VMs.
+    """
+    memCommitted = _memCommitted(cif)
+    resident = 0
+    for v in cif.vmContainer.values():
+        if v.conf['pid'] == '0':
+            continue
+        try:
+            with open('/proc/' + v.conf['pid'] + '/statm') as statmfile:
+                resident += int(statmfile.read().split()[1])
+        except:
+            pass
+    resident *= cpuarch.PAGE_SIZE_BYTES
+    meminfo = utils.readMemInfo()
+    freeOrCached = (meminfo['MemFree'] +
+                    meminfo['Cached'] + meminfo['Buffers']) * Kbytes
+    return freeOrCached + resident - memCommitted - \
+        config.getint('vars', 'host_mem_reserve') * Mbytes
+
+
+def _memFree():
+    """
+    Return the actual free mem on host.
+    """
+    meminfo = utils.readMemInfo()
+    return (meminfo['MemFree'] +
+            meminfo['Cached'] + meminfo['Buffers']) * Kbytes
+
+
+def _countVms(cif):
+    count = active = incoming = outgoing = 0
+    for vmId, v in cif.vmContainer.items():
+        try:
+            count += 1
+            status = v.lastStatus
+            if status == vmstatus.UP:
+                active += 1
+            elif status == vmstatus.MIGRATION_DESTINATION:
+                incoming += 1
+            elif status == vmstatus.MIGRATION_SOURCE:
+                outgoing += 1
+        except:
+            logging.error(vmId + ': Lost connection to VM')
+    return count, active, incoming + outgoing, incoming, outgoing
+
+
+def _getHaInfo(haClient):
+    """
+    Return Hosted Engine HA information for this host.
+    """
+    i = {
+        'configured': False,
+        'active': False,
+        'score': 0,
+        'globalMaintenance': False,
+        'localMaintenance': False,
+    }
+    if haClient:
+        try:
+            instance = haClient.HAClient()
+            host_id = instance.get_local_host_id()
+
+            # If a host id is available, consider HA configured
+            if host_id:
+                i['configured'] = True
+            else:
+                return i
+
+            stats = instance.get_all_stats()
+            if 0 in stats:
+                i['globalMaintenance'] = stats[0].get(
+                    haClient.HAClient.GlobalMdFlags.MAINTENANCE, False)
+            if host_id in stats:
+                i['active'] = stats[host_id]['live-data']
+                i['score'] = stats[host_id]['score']
+                i['localMaintenance'] = stats[host_id]['maintenance']
+        except IOError as ex:
+            if ex.errno == errno.ENOENT:
+                logging.error(
+                    ("failed to retrieve Hosted Engine HA score '{0}'"
+                     "Is the Hosted Engine setup finished?")
+                    .format(str(ex))
+                )
+            else:
+                logging.exception(
+                    "failed to retrieve Hosted Engine HA score"
+                )
+        except Exception:
+            logging.exception("failed to retrieve Hosted Engine HA info")
+    return i
+
+
+def getStats(irs, cif, haClient):
+    """
+    Retreive host internal statistics
+    """
+    hooks.before_get_stats()
+    stats = {}
+
+    first_sample, last_sample, _ = sampling.host_samples.stats()
+    decStats = hoststats.produce(first_sample, last_sample)
+
+    if irs:
+        decStats['storageDomains'] = irs.repoStats()
+        del decStats['storageDomains']['status']
+    else:
+        decStats['storageDomains'] = {}
+
+    for var in decStats:
+        stats[var] = utils.convertToStr(decStats[var])
+
+    stats['memAvailable'] = _memAvailable(cif) / Mbytes
+    stats['memCommitted'] = _memCommitted(cif) / Mbytes
+    stats['memFree'] = _memFree() / Mbytes
+    stats['swapTotal'], stats['swapFree'] = _readSwapTotalFree()
+    (stats['vmCount'], stats['vmActive'], stats['vmMigrating'],
+     stats['incomingVmMigrations'], stats['outgoingVmMigrations']) = \
+        _countVms(cif)
+    (tm_year, tm_mon, tm_day, tm_hour, tm_min, tm_sec,
+     dummy, dummy, dummy) = time.gmtime(time.time())
+    stats['dateTime'] = '%02d-%02d-%02dT%02d:%02d:%02d GMT' % (
+        tm_year, tm_mon, tm_day, tm_hour, tm_min, tm_sec)
+    stats['momStatus'] = cif.mom.getStatus()
+    stats.update(cif.mom.getKsmStats())
+    stats['netConfigDirty'] = str(cif._netConfigDirty)
+    stats['haStats'] = _getHaInfo()
+    if stats['haStats']['configured']:
+        # For backwards compatibility, will be removed in the future
+        stats['haScore'] = stats['haStats']['score']
+
+    stats = hooks.after_get_stats(stats)
+    return stats
diff --git a/vdsm/API.py b/vdsm/API.py
index e7aac09..e41c29d 100644
--- a/vdsm/API.py
+++ b/vdsm/API.py
@@ -21,27 +21,23 @@
 # pylint: disable=R0904
 
 import os
-import time
 import logging
-import errno
 
 from vdsm.network.errors import ConfigNetworkError
 
 from vdsm import commands
-from vdsm import cpuarch
 from vdsm import utils
 from clientIF import clientIF
 from vdsm import constants
 from vdsm import exception
 from vdsm import hooks
+from vdsm import host
 from vdsm import hostdev
 from vdsm import response
 from vdsm import supervdsm
 from vdsm import jobs
 from vdsm import v2v
-from vdsm.virt import hoststats
 from vdsm.virt import vmstatus
-from vdsm.virt import sampling
 from vdsm.virt import secret
 import storage.misc
 import storage.clusterlock
@@ -51,7 +47,7 @@
 from virt.vmdevices import graphics
 from virt.vmdevices import hwclass
 from vdsm.compat import pickle
-from vdsm.define import doneCode, errCode, Kbytes, Mbytes
+from vdsm.define import doneCode, errCode
 import caps
 from vdsm.config import config
 
@@ -1333,48 +1329,9 @@
         """
         Report host statistics.
         """
-
-        def _readSwapTotalFree():
-            meminfo = utils.readMemInfo()
-            return meminfo['SwapTotal'] / 1024, meminfo['SwapFree'] / 1024
-
-        hooks.before_get_stats()
-        stats = {}
-
-        first_sample, last_sample, _ = sampling.host_samples.stats()
-        decStats = hoststats.produce(first_sample, last_sample)
-
-        if self._irs:
-            decStats['storageDomains'] = self._irs.repoStats()
-            del decStats['storageDomains']['status']
-        else:
-            decStats['storageDomains'] = {}
-
-        for var in decStats:
-            stats[var] = utils.convertToStr(decStats[var])
-
-        stats['memAvailable'] = self._memAvailable() / Mbytes
-        stats['memCommitted'] = self._memCommitted() / Mbytes
-        stats['memFree'] = self._memFree() / Mbytes
-        stats['swapTotal'], stats['swapFree'] = _readSwapTotalFree()
-        (stats['vmCount'], stats['vmActive'], stats['vmMigrating'],
-         stats['incomingVmMigrations'], stats['outgoingVmMigrations']) = \
-            self._countVms()
-        (tm_year, tm_mon, tm_day, tm_hour, tm_min, tm_sec,
-         dummy, dummy, dummy) = time.gmtime(time.time())
-        stats['dateTime'] = '%02d-%02d-%02dT%02d:%02d:%02d GMT' % (
-            tm_year, tm_mon, tm_day, tm_hour, tm_min, tm_sec)
-        stats['momStatus'] = self._cif.mom.getStatus()
-        stats.update(self._cif.mom.getKsmStats())
-
-        stats['netConfigDirty'] = str(self._cif._netConfigDirty)
-        stats['haStats'] = self._getHaInfo()
-        if stats['haStats']['configured']:
-            # For backwards compatibility, will be removed in the future
-            stats['haScore'] = stats['haStats']['score']
-
-        stats = hooks.after_get_stats(stats)
-        return {'status': doneCode, 'info': stats}
+        return {'status': doneCode, 'info': host.getStats(self._irs,
+                                                          self._cif,
+                                                          haClient)}
 
     def setLogLevel(self, level):
         """
@@ -1581,112 +1538,6 @@
 
     def extend_image_ticket(self, uuid, timeout):
         return self._irs.extend_image_ticket(uuid, timeout)
-
-    # take a rough estimate on how much free mem is available for new vm
-    # memTotal = memFree + memCached + mem_used_by_non_qemu + resident  .
-    # simply returning (memFree + memCached) is not good enough, as the
-    # resident set size of qemu processes may grow - up to  memCommitted.
-    # Thus, we deduct the growth potential of qemu processes, which is
-    # (memCommitted - resident)
-
-    def _memAvailable(self):
-        """
-        Return an approximation of available memory for new VMs.
-        """
-        memCommitted = self._memCommitted()
-        resident = 0
-        for v in self._cif.vmContainer.values():
-            if v.conf['pid'] == '0':
-                continue
-            try:
-                with open('/proc/' + v.conf['pid'] + '/statm') as statmfile:
-                    resident += int(statmfile.read().split()[1])
-            except:
-                pass
-        resident *= cpuarch.PAGE_SIZE_BYTES
-        meminfo = utils.readMemInfo()
-        freeOrCached = (meminfo['MemFree'] +
-                        meminfo['Cached'] + meminfo['Buffers']) * Kbytes
-        return freeOrCached + resident - memCommitted - \
-            config.getint('vars', 'host_mem_reserve') * Mbytes
-
-    def _memFree(self):
-        """
-        Return the actual free mem on host.
-        """
-        meminfo = utils.readMemInfo()
-        return (meminfo['MemFree'] +
-                meminfo['Cached'] + meminfo['Buffers']) * Kbytes
-
-    def _memCommitted(self):
-        """
-        Return the amount of memory (Mb) committed for VMs
-        """
-        committed = 0
-        for v in self._cif.vmContainer.values():
-            committed += v.memCommitted
-        return committed
-
-    def _countVms(self):
-        count = active = incoming = outgoing = 0
-        for vmId, v in self._cif.vmContainer.items():
-            try:
-                count += 1
-                status = v.lastStatus
-                if status == vmstatus.UP:
-                    active += 1
-                elif status == vmstatus.MIGRATION_DESTINATION:
-                    incoming += 1
-                elif status == vmstatus.MIGRATION_SOURCE:
-                    outgoing += 1
-            except:
-                self.log.error(vmId + ': Lost connection to VM')
-        return count, active, incoming + outgoing, incoming, outgoing
-
-    def _getHaInfo(self):
-        """
-        Return Hosted Engine HA information for this host.
-        """
-        i = {
-            'configured': False,
-            'active': False,
-            'score': 0,
-            'globalMaintenance': False,
-            'localMaintenance': False,
-        }
-        if haClient:
-            try:
-                instance = haClient.HAClient()
-                host_id = instance.get_local_host_id()
-
-                # If a host id is available, consider HA configured
-                if host_id:
-                    i['configured'] = True
-                else:
-                    return i
-
-                stats = instance.get_all_stats()
-                if 0 in stats:
-                    i['globalMaintenance'] = stats[0].get(
-                        haClient.HAClient.GlobalMdFlags.MAINTENANCE, False)
-                if host_id in stats:
-                    i['active'] = stats[host_id]['live-data']
-                    i['score'] = stats[host_id]['score']
-                    i['localMaintenance'] = stats[host_id]['maintenance']
-            except IOError as ex:
-                if ex.errno == errno.ENOENT:
-                    self.log.error(
-                        ("failed to retrieve Hosted Engine HA score '{0}'"
-                         "Is the Hosted Engine setup finished?")
-                        .format(str(ex))
-                    )
-                else:
-                    self.log.exception(
-                        "failed to retrieve Hosted Engine HA score"
-                    )
-            except Exception:
-                self.log.exception("failed to retrieve Hosted Engine HA info")
-        return i
 
 
 class SDM(APIBase):


-- 
To view, visit https://gerrit.ovirt.org/56874
To unsubscribe, visit https://gerrit.ovirt.org/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I626a673c429d486ac78b309c2f31cb58af8852b2
Gerrit-PatchSet: 1
Gerrit-Project: vdsm
Gerrit-Branch: master
Gerrit-Owner: Yaniv Bronhaim <ybronhei at redhat.com>


More information about the vdsm-patches mailing list