Change in vdsm[master]: sampling: split up stats production

fromani at redhat.com fromani at redhat.com
Fri Jan 9 12:34:51 UTC 2015


Francesco Romani has uploaded a new change for review.

Change subject: sampling: split up stats production
......................................................................

sampling: split up stats production

VmStatsThread does both sampling gathering
and statistics production.

We want to add another, more efficient, way
to gather samples, but we don't want to duplicate
the code to produce stats.

So, this patch extracts the stats generation
code in a common mixin class, to be shared by
both new and old implementations.

Change-Id: I1a1d42b10714fa69c78f58ffaab7f7a32aed47ba
Signed-off-by: Francesco Romani <fromani at redhat.com>
---
M vdsm/virt/vm.py
1 file changed, 264 insertions(+), 229 deletions(-)


  git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/30/36730/1

diff --git a/vdsm/virt/vm.py b/vdsm/virt/vm.py
index c78c9cf..132f09f 100644
--- a/vdsm/virt/vm.py
+++ b/vdsm/virt/vm.py
@@ -181,9 +181,271 @@
                               ['uuid', 'path', 'allocation'])
 
 
-class VmStatsThread(AdvancedStatsThread):
+class VmStatsMixin(object):
     MBPS_TO_BPS = 10 ** 6 / 8
 
+    """
+    Translates vm samples into stats.
+    """
+
+    def __init__(self, vm):
+        self._vm = vm
+
+    def get(self):
+        stats = {}
+
+        try:
+            stats['statsAge'] = time.time() - self.getLastSampleTime()
+        except TypeError:
+            self._log.debug("Stats age not available")
+            stats['statsAge'] = -1.0
+
+        sCpu, eCpu, cpuInterval = self._getSampleCpu()
+        self._getCpuStats(stats, sCpu, eCpu, cpuInterval)
+
+        sNet, eNet, netInterval = self._getSampleNet()
+        self._getNetworkStats(stats, sNet, eNet, netInterval)
+
+        sDisk, eDisk, diskInterval = self._getSampleDisk()
+        self._getDiskStats(stats, sDisk, eDisk, diskInterval)
+
+        self._getBalloonStats(stats, self._getSampleBalloon())
+        self._getVmJobs(stats, self._getSampleVmJobs())
+        self._getNumaStats(stats, self._getSampleVcpu())
+
+        cpuTuneSample = self._getSampleCpuTune()
+        self._getCpuTuneInfo(stats, cpuTuneSample)
+        self._getCpuCount(stats, cpuTuneSample)
+        self._getUserCpuTuneInfo(stats, cpuTuneSample)
+
+        self._getIoTuneStats(stats)
+
+        return stats
+
+    def _getIoTuneStats(self, stats):
+        """
+        Collect the current ioTune settings for all disks VDSM knows about.
+
+        This assumes VDSM always has the correct info and nobody else is
+        touching the device without telling VDSM about it.
+
+        TODO: We might want to move to XML parsing (first update) and events
+        once libvirt supports them:
+        https://bugzilla.redhat.com/show_bug.cgi?id=1114492
+        """
+        ioTuneInfo = []
+
+        for disk in self._vm.getDiskDevices():
+            if "ioTune" in disk.specParams:
+                ioTuneInfo.append({
+                    "name": disk.name,
+                    "path": disk.path,
+                    "ioTune": disk.specParams["ioTune"]
+                })
+
+        stats['ioTune'] = ioTuneInfo
+
+    def _diff(self, prev, curr, val):
+        return prev[val] - curr[val]
+
+    def _usagePercentage(self, val, sampleInterval):
+        return 100 * val / sampleInterval / 1000 ** 3
+
+    def _getCpuStats(self, stats, sInfo, eInfo, interval):
+        stats['cpuUser'] = 0.0
+        stats['cpuSys'] = 0.0
+
+        if sInfo is None:
+            return
+
+        try:
+            stats['cpuSys'] = self._usagePercentage(
+                self._diff(eInfo, sInfo, 'user_time') +
+                self._diff(eInfo, sInfo, 'system_time'),
+                interval)
+            stats['cpuUser'] = self._usagePercentage(
+                self._diff(eInfo, sInfo, 'cpu_time')
+                - self._diff(eInfo, sInfo, 'user_time')
+                - self._diff(eInfo, sInfo, 'system_time'),
+                interval)
+
+        except (TypeError, ZeroDivisionError) as e:
+            self._log.exception("CPU stats not available: %s", e)
+
+    def _getBalloonStats(self, stats, sample):
+        max_mem = int(self._vm.conf.get('memSize')) * 1024
+
+        for dev in self._vm.getBalloonDevicesConf():
+            if dev['specParams']['model'] != 'none':
+                balloon_target = dev.get('target', max_mem)
+                break
+        else:
+            balloon_target = None
+
+        stats['balloonInfo'] = {}
+
+        # Do not return any balloon status info before we get all data
+        # MOM will ignore VMs with missing balloon information instead
+        # using incomplete data and computing wrong balloon targets
+        if balloon_target is not None and sample is not None:
+            stats['balloonInfo'].update({
+                'balloon_max': str(max_mem),
+                'balloon_min': str(
+                    int(self._vm.conf.get('memGuaranteedSize', '0')) * 1024),
+                'balloon_cur': str(sample),
+                'balloon_target': str(balloon_target)
+            })
+
+    def _getCpuTuneInfo(self, stats, sample):
+        # Handling the case when not enough samples exist
+        if sample is None:
+            return
+
+        # Handling the case where quota is not set, setting to 0.
+        # According to libvirt API:"A quota with value 0 means no value."
+        # The value does not have to be present in some transient cases
+        if sample.get('vcpu_quota', _NO_CPU_QUOTA) != _NO_CPU_QUOTA:
+            stats['vcpuQuota'] = str(sample['vcpu_quota'])
+
+        # Handling the case where period is not set, setting to 0.
+        # According to libvirt API:"A period with value 0 means no value."
+        # The value does not have to be present in some transient cases
+        if sample.get('vcpu_period', _NO_CPU_PERIOD) != _NO_CPU_PERIOD:
+            stats['vcpuPeriod'] = sample['vcpu_period']
+
+    def _getCpuCount(self, stats, sample):
+        # Handling the case when not enough samples exist
+        if sample is None:
+            return
+
+        if 'vcpuCount' in sample:
+            vcpuCount = sample['vcpuCount']
+            if vcpuCount != -1:
+                stats['vcpuCount'] = vcpuCount
+            else:
+                self._log.error('Failed to get VM cpu count')
+
+    def _getUserCpuTuneInfo(self, stats, sample):
+        # Handling the case when not enough samples exist
+        if sample is None:
+            return
+
+        if 'vcpuLimit' in sample:
+            value = sample['vcpuLimit']
+            stats['vcpuUserLimit'] = value
+
+    @classmethod
+    def _getNicStats(cls, name, model, mac,
+                     start_sample, end_sample, interval):
+        ifSpeed = [100, 1000][model in ('e1000', 'virtio')]
+
+        ifStats = {'macAddr': mac,
+                   'name': name,
+                   'speed': str(ifSpeed),
+                   'state': 'unknown'}
+
+        ifStats['rxErrors'] = str(end_sample[2])
+        ifStats['rxDropped'] = str(end_sample[3])
+        ifStats['txErrors'] = str(end_sample[6])
+        ifStats['txDropped'] = str(end_sample[7])
+
+        ifRxBytes = (100.0 *
+                     ((end_sample[0] - start_sample[0]) % 2 ** 32) /
+                     interval / ifSpeed / cls.MBPS_TO_BPS)
+        ifTxBytes = (100.0 *
+                     ((end_sample[4] - start_sample[4]) % 2 ** 32) /
+                     interval / ifSpeed / cls.MBPS_TO_BPS)
+
+        ifStats['rxRate'] = '%.1f' % ifRxBytes
+        ifStats['txRate'] = '%.1f' % ifTxBytes
+
+        return ifStats
+
+    def _getNetworkStats(self, stats, sInfo, eInfo, interval):
+        stats['network'] = {}
+
+        if sInfo is None:
+            return
+
+        for nic in self._vm.getNicDevices():
+            if nic.name.startswith('hostdev'):
+                continue
+
+            # may happen if nic is a new hot-plugged one
+            if nic.name not in sInfo or nic.name not in eInfo:
+                continue
+
+            stats['network'][nic.name] = self._getNicStats(
+                nic.name, nic.nicModel, nic.macAddr,
+                sInfo[nic.name], eInfo[nic.name], interval)
+
+    def _getDiskStats(self, stats, sInfo, eInfo, interval):
+        for vmDrive in self._vm.getDiskDevices():
+            dStats = {}
+            try:
+                dStats = {'truesize': str(vmDrive.truesize),
+                          'apparentsize': str(vmDrive.apparentsize),
+                          'readLatency': '0',
+                          'writeLatency': '0',
+                          'flushLatency': '0'}
+                if isVdsmImage(vmDrive):
+                    dStats['imageID'] = vmDrive.imageID
+                elif "GUID" in vmDrive:
+                    dStats['lunGUID'] = vmDrive.GUID
+                if (sInfo and vmDrive.name in sInfo and
+                        eInfo and vmDrive.name in eInfo):
+                    # will be None if sampled during recovery
+                    dStats.update(_calcDiskRate(vmDrive, sInfo, eInfo,
+                                                interval))
+                    dStats.update(_calcDiskLatency(vmDrive, sInfo, eInfo))
+
+            except (AttributeError, TypeError, ZeroDivisionError):
+                self._log.exception("Disk %s stats not available",
+                                    vmDrive.name)
+
+            stats[vmDrive.name] = dStats
+
+    def _getNumaStats(self, stats, sample):
+        vmNumaNodeRuntimeMap = numaUtils.getVmNumaNodeRuntimeInfo(
+            self._vm, sample)
+        if vmNumaNodeRuntimeMap:
+            stats['vNodeRuntimeInfo'] = vmNumaNodeRuntimeMap
+
+    def _getVmJobs(self, stats, sample):
+        if sample is not None:
+            # If we are unable to collect stats we must not return anything at
+            # all since an empty dictionary would be interpreted as vm jobs
+            # finishing.
+            stats['vmJobs'] = sample
+
+    # interface to be implemented by concrete classes:
+
+    def getLastSampleTime(self):
+        raise NotImplementedError
+
+    def _getSampleCpu(self):
+        raise NotImplementedError
+
+    def _getSampleNet(self):
+        raise NotImplementedError
+
+    def _getSampleDisk(self):
+        raise NotImplementedError
+
+    def _getSampleBalloon(self):
+        raise NotImplementedError
+
+    def _getSampleVmJobs(self):
+        raise NotImplementedError
+
+    def _getSampleVcpu(self):
+        raise NotImplementedError
+
+    def _getSampleCpuTune(self):
+        raise NotImplementedError
+
+
+class VmStatsThread(AdvancedStatsThread, VmStatsMixin):
     # This flag will prevent excessive log flooding when running
     # on libvirt with no support for metadata xml elements.
     #
@@ -195,7 +457,7 @@
 
     def __init__(self, vm):
         AdvancedStatsThread.__init__(self, log=vm.log, daemon=True)
-        self._vm = vm
+        VmStatsMixin.__init__(self, vm)
 
         self.highWrite = (
             AdvancedStatsFunction(
@@ -449,202 +711,6 @@
 
         return infos
 
-    def _getIoTuneStats(self, stats):
-        """
-        Collect the current ioTune settings for all disks VDSM knows about.
-
-        This assumes VDSM always has the correct info and nobody else is
-        touching the device without telling VDSM about it.
-
-        TODO: We might want to move to XML parsing (first update) and events
-        once libvirt supports them:
-        https://bugzilla.redhat.com/show_bug.cgi?id=1114492
-        """
-        ioTuneInfo = []
-
-        for disk in self._vm.getDiskDevices():
-            if "ioTune" in disk.specParams:
-                ioTuneInfo.append({
-                    "name": disk.name,
-                    "path": disk.path,
-                    "ioTune": disk.specParams["ioTune"]
-                })
-
-        stats['ioTune'] = ioTuneInfo
-
-    def _diff(self, prev, curr, val):
-        return prev[val] - curr[val]
-
-    def _usagePercentage(self, val, sampleInterval):
-        return 100 * val / sampleInterval / 1000 ** 3
-
-    def _getCpuStats(self, stats, sInfo, eInfo, interval):
-        stats['cpuUser'] = 0.0
-        stats['cpuSys'] = 0.0
-
-        if sInfo is None:
-            return
-
-        try:
-            stats['cpuSys'] = self._usagePercentage(
-                self._diff(eInfo, sInfo, 'user_time') +
-                self._diff(eInfo, sInfo, 'system_time'),
-                interval)
-            stats['cpuUser'] = self._usagePercentage(
-                self._diff(eInfo, sInfo, 'cpu_time')
-                - self._diff(eInfo, sInfo, 'user_time')
-                - self._diff(eInfo, sInfo, 'system_time'),
-                interval)
-
-        except (TypeError, ZeroDivisionError) as e:
-            self._log.exception("CPU stats not available: %s", e)
-
-    def _getBalloonStats(self, stats, sample):
-        max_mem = int(self._vm.conf.get('memSize')) * 1024
-
-        for dev in self._vm.getBalloonDevicesConf():
-            if dev['specParams']['model'] != 'none':
-                balloon_target = dev.get('target', max_mem)
-                break
-        else:
-            balloon_target = None
-
-        stats['balloonInfo'] = {}
-
-        # Do not return any balloon status info before we get all data
-        # MOM will ignore VMs with missing balloon information instead
-        # using incomplete data and computing wrong balloon targets
-        if balloon_target is not None and sample is not None:
-            stats['balloonInfo'].update({
-                'balloon_max': str(max_mem),
-                'balloon_min': str(
-                    int(self._vm.conf.get('memGuaranteedSize', '0')) * 1024),
-                'balloon_cur': str(sample),
-                'balloon_target': str(balloon_target)
-            })
-
-    def _getCpuTuneInfo(self, stats, sample):
-        # Handling the case when not enough samples exist
-        if sample is None:
-            return
-
-        # Handling the case where quota is not set, setting to 0.
-        # According to libvirt API:"A quota with value 0 means no value."
-        # The value does not have to be present in some transient cases
-        if sample.get('vcpu_quota', _NO_CPU_QUOTA) != _NO_CPU_QUOTA:
-            stats['vcpuQuota'] = str(sample['vcpu_quota'])
-
-        # Handling the case where period is not set, setting to 0.
-        # According to libvirt API:"A period with value 0 means no value."
-        # The value does not have to be present in some transient cases
-        if sample.get('vcpu_period', _NO_CPU_PERIOD) != _NO_CPU_PERIOD:
-            stats['vcpuPeriod'] = sample['vcpu_period']
-
-    def _getCpuCount(self, stats, sample):
-        # Handling the case when not enough samples exist
-        if sample is None:
-            return
-
-        if 'vcpuCount' in sample:
-            vcpuCount = sample['vcpuCount']
-            if vcpuCount != -1:
-                stats['vcpuCount'] = vcpuCount
-            else:
-                self._log.error('Failed to get VM cpu count')
-
-    def _getUserCpuTuneInfo(self, stats, sample):
-        # Handling the case when not enough samples exist
-        if sample is None:
-            return
-
-        if 'vcpuLimit' in sample:
-            value = sample['vcpuLimit']
-            stats['vcpuUserLimit'] = value
-
-    @classmethod
-    def _getNicStats(cls, name, model, mac,
-                     start_sample, end_sample, interval):
-        ifSpeed = [100, 1000][model in ('e1000', 'virtio')]
-
-        ifStats = {'macAddr': mac,
-                   'name': name,
-                   'speed': str(ifSpeed),
-                   'state': 'unknown'}
-
-        ifStats['rxErrors'] = str(end_sample[2])
-        ifStats['rxDropped'] = str(end_sample[3])
-        ifStats['txErrors'] = str(end_sample[6])
-        ifStats['txDropped'] = str(end_sample[7])
-
-        ifRxBytes = (100.0 *
-                     ((end_sample[0] - start_sample[0]) % 2 ** 32) /
-                     interval / ifSpeed / cls.MBPS_TO_BPS)
-        ifTxBytes = (100.0 *
-                     ((end_sample[4] - start_sample[4]) % 2 ** 32) /
-                     interval / ifSpeed / cls.MBPS_TO_BPS)
-
-        ifStats['rxRate'] = '%.1f' % ifRxBytes
-        ifStats['txRate'] = '%.1f' % ifTxBytes
-
-        return ifStats
-
-    def _getNetworkStats(self, stats, sInfo, eInfo, interval):
-        stats['network'] = {}
-
-        if sInfo is None:
-            return
-
-        for nic in self._vm.getNicDevices():
-            if nic.name.startswith('hostdev'):
-                continue
-
-            # may happen if nic is a new hot-plugged one
-            if nic.name not in sInfo or nic.name not in eInfo:
-                continue
-
-            stats['network'][nic.name] = self._getNicStats(
-                nic.name, nic.nicModel, nic.macAddr,
-                sInfo[nic.name], eInfo[nic.name], interval)
-
-    def _getDiskStats(self, stats, sInfo, eInfo, interval):
-        for vmDrive in self._vm.getDiskDevices():
-            dStats = {}
-            try:
-                dStats = {'truesize': str(vmDrive.truesize),
-                          'apparentsize': str(vmDrive.apparentsize),
-                          'readLatency': '0',
-                          'writeLatency': '0',
-                          'flushLatency': '0'}
-                if isVdsmImage(vmDrive):
-                    dStats['imageID'] = vmDrive.imageID
-                elif "GUID" in vmDrive:
-                    dStats['lunGUID'] = vmDrive.GUID
-                if (sInfo and vmDrive.name in sInfo and
-                        eInfo and vmDrive.name in eInfo):
-                    # will be None if sampled during recovery
-                    dStats.update(_calcDiskRate(vmDrive, sInfo, eInfo,
-                                                interval))
-                    dStats.update(_calcDiskLatency(vmDrive, sInfo, eInfo))
-
-            except (AttributeError, TypeError, ZeroDivisionError):
-                self._log.exception("Disk %s stats not available",
-                                    vmDrive.name)
-
-            stats[vmDrive.name] = dStats
-
-    def _getNumaStats(self, stats, sample):
-        vmNumaNodeRuntimeMap = numaUtils.getVmNumaNodeRuntimeInfo(
-            self._vm, sample)
-        if vmNumaNodeRuntimeMap:
-            stats['vNodeRuntimeInfo'] = vmNumaNodeRuntimeMap
-
-    def _getVmJobs(self, stats, sample):
-        if sample is not None:
-            # If we are unable to collect stats we must not return anything at
-            # all since an empty dictionary would be interpreted as vm jobs
-            # finishing.
-            stats['vmJobs'] = sample
-
     def _getSampleCpu(self):
         return self.sampleCpu.getStats()
 
@@ -665,37 +731,6 @@
 
     def _getSampleCpuTune(self):
         return self.sampleCpuTune.getLastSample()
-
-    def get(self):
-        stats = {}
-
-        try:
-            stats['statsAge'] = time.time() - self.getLastSampleTime()
-        except TypeError:
-            self._log.debug("Stats age not available")
-            stats['statsAge'] = -1.0
-
-        sCpu, eCpu, cpuInterval = self._getSampleCpu()
-        self._getCpuStats(stats, sCpu, eCpu, cpuInterval)
-
-        sNet, eNet, netInterval = self._getSampleNet()
-        self._getNetworkStats(stats, sNet, eNet, netInterval)
-
-        sDisk, eDisk, diskInterval = self._getSampleDisk()
-        self._getDiskStats(stats, sDisk, eDisk, diskInterval)
-
-        self._getBalloonStats(stats, self._getSampleBalloon())
-        self._getVmJobs(stats, self._getSampleVmJobs())
-        self._getNumaStats(stats, self._getSampleVcpu())
-
-        cpuTuneSample = self._getSampleCpuTune()
-        self._getCpuTuneInfo(stats, cpuTuneSample)
-        self._getCpuCount(stats, cpuTuneSample)
-        self._getUserCpuTuneInfo(stats, cpuTuneSample)
-
-        self._getIoTuneStats(stats)
-
-        return stats
 
     def handleStatsException(self, ex):
         # We currently handle only libvirt exceptions


-- 
To view, visit http://gerrit.ovirt.org/36730
To unsubscribe, visit http://gerrit.ovirt.org/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I1a1d42b10714fa69c78f58ffaab7f7a32aed47ba
Gerrit-PatchSet: 1
Gerrit-Project: vdsm
Gerrit-Branch: master
Gerrit-Owner: Francesco Romani <fromani at redhat.com>


More information about the vdsm-patches mailing list