Change in vdsm[master]: Query interface for all VM fields

vfeenstr at redhat.com vfeenstr at redhat.com
Mon Jun 16 14:50:29 UTC 2014


Vinzenz Feenstra has uploaded a new change for review.

Change subject: Query interface for all VM fields
......................................................................

Query interface for all VM fields

Change-Id: Iddb4657d738ca0406950d07b84a7c85721e80b78
Bug-Url: https://bugzilla.redhat.com/show_bug.cgi?id=1083070
Signed-off-by: Vinzenz Feenstra <vfeenstr at redhat.com>
---
A lib/vdsm/trackable.py
M vdsm/API.py
M vdsm/rpc/vdsmapi-schema.json
M vdsm/virt/vm.py
4 files changed, 511 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/19/28819/1

diff --git a/lib/vdsm/trackable.py b/lib/vdsm/trackable.py
new file mode 100644
index 0000000..7455025
--- /dev/null
+++ b/lib/vdsm/trackable.py
@@ -0,0 +1,223 @@
+#
+# Copyright 2014 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+#
+# Refer to the README and COPYING files for full details of the license
+#
+"""
+    Implementation for Trackable, a wrapper for objects whose values should be
+    trackable for changes
+"""
+
+import collections
+import threading
+
+
+class ChangeCounter(object):
+    def __init__(self):
+        self._counter_lock = threading.Lock()
+        self._current = 0
+
+    def next(self):
+        with self._counter_lock:
+            self._current += 1
+            return self._current
+
+
+_counter = ChangeCounter()
+
+
+class Trackable(object):
+    """
+        Trackable implements common functionality of TrackableMapping and
+        TrackableSequence and it is not required to be used.
+    """
+    def __init__(self):
+        super(Trackable, self).__init__()
+        self._last_changed = _counter.next()
+        self._item_last_changed = {}
+        self._tracked = None
+        self._watcher = None
+        self._track_depth = 0
+
+    def _set_item(self, key, value):
+        needs_update = True
+        if key in self._tracked:
+            old = self._tracked[key]
+            if isinstance(old, Trackable):
+                old._update(value)
+                # if an item is updated through _update it does not need to
+                # we are not propagating the update, this would bubble up on
+                # it's own
+                needs_update = False
+            elif old == value:
+                    # Not updated, no change required
+                    needs_update = False
+            else:
+                self._tracked.__setitem__(key, _wrap(value, self,
+                                                     self._track_depth))
+        else:
+            self._tracked.__setitem__(key, _wrap(value, self,
+                                                 self._track_depth))
+        if needs_update:
+            self._on_item_changed(key)
+
+    def _on_item_removed(self, key):
+            changed = _counter.next()
+            self._last_changed = changed
+            if self._watcher:
+                self._on_subitem_changed(changed)
+            del self._item_last_changed[key]
+
+    def _on_item_changed(self, key):
+        changed = _counter.next()
+        self._last_changed = changed
+        if self._watcher:
+            self._on_subitem_changed(changed)
+        self._item_last_changed[key] = changed
+
+    def _on_subitem_changed(self, changed):
+        self._last_changed = changed
+        if self._watcher:
+            self._watcher._on_subitem_changed(changed)
+
+    def changed(self, since, key=None):
+        if not key:
+            return self._last_changed > since
+        if isinstance(self._tracked.get(key, None), Trackable):
+            return self._tracked[key].changed(since=since)
+        return self._item_last_changed[key] > since
+
+    def last_changed(self):
+        return self._last_changed
+
+
+def _wrap(v, watcher, track_depth):
+    if track_depth == 0 or isinstance(v, Trackable):
+
+        return v
+    elif isinstance(v, collections.MutableSequence):
+        return TrackableSequence(tracked=v, _watcher=watcher,
+                                 track_depth=track_depth - 1)
+    elif isinstance(v, collections.MutableMapping):
+        return TrackableMapping(tracked=v, _watcher=watcher,
+                                track_depth=track_depth - 1)
+    return v
+
+
+class TrackableSequence(collections.MutableSequence, Trackable):
+    """
+        TrackableSequence implements a mutable sequence interface to
+        track changes to the elements in the list.
+    """
+    def __init__(self, tracked=None, track_depth=0, _watcher=None):
+        super(TrackableSequence, self).__init__()
+        self._tracked = tracked.__class__() if tracked else []
+        self._track_depth = track_depth
+        self._watcher = _watcher
+        if tracked:
+            for elem in tracked:
+                self.append(elem)
+
+    def insert(self, index, value):
+        self._tracked.insert(index, _wrap(value, self, self._track_depth))
+        self._on_item_changed(index)
+
+    def _update(self, value):
+        # lists when different need a full update
+        if self._tracked != value:
+            self._item_last_changed = {}
+            self._tracked = self._tracked.__class__()
+            for elem in value:
+                self.append(elem)
+
+    def __setitem__(self, key, value):
+        self._set_item(key, value)
+
+    def __getitem__(self, key):
+        return self._tracked.__getitem__(key)
+
+    def __delitem__(self, idx):
+        self._tracked.__delitem__(idx)
+        if not isinstance(self._tracked, Trackable):
+            self._on_item_removed(idx)
+
+    def __len__(self):
+        return self._tracked.__len__()
+
+    def __iter__(self):
+        return self._tracked.__iter__()
+
+    def __hash__(self):
+        return self._tracked.__hash__()
+
+
+class TrackableMapping(Trackable, collections.MutableMapping):
+    """
+        TrackableMapping implements a mutable mapping interface to track
+        changes in the keys/values in a mapping and acts like a proxy thereof
+    """
+    def __init__(self, tracked=None, track_depth=0, _watcher=None):
+        """
+        :param tracked:     The object which should be tracked usually a dict
+        :param track_depth: The depth the tracking should be applied
+                            the default value is 0 which means only the current
+                            objects keys are tracked.
+        :return:
+        """
+        super(TrackableMapping, self).__init__()
+        self._tracked = tracked.__class__() if tracked else {}
+        self._track_depth = track_depth
+        self._watcher = _watcher
+        if tracked:
+            for k, v in tracked.iteritems():
+                self[k] = v
+
+    def update(self, value, filter_if=None):
+        for k in value.keys():
+            if filter and filter(k):
+                continue
+            self[k] = value[k]
+
+    def _update(self, value):
+        for k in self._tracked.keys():
+            if k not in value:
+                del self[k]
+            elif self._tracked[k] != value[k]:
+                self[k] = value[k]
+        for k in value.keys():
+            if k not in self._tracked:
+                self[k] = value[k]
+
+    def __setitem__(self, key, value):
+        self._set_item(key, value)
+
+    def __getitem__(self, key):
+        return self._tracked.__getitem__(key)
+
+    def __delitem__(self, key):
+        self._tracked.__delitem__(key)
+        if not isinstance(self._tracked, Trackable):
+            self._on_item_removed(key)
+
+    def __len__(self):
+        return self._tracked.__len__()
+
+    def __iter__(self):
+        return self._tracked.__iter__()
+
+    def __hash__(self):
+        return self._tracked.__hash__()
\ No newline at end of file
diff --git a/vdsm/API.py b/vdsm/API.py
index ff2627e..5464d5f 100644
--- a/vdsm/API.py
+++ b/vdsm/API.py
@@ -353,6 +353,20 @@
             return errCode['noVM']
         return v.migrateStatus()
 
+    def query(self, fields=None, changedSince=-1, runHooks=True):
+        v = self._cif.vmContainer.get(self._UUID)
+        if not v:
+            return errCode['noVM']
+
+        if runHooks:
+            hooks.before_get_vm_stats()
+        stats = v.query(fields=fields, changedSince=changedSince).copy()
+        stats['vmId'] = self._UUID
+        if runHooks:
+            stats = hooks.after_get_vm_stats([stats])[0]
+        return {'status': doneCode, 'dataList': [stats]}
+
+
     def getStats(self, runHooks=True):
         """
         Obtain statistics of the specified VM
@@ -1194,6 +1208,21 @@
             self.log.error("failed to retrieve hardware info", exc_info=True)
             return errCode['hwInfoErr']
 
+    def queryVms(self, vmIDs=None, fields=None, changedSince=-1):
+        hooks.before_get_all_vm_stats()
+        vms = self.getVMList()
+        statsList = []
+        for s in vms['vmList']:
+            if vmIDs and s['vmId'] not in vmIDs:
+                continue
+            response = VM(s['vmId']).query(fields=fields,
+                                           changedSince=changedSince,
+                                           runHooks=False)
+            if response:
+                statsList.append(response['dataList'][0])
+        statsList = hooks.after_get_all_vm_stats(statsList)
+        return {'status': doneCode, 'dataList': statsList}
+
     def getAllVmStats(self):
         """
         Get statistics of all running VMs.
diff --git a/vdsm/rpc/vdsmapi-schema.json b/vdsm/rpc/vdsmapi-schema.json
index af563c0..dc475de 100644
--- a/vdsm/rpc/vdsmapi-schema.json
+++ b/vdsm/rpc/vdsmapi-schema.json
@@ -3190,6 +3190,248 @@
 {'type': 'VmShortStatus',
  'data': {'vmId': 'UUID', 'status': 'VmStatus'}}
 
+
+##
+# @VmData:
+#
+# All available queryable data fields of a VM instance.
+#
+# @acpiEnable:              #optional Indicates if ACPI is enabled inside the
+#                           VM
+#
+# @appsList:                #optional A list of installed applications with
+#                           their versions
+#
+# @balloonInfo:             #optional Guest memory balloon information
+#
+# @boot:                    #optional An alias for the type of device used to
+#                           boot the VM
+#
+# @cdrom:                   #optional The path to an ISO image used in the VM's
+#                           CD-ROM device
+#
+# @clientIp:                #optional The IP address of the client connected to
+#                           the display
+#
+# @copyPasteEnable:         #optional Specify if copy and paste is enabled.
+#                           Currently relevant for @qxl devices only.
+#
+# @cpuShares:               #optional The host scheduling priority (relative to
+#                           other VMs). In case both cpuShares and nice are
+#                           present, cpuShares will be used.
+#
+# @cpuSys:                  #optional Ratio of CPU time spent by qemu on other
+#                           than guest time
+#
+# @cpuType:                 #optional The type of CPU being emulated
+#                           special values 'hostPassthrough' and 'hostModel'
+#                           are reserved for host-passthrough and host-mode cpu
+#                           mode
+#
+# @cpuUser:                 #optional Ratio of CPU time spent by the guest VM
+#
+# @custom:                  #optional A dictionary of custom, free-form
+#                           properties
+#
+# @devices:                 #optional An array of VM devices present
+#
+# @disks:                   #optional Disk bandwidth/utilization statistics
+#
+# @disksUsage:              #optional Info about mounted filesystems as
+#                           reported by the agent
+#
+# @display:                 #optional The type of display
+#
+# @displayInfo:             #optional Display and graphics device informations.
+#
+# @displayIp:               #optional The IP address to use for accessing the
+#                           VM display
+#
+# @displayPort:             #optional The port in use for unencrypted display
+#                           data
+#
+# @displaySecurePort:       #optional The port in use for encrypted display
+#                           data
+#
+# @displayType:             #optional The type of display in use
+#
+# @elapsedTime:             #optional The number of seconds that the VM has
+#                           been running
+#
+# @emulatedMachine:         #optional The machine specification being emulated
+#
+# @exitCode:                #optional The exit code f the VM process has ended
+#
+# @exitMessage:             #optional Explains the reason that the VM process
+#                           has exited
+#
+# @guestCPUCount:           #optional The number of CPU cores are visible as
+#                           online on the guest OS. This value is -1 if not
+#                           supported to report
+#
+# @guestFQDN:               #optional Fully qualified domain name of the guest
+#                           OS. (Reported by the guest agent)
+#
+# @guestIPs:                #optional A space separated string of assigned IPv4
+#                           addresses
+#
+# @keyboardLayout:          #optional The keyboard layout string (eg. 'en-us')
+#
+# @kvmEnable:               #optional Indicates if KVM hardware acceleration is
+#                           enabled
+#
+# @maxVCpus:                #optional Maximum number of CPU available for the
+#                           guest. It is the upper boundry for hot plug CPU
+#                           action
+#
+# @memGuaranteedSize:       #optional The amount of memory guaranteed to the VM
+#                           in MB
+#
+# @memSize:                 #optional The amount of memory assigned to the VM
+#                           in MB
+#
+# @memUsage:                #optional The percent of memory in use by the guest
+#
+# @memoryStats:             #optional Memory statistics as reported by the
+#                           guest agent
+#
+# @migrationProgress:       #optional Indicates the percentage progress of a
+#                           Migration, when there is one active.
+#
+# @monitorResponse:         #optional Indicates if the qemu monitor is
+#                           responsive
+#
+# @netIfaces:               #optional Network device address info as reported
+#                           by the agent
+#
+# @network:                 #optional Network bandwidth/utilization statistics
+#
+# @nicModel:                #optional A comma-separated list of NIC models in
+#                           use by the VM
+#
+# @nice:                    #optional The host scheduling priority
+#
+# @pauseCode:               #optional Indicates the reason a VM has been paused
+#
+# @pid:                     #optional The process ID of the underlying qemu
+#                           process
+#
+# @serial:                  #optional Serial number for the VM.
+#
+# @session:                 #optional The current state of user interaction
+#                           with the VM
+#
+# @smp:                     #optional The number of CPUs presented to the VM
+#
+# @smpCoresPerSocket:       #optional Indicates the number of CPU cores per
+#                           socket
+#
+# @smpThreadsPerCore:       #optional Indicates the number of CPU threads per
+#                           core
+#
+# @statsAge:                #optional The age of these statistics in seconds
+#
+# @status:                  #optional The current VM status
+#
+# @timeOffset:              #optional The time difference from host to the VM
+#                           in seconds
+#
+# @transparentHugePages:    #optional Indicates if the Transparent Huge Pages
+#                           feature is enabled for this virtual machine
+#
+# @username:                #optional The username associated with the current
+#                           session
+#
+# @vNodeRuntimeInfo:        #optional Information about the vm numa node
+#                           runtime pinning to host numa node.
+#
+# @vmId:                    #optional The VM UUID
+#
+# @vmJobs:                  #optional Info about active vm jobs
+#
+# @vmName:                  #optional The VM name
+#
+# @vmType:                  #optional The type of VM
+#
+# @watchdogEvent:           #optional Information about the most recent
+#                           watchdog event
+#
+# Since: 4.15.0
+{'type': 'VmQueryResult',
+ 'data': {
+    '*acpiEnable': 'bool', '*appsList': ['str'], '*balloonInfo': 'BalloonInfo',
+    '*boot': 'VmBootMode', '*cdrom': 'str', '*clientIp': 'str',
+    '*copyPasteEnable': 'bool', '*cpuShares': 'str', '*cpuSys': 'float',
+    '*cpuType': 'str', '*cpuUser': 'float', '*custom': 'StringMap',
+    '*devices': ['VmDevice'], '*disks': 'VmDiskStatsMap',
+    '*disksUsage': ['GuestMountInfo'], '*display': 'VmDisplayType',
+    '*displayInfo': ['VmDisplayInfo'], '*displayIp': 'str',
+    '*displayPort': 'uint', '*displaySecurePort': 'uint',
+    '*displayType': 'VmDisplayType', '*elapsedTime': 'uint',
+    '*emulatedMachine': 'str', '*exitCode': 'int', '*exitMessage': 'str',
+    '*guestCPUCount': 'int', '*guestFQDN': 'str', '*guestIPs': 'str',
+    '*keyboardLayout': 'str', '*kvmEnable': 'bool', '*maxVCpus': 'uint',
+    '*memGuaranteedSize': 'uint', '*memSize': 'uint', '*memUsage': 'uint',
+    '*memoryStats': 'GuestMemoryStats', '*migrationProgress': 'uint',
+    '*monitorResponse': 'int', '*netIfaces': ['GuestNetworkDeviceInfo'],
+    '*network': 'NetworkInterfaceStatsMap', '*nicModel': 'str', '*nice': 'int',
+    '*pauseCode': 'str', '*pid': 'uint', '*serial': 'str',
+    '*session': 'GuestSessionState', '*smp': 'uint',
+    '*smpCoresPerSocket': 'uint', '*smpThreadsPerCore': 'uint',
+    '*statsAge': 'float', '*status': 'VmStatus', '*timeOffset': 'uint',
+    '*transparentHugePages': 'bool', '*username': 'str',
+    '*vNodeRuntimeInfo': 'VmNumaNodeRuntimeInfoMap', '*vmId': 'UUID',
+    '*vmJobs': 'VmJobsMap', '*vmName': 'str', '*vmType': 'VmType',
+    '*watchdogEvent': 'WatchdogEvent'}}
+
+##
+# @VM.query:
+#
+# Get statistics about a running virtual machine.
+#
+# @vmID:            The UUID of the VM
+#
+# @fields:          #optional A list of strings which specify the keys which
+#                   should be queried. All fields will be returned if this
+#                   value is not specified.
+#
+# @changedSince:    #optional If specified, all modifications since the given
+#                   change id are returned. Values for fields where the value
+#                   has not changed since, will not be returned.
+#
+# Returns:
+# An array containing a single VmData record
+#
+# Since: 4.10.0
+##
+{'command': {'class': 'VM', 'name': 'getStats'},
+ 'data': {'vmID': 'UUID', '*fields': ['string'], '*changedSince': 'uint'},
+ 'returns': ['VmData']}
+
+##
+# @Host.queryVms:
+#
+# Query all specified virtual machines for the specified fields.
+#
+# @vmIDs:           #optional A list of VM UUIDs. If not set, all VMs are used.
+#
+# @fields:          #optional A list of strings which specify the keys which
+#                   should be queried. All fields will be returned if this
+#                   value is not specified.
+#
+# @changedSince:    #optional If specified, all modifications since the given
+#                   change id are returned. Values for fields where the value
+#                   has not changed since, will not be returned.
+#
+# Returns:
+# A list of query results for all VMs in vmIDs or all VMs if vmIDs not set
+#
+# Since: 4.15.0
+##
+{'command': {'class': 'Host', 'name': 'queryAllVm'},
+ 'data': {'*vmIDs': ['UUID'], '*fields': ['string'], '*changedSince': 'uint'},
+ 'returns': ['VmData']}
+
 ##
 # @VmDefinition:
 #
diff --git a/vdsm/virt/vm.py b/vdsm/virt/vm.py
index 4b00c51..839f72c 100644
--- a/vdsm/virt/vm.py
+++ b/vdsm/virt/vm.py
@@ -41,6 +41,7 @@
 from vdsm import libvirtconnection
 from vdsm import netinfo
 from vdsm import qemuimg
+from vdsm import trackable
 from vdsm import utils
 from vdsm.compat import pickle
 from vdsm.config import config
@@ -1773,6 +1774,8 @@
         self._watchdogEvent = {}
         self.sdIds = []
         self.arch = caps.getTargetArch()
+        self._queryCacheLock = threading.Lock()
+        self._queryCache = trackable.TrackableMapping()
 
         if (self.arch not in ['ppc64', 'x86_64']):
             raise RuntimeError('Unsupported architecture: %s' % self.arch)
@@ -2663,6 +2666,20 @@
             stats["watchdogEvent"] = self._watchdogEvent
         return stats
 
+    def query(self, fields=None, changedSince=-1, runHooks=False):
+        # First we need to update the query cache
+        stats = self.getStats()
+        with self._queryCacheLock:
+            self._queryCache.update(stats)
+            self._queryCache.update(self.conf, lambda x: x.startswith('_'))
+
+        # Now we build up the response
+        result = {}
+        for k in (fields or self._queryCache.keys()):
+            if self._queryCache[k].changed(changedSince):
+                result[k] = self._queryCache[k]
+        return result
+
     def _getStatsInternal(self):
         """
         used by API.Vm.getStats


-- 
To view, visit http://gerrit.ovirt.org/28819
To unsubscribe, visit http://gerrit.ovirt.org/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Iddb4657d738ca0406950d07b84a7c85721e80b78
Gerrit-PatchSet: 1
Gerrit-Project: vdsm
Gerrit-Branch: master
Gerrit-Owner: Vinzenz Feenstra <vfeenstr at redhat.com>


More information about the vdsm-patches mailing list