[NEW PATCH] [WIP] Initial DRBD implementation (via gerrit-bot)

Federico Simoncelli fsimonce at redhat.com
Tue Sep 13 16:01:03 UTC 2011


New patch submitted by Federico Simoncelli (fsimonce at redhat.com)

You can review this change at: http://gerrit.usersys.redhat.com/923

commit 640d39a7ec52d2aaea50df2e91767ad3c3a37279
Author: Federico Simoncelli <fsimonce at redhat.com>
Date:   Fri Sep 9 10:01:51 2011 +0000

    [WIP] Initial DRBD implementation
    
    Change-Id: I33bb867ba6c7cfca54d31334554cb37db11aeeea

diff --git a/configure.ac b/configure.ac
index a279c29..1976723 100644
--- a/configure.ac
+++ b/configure.ac
@@ -61,6 +61,8 @@ AC_PATH_PROG([CP_PATH], [cp], [/bin/cp])
 AC_PATH_PROG([DD_PATH], [dd], [/bin/dd])
 AC_PATH_PROG([DMIDECODE_PATH], [dmidecode], [/usr/sbin/dmidecode])
 AC_PATH_PROG([DMSETUP_PATH], [dmsetup], [/sbin/dmsetup])
+AC_PATH_PROG([DRBDMETA_PATH], [drbdmeta], [/sbin/drbdmeta])
+AC_PATH_PROG([DRBDSETUP_PATH], [drbdsetup], [/sbin/drbdsetup])
 AC_PATH_PROG([ECHO_PATH], [echo], [/bin/echo])
 AC_PATH_PROG([FSCK_PATH], [fsck], [/sbin/fsck])
 AC_PATH_PROG([FUSER_PATH], [fuser], [/sbin/fuser])
diff --git a/vdsm.spec.in b/vdsm.spec.in
index f1a3473..3eb1ee2 100644
--- a/vdsm.spec.in
+++ b/vdsm.spec.in
@@ -289,6 +289,7 @@ machines without running real guests.
 %{_datadir}/%{vdsm_name}/storage/fileUtils.py*
 %{_datadir}/%{vdsm_name}/storage/misc.py*
 %{_datadir}/%{vdsm_name}/storage/lvm.py*
+%{_datadir}/%{vdsm_name}/storage/drbd.py*
 %{_datadir}/%{vdsm_name}/storage/resourceFactories.py*
 %{_datadir}/%{vdsm_name}/storage/outOfProcess.py*
 %{_datadir}/%{vdsm_name}/storage/processPool.py*
diff --git a/vdsm/constants.py.in.in b/vdsm/constants.py.in.in
index 55b05df..149089d 100644
--- a/vdsm/constants.py.in.in
+++ b/vdsm/constants.py.in.in
@@ -77,6 +77,9 @@ EXT_DELNETWORK = '@VDSMDIR@/delNetwork'
 EXT_DMIDECODE = '@DMIDECODE_PATH@'
 EXT_DMSETUP = '@DMSETUP_PATH@'
 
+EXT_DRBDMETA = '@DRBDMETA_PATH@'
+EXT_DRBDSETUP = '@DRBDSETUP_PATH@'
+
 EXT_EDITNETWORK = '@VDSMDIR@/editNetwork'
 
 EXT_FENCE_PREFIX = '/usr/bin/fence_'
diff --git a/vdsm/storage/Makefile.am b/vdsm/storage/Makefile.am
index 766687a..d4ac646 100644
--- a/vdsm/storage/Makefile.am
+++ b/vdsm/storage/Makefile.am
@@ -15,6 +15,7 @@ dist_vdsmstorage_DATA = \
 	blockVolume.py \
 	devicemapper.py \
 	dispatcher.py \
+	drbd.py \
 	fileSD.py \
 	fileUtils.py \
 	fileVolume.py \
diff --git a/vdsm/storage/drbd.py b/vdsm/storage/drbd.py
new file mode 100644
index 0000000..a7afef1
--- /dev/null
+++ b/vdsm/storage/drbd.py
@@ -0,0 +1,277 @@
+#
+# Copyright 2009-2011 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+#
+# Refer to the README and COPYING files for full details of the license
+#
+
+import random
+
+import utils
+import constants
+import threading
+import storage_exception as se
+
+DRBD_VERSION = "v08"
+DRBD_MAXUUID = 256**8
+DRBD_MAXMINOR = 255
+
+KNOWN_VALUES = (
+    "Configured",
+    "Unconfigured",
+    "NA",
+)
+
+CONN_STATE_VALUES = (
+    "StandAlone",
+    "Disconnecting",
+    "Unconnected",
+    "Timeout",
+    "BrokenPipe",
+    "NetworkFailure",
+    "ProtocolError",
+    "WFConnection",
+    "WFReportParams",
+    "TearDown",
+    "Connected",
+    "StartingSyncS",
+    "StartingSyncT",
+    "WFBitMapS",
+    "WFBitMapT",
+    "WFSyncUUID",
+    "SyncSource",
+    "SyncTarget",
+    "PausedSyncS",
+    "PausedSyncT",
+    "VerifyS",
+    "VerifyT",
+    "Ahead",
+    "Behind",
+)
+
+ROLE_STATE_VALUES = (
+    "Primary",
+    "Secondary",
+    "Unknown",
+)
+
+DISK_STATE_VALUES = (
+    "Diskless",
+    "Attaching",
+    "Failed",
+    "Negotiating",
+    "Inconsistent",
+    "Outdated",
+    "DUnknown",
+    "Consistent",
+    "UpToDate",
+)
+
+class ReplicatedDevice(object):
+    @classmethod
+    def newUUID(cls):
+        return "%016X" % (random.randint(0, DRBD_MAXUUID),)
+
+    @classmethod
+    def readUUID(cls, device):
+        rc, out, err = utils.execCmd(
+                        [constants.EXT_DRBDMETA, "0", DRBD_VERSION,
+                         device, "internal", "read-dev-uuid"])
+
+        if rc != 0 or len(out) != 1:
+            raise se.ReplicatedDeviceError()
+
+        return out[0]
+
+    @classmethod
+    def writeUUID(cls, device, uuid):
+        rc, out, err = utils.execCmd(
+                        [constants.EXT_DRBDMETA, "0", DRBD_VERSION,
+                         device, "internal", "write-dev-uuid", uuid])
+        if rc != 0:
+            raise se.ReplicatedDeviceError()
+
+    @classmethod
+    def create(cls, device):
+        # Checking if the device was already initialized
+        try:
+            uuid = cls.readUUID(device)
+            raise se.ReplicatedDeviceInitialized(uuid)
+        except se.ReplicatedDeviceError:
+            pass
+
+        rc, out, err = utils.execCmd(
+                        [constants.EXT_DRBDMETA, "0", DRBD_VERSION,
+                         device, "internal", "create-md"])
+        if rc != 0:
+            raise se.ReplicatedDeviceError()
+
+        uuid = cls.newUUID()
+        cls.writeUUID(device, uuid)
+
+        if uuid != cls.readUUID(device):
+            raise se.ReplicatedDeviceError()
+
+        return uuid
+
+class ReplicatedStatus(dict):
+    _statusMap = {
+    # format:
+    #   "inkey"         ("outkey", default, "_decodeMethod", methodArgs...),
+        "_minor":       ("minor", None, "_decodeMinor"),
+        "_res_name":    ("resName", "", "_decodeValue", str),
+        "_volume":      ("volume", 0, "_decodeValue", int),
+        "_known":       ("known", "NA", "_decodeEnum", KNOWN_VALUES),
+        "_cstate":      ("connState",
+                         "Unconnected", "_decodeEnum", CONN_STATE_VALUES),
+        "_role":        ("roleState",
+                         "Unknown", "_decodeEnum", ROLE_STATE_VALUES),
+        "_peer":        ("peerState",
+                         "Unknown", "_decodeEnum", ROLE_STATE_VALUES),
+        "_disk":        ("diskState",
+                         "DUnknown", "_decodeEnum", DISK_STATE_VALUES),
+        "_pdisk":       ("peerDiskState",
+                         "DUnknown", "_decodeEnum", DISK_STATE_VALUES),
+        # This is a drbd bug, the same key is called in two different ways
+        # depending on the state of the resource. Remove me in the future.
+        "_pdsk":        ("peerDiskState",
+                         "DUnknown", "_decodeEnum", DISK_STATE_VALUES),
+        "_flags_susp":  ("flagsSusp", False, "_decodeFlags"),
+        "_flags_aftr_isp": ("flagsAfterIsp", False, "_decodeFlags"),
+        "_flags_peer_isp": ("flagsPeerIsp", False, "_decodeFlags"),
+        "_flags_user_isp": ("flagsUserIsp", False, "_decodeFlags"),
+        "_resynced_percent": ("resyncedPercent", 0, "_decodeValue", int),
+    }
+
+    def __init__(self, *args, **kwargs):
+        dict.__init__(self, *args, **kwargs)
+
+        for key, description in self._statusMap.items():
+            dict.__setitem__(self, description[0], description[1])
+
+    def __setitem__(self, item, value):
+        if item not in self._statusMap:
+            raise KeyError(item)
+
+        description = self._statusMap[item]
+
+        # Validating key value
+        validate = getattr(self, description[2])
+        value = validate(value, description[1], *description[3:])
+
+        dict.__setitem__(self, description[0], value)
+
+    def _decodeMinor(self, value, default):
+        value = int(value)
+        return value if value != 4294967295 else default
+
+    def _decodeValue(self, value, default, newtype):
+        if value == "": return default
+        return newtype(value)
+
+    def _decodeEnum(self, value, default, enums):
+        if value in enums:
+            return value
+        elif default in enums:
+            return default
+        raise ValueError(default)
+
+    def _decodeFlags(self, value, default):
+        if value == "1": return True
+        return default
+
+class ReplicatedResource(object):
+    minorLock = threading.Lock()
+
+    def __init__(self, resource):
+        self.resource = resource
+        self.status = ReplicatedResource.info(resource)
+
+    @classmethod
+    def attach(cls, resource, device, volume="0"):
+        rc, out, err = utils.execCmd(
+                        [constants.EXT_DRBDSETUP, "new-resource",
+                         resource])
+        if rc != 0:
+            raise se.ReplicatedResourceError()
+
+        with cls.minorLock:
+            minor = str(cls.getFreeMinor())
+
+            rc, out, err = utils.execCmd(
+                            [constants.EXT_DRBDSETUP, "new-minor",
+                             resource, minor, volume])
+            if rc != 0:
+                raise se.ReplicatedResourceError()
+
+        rc, out, err = utils.execCmd(
+                        [constants.EXT_DRBDMETA, minor, DRBD_VERSION,
+                         device, "internal", "apply-al"])
+        if rc != 0:
+            raise se.ReplicatedResourceError()
+
+        rc, out, err = utils.execCmd(
+                        [constants.EXT_DRBDSETUP, "attach",
+                         resource, minor, device, device, "internal"])
+        if rc != 0:
+            raise se.ReplicatedResourceError()
+
+    @classmethod
+    def getFreeMinor(cls):
+        usedminors = set(map(lambda x: x['minor'], cls.info()))
+
+        for minor in xrange(1, DRBD_MAXMINOR):
+            if minor not in usedminors: return minor
+
+        raise se.ReplicatedResourceError()
+
+    @classmethod
+    def info(cls, resource=None):
+        target = resource if resource else "all"
+
+        rc, out, err = utils.execCmd(
+                        [constants.EXT_DRBDSETUP, "sh-status", target])
+        if rc != 20:
+            raise se.ReplicatedResourceError()
+
+        result = []
+        status = ReplicatedStatus()
+
+        for line in filter(lambda x: len(x) > 0, out):
+            if line == "_sh_status_process":
+                result.append(status)
+                status = ReplicatedStatus()
+                continue
+
+            key, value = line.split("=", 1)
+            status[key] = value
+
+        if resource:
+            if len(result) == 1: return result[0]
+            else: raise se.ReplicatedResourceError()
+
+        return result
+
+    # XXX: Using the force flags might lead to data corruption! It should be
+    #      used only during the device initialization or in an unrecoverable
+    #      split-brain situation.
+    def primary(self, force=False):
+        force_opt = ["--force"] if force else []
+
+        rc, out, err = utils.execCmd([constants.EXT_DRBDSETUP,
+                                      "primary", self.resource] + force_opt)
+        if rc != 0:
+            raise se.ReplicatedResourceError()
diff --git a/vdsm/storage/hsm.py b/vdsm/storage/hsm.py
index 85fa5da..93623f2 100644
--- a/vdsm/storage/hsm.py
+++ b/vdsm/storage/hsm.py
@@ -36,6 +36,7 @@ import sd
 import blockSD
 import spm
 import lvm
+import drbd
 import fileUtils
 import multipath
 from sdf import StorageDomainFactory as SDF
@@ -1213,6 +1214,53 @@ class HSM:
                 domClass=domClass, typeSpecificArg=typeSpecificArg, version=domVersion)
 
 
+    def public_createReplicatedDevice(self, device):
+        """
+        Creates a new replicated device (drbd).
+
+        :param device: The block device to be used.
+        :type device: str
+
+        :returns: The new device uuid.
+        :rtype: str
+        """
+        newUUID = drbd.ReplicatedDevice.create(device)
+        return dict(uuid=newUUID)
+
+    def public_getReplicatedResourcesList(self):
+        """
+        List all the replicated resources in the system (drbd).
+
+        :returns: The replicated resources list.
+        :rtype: list
+        """
+        resources = drbd.ReplicatedResource.info()
+        return dict(resList=map(lambda x: x['resName'], resources))
+
+    def public_getReplicatedResourceInfo(self, rdUUID):
+        """
+        Gets the info of a replicated resource (drbd).
+
+        :param rdUUID: The resource UUID.
+        :type rdUUID: UUID
+
+        :returns: The replicated resources information.
+        :rtype: dict
+        """
+        resource = dict(drbd.ReplicatedResource.info(rdUUID))
+        return dict(resInfo=resource)
+
+    def public_attachReplicatedResource(self, rdUUID, device):
+        """
+        Attaches a replicated resource (drbd) to a block device.
+
+        :param rdUUID: The resource UUID.
+        :type rdUUID: UUID
+        :param device: The block device to be used.
+        :type device: str
+        """
+        drbd.ReplicatedResource.attach(rdUUID, device)
+
     def public_validateStorageDomain(self, sdUUID, options = None):
         """
         Validates that the storage domain is accessible.
diff --git a/vdsm/storage/storage_exception.py b/vdsm/storage/storage_exception.py
index 0a0ab22..f297de9 100644
--- a/vdsm/storage/storage_exception.py
+++ b/vdsm/storage/storage_exception.py
@@ -1315,6 +1315,18 @@ class ResourceAcqusitionFailed(GeneralException):
     code = 855
     message = "Could not acquire resource. Probably resource factory threw an exception."
 
+#################################################
+#  Replicated Device Exceptions
+#################################################
+
+class ReplicatedDeviceError(StorageException):
+    code = 900
+    message = "Replicated device general error"
+
+class ReplicatedResourceError(StorageException):
+    code = 901
+    message = "Replicated resource general error"
+
 if __name__ == "__main__":
     import types
     codes = {}
diff --git a/vdsm/sudoers.vdsm.in.in b/vdsm/sudoers.vdsm.in.in
index 106c498..20dff84 100644
--- a/vdsm/sudoers.vdsm.in.in
+++ b/vdsm/sudoers.vdsm.in.in
@@ -27,6 +27,8 @@ Cmnd_Alias VDSM_STORAGE = @MOUNT_PATH@, @UMOUNT_PATH@, \
     @SCSI_ID_PATH@, \
     @ISCSIADM_PATH@ *, \
     @LVM_PATH@, \
+    @DRBDMETA_PATH@, \
+    @DRBDSETUP_PATH@, \
     @CAT_PATH@ /sys/block/*/device/../../*, \
     @CAT_PATH@ /sys/devices/platform/host*, \
     @CAT_PATH@ /etc/iscsi/iscsid.conf, \
diff --git a/vdsm_cli/vdsClient.py b/vdsm_cli/vdsClient.py
index 09b73b3..187a729 100644
--- a/vdsm_cli/vdsClient.py
+++ b/vdsm_cli/vdsClient.py
@@ -365,6 +365,38 @@ class service:
             return dom['status']['code'], dom['status']['message']
         return 0, ''
 
+    def createReplicatedDevice(self, args):
+        validateArgTypes(args, [str])
+        dev = self.s.createReplicatedDevice(*args)
+        if dev['status']['code']:
+            return dev['status']['code'], dev['status']['message']
+        return 0, dev['uuid']
+
+    def getReplicatedResourcesList(self, args):
+        res = self.s.getReplicatedResourcesList(*args)
+        if res['status']['code']:
+            return res['status']['code'], res['status']['message']
+        for res in res['resList']:
+            print res
+        return 0, ''
+
+    def getReplicatedResourceInfo(self, args):
+        rdUUID = args[0]
+        res = self.s.getReplicatedResourceInfo(rdUUID)
+        if res['status']['code']:
+            return res['status']['code'], res['status']['message']
+        for key, value in res['resInfo'].items():
+            print key, "=", value
+        return 0, ''
+
+    def attachReplicatedResource(self, args):
+        rdUUID = args[0]
+        device = args[1]
+        res = self.s.attachReplicatedResource(rdUUID, device)
+        if res['status']['code']:
+            return res['status']['code'], res['status']['message']
+        return 0, ''
+
     def setStorageDomainDescription(self, args):
         sdUUID = args[0]
         descr = args[1]
@@ -1725,6 +1757,22 @@ if __name__ == '__main__':
                            ('<storage type> <domain UUID> <domain name> <param> <domType> <version>',
                             'Creates new storage domain'
                             )),
+            'createReplicatedDevice' : ( serv.createReplicatedDevice,
+                           ('<device>',
+                            'Creates new replicated device'
+                            )),
+            'getReplicatedResourcesList' : ( serv.getReplicatedResourcesList,
+                           ('',
+                            'Get replicated resources list'
+                            )),
+            'getReplicatedResourceInfo' : ( serv.getReplicatedResourceInfo,
+                           ('<rdUUID>',
+                            'Get replicated device info.'
+                            )),
+            'attachReplicatedResource' : ( serv.attachReplicatedResource,
+                           ('<rdUUID> <device>',
+                            'Attach a replicated resource to a block device.'
+                            )),
             'setStorageDomainDescription' : ( serv.setStorageDomainDescription,
                            ('<domain UUID> <descr>',
                             'Set storage domain description'




More information about the vdsm-patches mailing list