Change in vdsm[master]: lib: Add vdsm.concurrent library

nsoffer at redhat.com nsoffer at redhat.com
Mon Mar 9 00:08:37 UTC 2015


Nir Soffer has uploaded a new change for review.

Change subject: lib: Add vdsm.concurrent library
......................................................................

lib: Add vdsm.concurrent library

Move misc.tmap to new module. We will probably need to move some other
utilities to the new module later.  The tmap utility will be used to
simplify fc-scan and multipath-resize helpers.

tmap() was moved as is from misc for easier review. The next patch will
clean it up a little bit. The tests were rewritten in a simpler way.

Change-Id: Iee90f4c201293c3df048cc4ad7123a837d236a38
Signed-off-by: Nir Soffer <nsoffer at redhat.com>
---
M debian/vdsm-python.install
M lib/vdsm/Makefile.am
A lib/vdsm/concurrent.py
M tests/Makefile.am
A tests/concurrentTests.py
M tests/miscTests.py
M vdsm.spec.in
M vdsm/storage/misc.py
8 files changed, 117 insertions(+), 71 deletions(-)


  git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/64/38464/1

diff --git a/debian/vdsm-python.install b/debian/vdsm-python.install
index 4fab53e..ab44430 100644
--- a/debian/vdsm-python.install
+++ b/debian/vdsm-python.install
@@ -3,6 +3,7 @@
 ./usr/lib/python2.7/dist-packages/vdsm/__init__.py
 ./usr/lib/python2.7/dist-packages/vdsm/cmdutils.py
 ./usr/lib/python2.7/dist-packages/vdsm/compat.py
+./usr/lib/python2.7/dist-packages/vdsm/concurrent.py
 ./usr/lib/python2.7/dist-packages/vdsm/config.py
 ./usr/lib/python2.7/dist-packages/vdsm/constants.py
 ./usr/lib/python2.7/dist-packages/vdsm/define.py
diff --git a/lib/vdsm/Makefile.am b/lib/vdsm/Makefile.am
index f677284..ff5c145 100644
--- a/lib/vdsm/Makefile.am
+++ b/lib/vdsm/Makefile.am
@@ -25,6 +25,7 @@
 	__init__.py \
 	cmdutils.py \
 	compat.py \
+	concurrent.py \
 	define.py \
 	exception.py \
 	executor.py \
diff --git a/lib/vdsm/concurrent.py b/lib/vdsm/concurrent.py
new file mode 100644
index 0000000..f4886b1
--- /dev/null
+++ b/lib/vdsm/concurrent.py
@@ -0,0 +1,60 @@
+#
+# Copyright 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+#
+# Refer to the README and COPYING files for full details of the license
+#
+
+import logging
+import threading
+
+log = logging.getLogger("vds.concurrent")
+
+
+def tmap(func, iterable):
+    resultsDict = {}
+    error = [None]
+
+    def wrapper(f, arg, index):
+        try:
+            resultsDict[index] = f(arg)
+        except Exception as e:
+            # We will throw the last error received
+            # we can only throw one error, and the
+            # last one is as good as any. This shouldn't
+            # happen. Wrapped methods should not throw
+            # exceptions, if this happens it's a bug
+            log.error("tmap caught an unexpected error", exc_info=True)
+            error[0] = e
+            resultsDict[index] = None
+
+    threads = []
+    for i, arg in enumerate(iterable):
+        t = threading.Thread(target=wrapper, args=(func, arg, i))
+        threads.append(t)
+        t.start()
+
+    for t in threads:
+        t.join()
+
+    results = [None] * len(resultsDict)
+    for i, result in resultsDict.iteritems():
+        results[i] = result
+
+    if error[0] is not None:
+        raise error[0]
+
+    return tuple(results)
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 3a9b554..bb05a97 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -29,6 +29,7 @@
 	cPopenTests.py \
 	capsTests.py \
 	clientifTests.py \
+	concurrentTests.py \
 	configNetworkTests.py \
 	cpuProfileTests.py \
 	deviceTests.py \
diff --git a/tests/concurrentTests.py b/tests/concurrentTests.py
new file mode 100644
index 0000000..46ad44c
--- /dev/null
+++ b/tests/concurrentTests.py
@@ -0,0 +1,53 @@
+#
+# Copyright 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+#
+# Refer to the README and COPYING files for full details of the license
+#
+
+import time
+import random
+
+from testlib import VdsmTestCase
+
+from vdsm import concurrent
+
+
+class TMapTests(VdsmTestCase):
+
+    def test_results(self):
+        values = tuple(range(10))
+        results = concurrent.tmap(lambda x: x, values)
+        self.assertEqual(results, values)
+
+    def test_results_order(self):
+        def func(x):
+            time.sleep(x)
+            return x
+        values = tuple(random.random() * 0.1 for x in range(10))
+        results = concurrent.tmap(func, values)
+        self.assertEqual(results, values)
+
+    def test_concurrency(self):
+        start = time.time()
+        concurrent.tmap(time.sleep, [0.1] * 10)
+        elapsed = time.time() - start
+        self.assertTrue(0.1 < elapsed < 0.2)
+
+    def test_error(self):
+        def func(x):
+            raise RuntimeError()
+        self.assertRaises(RuntimeError, concurrent.tmap, func, range(10))
diff --git a/tests/miscTests.py b/tests/miscTests.py
index e17954f..31f64fa 100644
--- a/tests/miscTests.py
+++ b/tests/miscTests.py
@@ -164,41 +164,6 @@
         self.flag.set()
 
 
-class TMap(TestCaseBase):
-
-    def test(self):
-        def dummy(arg):
-            # This will cause some of the operations to take longer
-            # thus testing the result reordering mechanism
-            if len(arg) % 2:
-                time.sleep(1)
-            return arg
-
-        data = """Stephen Fry: Well next week I shall be examining the claims
-                  of a man who says that in a previous existence he was
-                  Education Secretary Kenneth Baker and I shall be talking to a
-                  woman who claims she can make flowers grow just by planting
-                  seeds in soil and watering them. Until then, wait very
-                  quietly in your seats please. Goodnight."""
-        # (C) BBC - A Bit of Fry and Laury
-        data = data.split()
-        self.assertEquals(list(misc.tmap(dummy, data)), data)
-
-    def testErrMethod(self):
-        exceptionStr = ("It's time to kick ass and chew bubble gum... "
-                        "and I'm all outta gum.")
-
-        def dummy(arg):
-            raise Exception(exceptionStr)
-        try:
-            misc.tmap(dummy, [1, 2, 3, 4])
-        except Exception as e:
-            self.assertEquals(str(e), exceptionStr)
-            return
-        else:
-            self.fail("tmap did not throw an exception")
-
-
 class ITMap(TestCaseBase):
 
     def testMoreArgsThanThreads(self):
diff --git a/vdsm.spec.in b/vdsm.spec.in
index 576ea97..65fda22 100644
--- a/vdsm.spec.in
+++ b/vdsm.spec.in
@@ -1292,6 +1292,7 @@
 %{python_sitelib}/%{vdsm_name}/__init__.py*
 %{python_sitelib}/%{vdsm_name}/cmdutils.py*
 %{python_sitelib}/%{vdsm_name}/compat.py*
+%{python_sitelib}/%{vdsm_name}/concurrent.py*
 %{python_sitelib}/%{vdsm_name}/config.py*
 %{python_sitelib}/%{vdsm_name}/constants.py*
 %{python_sitelib}/%{vdsm_name}/define.py*
diff --git a/vdsm/storage/misc.py b/vdsm/storage/misc.py
index 08d1f87..eb484c7 100644
--- a/vdsm/storage/misc.py
+++ b/vdsm/storage/misc.py
@@ -753,42 +753,6 @@
     return helper
 
 
-def tmap(func, iterable):
-    resultsDict = {}
-    error = [None]
-
-    def wrapper(f, arg, index):
-        try:
-            resultsDict[index] = f(arg)
-        except Exception as e:
-            # We will throw the last error received
-            # we can only throw one error, and the
-            # last one is as good as any. This shouldn't
-            # happen. Wrapped methods should not throw
-            # exceptions, if this happens it's a bug
-            log.error("tmap caught an unexpected error", exc_info=True)
-            error[0] = e
-            resultsDict[index] = None
-
-    threads = []
-    for i, arg in enumerate(iterable):
-        t = threading.Thread(target=wrapper, args=(func, arg, i))
-        threads.append(t)
-        t.start()
-
-    for t in threads:
-        t.join()
-
-    results = [None] * len(resultsDict)
-    for i, result in resultsDict.iteritems():
-        results[i] = result
-
-    if error[0] is not None:
-        raise error[0]
-
-    return tuple(results)
-
-
 def getfds():
     return [int(fd) for fd in os.listdir("/proc/self/fd")]
 


-- 
To view, visit https://gerrit.ovirt.org/38464
To unsubscribe, visit https://gerrit.ovirt.org/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Iee90f4c201293c3df048cc4ad7123a837d236a38
Gerrit-PatchSet: 1
Gerrit-Project: vdsm
Gerrit-Branch: master
Gerrit-Owner: Nir Soffer <nsoffer at redhat.com>


More information about the vdsm-patches mailing list