Repository :
http://git.fedorahosted.org/cgit/copr.git
On branch : master
---------------------------------------------------------------
commit d74280948cb6ca5aabccfd1d743df19d8fd7ed66
Author: Miroslav Suchý <msuchy(a)redhat.com>
Date: Tue Feb 25 09:47:10 2014 +0100
[backend] ensure that only one createrepo is running at the same time
so multiple instances (which consumes 2+GB RAM each) does not take server down
---------------------------------------------------------------
backend/dispatcher.py | 4 +++-
backend/mockremote.py | 11 ++++++++---
copr-be.py | 4 +++-
3 files changed, 14 insertions(+), 5 deletions(-)
diff --git a/backend/dispatcher.py b/backend/dispatcher.py
index 77bf2ed..436b9e4 100644
--- a/backend/dispatcher.py
+++ b/backend/dispatcher.py
@@ -103,7 +103,7 @@ class WorkerCallback(object):
class Worker(multiprocessing.Process):
def __init__(self, opts, jobs, events, worker_num,
- ip=None, create=True, callback=None):
+ ip=None, create=True, callback=None, lock=None):
# base class initialization
multiprocessing.Process.__init__(self, name="worker-builder")
@@ -118,6 +118,7 @@ class Worker(multiprocessing.Process):
self.kill_received = False
self.callback = callback
self.create = create
+ self.lock = lock
self.frontend_callback = FrontendCallback(opts)
if not self.callback:
self.logfile = os.path.join(
@@ -469,6 +470,7 @@ class Worker(multiprocessing.Process):
recurse=True,
repos=chroot_repos,
macros=macros,
+ lock=lock,
buildroot_pkgs=job.buildroot_pkgs,
callback=mockremote.CliLogCallBack(
quiet=True, logfn=chrootlogfile))
diff --git a/backend/mockremote.py b/backend/mockremote.py
index ce1412d..20b2870 100755
--- a/backend/mockremote.py
+++ b/backend/mockremote.py
@@ -62,15 +62,19 @@ class SortedOptParser(optparse.OptionParser):
return optparse.OptionParser.format_help(self, formatter=None)
-def createrepo(path):
+def createrepo(path, lock):
if os.path.exists(path + '/repodata/repomd.xml'):
comm = ['/usr/bin/createrepo_c', '--database',
'--ignore-lock',
'--update', path]
else:
comm = ['/usr/bin/createrepo_c', '--database',
'--ignore-lock', path]
+ if lock:
+ lock.acquire()
cmd = subprocess.Popen(comm,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = cmd.communicate()
+ if lock:
+ lock.release()
return cmd.returncode, out, err
@@ -529,7 +533,7 @@ class MockRemote(object):
destdir=DEF_DESTDIR, chroot=DEF_CHROOT, cont=False,
recurse=False, repos=DEF_REPOS, callback=None,
remote_basedir=DEF_REMOTE_BASEDIR, remote_tempdir=None,
- macros=DEF_MACROS,
+ macros=DEF_MACROS, lock=None,
buildroot_pkgs=DEF_BUILDROOT_PKGS):
self.destdir = destdir
@@ -541,6 +545,7 @@ class MockRemote(object):
self.remote_basedir = remote_basedir
self.remote_tempdir = remote_tempdir
self.macros = macros
+ self.lock = lock
if not self.callback:
self.callback = DefaultCallBack()
@@ -659,7 +664,7 @@ class MockRemote(object):
built_pkgs.append(pkg)
# createrepo with the new pkgs
for d in [self.destdir, chroot_dir]:
- rc, out, err = createrepo(d)
+ rc, out, err = createrepo(d, self.lock)
if err.strip():
self.callback.error(
"Error making local repo: {0}".format(d))
diff --git a/copr-be.py b/copr-be.py
index f253920..e5f3554 100755
--- a/copr-be.py
+++ b/copr-be.py
@@ -270,8 +270,10 @@ class CoprBackend(object):
self.event("Spinning up more workers for jobs")
for _ in range(self.opts.num_workers - len(self.workers)):
worker_num = len(self.workers) + 1
+ lock = multiprocessing.Lock()
w = Worker(
- self.opts, self.jobs, self.events, worker_num)
+ self.opts, self.jobs, self.events, worker_num,
+ lock=lock)
self.workers.append(w)
w.start()
self.event("Finished starting worker processes")