Repository :
http://git.fedorahosted.org/cgit/copr.git
On branch : master
---------------------------------------------------------------
commit 1fecb6d004b4a38f6bef280356b13b7e49754e35
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Fri Apr 17 17:51:17 2015 +0200
[backend] job_grab: postpone build is we already serving more builds
than`max_vm_per_user` option
---------------------------------------------------------------
backend/backend/daemons/job_grab.py | 27 ++++++++++++++-------------
backend/backend/mockremote/builder.py | 2 +-
backend/copr-backend.spec | 8 ++++----
backend/tests/deamons/test_job_grab.py | 22 +++++++++++++++++-----
backend/tests/mockremote/test_builder.py | 2 +-
5 files changed, 37 insertions(+), 24 deletions(-)
diff --git a/backend/backend/daemons/job_grab.py b/backend/backend/daemons/job_grab.py
index 31d694d..246e3e0 100644
--- a/backend/backend/daemons/job_grab.py
+++ b/backend/backend/daemons/job_grab.py
@@ -57,7 +57,7 @@ class CoprJobGrab(Process):
self.task_queues_by_arch = {}
self.task_queues_by_group = {}
- self.added_jobs = set()
+ self.added_jobs_dict = dict() # task_id -> task dict
self.lock = lock
self.frontend_client = frontend_client
@@ -68,8 +68,6 @@ class CoprJobGrab(Process):
self.log = get_redis_logger(self.opts, "backend.job_grab",
"job_grab")
- self.vm_manager = VmManager(opts=self.opts, logger=self.log)
-
def connect_queues(self):
"""
Connects to the retask queues. One queue per builders group.
@@ -109,7 +107,7 @@ class CoprJobGrab(Process):
"""
count = 0
if "task_id" in task:
- if task["task_id"] not in self.added_jobs:
+ if task["task_id"] not in self.added_jobs_dict:
arch = task["chroot"].split("-")[2]
if arch not in self.task_queues_by_arch:
raise CoprJobGrabError("No builder group for architecture: {},
task: {}"
@@ -117,11 +115,15 @@ class CoprJobGrab(Process):
username = task["project_owner"]
group_id = int(self.arch_to_group_id_map[arch])
- if not self.vm_manager.can_user_acquire_more_vm(username, group_id):
- self.log.debug("User can not acquire more VM, don't schedule
more tasks")
+ active_jobs_count = len([t for t_id, t in self.added_jobs_dict.items()
+ if t["project_owner"] == username])
+
+ if active_jobs_count >
self.opts.build_groups[group_id]["max_vm_per_user"]:
+ self.log.debug("User can not acquire more VM (active builds
#{}), "
+ "don't schedule more
tasks".format(active_jobs_count))
return 0
- self.added_jobs.add(task["task_id"])
+ self.added_jobs_dict[task["task_id"]] = task
task_obj = Task(task)
self.task_queues_by_arch[arch].enqueue(task_obj)
@@ -208,12 +210,12 @@ class CoprJobGrab(Process):
self.log.info("Rescheduling task `{}`".format(task_id))
self.frontend_client.reschedule_build(msg["build_id"],
msg["chroot"])
- if task_id not in self.added_jobs:
+ if task_id not in self.added_jobs_dict:
self.log.debug("Task `{}` not present in added jobs, msg ignored:
{}".format(task_id, msg))
return
if action in ["remove", "reschedule"]:
- self.added_jobs.remove(task_id)
+ self.added_jobs_dict.pop(task_id)
self.log.info("Removed task `{}` from
added_jobs".format(task_id))
except Exception as err:
@@ -221,9 +223,9 @@ class CoprJobGrab(Process):
.format(raw, err))
def log_queue_info(self):
- if self.added_jobs:
- self.log.debug("Added jobs after remove and load:
{}".format(self.added_jobs))
- self.log.debug("# of executed jobs:
{}".format(len(self.added_jobs)))
+ if self.added_jobs_dict:
+ self.log.debug("Added jobs after remove and load:
{}".format(self.added_jobs_dict))
+ self.log.debug("# of executed jobs:
{}".format(len(self.added_jobs_dict)))
for group, queue in self.task_queues_by_group.items():
if queue.length > 0:
@@ -236,7 +238,6 @@ class CoprJobGrab(Process):
setproctitle("CoprJobGrab")
self.connect_queues()
self.listen_to_pubsub()
- self.vm_manager.post_init()
self.log.info("JobGrub started.")
try:
diff --git a/backend/backend/mockremote/builder.py
b/backend/backend/mockremote/builder.py
index 6fd9b62..8016fdf 100644
--- a/backend/backend/mockremote/builder.py
+++ b/backend/backend/mockremote/builder.py
@@ -331,7 +331,7 @@ class Builder(object):
remote_src = "{0}@{1}:{2}".format(self.opts.build_user, self.hostname,
rpd)
ssh_opts = "'ssh -o PasswordAuthentication=no -o
StrictHostKeyChecking=no'"
- rsync_log_filepath = os.path.join(destdir,
"build-{}.rsync.log".format(self.job.build_id))
+ rsync_log_filepath = os.path.join(destdir,
"build-{:08d}.rsync.log".format(self.job.build_id))
command = "{} -avH -e {} {} {}/ &> {}".format(
rsync, ssh_opts, remote_src, destdir,
rsync_log_filepath)
diff --git a/backend/copr-backend.spec b/backend/copr-backend.spec
index 9d04321..708a983 100644
--- a/backend/copr-backend.spec
+++ b/backend/copr-backend.spec
@@ -177,12 +177,12 @@ cp -a conf/logstash/copr_backend.conf
%{buildroot}%{_pkgdocdir}/examples/%{_sysc
%check
-# redis-server --port 7777 &> /dev/null &
+redis-server --port 7777 &> /dev/null &
-#PYTHONPATH=backend:run:$PYTHONPATH python -B -m pytest \
-# -s -v --cov-report term-missing --cov ./backend --cov ./run ./tests/
+PYTHONPATH=backend:run:$PYTHONPATH python -B -m pytest \
+ -s -v --cov-report term-missing --cov ./backend --cov ./run ./tests/
-# kill %1
+kill %1
%pre
getent group copr >/dev/null || groupadd -r copr
diff --git a/backend/tests/deamons/test_job_grab.py
b/backend/tests/deamons/test_job_grab.py
index c6e9735..a5736fe 100644
--- a/backend/tests/deamons/test_job_grab.py
+++ b/backend/tests/deamons/test_job_grab.py
@@ -88,8 +88,11 @@ class TestJobGrab(object):
logfile=self.log_file,
verbose=False,
build_groups=[
- {"id": 0, "name": "x86", "archs":
["i386", "i686", "x86_64"]},
- {"id": 1, "name": "arm", "archs":
["armv7"]},
+ {"id": 0, "name": "x86",
+ "archs": ["i386", "i686",
"x86_64"],
+ "max_vm_per_user": 5},
+ {"id": 1, "name": "arm", "archs":
["armv7"],
+ "max_vm_per_user": 5},
],
destdir="/dev/null",
frontend_base_url="http://example.com/",
@@ -165,19 +168,28 @@ class TestJobGrab(object):
assert not mc_rc.pubsub.called
self.jg.listen_to_pubsub()
-
assert mc_rc.pubsub.called
assert mc_rc.pubsub.return_value.subscribe.called
def test_route_build_task_skip_added(self, init_jg):
- self.jg.added_jobs.add(12345)
- self.jg.added_jobs.add(12346)
+ for d in [self.task_dict_1, self.task_dict_2]:
+ self.jg.added_jobs_dict[d["task_id"]] = d
assert self.jg.route_build_task(self.task_dict_1) == 0
assert self.jg.route_build_task(self.task_dict_2) == 0
for obj in self.jg.task_queues_by_arch.values():
assert not obj.enqueue.called
+ def test_route_build_task_skip_too_much_added(self, init_jg):
+ for i in range(10):
+ task = dict(self.task_dict_1)
+ task["task_id"] = 1000 + i
+ self.jg.added_jobs_dict[task["task_id"]] = task
+
+ assert self.jg.route_build_task(self.task_dict_1) == 0
+ for obj in self.jg.task_queues_by_arch.values():
+ assert not obj.enqueue.called
+
def test_route_build_task_missing_task_ud(self, init_jg):
assert self.jg.route_build_task({"task": "wrong_key"}) == 0
for obj in self.jg.task_queues_by_arch.values():
diff --git a/backend/tests/mockremote/test_builder.py
b/backend/tests/mockremote/test_builder.py
index 1ce13af..9f42a26 100644
--- a/backend/tests/mockremote/test_builder.py
+++ b/backend/tests/mockremote/test_builder.py
@@ -702,7 +702,7 @@ class TestBuilder(object):
expected_arg = (
"/usr/bin/rsync -avH -e 'ssh -o PasswordAuthentication=no -o
StrictHostKeyChecking=no'"
"
copr_builder@example.com:/tmp/copr-backend-test-tmp/build/results/fedora-20-i386/foovar-2.41.f21
"
- "'/tmp/copr-backend-test'/ &>
'/tmp/copr-backend-test'/build-12345.rsync.log")
+ "'/tmp/copr-backend-test'/ &>
'/tmp/copr-backend-test'/build-00012345.rsync.log")
assert mc_popen.call_args[0][0] == expected_arg