Repository :
http://git.fedorahosted.org/cgit/copr.git
On branch : master
---------------------------------------------------------------
commit be7d2f1ab808d2b1531f560510336e8f375a9ca3
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Fri Apr 17 19:26:05 2015 +0200
[backend] [vmm] terminate VM with state IN_USE only when builder process is missed
---------------------------------------------------------------
backend/backend/daemons/dispatcher.py | 2 +
backend/backend/daemons/vm_master.py | 35 +++++++++++++------------
backend/backend/helpers.py | 2 +-
backend/backend/vm_manage/event_handle.py | 2 +-
backend/backend/vm_manage/manager.py | 4 +++
backend/tests/deamons/test_vm_master.py | 6 +++-
backend/tests/vm_manager/test_event_handle.py | 5 +--
7 files changed, 32 insertions(+), 24 deletions(-)
diff --git a/backend/backend/daemons/dispatcher.py
b/backend/backend/daemons/dispatcher.py
index 2931cd2..08566ca 100644
--- a/backend/backend/daemons/dispatcher.py
+++ b/backend/backend/daemons/dispatcher.py
@@ -258,6 +258,7 @@ class Worker(multiprocessing.Process):
# Checking whether the build is not cancelled
if not self.starting_build(job):
+ self.log.info("Couldn't start job: {}".format(job))
return
# Checking whether to build or skip
@@ -386,6 +387,7 @@ class Worker(multiprocessing.Process):
job = self.obtain_job()
if not job:
+ self.update_process_title(suffix="trying to acquire job")
time.sleep(self.opts.sleeptime)
return
diff --git a/backend/backend/daemons/vm_master.py b/backend/backend/daemons/vm_master.py
index bc4080f..db32ef7 100644
--- a/backend/backend/daemons/vm_master.py
+++ b/backend/backend/daemons/vm_master.py
@@ -54,8 +54,22 @@ class VmMaster(Process):
.format(vmd.vm_name, not_re_acquired_in))
self.vmm.start_vm_termination(vmd.vm_name,
allowed_pre_state=VmStates.READY)
- def check_one_vm_for_dead_builder(self, vmd):
+ def request_build_reschedule(self, vmd):
+ self.log.info("trying to publish reschedule")
+ vmd_dict = vmd.to_dict()
+ if all(x in vmd_dict for x in ["build_id", "task_id",
"chroot"]):
+ request = {
+ "action": "reschedule",
+ "build_id": vmd.build_id,
+ "task_id": vmd.task_id,
+ "chroot": vmd.chroot,
+ }
+ self.log.info("trying to publish reschedule: {}".format(request))
+ self.vmm.rc.publish(JOB_GRAB_TASK_END_PUBSUB, json.dumps(request))
+ # else:
+ # self.log.info("Failed to release VM: {}".format(vmd.vm_name))
+ def check_one_vm_for_dead_builder(self, vmd):
in_use_since = vmd.get_field(self.vmm.rc, "in_use_since")
pid = vmd.get_field(self.vmm.rc, "used_by_pid")
@@ -71,22 +85,9 @@ class VmMaster(Process):
if psutil.pid_exists(pid) and vmd.vm_name in psutil.Process(pid).cmdline[0]:
return
- self.log.info("Process `{}` not exists anymore, releasing VM: {}
".format(pid, vmd.vm_name))
- # from celery.contrib import rdb; rdb.set_trace()
- if self.vmm.release_vm(vmd.vm_name):
- self.log.info("trying to publish reschedule")
- vmd_dict = vmd.to_dict()
- if all(x in vmd_dict for x in ["build_id", "task_id",
"chroot"]):
- request = {
- "action": "reschedule",
- "build_id": vmd.build_id,
- "task_id": vmd.task_id,
- "chroot": vmd.chroot,
- }
- self.log.info("trying to publish reschedule:
{}".format(request))
- self.vmm.rc.publish(JOB_GRAB_TASK_END_PUBSUB, json.dumps(request))
- else:
- self.log.info("Failed to release VM: {}".format(vmd.vm_name))
+ self.log.info("Process `{}` not exists anymore, terminating VM: {}
".format(pid, vmd.vm_name))
+ self.vmm.start_vm_termination(vmd.vm_name, allowed_pre_state=VmStates.IN_USE)
+ self.request_build_reschedule(vmd)
def remove_vm_with_dead_builder(self):
# TODO: rewrite build manage at backend and move functionality there
diff --git a/backend/backend/helpers.py b/backend/backend/helpers.py
index e6571d6..94eeab6 100644
--- a/backend/backend/helpers.py
+++ b/backend/backend/helpers.py
@@ -151,7 +151,7 @@ class BackendConfigReader(object):
cp, "backend",
"group{}_max_vm_total".format(group_id),
# default=16, mode="int"),
default=8, mode="int"),
- "max_vm_per_user": _get_conf(
+ "max_vm_per_user": _get_conf(
cp, "backend",
"group{}_max_vm_per_user".format(group_id),
default=4, mode="int"),
"max_builds_per_vm": _get_conf(
diff --git a/backend/backend/vm_manage/event_handle.py
b/backend/backend/vm_manage/event_handle.py
index 88c3e69..c12076f 100644
--- a/backend/backend/vm_manage/event_handle.py
+++ b/backend/backend/vm_manage/event_handle.py
@@ -104,7 +104,7 @@ class EventHandler(Process):
self.log.debug("recording check fail: {}".format(msg))
self.lua_scripts["record_failure"](keys=[vmd.vm_key])
too_much_fails = int(vmd.get_field(self.vmm.rc, "check_fails") or
0) > Thresholds.max_check_fails
- if too_much_fails:
+ if too_much_fails and vmd.state != VmStates.IN_USE:
self.log.info("check fail threshold reached: {}, terminating:
{}"
.format(check_fails_count, msg))
self.vmm.start_vm_termination(vmd.vm_name)
diff --git a/backend/backend/vm_manage/manager.py b/backend/backend/vm_manage/manager.py
index 546b360..ab954bd 100644
--- a/backend/backend/vm_manage/manager.py
+++ b/backend/backend/vm_manage/manager.py
@@ -84,6 +84,10 @@ end
terminate_vm_lua = """
local old_state = redis.call("HGET", KEYS[1], "state")
+if old_state == "in_use" and ARGV[1] ~= "in_use" then
+ return "Termination of VM in in_use state are forbidden"
+end
+
if ARGV[1] and ARGV[1] ~= "None" and old_state ~= ARGV[1] then
return "Old state != `allowed_pre_state`"
elseif old_state == "terminating" and ARGV[1] ~= "terminating" then
diff --git a/backend/tests/deamons/test_vm_master.py
b/backend/tests/deamons/test_vm_master.py
index b31e397..be8c5f8 100644
--- a/backend/tests/deamons/test_vm_master.py
+++ b/backend/tests/deamons/test_vm_master.py
@@ -200,8 +200,10 @@ class TestVmMaster(object):
def test_remove_vm_with_dead_builder(self, mc_time, add_vmd, mc_psutil):
mc_time.time.return_value = time.time()
+ self.vm_master.log = MagicMock()
- self.vmm.release_vm = types.MethodType(MagicMock(), self.vmm)
+ self.vmm.start_vm_termination = MagicMock()
+ self.vmm.start_vm_termination.return_value = "OK"
for idx, vmd in enumerate([self.vmd_a1, self.vmd_a2,
self.vmd_b1, self.vmd_b2, self.vmd_b3]):
@@ -245,7 +247,7 @@ class TestVmMaster(object):
self.vm_master.remove_vm_with_dead_builder()
msg_list = self.rcv_from_ps_message_bus()
-
+ print(self.vm_master.log.call_args_list)
assert set(["2", "4"]) ==
set([json.loads(m["data"])["build_id"] for m in msg_list])
def test_check_vms_health(self, mc_time, add_vmd):
diff --git a/backend/tests/vm_manager/test_event_handle.py
b/backend/tests/vm_manager/test_event_handle.py
index fe0ef1c..b5bb82f 100644
--- a/backend/tests/vm_manager/test_event_handle.py
+++ b/backend/tests/vm_manager/test_event_handle.py
@@ -243,11 +243,10 @@ class TestEventHandle(object):
assert self.vmd.get_field(self.rc, "state") == VmStates.IN_USE
assert int(self.vmd.get_field(self.rc, "check_fails")) == 2
- # when threshold exceeded request termination
+ # when threshold exceeded request termination do NOT terminate it
self.eh.on_health_check_result(msg)
assert self.vmd.get_field(self.rc, "state") == VmStates.IN_USE
-
- assert self.vmm.start_vm_termination.called
+ assert not self.vmm.start_vm_termination.called
def test_health_check_result_on_wrong_states(self):
self.vmd = VmDescriptor(self.vm_ip, self.vm_name, self.group, VmStates.GOT_IP)