Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
commit da03bd134432633acfd3699a5ce725bec8f813b5 Author: Valentin Gologuzov vgologuz@redhat.com Date: Mon Dec 8 23:24:39 2014 +0100
[backend] removed spawn_vars options, to be able spawn VMs in advance
backend/backend/daemons/dispatcher.py | 18 ++++++---------- backend/backend/helpers.py | 1 - backend/conf/copr-be.conf.example | 13 +----------- backend/conf/copr-be.local.conf | 13 +----------- backend/tests/deamons/test_backend.py | 2 - backend/tests/deamons/test_dispatcher.py | 31 +++++++++-------------------- 6 files changed, 19 insertions(+), 59 deletions(-)
diff --git a/backend/backend/daemons/dispatcher.py b/backend/backend/daemons/dispatcher.py index 9b826fd..b995d22 100644 --- a/backend/backend/daemons/dispatcher.py +++ b/backend/backend/daemons/dispatcher.py @@ -288,7 +288,7 @@ class Worker(multiprocessing.Process):
return ipaddr
- def spawn_instance(self, job): + def spawn_instance(self): """ Spawn new VM, executing the following steps:
@@ -306,18 +306,12 @@ class Worker(multiprocessing.Process): # Ansible playbook python API does not work here, dunno why. See: # https://groups.google.com/forum/#!topic/ansible-project/DNBD2oHv5k8
- extra_vars = {} - for var in self.opts.spawn_vars: - if hasattr(job, var): - extra_vars[var] = getattr(job, var) - try: spawn_playbook = self.opts.build_groups[self.group_id]["spawn_playbook"] except KeyError: return None
- spawn_args = "-c ssh {0} {1}".format( - spawn_playbook, ans_extra_vars_encode(extra_vars, "copr_task")) + spawn_args = "-c ssh {}".format(spawn_playbook)
# TODO: replace with for i in range(MAX_SPAWN_TRIES): ... else raise FatalError i = 0 @@ -431,7 +425,7 @@ class Worker(multiprocessing.Process): return True return False
- def spawn_instance_with_check(self, job): + def spawn_instance_with_check(self): """ Wrapper around self.spawn_instance() with exception checking
@@ -444,7 +438,7 @@ class Worker(multiprocessing.Process): - :py:class:`AnsibleError`: failure during anible command execution """ try: - ip = self.spawn_instance(job) + ip = self.spawn_instance() if not ip: # TODO: maybe add specific exception? raise CoprWorkerError( @@ -615,6 +609,8 @@ class Worker(multiprocessing.Process):
vm_ip = None while not self.kill_received: + if self.opts.spawn_in_advance: + vm_ip = self.spawn_instance_with_check()
job = self.obtain_job() if not job: @@ -622,7 +618,7 @@ class Worker(multiprocessing.Process): continue
if not vm_ip: - vm_ip = self.spawn_instance_with_check(job) + vm_ip = self.spawn_instance_with_check()
try: self.do_job(vm_ip, job) diff --git a/backend/backend/helpers.py b/backend/backend/helpers.py index 5948888..cfb03cb 100644 --- a/backend/backend/helpers.py +++ b/backend/backend/helpers.py @@ -152,7 +152,6 @@ class BackendConfigReader(object): cp, "backend", "verbose", False, mode="bool") opts.worker_logdir = _get_conf( cp, "backend", "worker_logdir", "/var/log/copr/workers/") - opts.spawn_vars = _get_conf(cp, "backend", "spawn_vars", "").split(",") opts.terminate_vars = _get_conf(cp, "backend", "terminate_vars", "").split(",")
opts.prune_days = _get_conf(cp, "backend", "prune_days", None, mode="int") diff --git a/backend/conf/copr-be.conf.example b/backend/conf/copr-be.conf.example index fe76c37..97fe92f 100644 --- a/backend/conf/copr-be.conf.example +++ b/backend/conf/copr-be.conf.example @@ -43,18 +43,7 @@ group0_terminate_playbook=/srv/copr-work/provision/terminatepb-PC.yml group0_max_workers=8
-# What variables will be send to spawn_playbook. Expects comma separated list -# of keywords. For example 'spawn_vars=chroot' will create -# 'copr_task["chroot"]' variable accessible from spawn_playbook. -# Currently supported parameters -# - chroot -# string representation of expect chroot to be alloced, e.g. -# fedora-rawhide-ppc64 -# -# default is None (empty) -#spawn_vars=chroot - -# The same option as 'spawn_vars' above, except that variables will be send to +# That variables will be send to # terminate_playbook ansible script. # Currently supported parameters: # - ip diff --git a/backend/conf/copr-be.local.conf b/backend/conf/copr-be.local.conf index f59fb71..4fc5f47 100644 --- a/backend/conf/copr-be.local.conf +++ b/backend/conf/copr-be.local.conf @@ -42,18 +42,7 @@ group0_terminate_playbook=/srv/copr-work/provision/terminatepb-PC.yml group0_max_workers=8
-# What variables will be send to spawn_playbook. Expects comma separated list -# of keywords. For example 'spawn_vars=chroot' will create -# 'copr_task["chroot"]' variable accessible from spawn_playbook. -# Currently supported parameters -# - chroot -# string representation of expect chroot to be alloced, e.g. -# fedora-rawhide-ppc64 -# -# default is None (empty) -#spawn_vars=chroot - -# The same option as 'spawn_vars' above, except that variables will be send to +# That variables will be send to # terminate_playbook ansible script. # Currently supported parameters: # - ip diff --git a/backend/tests/deamons/test_backend.py b/backend/tests/deamons/test_backend.py index 4fb81b2..4818b40 100644 --- a/backend/tests/deamons/test_backend.py +++ b/backend/tests/deamons/test_backend.py @@ -322,8 +322,6 @@ class TestBackend(object): self.pwd.getpwnam.return_value.pw_uid = 9
run_backend(self.run_opts) - - print() ddc = mc_daemon_context.call_args[1]
assert ddc["signal_map"] == {1: u'terminate', 15: u'terminate'} diff --git a/backend/tests/deamons/test_dispatcher.py b/backend/tests/deamons/test_dispatcher.py index 1b6a2a2..11a4dbf 100644 --- a/backend/tests/deamons/test_dispatcher.py +++ b/backend/tests/deamons/test_dispatcher.py @@ -88,7 +88,6 @@ class TestDispatcher(object): "name": "3" } }, - spawn_vars=["chroot"], terminate_vars=[],
fedmsg_enabled=False, @@ -101,10 +100,7 @@ class TestDispatcher(object): ) self.job = BuildJob(self.task, self.opts)
- self.try_spawn_args = ( - '-c ssh {} ' - '--extra-vars='{{"copr_task": {{"chroot": "{}"}}}}'' - ).format(self.spawn_pb, self.CHROOT) + self.try_spawn_args = '-c ssh {}'.format(self.spawn_pb)
self.worker_num = 2 self.group_id = "3" @@ -155,7 +151,7 @@ class TestDispatcher(object): self.worker.spawn_instance = MagicMock() self.worker.spawn_instance.return_value = self.vm_ip
- assert self.vm_ip == self.worker.spawn_instance_with_check(self.job) + assert self.vm_ip == self.worker.spawn_instance_with_check()
def test_spawn_instance_with_check_no_ip(self, init_worker): self.worker.spawn_instance = MagicMock() @@ -163,7 +159,7 @@ class TestDispatcher(object):
# with pytest.raises(ansible.errors.AnsibleError): with pytest.raises(CoprWorkerError): - self.worker.spawn_instance_with_check(self.job) + self.worker.spawn_instance_with_check()
def test_spawn_instance_with_check_ansible_error_reraised(self, init_worker): self.worker.spawn_instance = MagicMock() @@ -171,14 +167,14 @@ class TestDispatcher(object):
# with pytest.raises(): with pytest.raises(AnsibleError): - self.worker.spawn_instance_with_check(self.job) + self.worker.spawn_instance_with_check()
def test_spawn_instance_missing_playbook_for_group_id(self, init_worker): self.worker.try_spawn = MagicMock() self.worker.validate_new_vm = MagicMock() self.worker.group_id = "175"
- assert self.worker.spawn_instance(self.job) is None + assert self.worker.spawn_instance() is None assert not self.worker.try_spawn.called assert not self.worker.validate_new_vm.called
@@ -187,7 +183,7 @@ class TestDispatcher(object): self.worker.try_spawn.return_value = self.vm_ip self.worker.validate_new_vm = MagicMock()
- assert self.worker.spawn_instance(self.job) == self.vm_ip + assert self.worker.spawn_instance() == self.vm_ip assert self.worker.try_spawn.called assert self.worker.validate_new_vm.called
@@ -199,7 +195,7 @@ class TestDispatcher(object): ] self.worker.validate_new_vm = MagicMock()
- assert self.worker.spawn_instance(self.job) == self.vm_ip + assert self.worker.spawn_instance() == self.vm_ip
assert len(self.worker.try_spawn.call_args_list) == 2 assert len(self.worker.validate_new_vm.call_args_list) == 1 @@ -214,7 +210,7 @@ class TestDispatcher(object): None, ]
- assert self.worker.spawn_instance(self.job) == self.vm_ip + assert self.worker.spawn_instance() == self.vm_ip
assert len(self.worker.try_spawn.call_args_list) == 2 assert len(self.worker.validate_new_vm.call_args_list) == 2 @@ -225,17 +221,12 @@ class TestDispatcher(object): self.worker.try_spawn.return_value = self.vm_ip self.worker.validate_new_vm = MagicMock()
- assert self.worker.spawn_instance(self.job) == self.vm_ip + assert self.worker.spawn_instance() == self.vm_ip assert self.worker.try_spawn.called assert self.worker.validate_new_vm.called
assert self.worker.try_spawn.call_args == mock.call(self.try_spawn_args)
- self.worker.try_spawn.reset_mock() - self.worker.opts.spawn_vars = [] - assert self.worker.spawn_instance(self.job) == self.vm_ip - assert self.worker.try_spawn.call_args == mock.call('-c ssh /spawn.yml ') - def test_try_spawn_ansible_error_no_result(self, init_worker): mc_run_ans = MagicMock() self.worker.run_ansible_playbook = mc_run_ans @@ -397,9 +388,7 @@ class TestDispatcher(object):
assert mc_subprocess.check_output.called_once assert mc_subprocess.check_output.call_args == mock.call( - 'ansible-playbook -c ssh /spawn.yml ' - '--extra-vars='{"copr_task": {"chroot": "fedora-20-x86_64"}}'', - shell=True) + 'ansible-playbook -c ssh /spawn.yml', shell=True)
copr-commits@lists.fedorahosted.org