Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
commit 0194a5831f594a1f4ae5b243f17f38fde59d1bd2 Author: Valentin Gologuzov vgologuz@redhat.com Date: Wed Dec 10 16:43:00 2014 +0100
[backend] added option to control ansible ssh transport, changed by default to `paramiko` [frontend] bugfix api create new
backend/backend/daemons/dispatcher.py | 13 +++++---- backend/backend/helpers.py | 6 ++++ backend/backend/mockremote/__init__.py | 20 ++++++++------- backend/backend/mockremote/builder.py | 26 ++++++++++--------- backend/tests/deamons/test_dispatcher.py | 1 + backend/tests/mockremote/test_builder.py | 8 ++++++ .../coprs/views/api_ns/api_general.py | 17 +++++++----- 7 files changed, 57 insertions(+), 34 deletions(-)
diff --git a/backend/backend/daemons/dispatcher.py b/backend/backend/daemons/dispatcher.py index b995d22..fe046f8 100644 --- a/backend/backend/daemons/dispatcher.py +++ b/backend/backend/daemons/dispatcher.py @@ -232,14 +232,15 @@ class Worker(multiprocessing.Process): """ # we were getting some dead instances # that's why I'm testing the connectivity here - connection = ansible.runner.Runner( + runner_options = dict( remote_user="root", host_list="{},".format(ipaddr), pattern=ipaddr, forks=1, - transport="ssh", + transport=self.opts.ssh.transport, timeout=500 ) + connection = ansible.runner.Runner(**runner_options) connection.module_name = "shell" connection.module_args = "echo hello"
@@ -252,8 +253,9 @@ class Worker(multiprocessing.Process):
if ipaddr not in res.get("contacted", {}): self.callback.log( - "Worker is not responding to" - "the testing playbook. Terminating it.") + "Worker is not responding to the testing playbook. Terminating it." + "Runner options: {}".format(runner_options) + + "Ansible raw response:\n{}".format(res)) raise CoprWorkerSpawnFailError("Created VM ({}) was unresponsive " "and therefore terminated".format(ipaddr))
@@ -469,8 +471,7 @@ class Worker(multiprocessing.Process): """ self._announce_start(job) self.callback.log( - "Skipping: package {0} has been already built before." - .format(' '.join(job.pkg))) + "Skipping: package {0} has been already built before.".format(job.pkg)) job.status = BuildStatus.SKIPPED # skipped self._announce_end(job)
diff --git a/backend/backend/helpers.py b/backend/backend/helpers.py index cfb03cb..f7e149f 100644 --- a/backend/backend/helpers.py +++ b/backend/backend/helpers.py @@ -160,6 +160,12 @@ class BackendConfigReader(object): opts.spawn_in_advance = _get_conf( cp, "backend", "spawn_in_advance", False, mode="bool")
+ # ssh options + opts.ssh = Bunch() + # TODO: ansible Runner show some magic bugs with transport "ssh", using paramiko + opts.ssh.transport = _get_conf( + cp, "ssh", "transport", "paramiko") + # thoughts for later # ssh key for connecting to builders? # cloud key stuff? diff --git a/backend/backend/mockremote/__init__.py b/backend/backend/mockremote/__init__.py index 61c4233..380477e 100755 --- a/backend/backend/mockremote/__init__.py +++ b/backend/backend/mockremote/__init__.py @@ -134,15 +134,17 @@ class MockRemote(object): self.callback = DefaultCallBack()
self.callback.log("Setting up builder: {0}".format(builder_host)) - self.builder = Builder(hostname=builder_host, - username=self.opts.build_user, - chroot=self.job.chroot, - timeout=self.job.timeout or DEF_TIMEOUT, - buildroot_pkgs=self.job.buildroot_pkgs, - callback=self.callback, - remote_basedir=self.opts.remote_basedir, - remote_tempdir=self.opts.remote_tempdir, - macros=self.macros, repos=self.repos) + self.builder = Builder( + opts=self.opts, + hostname=builder_host, + username=self.opts.build_user, + chroot=self.job.chroot, + timeout=self.job.timeout or DEF_TIMEOUT, + buildroot_pkgs=self.job.buildroot_pkgs, + callback=self.callback, + remote_basedir=self.opts.remote_basedir, + remote_tempdir=self.opts.remote_tempdir, + macros=self.macros, repos=self.repos) self.builder.check()
if not self.job.chroot: diff --git a/backend/backend/mockremote/builder.py b/backend/backend/mockremote/builder.py index 02f489b..cdcb3bd 100644 --- a/backend/backend/mockremote/builder.py +++ b/backend/backend/mockremote/builder.py @@ -13,12 +13,14 @@ from ..constants import mockchain, rsync
class Builder(object):
- def __init__(self, hostname, username, + def __init__(self, opts, hostname, username, timeout, chroot, buildroot_pkgs, callback, remote_basedir, remote_tempdir=None, macros=None, repos=None):
+ # TODO: remove fields obtained from opts + self.opts = opts self.hostname = hostname self.username = username self.timeout = timeout @@ -33,9 +35,8 @@ class Builder(object): self._remote_tempdir = remote_tempdir self._remote_basedir = remote_basedir # if we're at this point we've connected and done stuff on the host - self.conn = _create_ans_conn( - self.hostname, self.username, self.timeout) - self.root_conn = _create_ans_conn(self.hostname, "root", self.timeout) + self.conn = self._create_ans_conn() + self.root_conn = self._create_ans_conn(username="root")
# self.callback.log("Created builder: {}".format(self.__dict__))
@@ -75,6 +76,15 @@ class Builder(object): def tempdir(self, value): self._remote_tempdir = value
+ def _create_ans_conn(self, username=None): + ans_conn = Runner(remote_user=username or self.username, + host_list=self.hostname + ",", + pattern=self.hostname, + forks=1, + transport=self.opts.ssh.transport, + timeout=self.timeout) + return ans_conn + def _run_ansible(self, cmd, module_name=None, as_root=False): """ Executes single ansible module @@ -356,14 +366,6 @@ class Builder(object): raise BuilderError(msg)
-def _create_ans_conn(hostname, username, timeout): - ans_conn = Runner(remote_user=username, - host_list=hostname + ",", - pattern=hostname, - forks=1, - transport="ssh", - timeout=timeout) - return ans_conn
def get_ans_results(results, hostname): diff --git a/backend/tests/deamons/test_dispatcher.py b/backend/tests/deamons/test_dispatcher.py index 49d5820..9a90daf 100644 --- a/backend/tests/deamons/test_dispatcher.py +++ b/backend/tests/deamons/test_dispatcher.py @@ -78,6 +78,7 @@ class TestDispatcher(object): self.spawn_pb = "/spawn.yml" self.terminate_pb = "/terminate.yml" self.opts = Bunch( + ssh=Bunch(transport="paramiko"), spawn_in_advance=False, frontend_url="http://example.com/", frontend_auth="12345678", diff --git a/backend/tests/mockremote/test_builder.py b/backend/tests/mockremote/test_builder.py index 5cfd2bf..39eb0ed 100644 --- a/backend/tests/mockremote/test_builder.py +++ b/backend/tests/mockremote/test_builder.py @@ -3,6 +3,7 @@ import copy
from collections import defaultdict from pprint import pprint +from bunch import Bunch from backend.exceptions import BuilderError, BuilderTimeOutError
import tempfile @@ -57,8 +58,15 @@ class TestBuilder(object): STDOUT = "stdout" STDERR = "stderr"
+ opts = Bunch( + ssh=Bunch( + transport="paramiko" + ) + ) + def get_test_builder(self): builder = Builder( + opts=self.opts, hostname=self.BUILDER_HOSTNAME, username=self.BUILDER_USER, timeout=self.BUILDER_TIMEOUT, diff --git a/frontend/coprs_frontend/coprs/views/api_ns/api_general.py b/frontend/coprs_frontend/coprs/views/api_ns/api_general.py index faf08cc..409598d 100644 --- a/frontend/coprs_frontend/coprs/views/api_ns/api_general.py +++ b/frontend/coprs_frontend/coprs/views/api_ns/api_general.py @@ -84,13 +84,16 @@ def api_new_copr(username): infos = []
auto_createrepo = True - dct = json.loads(flask.request.data) - if "auto_createrepo" in dct: - val = dct["auto_createrepo"] - if isinstance(val, bool): - auto_createrepo = val - elif str(val).lower() in ["false", "no"]: - auto_createrepo = False + try: + dct = json.loads(flask.request.data) + if "auto_createrepo" in dct: + val = dct["auto_createrepo"] + if isinstance(val, bool): + auto_createrepo = val + elif str(val).lower() in ["false", "no"]: + auto_createrepo = False + except Exception: + pass
try: copr = coprs_logic.CoprsLogic.add(
copr-commits@lists.fedorahosted.org