[copr] master: [frontend] [rhbz:#1133650] RFE: copr frontend on page of build details, results section should show multiple links that link directly for every chroot directory (8692f76)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 8692f763c03de97198e79252db68f6ecf65f0f4e
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Mon Jan 19 12:09:41 2015 +0100
[frontend] [rhbz:#1133650] RFE: copr frontend on page of build details, results section should show multiple links that link directly for every chroot directory
>---------------------------------------------------------------
frontend/coprs_frontend/coprs/models.py | 11 +++++++++++
.../coprs/templates/coprs/detail/build.html | 7 ++++++-
2 files changed, 17 insertions(+), 1 deletions(-)
diff --git a/frontend/coprs_frontend/coprs/models.py b/frontend/coprs_frontend/coprs/models.py
index f9fcf1d..5af3310 100644
--- a/frontend/coprs_frontend/coprs/models.py
+++ b/frontend/coprs_frontend/coprs/models.py
@@ -371,6 +371,17 @@ class Build(db.Model, helpers.Serializer):
return self.state in ["succeeded", "canceled", "skipped"]
+ @property
+ def src_pkg_name(self):
+ """
+ Extract source package name from URL
+ """
+ src_rpm_name = self.pkgs.split("/")[-1]
+ if src_rpm_name.endswith(".src.rpm"):
+ return src_rpm_name[:-8]
+ else:
+ return src_rpm_name
+
class MockChroot(db.Model, helpers.Serializer):
diff --git a/frontend/coprs_frontend/coprs/templates/coprs/detail/build.html b/frontend/coprs_frontend/coprs/templates/coprs/detail/build.html
index 0b231e3..9dc0bb9 100644
--- a/frontend/coprs_frontend/coprs/templates/coprs/detail/build.html
+++ b/frontend/coprs_frontend/coprs/templates/coprs/detail/build.html
@@ -49,7 +49,12 @@
<dt> Chroots:</dt>
{% for chroot in build.build_chroots %} <tr>
<dd>
- {{ chroot.name }}:
+ {% if build.results %}
+ <a href="{{ build.results }}/{{ chroot.name }}/{{ build.src_pkg_name }}">{{ chroot.name }}</a>
+ {% else %}
+ {{ chroot.name }}
+ {% endif %}
+ :
<span class="build-{{chroot.state}}"
alt="{{chroot.state|build_state_description}}">
{{ chroot.state }}
9 years, 3 months
[copr] master: [backend] [rhbz:#1117446] add a build id tagfile into the package directory (c0daa86)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit c0daa8647215b2ca99abb0f3d55ba6fc04f448dc
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Fri Jan 16 14:58:35 2015 +0100
[backend] [rhbz:#1117446] add a build id tagfile into the package directory
>---------------------------------------------------------------
backend/backend/mockremote/__init__.py | 29 +++++++++++++++-----------
backend/tests/mockremote/test_mockremote.py | 16 ++++++++++++++-
2 files changed, 32 insertions(+), 13 deletions(-)
diff --git a/backend/backend/mockremote/__init__.py b/backend/backend/mockremote/__init__.py
index c835910..a57290b 100755
--- a/backend/backend/mockremote/__init__.py
+++ b/backend/backend/mockremote/__init__.py
@@ -290,8 +290,8 @@ class MockRemote(object):
os.unlink(os.path.join(p_path, "fail"))
# mkdir to download results
- if not os.path.exists(self.chroot_dir):
- os.makedirs(self.chroot_dir)
+ if not os.path.exists(p_path):
+ os.makedirs(p_path)
self.mark_dir_with_build_id()
@@ -362,13 +362,18 @@ class MockRemote(object):
return build_details
def mark_dir_with_build_id(self):
- pass
- # # adding info file with
- # TODO: add self.build_id
- #
- # try:
- # with open(os.path.join(get_target_dir(self.chroot_dir, pkg), "build.info"), 'w') as info_file:
- # info_file.write("build_id={}\n".format(self.build_id))
- # except IOError:
- # pass
- # checking where to stick stuff
+ """
+ Places "build.info" which contains job build_id
+ into the directory with downloaded files.
+
+ """
+ info_file_path = os.path.join(self._get_pkg_destpath(self.job.pkg),
+ "build.info")
+ self.callback.log("mark build dir with build_id, ")
+ try:
+ with open(info_file_path, 'w') as info_file:
+ info_file.writelines(["build_id={}".format(self.job.build_id),])
+
+ except Exception as error:
+ msg = "Failed to mark build {} with build_id".format(error)
+ self.callback.log(msg)
diff --git a/backend/tests/mockremote/test_mockremote.py b/backend/tests/mockremote/test_mockremote.py
index b270fef..b435eda 100644
--- a/backend/tests/mockremote/test_mockremote.py
+++ b/backend/tests/mockremote/test_mockremote.py
@@ -227,7 +227,10 @@ class TestMockRemote(object):
def test_prepare_build_dir_creates_dirs(self, f_mock_remote):
self.mr.prepare_build_dir()
- assert os.path.exists(self.DESTDIR_CHROOT)
+ target_dir = os.path.join(
+ self.DESTDIR, self.CHROOT,
+ "{}-{}".format(self.PKG_NAME, self.PKG_VERSION))
+ assert os.path.exists(target_dir)
def test_build_pkg_and_process_results(self, f_mock_remote):
self.mr.on_success_build = MagicMock()
@@ -282,8 +285,19 @@ class TestMockRemote(object):
def test_mark_dir_with_build_id(self, f_mock_remote):
# TODO: create real test
+ os.makedirs(os.path.join(self.DESTDIR_CHROOT,
+ "{}-{}".format(self.PKG_NAME, self.PKG_VERSION)))
+
+ info_file_path = os.path.join(self.DESTDIR_CHROOT,
+ "{}-{}".format(self.PKG_NAME, self.PKG_VERSION),
+ "build.info")
+ assert not os.path.exists(info_file_path)
self.mr.mark_dir_with_build_id()
+ assert os.path.exists(info_file_path)
+ with open(info_file_path) as handle:
+ assert str(self.JOB.build_id) in handle.read()
+
def test_build_pkgs_ignore_error_once(self, f_mock_remote):
err_msg = "error message"
build_details = MagicMock()
9 years, 3 months
[copr] master: [backend] Updated unittests to reflect latest changes. (db40dfe)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit db40dfeafcda723ba0a3a34b2d87543659aa345d
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Fri Jan 16 13:14:12 2015 +0100
[backend] Updated unittests to reflect latest changes.
>---------------------------------------------------------------
backend/backend/mockremote/builder.py | 2 +-
backend/tests/deamons/test_dispatcher.py | 18 +++---
backend/tests/mockremote/test_builder.py | 82 ++++++++++++++++++++----------
3 files changed, 65 insertions(+), 37 deletions(-)
diff --git a/backend/backend/mockremote/builder.py b/backend/backend/mockremote/builder.py
index 1f3b800..6e90c0d 100644
--- a/backend/backend/mockremote/builder.py
+++ b/backend/backend/mockremote/builder.py
@@ -324,7 +324,7 @@ class Builder(object):
"see logs dir. Original error: {}".format(error))
if cmd.returncode != 0:
raise BuilderError(msg="Failed to download from builder due to rsync error, "
- "see logs dir. Rsync return_code={}".format(cmd.returncode))
+ "see logs dir.", return_code=cmd.returncode)
def check(self):
# do check of host
diff --git a/backend/tests/deamons/test_dispatcher.py b/backend/tests/deamons/test_dispatcher.py
index a78cdbf..16a1f8c 100644
--- a/backend/tests/deamons/test_dispatcher.py
+++ b/backend/tests/deamons/test_dispatcher.py
@@ -303,20 +303,20 @@ class TestDispatcher(object):
self.worker.validate_vm()
assert mc_ans_conn.run.called
- def _test_terminate_instance(self, init_worker, reg_vm):
+ def test_terminate_instance(self, init_worker, reg_vm):
mc_run_ans = MagicMock()
self.worker.run_ansible_playbook = mc_run_ans
self.worker.terminate_instance()
assert mc_run_ans.called
expected_call = mock.call(
- "-c ssh -i '{},' {} ".format(self.vm_ip, self.terminate_pb),
+ "-c ssh {} ".format(self.terminate_pb),
'terminate instance')
assert expected_call == mc_run_ans.call_args
assert self.worker.vm_ip is None
assert self.worker.vm_name is None
- def _test_terminate_instance_with_vm_name(self, init_worker, reg_vm):
+ def test_terminate_instance_with_vm_name(self, init_worker, reg_vm):
mc_run_ans = MagicMock()
self.worker.run_ansible_playbook = mc_run_ans
self.opts.terminate_vars = ["vm_name"]
@@ -324,15 +324,15 @@ class TestDispatcher(object):
self.worker.terminate_instance()
assert mc_run_ans.called
expected_call = mock.call(
- '-c ssh -i \'{},\' {} --extra-vars=\'{{"copr_task": {{"vm_name": "{}"}}}}\''
- .format(self.vm_ip, self.terminate_pb, self.vm_name),
+ '-c ssh {} --extra-vars=\'{{"copr_task": {{"vm_name": "{}"}}}}\''
+ .format(self.terminate_pb, self.vm_name),
'terminate instance')
assert expected_call == mc_run_ans.call_args
assert self.worker.vm_ip is None
assert self.worker.vm_name is None
- def _test_terminate_instance_with_ip_and_vm_name(self, init_worker, reg_vm):
+ def test_terminate_instance_with_ip_and_vm_name(self, init_worker, reg_vm):
mc_run_ans = MagicMock()
self.worker.run_ansible_playbook = mc_run_ans
self.opts.terminate_vars = ["ip", "vm_name"]
@@ -341,16 +341,16 @@ class TestDispatcher(object):
assert mc_run_ans.called
expected_call = mock.call(
- '-c ssh -i \'{},\' {} --extra-vars=\''
+ '-c ssh {} --extra-vars=\''
'{{"copr_task": {{"vm_name": "{}", "ip": "{}"}}}}\''
- .format(self.vm_ip, self.terminate_pb, self.vm_name, self.vm_ip),
+ .format(self.terminate_pb, self.vm_name, self.vm_ip),
'terminate instance')
assert expected_call == mc_run_ans.call_args
assert self.worker.vm_ip is None
assert self.worker.vm_name is None
- def _test_terminate_instance_missed_playbook(self, init_worker, reg_vm):
+ def test_terminate_instance_missed_playbook(self, init_worker, reg_vm):
mc_run_ans = MagicMock()
self.worker.run_ansible_playbook = mc_run_ans
self.worker.group_id = "322"
diff --git a/backend/tests/mockremote/test_builder.py b/backend/tests/mockremote/test_builder.py
index 2ced04b..2f2ff3b 100644
--- a/backend/tests/mockremote/test_builder.py
+++ b/backend/tests/mockremote/test_builder.py
@@ -21,6 +21,7 @@ else:
from mock import patch, MagicMock
import pytest
+from types import MethodType
import backend.mockremote.builder as builder_module
from backend.mockremote.builder import Builder
@@ -439,57 +440,76 @@ class TestBuilder(object):
builder.modify_mock_chroot_config()
def test_modify_base_buildroot_on_error(self, ):
+ storage = []
+
+ def fake_run_ansible(self, cmd, *args, **kwargs):
+ storage.append(cmd)
+ raise AnsibleCallError("", "", "", True)
+
builder = self.get_test_builder()
+ builder.run_ansible_with_check = MethodType(fake_run_ansible, builder)
br_pkgs = "foo bar"
builder.buildroot_pkgs = br_pkgs
- builder.root_conn.run.return_value = {
- "contacted": {self.BUILDER_HOSTNAME: {"rc": 1, "stdout": None}},
- "dark": {}
- }
with pytest.raises(BuilderError) as err:
builder.modify_mock_chroot_config()
-
assert_in_log("putting {} into minimal buildroot of fedora-20-i386".format(br_pkgs),
self._cb_log)
- assert not builder.conn.run.called
- assert builder.root_conn.run.called
-
expected = (
"dest=/etc/mock/{}.cfg "
"line=\"config_opts['chroot_setup_cmd'] = 'install @buildsys-build {}'\" "
"regexp=\"^.*chroot_setup_cmd.*$\""
).format(self.BUILDER_CHROOT, br_pkgs)
- assert builder.root_conn.module_args == expected
+ assert any([expected in r for r in storage])
assert_in_log("Error: ", self._cb_log[1:])
def test_modify_base_buildroot(self, ):
+ storage = []
+
+ def fake_run_ansible(self, cmd, *args, **kwargs):
+ storage.append(cmd)
+
builder = self.get_test_builder()
- # TODO: rewrite test to use builder.run_ansible_with_check
+ builder.run_ansible_with_check = MethodType(fake_run_ansible, builder)
br_pkgs = "foo bar"
builder.buildroot_pkgs = br_pkgs
+ builder.modify_mock_chroot_config()
+ assert_in_log("putting {} into minimal buildroot of fedora-20-i386".format(br_pkgs),
+ self._cb_log)
+
+ expected = (
+ "dest=/etc/mock/{}.cfg "
+ "line=\"config_opts['chroot_setup_cmd'] = 'install @buildsys-build {}'\" "
+ "regexp=\"^.*chroot_setup_cmd.*$\""
+ ).format(self.BUILDER_CHROOT, br_pkgs)
+
+ assert any([expected in r for r in storage])
+
+ def test_modify_chroot_disable_networking(self):
+ storage = []
+
+ def fake_run_ansible(self, cmd, *args, **kwargs):
+ storage.append(cmd)
+
+ builder = self.get_test_builder()
+ builder.run_ansible_with_check = MethodType(fake_run_ansible, builder)
builder.root_conn.run.return_value = {
"contacted": {self.BUILDER_HOSTNAME: {"rc": 0, "stdout": None}},
"dark": {}
}
+ self.job.enable_net = False
+ # net should be disabled
builder.modify_mock_chroot_config()
- assert_in_log("putting {} into minimal buildroot of fedora-20-i386".format(br_pkgs),
- self._cb_log)
-
- assert not builder.conn.run.called
- assert builder.root_conn.run.called
- #
- # expected = (
- # "dest=/etc/mock/{}.cfg "
- # "line=\"config_opts['chroot_setup_cmd'] = 'install @buildsys-build {}'\" "
- # "regexp=\"^.*chroot_setup_cmd.*$\""
- # ).format(self.BUILDER_CHROOT, br_pkgs)
- # assert builder.root_conn.module_args == expected
+ expected = (
+ 'dest=/etc/mock/fedora-20-i386.cfg '
+ 'line="config_opts[\'use_host_resolv\'] = False" '
+ 'regexp="^.*user_host_resolv.*$"')
+ assert any([expected in r for r in storage])
def test_collect_build_packages(self):
builder = self.get_test_builder()
@@ -703,7 +723,7 @@ class TestBuilder(object):
builder.run_command_and_wait(build_cmd)
@mock.patch("backend.mockremote.builder.Popen")
- def _test_download(self, mc_popen):
+ def test_download(self, mc_popen):
builder = self.get_test_builder()
for ret_code, expected_success in [(0, True), (1, False), (23, False)]:
@@ -712,21 +732,29 @@ class TestBuilder(object):
mc_cmd.returncode = ret_code
mc_popen.return_value = mc_cmd
if expected_success:
- assert builder.download(self.BUILDER_PKG, self.BUILDER_REMOTE_BASEDIR) == self.STDOUT
+ builder.download(self.BUILDER_PKG, self.BUILDER_REMOTE_BASEDIR)
else:
with pytest.raises(BuilderError) as err:
builder.download(self.BUILDER_PKG, self.BUILDER_REMOTE_BASEDIR)
assert err.value.return_code == ret_code
- assert err.value.stderr == self.STDERR
- assert err.value.stdout == self.STDOUT
+ # assert err.value.stderr == self.STDERR
+ # assert err.value.stdout == self.STDOUT
+ #import ipdb; ipdb.set_trace()
expected_arg = (
"/usr/bin/rsync -avH -e 'ssh -o PasswordAuthentication=no -o StrictHostKeyChecking=no'"
" copr_builder@example.com:/tmp/copr-backend-test-tmp/build/results/fedora-20-i386/foovar-2.41.f21 "
- "'/tmp/copr-backend-test'/")
+ "'/tmp/copr-backend-test'/ &> '/tmp/copr-backend-test'/build-12345.rsync.log")
assert mc_popen.call_args[0][0] == expected_arg
+ @mock.patch("backend.mockremote.builder.Popen")
+ def test_download_popen_error(self, mc_popen):
+ builder = self.get_test_builder()
+ mc_popen.side_effect = IOError()
+ with pytest.raises(BuilderError):
+ builder.download(self.BUILDER_PKG, self.BUILDER_REMOTE_BASEDIR)
+
def test_build(self):
builder = self.get_test_builder()
builder.modify_mock_chroot_config = MagicMock()
9 years, 3 months
[copr] master: [backend] cleanup example config (e39a9f1)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit e39a9f127ba2b9cc31215743673179f402341c54
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Thu Jan 15 11:39:05 2015 +0100
[backend] cleanup example config
>---------------------------------------------------------------
backend/conf/copr-be.conf.example | 3 ---
1 files changed, 0 insertions(+), 3 deletions(-)
diff --git a/backend/conf/copr-be.conf.example b/backend/conf/copr-be.conf.example
index 97fe92f..edfe14d 100644
--- a/backend/conf/copr-be.conf.example
+++ b/backend/conf/copr-be.conf.example
@@ -63,9 +63,6 @@ destdir=/var/lib/copr/public_html/results
# default is 10
sleeptime=30
-# default is 8
-num_workers=5
-
# path to log file
# default is /var/log/copr/backend.log
logfile=/var/log/copr/backend.log
9 years, 3 months
[copr] master: [backend] builder: use only one log file for rsync per build (e0781e7)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit e0781e755c84834495b1596788b18a96979ebb3f
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Thu Jan 15 13:37:03 2015 +0100
[backend] builder: use only one log file for rsync per build
>---------------------------------------------------------------
backend/backend/mockremote/__init__.py | 5 ++---
backend/backend/mockremote/builder.py | 8 +++-----
2 files changed, 5 insertions(+), 8 deletions(-)
diff --git a/backend/backend/mockremote/__init__.py b/backend/backend/mockremote/__init__.py
index f23ee15..c835910 100755
--- a/backend/backend/mockremote/__init__.py
+++ b/backend/backend/mockremote/__init__.py
@@ -317,8 +317,7 @@ class MockRemote(object):
# downloading
self.callback.start_download(self.pkg)
- download_stdout = self.builder.download(self.pkg, self.chroot_dir)
- self.callback.log("builder.download output {}".format(download_stdout))
+ self.builder.download(self.pkg, self.chroot_dir)
self.callback.end_download(self.pkg)
# add record to mockchain.log # TODO:
@@ -331,7 +330,7 @@ class MockRemote(object):
if build_error:
raise MockRemoteError("Error occurred during build {}: {}"
- .format(os.path.basename(self.pkg), build_error))
+ .format(os.path.basename(self.pkg), build_error))
self.on_success_build()
return build_details
diff --git a/backend/backend/mockremote/builder.py b/backend/backend/mockremote/builder.py
index 978998c..1f3b800 100644
--- a/backend/backend/mockremote/builder.py
+++ b/backend/backend/mockremote/builder.py
@@ -308,12 +308,10 @@ class Builder(object):
remote_src = "{0}@{1}:{2}".format(self.username, self.hostname, rpd)
ssh_opts = "'ssh -o PasswordAuthentication=no -o StrictHostKeyChecking=no'"
- stdout_filepath = os.path.join(destdir, "build-{}.rsync_out.log".format(self.job.build_id))
- stderr_filepath = os.path.join(destdir, "build-{}.rsync_err.log".format(self.job.build_id))
-
- command = "{} -avH -e {} {} {}/ > {} 2> {}".format(
+ rsync_log_filepath = os.path.join(destdir, "build-{}.rsync.log".format(self.job.build_id))
+ command = "{} -avH -e {} {} {}/ &> {}".format(
rsync, ssh_opts, remote_src, destdir,
- stdout_filepath, stderr_filepath)
+ rsync_log_filepath)
# dirty magic with Popen due to IO buffering
# see http://thraxil.org/users/anders/posts/2008/03/13/Subprocess-Hanging-PIPE-...
9 years, 3 months
[copr] master: [backend] dispatcher: run terminate_instance safely (84942d7)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 84942d701b079ef8013008f9d8a2336132c087a9
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Thu Jan 15 11:45:22 2015 +0100
[backend] dispatcher: run terminate_instance safely
>---------------------------------------------------------------
backend/backend/daemons/dispatcher.py | 7 ++++++-
1 files changed, 6 insertions(+), 1 deletions(-)
diff --git a/backend/backend/daemons/dispatcher.py b/backend/backend/daemons/dispatcher.py
index 902e6bd..58b3afe 100644
--- a/backend/backend/daemons/dispatcher.py
+++ b/backend/backend/daemons/dispatcher.py
@@ -372,7 +372,12 @@ class Worker(multiprocessing.Process):
playbook,
ans_extra_vars_encode(term_args, "copr_task"))
- self.run_ansible_playbook(args, "terminate instance")
+ try:
+ self.run_ansible_playbook(args, "terminate instance")
+ except Exception as error:
+ self.callback.log("Failed to terminate an instance: vm_name={}, vm_ip={}. Original error: {}"
+ .format(self.vm_name, self.vm_ip, error))
+
# TODO: should we check that machine was destroyed?
self.vm_ip = None
self.vm_name = None
9 years, 3 months
[copr] master: [backend] cleanup mockremote.builder (3ef8c64)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 3ef8c645e495ca30dea2968621610e7ae9b4d988
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Thu Jan 15 12:10:20 2015 +0100
[backend] cleanup mockremote.builder
>---------------------------------------------------------------
backend/backend/daemons/dispatcher.py | 2 +-
backend/backend/mockremote/builder.py | 82 +-------------------------------
2 files changed, 4 insertions(+), 80 deletions(-)
diff --git a/backend/backend/daemons/dispatcher.py b/backend/backend/daemons/dispatcher.py
index 19b2f19..902e6bd 100644
--- a/backend/backend/daemons/dispatcher.py
+++ b/backend/backend/daemons/dispatcher.py
@@ -11,7 +11,7 @@ import multiprocessing
import ansible
import ansible.runner
import ansible.utils
-from ansible import callbacks
+
from ansible.errors import AnsibleError
from setproctitle import setproctitle
diff --git a/backend/backend/mockremote/builder.py b/backend/backend/mockremote/builder.py
index 972cd73..978998c 100644
--- a/backend/backend/mockremote/builder.py
+++ b/backend/backend/mockremote/builder.py
@@ -6,11 +6,9 @@ import time
from ansible.runner import Runner
-# from ..exceptions import BuilderError, BuilderTimeOutError, AnsibleCallError, AnsibleResponseError
-from backend.exceptions import BuilderError, BuilderTimeOutError, AnsibleCallError, AnsibleResponseError
+from ..exceptions import BuilderError, BuilderTimeOutError, AnsibleCallError, AnsibleResponseError
-# from ..constants import mockchain, rsync
-from backend.constants import mockchain, rsync
+from ..constants import mockchain, rsync
class Builder(object):
@@ -176,21 +174,6 @@ class Builder(object):
self.callback.log(str(err))
raise
-
- # results = self._run_ansible(
- # "dest=/etc/mock/{0}.cfg"
- # " line=\"config_opts['chroot_setup_cmd'] = 'install @buildsys-build {1}'\""
- # " regexp=\"^.*chroot_setup_cmd.*$\""
- # .format(self.chroot, self.buildroot_pkgs),
- # module_name="lineinfile", as_root=True)
- #
- # is_err, err_results = check_for_ans_error(results, self.hostname)
- #
- # if is_err:
- # self.callback.log("Error: {0}".format(err_results))
- # myresults = get_ans_results(results, self.hostname)
- # self.callback.log("{0}".format(myresults))
-
def collect_built_packages(self, build_details, pkg):
self.callback.log("Listing built binary packages")
# self.conn.module_name = "shell"
@@ -208,10 +191,7 @@ class Builder(object):
def check_build_success(self, pkg):
successfile = os.path.join(self._get_remote_pkg_dir(pkg), "success")
ansible_test_results = self._run_ansible("/usr/bin/test -f {0}".format(successfile))
- # is_err, err_results = check_for_ans_error(ansible_test_results, self.hostname)
check_for_ans_error(ansible_test_results, self.hostname)
- # return err, is_err, out
- #return out
def check_if_pkg_local_or_http(self, pkg):
"""
@@ -307,26 +287,15 @@ class Builder(object):
# run the mockchain command async
ansible_build_results = self.run_command_and_wait(buildcmd) # now raises BuildTimeoutError
- # try:
- # except BuilderTimeOutError:
- # return False, "", "Timeout expired", build_details
check_for_ans_error(ansible_build_results, self.hostname) # on error raises AnsibleResponseError
- # is_err, err_results = check_for_ans_error(ansible_build_results, self.hostname)
- # if is_err:
- # return (False, err_results.get("stdout", ""),
- # err_results.get("stderr", ""), build_details)
# we know the command ended successfully but not if the pkg built
# successfully
self.check_build_success(pkg)
build_out = get_ans_results(ansible_build_results, self.hostname).get("stdout", "")
- # success = False
- # if not is_err:
- # success = True
self.collect_built_packages(build_details, pkg)
return build_details, build_out
- # return success, out, err, build_details
def download(self, pkg, destdir):
# download the pkg to destdir using rsync + ssh
@@ -346,11 +315,9 @@ class Builder(object):
rsync, ssh_opts, remote_src, destdir,
stdout_filepath, stderr_filepath)
-
# dirty magic with Popen due to IO buffering
# see http://thraxil.org/users/anders/posts/2008/03/13/Subprocess-Hanging-PIPE-...
# alternative: use tempfile.Tempfile as Popen stdout/stderr
-
try:
cmd = Popen(command, shell=True)
cmd.wait()
@@ -363,26 +330,12 @@ class Builder(object):
def check(self):
# do check of host
-
- # errors = []
-
try:
socket.gethostbyname(self.hostname)
except socket.gaierror:
raise BuilderError("{0} could not be resolved".format(
self.hostname))
- # res = self._run_ansible("/bin/rpm -q mock rsync")
- # check for mock/rsync from results
- # is_err, err_results = check_for_ans_error(res, self.hostname)
- #
- # if is_err:
- # if "rc" in err_results:
- # errors.append(
- # "Warning: {0} does not have mock or rsync installed"
- # .format(self.hostname))
- # else:
- # errors.append(err_results["msg"])
try:
# check_for_ans_error(res, self.hostname)
self.run_ansible_with_check("/bin/rpm -q mock rsync")
@@ -390,14 +343,7 @@ class Builder(object):
raise BuilderError(msg="Build host `{0}` does not have mock or rsync installed"
.format(self.hostname))
- # test for path existence for mockchain and chroot config for this
- # chroot
-
- # res = self._run_ansible(
- # "/usr/bin/test -f {}"
- # " && /usr/bin/test -f /etc/mock/{}.cfg"
- # .format(mockchain, self.chroot))
- #
+ # test for path existence for mockchain and chroot config for this chroot
try:
self.run_ansible_with_check("/usr/bin/test -f {0}".format(mockchain))
except AnsibleCallError:
@@ -412,23 +358,6 @@ class Builder(object):
.format(self.hostname, self.chroot))
- # is_err, err_results = check_for_ans_error(res, self.hostname)
- #
- # if is_err:
- # if "rc" in err_results:
- # errors.append(
- # "Warning: {0} lacks mockchain binary or mock config for chroot {1}".format(
- # self.hostname, self.chroot))
- # else:
- # errors.append(err_results["msg"])
- #
- # if not errors:
- # self.checked = True
- # else:
- # msg = "\n".join(errors)
- # raise BuilderError(msg)
-
-
def get_ans_results(results, hostname):
if hostname in results["dark"]:
return results["dark"][hostname]
@@ -452,10 +381,6 @@ def check_for_ans_error(results, hostname, err_codes=None, success_codes=None):
success_codes = [0]
if "dark" in results and hostname in results["dark"]:
- # err_results["msg"] = "Error: Could not contact/connect" \
- # " to {0}.".format(hostname)
- #
- # return True, err_results
raise AnsibleResponseError(
msg="Error: Could not contact/connect to {}.".format(hostname))
@@ -487,4 +412,3 @@ def check_for_ans_error(results, hostname, err_codes=None, success_codes=None):
if error:
raise AnsibleResponseError(**err_results)
- # return error, err_results
9 years, 3 months
[copr] master: [backend] Builder.download don't use Popen+PIPE.communicate with rsync, output redirected to the files. (be4fc27)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit be4fc27975e5bef6cce237e89bc0e1bfc6d539f1
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Tue Jan 13 20:21:02 2015 +0100
[backend] Builder.download don't use Popen+PIPE.communicate with rsync, output redirected to the files.
>---------------------------------------------------------------
backend/backend/daemons/dispatcher.py | 7 +++-
backend/backend/mockremote/__init__.py | 2 -
backend/backend/mockremote/builder.py | 40 +++++++++++++++++++----------
backend/copr-backend.spec | 2 -
backend/run/copr_be.py | 3 +-
backend/tests/deamons/test_dispatcher.py | 8 +++---
backend/tests/mockremote/test_builder.py | 2 +-
7 files changed, 38 insertions(+), 26 deletions(-)
diff --git a/backend/backend/daemons/dispatcher.py b/backend/backend/daemons/dispatcher.py
index 709ae4e..19b2f19 100644
--- a/backend/backend/daemons/dispatcher.py
+++ b/backend/backend/daemons/dispatcher.py
@@ -366,9 +366,12 @@ class Worker(multiprocessing.Process):
.format(self.group_id))
sys.exit(255)
- args = "-c ssh -i '{0},' {1} {2}".format(
- self.vm_ip, playbook,
+ # args = "-c ssh -i '{0},' {1} {2}".format(
+ args = "-c ssh {} {}".format(
+ # self.vm_ip,
+ playbook,
ans_extra_vars_encode(term_args, "copr_task"))
+
self.run_ansible_playbook(args, "terminate instance")
# TODO: should we check that machine was destroyed?
self.vm_ip = None
diff --git a/backend/backend/mockremote/__init__.py b/backend/backend/mockremote/__init__.py
index da95cf3..f23ee15 100755
--- a/backend/backend/mockremote/__init__.py
+++ b/backend/backend/mockremote/__init__.py
@@ -106,8 +106,6 @@ class MockRemote(object):
# :param bool recurse: if more than one pkg and it fails to build,
# try to build the rest and come back to it
"""
-
-
self.opts = Bunch(
do_sign=False,
frontend_base_url=None,
diff --git a/backend/backend/mockremote/builder.py b/backend/backend/mockremote/builder.py
index e06eab4..972cd73 100644
--- a/backend/backend/mockremote/builder.py
+++ b/backend/backend/mockremote/builder.py
@@ -6,9 +6,11 @@ import time
from ansible.runner import Runner
-from ..exceptions import BuilderError, BuilderTimeOutError, AnsibleCallError, AnsibleResponseError
+# from ..exceptions import BuilderError, BuilderTimeOutError, AnsibleCallError, AnsibleResponseError
+from backend.exceptions import BuilderError, BuilderTimeOutError, AnsibleCallError, AnsibleResponseError
-from ..constants import mockchain, rsync
+# from ..constants import mockchain, rsync
+from backend.constants import mockchain, rsync
class Builder(object):
@@ -327,27 +329,37 @@ class Builder(object):
# return success, out, err, build_details
def download(self, pkg, destdir):
- # download the pkg to destdir using rsync + ssh
+ # download the pkg to destdir using rsync + ssh
rpd = self._get_remote_pkg_dir(pkg)
# make spaces work w/our rsync command below :(
destdir = "'" + destdir.replace("'", "'\\''") + "'"
+
# build rsync command line from the above
remote_src = "{0}@{1}:{2}".format(self.username, self.hostname, rpd)
ssh_opts = "'ssh -o PasswordAuthentication=no -o StrictHostKeyChecking=no'"
- command = "{0} -avH -e {1} {2} {3}/".format(
- rsync, ssh_opts, remote_src, destdir)
- cmd = Popen(command, shell=True, stdout=PIPE, stderr=PIPE)
+ stdout_filepath = os.path.join(destdir, "build-{}.rsync_out.log".format(self.job.build_id))
+ stderr_filepath = os.path.join(destdir, "build-{}.rsync_err.log".format(self.job.build_id))
- # rsync results into opts.destdir
- stdout, stderr = cmd.communicate()
- if cmd.returncode == 0:
- return stdout
- else:
- raise BuilderError(
- msg="Failed to download files from builder",
- return_code=cmd.returncode, stdout=stdout, stderr=stderr)
+ command = "{} -avH -e {} {} {}/ > {} 2> {}".format(
+ rsync, ssh_opts, remote_src, destdir,
+ stdout_filepath, stderr_filepath)
+
+
+ # dirty magic with Popen due to IO buffering
+ # see http://thraxil.org/users/anders/posts/2008/03/13/Subprocess-Hanging-PIPE-...
+ # alternative: use tempfile.Tempfile as Popen stdout/stderr
+
+ try:
+ cmd = Popen(command, shell=True)
+ cmd.wait()
+ except Exception as error:
+ raise BuilderError(msg="Failed to download from builder due to rsync error, "
+ "see logs dir. Original error: {}".format(error))
+ if cmd.returncode != 0:
+ raise BuilderError(msg="Failed to download from builder due to rsync error, "
+ "see logs dir. Rsync return_code={}".format(cmd.returncode))
def check(self):
# do check of host
diff --git a/backend/copr-backend.spec b/backend/copr-backend.spec
index 3e1f693..ac29eaa 100644
--- a/backend/copr-backend.spec
+++ b/backend/copr-backend.spec
@@ -46,7 +46,6 @@ BuildRequires: python-six
BuildRequires: ansible >= 1.2
BuildRequires: python-IPy
BuildRequires: python-paramiko
-# BuildRequires: python-plumbum
BuildRequires: wget
Requires: obs-signd
@@ -73,7 +72,6 @@ Requires: fedmsg
Requires: gawk
Requires: crontabs
Requires: python-paramiko
-# Requires: python-plumbum
Requires(post): systemd
Requires(preun): systemd
diff --git a/backend/run/copr_be.py b/backend/run/copr_be.py
index 61f62b8..1ef9680 100755
--- a/backend/run/copr_be.py
+++ b/backend/run/copr_be.py
@@ -1,4 +1,5 @@
-#!/usr/bin/python -ttu
+#!/usr/bin/python
+# | -ttu
from __future__ import print_function
from __future__ import unicode_literals
diff --git a/backend/tests/deamons/test_dispatcher.py b/backend/tests/deamons/test_dispatcher.py
index 8365dd1..a78cdbf 100644
--- a/backend/tests/deamons/test_dispatcher.py
+++ b/backend/tests/deamons/test_dispatcher.py
@@ -303,7 +303,7 @@ class TestDispatcher(object):
self.worker.validate_vm()
assert mc_ans_conn.run.called
- def test_terminate_instance(self, init_worker, reg_vm):
+ def _test_terminate_instance(self, init_worker, reg_vm):
mc_run_ans = MagicMock()
self.worker.run_ansible_playbook = mc_run_ans
@@ -316,7 +316,7 @@ class TestDispatcher(object):
assert self.worker.vm_ip is None
assert self.worker.vm_name is None
- def test_terminate_instance_with_vm_name(self, init_worker, reg_vm):
+ def _test_terminate_instance_with_vm_name(self, init_worker, reg_vm):
mc_run_ans = MagicMock()
self.worker.run_ansible_playbook = mc_run_ans
self.opts.terminate_vars = ["vm_name"]
@@ -332,7 +332,7 @@ class TestDispatcher(object):
assert self.worker.vm_ip is None
assert self.worker.vm_name is None
- def test_terminate_instance_with_ip_and_vm_name(self, init_worker, reg_vm):
+ def _test_terminate_instance_with_ip_and_vm_name(self, init_worker, reg_vm):
mc_run_ans = MagicMock()
self.worker.run_ansible_playbook = mc_run_ans
self.opts.terminate_vars = ["ip", "vm_name"]
@@ -350,7 +350,7 @@ class TestDispatcher(object):
assert self.worker.vm_ip is None
assert self.worker.vm_name is None
- def test_terminate_instance_missed_playbook(self, init_worker, reg_vm):
+ def _test_terminate_instance_missed_playbook(self, init_worker, reg_vm):
mc_run_ans = MagicMock()
self.worker.run_ansible_playbook = mc_run_ans
self.worker.group_id = "322"
diff --git a/backend/tests/mockremote/test_builder.py b/backend/tests/mockremote/test_builder.py
index dc60375..2ced04b 100644
--- a/backend/tests/mockremote/test_builder.py
+++ b/backend/tests/mockremote/test_builder.py
@@ -703,7 +703,7 @@ class TestBuilder(object):
builder.run_command_and_wait(build_cmd)
@mock.patch("backend.mockremote.builder.Popen")
- def test_download(self, mc_popen):
+ def _test_download(self, mc_popen):
builder = self.get_test_builder()
for ret_code, expected_success in [(0, True), (1, False), (23, False)]:
9 years, 3 months
[copr] master: [frontend] UI to control `enable_net` option, DB schema changes (dd9eb93)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit dd9eb9307c620b4bb4cda4acf13dd8a17e2a06ce
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Mon Jan 5 16:19:02 2015 +0100
[frontend] UI to control `enable_net` option, DB schema changes
>---------------------------------------------------------------
... 2df28d2d072f_added_column_build_enable_net.py} | 17 ++++---
...0dab2e478_added_column_copr_build_enable_net.py | 26 ++++++++++
frontend/coprs_frontend/coprs/forms.py | 3 +
.../coprs_frontend/coprs/logic/builds_logic.py | 7 ++-
frontend/coprs_frontend/coprs/models.py | 7 +++
.../coprs/templates/coprs/_coprs_forms.html | 7 +++
.../templates/coprs/detail/_builds_forms.html | 49 +++++++++++++------
.../coprs/templates/coprs/detail/build.html | 5 ++
.../coprs/views/backend_ns/backend_general.py | 3 +-
.../coprs/views/coprs_ns/coprs_builds.py | 3 +-
.../coprs/views/coprs_ns/coprs_general.py | 2 +
11 files changed, 101 insertions(+), 28 deletions(-)
diff --git a/frontend/coprs_frontend/alembic/versions/5845661bb37d_foo.py b/frontend/coprs_frontend/alembic/versions/2df28d2d072f_added_column_build_enable_net.py
similarity index 50%
copy from frontend/coprs_frontend/alembic/versions/5845661bb37d_foo.py
copy to frontend/coprs_frontend/alembic/versions/2df28d2d072f_added_column_build_enable_net.py
index f37869f..8e19dff 100644
--- a/frontend/coprs_frontend/alembic/versions/5845661bb37d_foo.py
+++ b/frontend/coprs_frontend/alembic/versions/2df28d2d072f_added_column_build_enable_net.py
@@ -1,14 +1,14 @@
-"""add playgroung column
+"""added column Build.enable_net
-Revision ID: 5845661bb37d
-Revises: 498884ac47db
-Create Date: 2014-04-04 11:25:36.216132
+Revision ID: 2df28d2d072f
+Revises: 1d5b5b1b27f9
+Create Date: 2015-01-02 17:55:02.169421
"""
# revision identifiers, used by Alembic.
-revision = '5845661bb37d'
-down_revision = '498884ac47db'
+revision = '2df28d2d072f'
+down_revision = '1d5b5b1b27f9'
from alembic import op
import sqlalchemy as sa
@@ -16,11 +16,12 @@ import sqlalchemy as sa
def upgrade():
### commands auto generated by Alembic - please adjust! ###
- op.add_column('copr', sa.Column('playground', sa.Boolean(), nullable=True))
+ op.add_column('build', sa.Column('enable_net', sa.Boolean(), server_default='0', nullable=False))
### end Alembic commands ###
def downgrade():
### commands auto generated by Alembic - please adjust! ###
- op.drop_column('copr', 'playground')
+ op.drop_column('build', 'enable_net')
### end Alembic commands ###
+
diff --git a/frontend/coprs_frontend/alembic/versions/bd0dab2e478_added_column_copr_build_enable_net.py b/frontend/coprs_frontend/alembic/versions/bd0dab2e478_added_column_copr_build_enable_net.py
new file mode 100644
index 0000000..c1fd653
--- /dev/null
+++ b/frontend/coprs_frontend/alembic/versions/bd0dab2e478_added_column_copr_build_enable_net.py
@@ -0,0 +1,26 @@
+"""added column Copr.build_enable_net
+
+Revision ID: bd0dab2e478
+Revises: 2df28d2d072f
+Create Date: 2015-01-05 11:47:15.291386
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'bd0dab2e478'
+down_revision = '2df28d2d072f'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+ ### commands auto generated by Alembic - please adjust! ###
+ op.add_column('copr', sa.Column('build_enable_net', sa.Boolean(), server_default='1', nullable=False))
+ ### end Alembic commands ###
+
+
+def downgrade():
+ ### commands auto generated by Alembic - please adjust! ###
+ op.drop_column('copr', 'build_enable_net')
+ ### end Alembic commands ###
diff --git a/frontend/coprs_frontend/coprs/forms.py b/frontend/coprs_frontend/coprs/forms.py
index 6a644c2..a3ea066 100644
--- a/frontend/coprs_frontend/coprs/forms.py
+++ b/frontend/coprs_frontend/coprs/forms.py
@@ -126,6 +126,7 @@ class CoprFormFactory(object):
filters=[StringListFilter()])
disable_createrepo = wtforms.BooleanField(default=False)
+ build_enable_net = wtforms.BooleanField(default=True)
@property
def selected_chroots(self):
@@ -220,6 +221,8 @@ class BuildFormFactory(object):
max=constants.MAX_BUILD_TIMEOUT)],
default=constants.DEFAULT_BUILD_TIMEOUT)
+ enable_net = wtforms.BooleanField()
+
F.chroots_list = map(lambda x: x.name, active_chroots)
F.chroots_list.sort()
F.chroots_sets = {}
diff --git a/frontend/coprs_frontend/coprs/logic/builds_logic.py b/frontend/coprs_frontend/coprs/logic/builds_logic.py
index 0024874..b157a4a 100644
--- a/frontend/coprs_frontend/coprs/logic/builds_logic.py
+++ b/frontend/coprs_frontend/coprs/logic/builds_logic.py
@@ -115,7 +115,8 @@ class BuildsLogic(object):
@classmethod
def add(cls, user, pkgs, copr,
- repos=None, memory_reqs=None, timeout=None, chroots=None):
+ repos=None, chroots=None,
+ memory_reqs=None, timeout=None, enable_net=True):
if chroots is None:
chroots = []
coprs_logic.CoprsLogic.raise_if_unfinished_blocking_action(
@@ -133,7 +134,9 @@ class BuildsLogic(object):
pkgs=pkgs,
copr=copr,
repos=repos,
- submitted_on=int(time.time()))
+ submitted_on=int(time.time()),
+ enable_net=bool(enable_net),
+ )
if memory_reqs:
build.memory_reqs = memory_reqs
diff --git a/frontend/coprs_frontend/coprs/models.py b/frontend/coprs_frontend/coprs/models.py
index dc3239d..f9fcf1d 100644
--- a/frontend/coprs_frontend/coprs/models.py
+++ b/frontend/coprs_frontend/coprs/models.py
@@ -152,6 +152,10 @@ class Copr(db.Model, helpers.Serializer):
owner = db.relationship("User", backref=db.backref("coprs"))
mock_chroots = association_proxy("copr_chroots", "mock_chroot")
+ # enable networking for the builds by default
+ build_enable_net = db.Column(db.Boolean, default=True,
+ server_default="1", nullable=False)
+
__mapper_args__ = {
"order_by": created_on.desc()
}
@@ -275,6 +279,9 @@ class Build(db.Model, helpers.Serializer):
memory_reqs = db.Column(db.Integer, default=constants.DEFAULT_BUILD_MEMORY)
# maximum allowed time of build, build will fail if exceeded
timeout = db.Column(db.Integer, default=constants.DEFAULT_BUILD_TIMEOUT)
+ # enable networking during a build process
+ enable_net = db.Column(db.Boolean, default=False,
+ server_default="0", nullable=False)
# relations
user_id = db.Column(db.Integer, db.ForeignKey("user.id"))
diff --git a/frontend/coprs_frontend/coprs/templates/coprs/_coprs_forms.html b/frontend/coprs_frontend/coprs/templates/coprs/_coprs_forms.html
index 0165f39..568e97f 100644
--- a/frontend/coprs_frontend/coprs/templates/coprs/_coprs_forms.html
+++ b/frontend/coprs_frontend/coprs/templates/coprs/_coprs_forms.html
@@ -71,6 +71,13 @@
{{ form.disable_createrepo }}
</dd>
+ <dt><b>Default build options</b></dt>
+ <dd>
+ Enable network access during the build
+ {{ form.build_enable_net }}
+ </dd>
+
+
<dt><input type="submit" value="{% if copr %}Update{% else %}Create{% endif %}"></dt>
</dl>
</form>
diff --git a/frontend/coprs_frontend/coprs/templates/coprs/detail/_builds_forms.html b/frontend/coprs_frontend/coprs/templates/coprs/detail/_builds_forms.html
index 0fdf872..2ec2429 100644
--- a/frontend/coprs_frontend/coprs/templates/coprs/detail/_builds_forms.html
+++ b/frontend/coprs_frontend/coprs/templates/coprs/detail/_builds_forms.html
@@ -26,22 +26,39 @@
{% endfor %}
</table>
{% endfor %}
- {{ form.csrf_token }}
- {{ render_field(form.pkgs, label='URLs of packages to build', rows = 10, cols = 50) }}
- {% if g.user.proven %}
- {{ render_field(form.memory_reqs) }}
- {{ render_field(form.timeout) }}
- {% else %} {# once we pass the hidden attribute, the field will just be hidden, it seems #}
- {{ render_field(form.memory_reqs, hidden = True) }}
- {{ render_field(form.timeout, hidden = True) }}
- {% endif %}
- <dd>
- <p>
- You agree to build only <a href="https://fedorahosted.org/copr/wiki/UserDocs#WhatIcanbuildinCopr">allowed content</a> in Copr.
- Check if your <a href="https://fedoraproject.org/wiki/Licensing:Main?rd=Licensing#Good_Licenses">license</a> is allowed.
- </p>
- </dd>
- <dd><input type="submit" value="Build"></dd>
+
+ {{ form.csrf_token }}
+ {{ render_field(form.pkgs, label='URLs of packages to build', rows = 10, cols = 50) }}
+ {% if g.user.proven %}
+ {{ render_field(form.memory_reqs) }}
+ {{ render_field(form.timeout) }}
+ {% else %} {# once we pass the hidden attribute, the field will just be hidden, it seems #}
+ {{ render_field(form.memory_reqs, hidden = True) }}
+ {{ render_field(form.timeout, hidden = True) }}
+ {% endif %}
+
+ <dt class="field-label">Build options</dt>
+ <dd class="field-label-help">
+ For reproducible builds it's recommended to disable networking.
+ </dd>
+ <dd>
+ Enable networking
+ {{ form.enable_net(checked=copr.build_enable_net) }}
+ </dd>
+ <!---->
+ <!--<dd>-->
+ <!--Build timeout (in seconds)-->
+ <!--{{ form.timeout }}-->
+ <!--</dd>-->
+
+ <hr />
+ <dd>
+ <p>
+ You agree to build only <a href="https://fedorahosted.org/copr/wiki/UserDocs#WhatIcanbuildinCopr">allowed content</a> in Copr.
+ Check if your <a href="https://fedoraproject.org/wiki/Licensing:Main?rd=Licensing#Good_Licenses">license</a> is allowed.
+ </p>
+ </dd>
+ <dd><input type="submit" value="Build"></dd>
</dl>
</form>
{% endmacro %}
diff --git a/frontend/coprs_frontend/coprs/templates/coprs/detail/build.html b/frontend/coprs_frontend/coprs/templates/coprs/detail/build.html
index df826a7..0b231e3 100644
--- a/frontend/coprs_frontend/coprs/templates/coprs/detail/build.html
+++ b/frontend/coprs_frontend/coprs/templates/coprs/detail/build.html
@@ -93,6 +93,11 @@
<dd>
{{ build.started_on|time_ago(build.ended_on) }}
</dd>
+
+ <dt class="oneline"> Networking enabled: </dt>
+ <dd>
+ {{ build.enable_net }}
+ </dd>
</div>
<div>
<dt> Source: </dt>
diff --git a/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py b/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py
index eabb1db..c4df74a 100644
--- a/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py
+++ b/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py
@@ -39,7 +39,8 @@ def waiting():
"buildroot_pkgs": task.build.copr.buildroot_pkgs(task.mock_chroot),
"repos": task.build.repos,
"memory_reqs": task.build.memory_reqs,
- "timeout": task.build.timeout
+ "timeout": task.build.timeout,
+ "enable_net": task.build.enable_net,
}
for task in builds_logic.BuildsLogic.get_build_task_queue()
]
diff --git a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_builds.py b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_builds.py
index 3d01697..938142d 100644
--- a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_builds.py
+++ b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_builds.py
@@ -121,7 +121,8 @@ def copr_new_build(username, coprname):
user=flask.g.user,
pkgs=pkg,
copr=copr,
- chroots=chroots)
+ chroots=chroots,
+ enable_net=form.enable_net.data)
if flask.g.user.proven:
build.memory_reqs = form.memory_reqs.data
diff --git a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
index c5e765b..3f8f2c9 100644
--- a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
+++ b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
@@ -258,6 +258,8 @@ def copr_update(username, coprname):
copr.description = form.description.data
copr.instructions = form.instructions.data
copr.disable_createrepo = form.disable_createrepo.data
+ copr.build_enable_net = form.build_enable_net.data
+
coprs_logic.CoprChrootsLogic.update_from_names(
flask.g.user, copr, form.selected_chroots)
9 years, 3 months
[copr] master: [backend] disable networking only when required; python style exception handling in mockremote*; removed run/copr_mockremote (d6b9be0)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit d6b9be00d85a812d4f928d12f1c889d48ecdba1a
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Mon Jan 12 16:58:08 2015 +0100
[backend] disable networking only when required; python style exception handling in mockremote*; removed run/copr_mockremote
>---------------------------------------------------------------
backend/backend/daemons/dispatcher.py | 2 +
backend/backend/exceptions.py | 28 +++-
backend/backend/job.py | 1 +
backend/backend/mockremote/__init__.py | 70 +++++++---
backend/backend/mockremote/builder.py | 192 +++++++++++++++------------
backend/copr-backend.spec | 4 +-
backend/run/copr_mockremote.py | 165 -----------------------
backend/tests/deamons/test_dispatcher.py | 4 +-
backend/tests/mockremote/test_builder.py | 170 ++++++++++++------------
backend/tests/mockremote/test_mockremote.py | 19 ++-
10 files changed, 276 insertions(+), 379 deletions(-)
diff --git a/backend/backend/daemons/dispatcher.py b/backend/backend/daemons/dispatcher.py
index c71601e..709ae4e 100644
--- a/backend/backend/daemons/dispatcher.py
+++ b/backend/backend/daemons/dispatcher.py
@@ -584,6 +584,8 @@ class Worker(multiprocessing.Process):
macros=macros, opts=self.opts, lock=self.lock,
callback=CliLogCallBack(quiet=True, logfn=chroot_logfile),
)
+ mr.check()
+
build_details = mr.build_pkg()
job.update(build_details)
diff --git a/backend/backend/exceptions.py b/backend/backend/exceptions.py
index 164f4a7..2bb2a3b 100644
--- a/backend/backend/exceptions.py
+++ b/backend/backend/exceptions.py
@@ -8,21 +8,39 @@ class MockRemoteError(Exception):
class BuilderError(MockRemoteError):
+ def __init__(self, msg, return_code=None, stdout=None, stderr=None):
+ super(BuilderError, self).__init__(msg)
+
+ self.return_code = return_code
+ self.stdout = stdout
+ self.stderr = stderr
+
+ def __str__(self):
+ result = "BuildError: {}".format(self.msg)
+ if self.return_code:
+ result += "; return code: {}".format(self.return_code)
+ if self.stdout:
+ result += "; stdout: {}".format(self.stdout)
+ if self.stderr:
+ result += "; stderr: {}".format(self.stderr)
+ return result
+
+
+class AnsibleResponseError(BuilderError):
pass
class AnsibleCallError(BuilderError):
- def __init__(self, msg, ansible_errors, cmd, module_name, as_root):
- self.msg = "{}\nError: {}\nCall cmd: `{}`, module: `{}`, as root: {}".format(
- msg, ansible_errors, cmd, module_name, as_root
+ def __init__(self, msg, cmd, module_name, as_root, **kwargs):
+ self.msg = "{}\n Call cmd: `{}`, module: `{}`, as root: {}".format(
+ msg, cmd, module_name, as_root
)
- super(AnsibleCallError, self).__init__(self.msg)
+ super(AnsibleCallError, self).__init__(self.msg, **kwargs)
self.call_args = dict(
cmd=cmd,
module_name=module_name,
as_root=as_root,
)
- self.ansible_errors = ansible_errors
class BuilderTimeOutError(BuilderError):
diff --git a/backend/backend/job.py b/backend/backend/job.py
index 9ce7b22..2f4110e 100644
--- a/backend/backend/job.py
+++ b/backend/backend/job.py
@@ -27,6 +27,7 @@ class BuildJob(object):
self.timeout = worker_opts.timeout
self.memory_reqs = None
+ self.enable_net = True
self.project_owner = None
self.project_name = None
diff --git a/backend/backend/mockremote/__init__.py b/backend/backend/mockremote/__init__.py
index b889dad..da95cf3 100755
--- a/backend/backend/mockremote/__init__.py
+++ b/backend/backend/mockremote/__init__.py
@@ -39,7 +39,7 @@ from bunch import Bunch
from ..constants import DEF_REMOTE_BASEDIR, DEF_BUILD_TIMEOUT, DEF_REPOS, \
DEF_BUILD_USER, DEF_MACROS
-from ..exceptions import MockRemoteError
+from ..exceptions import MockRemoteError, BuilderError
# TODO: replace sign & createrepo with dependency injection
@@ -106,6 +106,8 @@ class MockRemote(object):
# :param bool recurse: if more than one pkg and it fails to build,
# try to build the rest and come back to it
"""
+
+
self.opts = Bunch(
do_sign=False,
frontend_base_url=None,
@@ -146,15 +148,23 @@ class MockRemote(object):
remote_basedir=self.opts.remote_basedir,
remote_tempdir=self.opts.remote_tempdir,
macros=self.macros, repos=self.repos)
- self.builder.check()
-
- if not self.job.chroot:
- raise MockRemoteError("No chroot specified!")
self.failed = []
self.finished = []
- # self.callback.log("self dict: {}".format(self.__dict__))
+ # self.callback.log("MockRemote: {}".format(self.__dict__))
+
+ def check(self):
+ """
+ Checks that MockRemote configuration and environment are correct.
+
+ :raises MockRemoteError: when configuration is wrong or
+ some expected resource is unavailable
+ """
+ if not self.job.chroot:
+ raise MockRemoteError("No chroot specified!")
+
+ self.builder.check()
@property
def chroot_dir(self):
@@ -236,7 +246,7 @@ class MockRemote(object):
if to_err_list:
r_log.write("\nstderr\n")
for to_err in to_err_list:
- r_log.write(to_err)
+ r_log.write(str(to_err))
fcntl.flock(r_log, fcntl.LOCK_UN)
r_log.close()
@@ -285,34 +295,48 @@ class MockRemote(object):
if not os.path.exists(self.chroot_dir):
os.makedirs(self.chroot_dir)
+ self.mark_dir_with_build_id()
+
def build_pkg_and_process_results(self):
self.prepare_build_dir()
- self.mark_dir_with_build_id()
# building
self.callback.start_build(self.pkg)
- b_status, b_out, b_err, build_details = self.builder.build(self.pkg)
- self.callback.log("builder.build output: {}".format((b_status, b_out, b_err, build_details)))
+ # b_status, b_out, b_err, build_details = self.builder.build(self.pkg)
+ build_stdout = ""
+ build_error = build_details = None
+ try:
+ build_details, build_stdout = self.builder.build(self.pkg)
+ self.callback.log("builder.build finished; details: {}\n stdout: {}".format(build_details, build_stdout))
+ except BuilderError as error:
+ self.callback.log("builder.build error building pkg `{}`: {}".format(self.pkg, error))
+ build_error = error
+
+ # TODO: try to use
+ # finally:
+
self.callback.end_build(self.pkg)
# downloading
self.callback.start_download(self.pkg)
- d_success, d_out, d_err = self.builder.download(self.pkg, self.chroot_dir)
- self.callback.log("builder.download output {}".format((d_success, d_out, d_err)))
- if not d_success:
- raise MockRemoteError(
- "Failure to download {0}: {1}".format(self.pkg, d_out + d_err))
-
+ download_stdout = self.builder.download(self.pkg, self.chroot_dir)
+ self.callback.log("builder.download output {}".format(download_stdout))
self.callback.end_download(self.pkg)
- self.log_to_file_safe(os.path.join(self.chroot_dir, "mockchain.log"),
- ["\n\n{0}\n\n".format(self.pkg), b_out], [b_err])
- if b_status:
- self.on_success_build()
- return build_details
+ # add record to mockchain.log # TODO:
+ if build_error is not None:
+ stderr_to_log = build_error.stderr
else:
- raise MockRemoteError("Error occurred during build{0}"
- .format(os.path.basename(self.pkg)))
+ stderr_to_log = ""
+ self.log_to_file_safe(os.path.join(self.chroot_dir, "mockchain.log"),
+ ["\n\n{0}\n\n".format(self.pkg), build_stdout], [stderr_to_log])
+
+ if build_error:
+ raise MockRemoteError("Error occurred during build {}: {}"
+ .format(os.path.basename(self.pkg), build_error))
+
+ self.on_success_build()
+ return build_details
def build_pkg(self):
"""Build pkg defined in self.job
diff --git a/backend/backend/mockremote/builder.py b/backend/backend/mockremote/builder.py
index 903a12e..e06eab4 100644
--- a/backend/backend/mockremote/builder.py
+++ b/backend/backend/mockremote/builder.py
@@ -6,7 +6,7 @@ import time
from ansible.runner import Runner
-from ..exceptions import BuilderError, BuilderTimeOutError, AnsibleCallError
+from ..exceptions import BuilderError, BuilderTimeOutError, AnsibleCallError, AnsibleResponseError
from ..constants import mockchain, rsync
@@ -32,7 +32,6 @@ class Builder(object):
self.buildroot_pkgs = buildroot_pkgs or ""
- self.checked = False
self._remote_tempdir = remote_tempdir
self._remote_basedir = remote_basedir
# if we're at this point we've connected and done stuff on the host
@@ -90,14 +89,16 @@ class Builder(object):
err_codes=None, success_codes=None):
results = self._run_ansible(cmd, module_name, as_root)
- error, err_results = check_for_ans_error(
- results, self.hostname, err_codes, success_codes)
- if error:
+ try:
+ check_for_ans_error(
+ results, self.hostname, err_codes, success_codes)
+ except AnsibleResponseError as response_error:
raise AnsibleCallError(
msg="Failed to execute ansible command",
- ansible_errors=err_results,
- cmd=cmd, module_name=module_name, as_root=as_root
+ cmd=cmd, module_name=module_name, as_root=as_root,
+ return_code=response_error.return_code,
+ stdout=response_error.stdout, stderr=response_error.stderr
)
return results
@@ -166,11 +167,12 @@ class Builder(object):
try:
self.run_ansible_with_check(buildroot_cmd.format(**kwargs),
module_name="lineinfile", as_root=True)
- # TODO: do it conditionally
- self.run_ansible_with_check(disable_networking_cmd.format(**kwargs),
- module_name="lineinfile", as_root=True)
+ if not self.job.enable_net:
+ self.run_ansible_with_check(disable_networking_cmd.format(**kwargs),
+ module_name="lineinfile", as_root=True)
except BuilderError as err:
self.callback.log(str(err))
+ raise
# results = self._run_ansible(
@@ -201,14 +203,13 @@ class Builder(object):
build_details["built_packages"] = list(results["contacted"].values())[0][u"stdout"]
self.callback.log("Packages:\n{}".format(build_details["built_packages"]))
- def check_build_success(self, pkg, results):
- myresults = get_ans_results(results, self.hostname)
- out = myresults.get("stdout", "")
- err = myresults.get("stderr", "")
+ def check_build_success(self, pkg):
successfile = os.path.join(self._get_remote_pkg_dir(pkg), "success")
- results = self._run_ansible("/usr/bin/test -f {0}".format(successfile))
- is_err, err_results = check_for_ans_error(results, self.hostname)
- return err, is_err, out
+ ansible_test_results = self._run_ansible("/usr/bin/test -f {0}".format(successfile))
+ # is_err, err_results = check_for_ans_error(ansible_test_results, self.hostname)
+ check_for_ans_error(ansible_test_results, self.hostname)
+ # return err, is_err, out
+ #return out
def check_if_pkg_local_or_http(self, pkg):
"""
@@ -273,8 +274,10 @@ class Builder(object):
break
if waited >= self.timeout:
- self.callback.log("Build timeout expired.")
- raise BuilderTimeOutError("Build timeout expired.")
+ msg = "Build timeout expired. Time limit: {}s, time spent: {}s".format(
+ self.timeout, waited)
+ self.callback.log(msg)
+ raise BuilderTimeOutError(msg)
time.sleep(10)
waited += 10
@@ -284,8 +287,6 @@ class Builder(object):
# build the pkg passed in
# add pkg to various lists
# check for success/failure of build
- # return success/failure,stdout,stderr of build command
- # returns success_bool, out, err
build_details = {}
self.modify_mock_chroot_config()
@@ -303,30 +304,30 @@ class Builder(object):
buildcmd = self.gen_mockchain_command(dest)
# run the mockchain command async
- try:
- results = self.run_command_and_wait(buildcmd)
- except BuilderTimeOutError:
- return False, "", "Timeout expired", build_details
-
- is_err, err_results = check_for_ans_error(results, self.hostname)
-
- if is_err:
- return (False, err_results.get("stdout", ""),
- err_results.get("stderr", ""), build_details)
+ ansible_build_results = self.run_command_and_wait(buildcmd) # now raises BuildTimeoutError
+ # try:
+ # except BuilderTimeOutError:
+ # return False, "", "Timeout expired", build_details
+ check_for_ans_error(ansible_build_results, self.hostname) # on error raises AnsibleResponseError
+ # is_err, err_results = check_for_ans_error(ansible_build_results, self.hostname)
+ # if is_err:
+ # return (False, err_results.get("stdout", ""),
+ # err_results.get("stderr", ""), build_details)
# we know the command ended successfully but not if the pkg built
# successfully
- err, is_err, out = self.check_build_success(pkg, results)
- success = False
- if not is_err:
- success = True
- self.collect_built_packages(build_details, pkg)
+ self.check_build_success(pkg)
+ build_out = get_ans_results(ansible_build_results, self.hostname).get("stdout", "")
- return success, out, err, build_details
+ # success = False
+ # if not is_err:
+ # success = True
+ self.collect_built_packages(build_details, pkg)
+ return build_details, build_out
+ # return success, out, err, build_details
def download(self, pkg, destdir):
# download the pkg to destdir using rsync + ssh
- # return success/failure, stdout, stderr
rpd = self._get_remote_pkg_dir(pkg)
# make spaces work w/our rsync command below :(
@@ -340,23 +341,18 @@ class Builder(object):
cmd = Popen(command, shell=True, stdout=PIPE, stderr=PIPE)
# rsync results into opts.destdir
- out, err = cmd.communicate()
- if cmd.returncode:
- success = False
+ stdout, stderr = cmd.communicate()
+ if cmd.returncode == 0:
+ return stdout
else:
- success = True
-
- return success, out, err
+ raise BuilderError(
+ msg="Failed to download files from builder",
+ return_code=cmd.returncode, stdout=stdout, stderr=stderr)
def check(self):
# do check of host
- # set checked if successful
- # return success/failure, errorlist
-
- if self.checked:
- return True, []
- errors = []
+ # errors = []
try:
socket.gethostbyname(self.hostname)
@@ -364,41 +360,61 @@ class Builder(object):
raise BuilderError("{0} could not be resolved".format(
self.hostname))
- res = self._run_ansible("/bin/rpm -q mock rsync")
+ # res = self._run_ansible("/bin/rpm -q mock rsync")
# check for mock/rsync from results
- is_err, err_results = check_for_ans_error(res, self.hostname)
-
- if is_err:
- if "rc" in err_results:
- errors.append(
- "Warning: {0} does not have mock or rsync installed"
- .format(self.hostname))
- else:
- errors.append(err_results["msg"])
+ # is_err, err_results = check_for_ans_error(res, self.hostname)
+ #
+ # if is_err:
+ # if "rc" in err_results:
+ # errors.append(
+ # "Warning: {0} does not have mock or rsync installed"
+ # .format(self.hostname))
+ # else:
+ # errors.append(err_results["msg"])
+ try:
+ # check_for_ans_error(res, self.hostname)
+ self.run_ansible_with_check("/bin/rpm -q mock rsync")
+ except AnsibleCallError:
+ raise BuilderError(msg="Build host `{0}` does not have mock or rsync installed"
+ .format(self.hostname))
# test for path existence for mockchain and chroot config for this
# chroot
- res = self._run_ansible(
- "/usr/bin/test -f {0}"
- " && /usr/bin/test -f /etc/mock/{1}.cfg"
- .format(mockchain, self.chroot))
+ # res = self._run_ansible(
+ # "/usr/bin/test -f {}"
+ # " && /usr/bin/test -f /etc/mock/{}.cfg"
+ # .format(mockchain, self.chroot))
+ #
+ try:
+ self.run_ansible_with_check("/usr/bin/test -f {0}".format(mockchain))
+ except AnsibleCallError:
+ raise BuilderError(msg="Build host `{}` missing mockchain binary `{}`"
+ .format(self.hostname, mockchain))
- is_err, err_results = check_for_ans_error(res, self.hostname)
+ try:
+ self.run_ansible_with_check("/usr/bin/test -f /etc/mock/{}.cfg"
+ .format(self.chroot))
+ except AnsibleCallError:
+ raise BuilderError(msg="Build host `{}` missing mock config for chroot `{}`"
+ .format(self.hostname, self.chroot))
- if is_err:
- if "rc" in err_results:
- errors.append(
- "Warning: {0} lacks mockchain binary or mock config for chroot {1}".format(
- self.hostname, self.chroot))
- else:
- errors.append(err_results["msg"])
- if not errors:
- self.checked = True
- else:
- msg = "\n".join(errors)
- raise BuilderError(msg)
+ # is_err, err_results = check_for_ans_error(res, self.hostname)
+ #
+ # if is_err:
+ # if "rc" in err_results:
+ # errors.append(
+ # "Warning: {0} lacks mockchain binary or mock config for chroot {1}".format(
+ # self.hostname, self.chroot))
+ # else:
+ # errors.append(err_results["msg"])
+ #
+ # if not errors:
+ # self.checked = True
+ # else:
+ # msg = "\n".join(errors)
+ # raise BuilderError(msg)
def get_ans_results(results, hostname):
@@ -415,7 +431,7 @@ def check_for_ans_error(results, hostname, err_codes=None, success_codes=None):
dict includes 'msg'
may include 'rc', 'stderr', 'stdout' and any other requested result codes
- :return tuple: (True or False, dict)
+ :raises AnsibleResponseError:
"""
if err_codes is None:
@@ -423,21 +439,21 @@ def check_for_ans_error(results, hostname, err_codes=None, success_codes=None):
if success_codes is None:
success_codes = [0]
- err_results = {}
-
if "dark" in results and hostname in results["dark"]:
- err_results["msg"] = "Error: Could not contact/connect" \
- " to {0}.".format(hostname)
-
- return True, err_results
+ # err_results["msg"] = "Error: Could not contact/connect" \
+ # " to {0}.".format(hostname)
+ #
+ # return True, err_results
+ raise AnsibleResponseError(
+ msg="Error: Could not contact/connect to {}.".format(hostname))
error = False
-
+ err_results = {}
if err_codes or success_codes:
if hostname in results["contacted"]:
if "rc" in results["contacted"][hostname]:
rc = int(results["contacted"][hostname]["rc"])
- err_results["rc"] = rc
+ err_results["return_code"] = rc
# check for err codes first
if rc in err_codes:
error = True
@@ -457,4 +473,6 @@ def check_for_ans_error(results, hostname, err_codes=None, success_codes=None):
if item in results["contacted"][hostname]:
err_results[item] = results["contacted"][hostname][item]
- return error, err_results
+ if error:
+ raise AnsibleResponseError(**err_results)
+ # return error, err_results
diff --git a/backend/copr-backend.spec b/backend/copr-backend.spec
index 7a99e1d..3e1f693 100644
--- a/backend/copr-backend.spec
+++ b/backend/copr-backend.spec
@@ -46,7 +46,7 @@ BuildRequires: python-six
BuildRequires: ansible >= 1.2
BuildRequires: python-IPy
BuildRequires: python-paramiko
-BuildRequires: python-plumbum
+# BuildRequires: python-plumbum
BuildRequires: wget
Requires: obs-signd
@@ -73,7 +73,7 @@ Requires: fedmsg
Requires: gawk
Requires: crontabs
Requires: python-paramiko
-Requires: python-plumbum
+# Requires: python-plumbum
Requires(post): systemd
Requires(preun): systemd
diff --git a/backend/run/copr_mockremote.py b/backend/run/copr_mockremote.py
deleted file mode 100644
index 1541881..0000000
--- a/backend/run/copr_mockremote.py
+++ /dev/null
@@ -1,165 +0,0 @@
-from __future__ import print_function
-from __future__ import unicode_literals
-from __future__ import division
-from __future__ import absolute_import
-
-import sys
-import os
-
-from backend.exceptions import MockRemoteError
-from backend.mockremote import MockRemote, DEF_CHROOT, DEF_BUILD_USER, DEF_REPOS, DEF_BUILD_TIMEOUT, DEF_DESTDIR
-from backend.helpers import SortedOptParser
-from backend.mockremote.callback import CliLogCallBack
-
-
-def read_list_from_file(fn):
- lst = []
- with open(fn) as f:
- for line in f.readlines():
- line = line.replace("\n", "")
- line = line.strip()
- if line.startswith("#"):
- continue
- lst.append(line)
-
- return lst
-
-
-def parse_args(args):
-
- parser = SortedOptParser(
- "mockremote -b hostname -u user -r chroot pkg pkg pkg")
- parser.add_option("-r", "--root", default=DEF_CHROOT, dest="chroot",
- help="chroot config name/base to use in the mock build")
- parser.add_option("-c", "--continue", default=False, action="store_true",
- dest="cont",
- help="if a pkg fails to build, continue to the next one")
- parser.add_option("-a", "--addrepo", default=DEF_REPOS, action="append",
- dest="repos",
- help="add these repo baseurls to the chroot's yum config")
- parser.add_option("--recurse", default=False, action="store_true",
- help="if more than one pkg and it fails to build,"
- " try to build the rest and come back to it")
- parser.add_option("--log", default=None, dest="logfile",
- help="log to the file named by this option,"
- " defaults to not logging")
- parser.add_option("-b", "--builder", dest="builder", default=None,
- help="builder to use")
- parser.add_option("-u", dest="user", default=DEF_BUILD_USER,
- help="user to run as/connect as on builder systems")
- parser.add_option("-t", "--timeout", dest="timeout", type="int",
- default=DEF_BUILD_TIMEOUT,
- help="maximum time in seconds a build can take to run")
- parser.add_option("--destdir", dest="destdir", default=DEF_DESTDIR,
- help="place to download all the results/packages")
- parser.add_option("--packages", dest="packages_file", default=None,
- help="file to read list of packages from")
- parser.add_option("--do-sign", dest="do_sign", default=False,
- help="enable package signing")
- parser.add_option("-q", "--quiet", dest="quiet", default=False,
- action="store_true",
- help="output very little to the terminal")
- parser.add_option("-f", "--front_url", dest="front_url",
- help="copr frontend url")
- parser.add_option("--results_url", dest="results_baseurl",
- help="backend base url for built packages")
-
- opts, args = parser.parse_args(args)
-
- if not opts.builder:
- sys.stderr.write("Must specify a system to build on")
- sys.exit(1)
-
- if opts.packages_file and os.path.exists(opts.packages_file):
- args.extend(read_list_from_file(opts.packages_file))
-
- # args = list(set(args)) # poor man's 'unique' - this also changes the order
- # :(
-
- if not args:
- sys.stderr.write("Must specify at least one pkg to build")
- sys.exit(1)
-
- if not opts.chroot:
- sys.stderr.write("Must specify a mock chroot")
- sys.exit(1)
-
- for url in opts.repos:
- if not (url.startswith("http://") or
- url.startswith("https://") or url.startswith("file://")):
-
- sys.stderr.write("Only http[s] or file urls allowed for repos")
- sys.exit(1)
-
- if not opts.front_url:
- print("No front url was specified, exiting", file=sys.stderr)
- sys.exit(1)
-
- return opts, args
-
-
-# FIXME
-# play with createrepo run at the end of each build
-# need to output the things that actually worked :)
-
-
-def main(args):
-
- # parse args
- opts, pkgs = parse_args(args)
-
- if not os.path.exists(opts.destdir):
- os.makedirs(opts.destdir)
-
- try:
- # setup our callback
- callback = CliLogCallBack(logfn=opts.logfile, quiet=opts.quiet)
- # our mockremote instance
-
- class JobClass(object):
- __slots__ = ["timeout", "destdir", "chroot", "pkgs"]
-
- job = JobClass()
- job.timeout = opts.timeout
- job.destdir = opts.destdir
- job.chroot = opts.chroot
- job.pkgs = opts.pkgs
-
- mr = MockRemote(
- job=job,
- builder_host=opts.builder,
- user=opts.user,
- # cont=opts.cont,
- # recurse=opts.recurse,
- repos=opts.repos,
- do_sign=opts.do_sign,
- callback=callback,
- front_url=opts.front_url,
- results_base_url=opts.results_baseurl,
- )
-
- # FIXMES
- # things to think about doing:
- # output the remote tempdir when you start up
- # output the number of pkgs
- # output where you're writing things to
- # consider option to sync over destdir to the remote system to use
- # as a local repo for the build
- #
-
- if not opts.quiet:
- print("Building {0} pkgs".format(len(pkgs)))
-
- mr.build_pkg()
-
- if not opts.quiet:
- print("Output written to: {0}".format(opts.destdir))
-
- except MockRemoteError as e:
- sys.stderr.write("Error on build:\n")
- sys.stderr.write("{0}\n".format(e))
- return
-
-
-if __name__ == "__main__":
- main(sys.argv[1:])
diff --git a/backend/tests/deamons/test_dispatcher.py b/backend/tests/deamons/test_dispatcher.py
index 5aef710..8365dd1 100644
--- a/backend/tests/deamons/test_dispatcher.py
+++ b/backend/tests/deamons/test_dispatcher.py
@@ -472,7 +472,7 @@ class TestDispatcher(object):
'started_on': None, 'submitted_on': None, 'chroot': 'fedora-20-x86_64',
'ended_on': None, 'built_packages': '', 'timeout': 1800, 'pkg_version': '',
'memory_reqs': None, 'buildroot_pkgs': None, 'id': self.job_build_id,
- 'pkg': self.SRC_PKG_URL}
+ 'pkg': self.SRC_PKG_URL, "enable_net": True}
]})
assert expected_call == self.worker_fe_callback.update.call_args
@@ -500,7 +500,7 @@ class TestDispatcher(object):
'started_on': self.job.started_on, 'submitted_on': None, 'chroot': 'fedora-20-x86_64',
'ended_on': self.job.ended_on, 'built_packages': '', 'timeout': 1800, 'pkg_version': '',
'memory_reqs': None, 'buildroot_pkgs': None, 'id': self.job_build_id,
- 'pkg': self.SRC_PKG_URL}
+ 'pkg': self.SRC_PKG_URL, "enable_net": True}
]})
assert expected_call == self.worker_fe_callback.update.call_args
diff --git a/backend/tests/mockremote/test_builder.py b/backend/tests/mockremote/test_builder.py
index 6f80130..dc60375 100644
--- a/backend/tests/mockremote/test_builder.py
+++ b/backend/tests/mockremote/test_builder.py
@@ -4,7 +4,7 @@ import copy
from collections import defaultdict
from pprint import pprint
from bunch import Bunch
-from backend.exceptions import BuilderError, BuilderTimeOutError, AnsibleCallError
+from backend.exceptions import BuilderError, BuilderTimeOutError, AnsibleCallError, AnsibleResponseError
import tempfile
import shutil
@@ -172,7 +172,7 @@ class TestBuilder(object):
}, self.BUILDER_HOSTNAME
],
"kwargs": {},
- "expected_return": (False, {}),
+ "expected_return": None,
"expected_exception": None
},
{
@@ -183,10 +183,8 @@ class TestBuilder(object):
}, self.BUILDER_HOSTNAME
],
"kwargs": {},
- "expected_return": (
- True,
- {"msg": "Error: Could not contact/connect to {}.".format(self.BUILDER_HOSTNAME)}),
- "expected_exception": None
+ "expected_return": None,
+ "expected_exception": AnsibleResponseError
},
{
"args": [
@@ -201,7 +199,7 @@ class TestBuilder(object):
}, self.BUILDER_HOSTNAME
],
"kwargs": {},
- "expected_return": (False, {"rc": 0}),
+ "expected_return": None,
"expected_exception": None
},
{
@@ -217,17 +215,10 @@ class TestBuilder(object):
}, self.BUILDER_HOSTNAME
],
"kwargs": {},
- "expected_return": (
- True, {
- "rc": 1,
- "msg": "rc 1 not in success_codes",
- "stdout": "stdout",
- "stderr": "stderr",
-
- }),
- "expected_exception": None
+ "expected_return": None,
+ "expected_exception": AnsibleResponseError
},
- {
+ { # 5
"args": [
{
"dark": {},
@@ -241,8 +232,8 @@ class TestBuilder(object):
}, self.BUILDER_HOSTNAME
],
"kwargs": {"success_codes": [0, 1]},
- "expected_return": (False, {"rc": 1}),
- "expected_exception": None
+ "expected_return": None,
+ "expected_exception": None,
},
{
"args": [
@@ -256,9 +247,8 @@ class TestBuilder(object):
}, self.BUILDER_HOSTNAME
],
"kwargs": {"err_codes": [2, 3]},
- "expected_return": (True, {"rc": 2, "msg": "rc 2 matched err_codes",
- "stdout": "stdout", "stderr": "stderr"}),
- "expected_exception": None
+ "expected_return": None,
+ "expected_exception": AnsibleResponseError
},
{
"args": [
@@ -273,11 +263,11 @@ class TestBuilder(object):
}, self.BUILDER_HOSTNAME
],
"kwargs": {},
- "expected_return": (True, {"msg": "results included failed as true",
- "stdout": "stdout", "stderr": "stderr"}),
- "expected_exception": None
+ "expected_return": None,
+ "expected_exception": AnsibleResponseError
}
]
+ # counter = 0
for case in cases:
if case["expected_exception"]:
with pytest.raises(case["expected_exception"]):
@@ -286,6 +276,9 @@ class TestBuilder(object):
result = tested_func(*case["args"], **case["kwargs"])
assert result == case["expected_return"]
+ # counter += 1
+ # print("\nCounter {} passed".format(counter))
+
def test_get_ans_results(self):
result_obj = "RESULT_STRING"
results = {"dark": {self.BUILDER_HOSTNAME: result_obj}, "contacted": {}}
@@ -303,7 +296,7 @@ class TestBuilder(object):
def test_check_not_repeated(self):
builder = self.get_test_builder()
- assert builder.check() == (True, [])
+ builder.check()
assert not builder.conn.called
assert not builder.root_conn.called
@@ -329,7 +322,6 @@ class TestBuilder(object):
builder.conn.run.side_effect = ans_run
with pytest.raises(BuilderError) as err:
- builder.checked = False
builder.check()
# import ipdb; ipdb.set_trace()
@@ -351,46 +343,37 @@ class TestBuilder(object):
builder.conn.run.side_effect = ans_run
with pytest.raises(BuilderError) as err:
- builder.checked = False
builder.check()
assert "/usr/bin/test -f /usr/bin/mockchain" in self.stage_ctx[1]["conn"].module_args
- assert "/usr/bin/test -f /etc/mock/{}.cfg".format(self.BUILDER_CHROOT) in \
- self.stage_ctx[1]["conn"].module_args
+ # assert "/usr/bin/test -f /etc/mock/{}.cfg".format(self.BUILDER_CHROOT) in \
+ # self.stage_ctx[1]["conn"].module_args
- assert "lacks mockchain binary or mock config for chroot " in \
- err.value.msg
+ assert "missing mockchain binary" in err.value.msg
- def test_check_missing_alt_err_messages(self):
+ def test_check_missing_mock_config(self):
builder = self.get_test_builder()
+ ret_map = {
+ 0: {"contacted": {self.BUILDER_HOSTNAME: {"rc": "0", }}},
+ 1: {"contacted": {self.BUILDER_HOSTNAME: {"rc": "0", }}},
+ 2: {"contacted": {self.BUILDER_HOSTNAME: {"failed": "fatal_2", }}}
+ }
+
def ans_run():
self.stage_ctx[self.stage]["conn"] = copy.deepcopy(builder.conn)
- ret_map = {
- 0: {"contacted": {self.BUILDER_HOSTNAME: {"failed": "fatal_1", }}},
- 1: {"contacted": {self.BUILDER_HOSTNAME: {"failed": "fatal_2", }}}
- }
self.stage += 1
return ret_map[self.stage - 1]
builder.conn.run.side_effect = ans_run
with pytest.raises(BuilderError) as err:
- builder.checked = False
builder.check()
assert "/usr/bin/test -f /usr/bin/mockchain" in self.stage_ctx[1]["conn"].module_args
assert "/usr/bin/test -f /etc/mock/{}.cfg".format(self.BUILDER_CHROOT) in \
- self.stage_ctx[1]["conn"].module_args
+ self.stage_ctx[2]["conn"].module_args
- assert "results included failed as true" in err.value.msg
-
- def test_check_set_checked(self):
- builder = self.get_test_builder()
-
- builder.conn.run.return_value = {"contacted": {self.BUILDER_HOSTNAME: {"rc": 0, }}}
- builder.checked = False
- builder.check()
- assert builder.checked
+ assert "missing mock config for chroot" in err.value.msg
def test_tempdir_nop_when_provided(self):
builder = self.get_test_builder()
@@ -464,7 +447,9 @@ class TestBuilder(object):
"dark": {}
}
- builder.modify_mock_chroot_config()
+ with pytest.raises(BuilderError) as err:
+ builder.modify_mock_chroot_config()
+
assert_in_log("putting {} into minimal buildroot of fedora-20-i386".format(br_pkgs),
self._cb_log)
@@ -554,7 +539,7 @@ class TestBuilder(object):
err_codes, success_codes)
assert expected_call_check == mc_check_for_ans_errror.call_args
- mc_check_for_ans_errror.return_value = (True, err_results)
+ mc_check_for_ans_errror.side_effect = AnsibleResponseError(msg="err message", **err_results)
with pytest.raises(AnsibleCallError):
builder.run_ansible_with_check(
@@ -566,16 +551,23 @@ class TestBuilder(object):
def test_check_build_success(self, mc_check_for_ans_errror):
builder = self.get_test_builder()
- mc_check_for_ans_errror.return_value = "is_err", "err_results"
+ builder.check_build_success(self.BUILDER_PKG)
- results = builder.check_build_success(self.BUILDER_PKG, {
- "contacted": {self.BUILDER_HOSTNAME: {"rc": 0,
- "stdout": "stdout",
- "stderr": "stderr"}},
- "dark": {}
- })
+ expected_ans_args = (
+ "/usr/bin/test -f "
+ "/tmp/copr-backend-test-tmp/build/results/"
+ "{}/{}/success"
+ ).format(self.BUILDER_CHROOT, self.BUILDER_PKG_BASE)
+ assert expected_ans_args == builder.conn.module_args
+
+ @mock.patch("backend.mockremote.builder.check_for_ans_error")
+ def test_check_build_exception(self, mc_check_for_ans_errror):
+ builder = self.get_test_builder()
- assert ("stderr", "is_err", "stdout") == results
+ mc_check_for_ans_errror.side_effect = AnsibleResponseError(msg="err msg")
+
+ with pytest.raises(BuilderError):
+ builder.check_build_success(self.BUILDER_PKG)
expected_ans_args = (
"/usr/bin/test -f "
@@ -584,6 +576,7 @@ class TestBuilder(object):
).format(self.BUILDER_CHROOT, self.BUILDER_PKG_BASE)
assert expected_ans_args == builder.conn.module_args
+
def test_check_if_pkg_local_or_http_local(self):
pkg_path = os.path.join(self.test_root_path, "{}.src.rpm".format(self.BUILDER_PKG_BASE))
with open(pkg_path, "w") as handle:
@@ -681,7 +674,7 @@ class TestBuilder(object):
mc_time.return_value = None
- with pytest.raises(BuilderTimeOutError):
+ with pytest.raises(BuilderTimeOutError) as error:
builder.run_command_and_wait(build_cmd)
@mock.patch("backend.mockremote.builder.time")
@@ -718,17 +711,21 @@ class TestBuilder(object):
mc_cmd.communicate.return_value = self.STDOUT, self.STDERR
mc_cmd.returncode = ret_code
mc_popen.return_value = mc_cmd
- success, stdout, stderr = builder.download(self.BUILDER_PKG, self.BUILDER_REMOTE_BASEDIR)
+ if expected_success:
+ assert builder.download(self.BUILDER_PKG, self.BUILDER_REMOTE_BASEDIR) == self.STDOUT
+ else:
+ with pytest.raises(BuilderError) as err:
+ builder.download(self.BUILDER_PKG, self.BUILDER_REMOTE_BASEDIR)
+ assert err.value.return_code == ret_code
+ assert err.value.stderr == self.STDERR
+ assert err.value.stdout == self.STDOUT
- expected = (
+ expected_arg = (
"/usr/bin/rsync -avH -e 'ssh -o PasswordAuthentication=no -o StrictHostKeyChecking=no'"
" copr_builder@example.com:/tmp/copr-backend-test-tmp/build/results/fedora-20-i386/foovar-2.41.f21 "
"'/tmp/copr-backend-test'/")
- assert mc_popen.call_args[0][0] == expected
- assert expected_success == success
- assert stdout == self.STDOUT
- assert stderr == self.STDERR
+ assert mc_popen.call_args[0][0] == expected_arg
def test_build(self):
builder = self.get_test_builder()
@@ -753,10 +750,9 @@ class TestBuilder(object):
builder.collect_built_packages = MagicMock()
- success, stdout, stderr, build_details = builder.build(self.BUILDER_PKG)
- assert success
+ build_details, stdout = builder.build(self.BUILDER_PKG)
assert stdout == self.STDOUT
- assert stderr == self.STDERR
+
assert builder.modify_mock_chroot_config.called
assert builder.check_if_pkg_local_or_http.called
assert builder.run_command_and_wait.called
@@ -766,22 +762,23 @@ class TestBuilder(object):
# test providing version
builder.get_package_version.return_value = "srpm_version"
- success, stdout, stderr, build_details = builder.build(self.BUILDER_PKG)
+ build_details, stdout = builder.build(self.BUILDER_PKG)
assert "pkg_version" in build_details
assert build_details["pkg_version"] == "srpm_version"
# test timeout handle
builder.run_command_and_wait.side_effect = BuilderTimeOutError("msg")
- success, stdout, stderr, build_details = builder.build(self.BUILDER_PKG)
- assert not success
- assert stderr == "Timeout expired"
- assert build_details["pkg_version"] == "srpm_version"
+
+ with pytest.raises(BuilderError) as error:
+ builder.build(self.BUILDER_PKG)
+
+ assert error.value.msg == "msg"
+ # assert build_details["pkg_version"] == "srpm_version"
# remove timeout
builder.run_command_and_wait.side_effect = None
- success, stdout, stderr, build_details = builder.build(self.BUILDER_PKG)
- assert success
+ build_details, stdout = builder.build(self.BUILDER_PKG)
# error inside wait result
unsuccessful_wait_result = {
@@ -791,25 +788,24 @@ class TestBuilder(object):
"dark": {}
}
builder.run_command_and_wait.return_value = unsuccessful_wait_result
- success, stdout, stderr, build_details = builder.build(self.BUILDER_PKG)
- assert not success
+ with pytest.raises(BuilderError):
+ builder.build(self.BUILDER_PKG)
# make wait result successful again
builder.run_command_and_wait.return_value = successful_wait_result
- # error during build check
- builder.check_build_success.return_value = (self.STDERR, True, self.STDOUT)
- assert not success
- assert stdout == self.STDOUT
- assert stderr == self.STDERR
-
- # revert to successful check build
- builder.check_build_success.return_value = (self.STDERR, False, self.STDOUT)
+ # # error during build check
+ # builder.check_build_success.return_value = (self.STDERR, True, self.STDOUT)
+ # assert not success
+ # assert stdout == self.STDOUT
+ # assert stderr == self.STDERR
+ #
+ # # revert to successful check build
+ # builder.check_build_success.return_value = (self.STDERR, False, self.STDOUT)
# check update build details
def upd(bd, pkg):
bd["foo"] = "bar"
builder.collect_built_packages.side_effect = upd
- success, stdout, stderr, build_details = builder.build(self.BUILDER_PKG)
- assert success
+ build_details, stdout = builder.build(self.BUILDER_PKG)
assert build_details["foo"] == "bar"
diff --git a/backend/tests/mockremote/test_mockremote.py b/backend/tests/mockremote/test_mockremote.py
index 5c88f94..b270fef 100644
--- a/backend/tests/mockremote/test_mockremote.py
+++ b/backend/tests/mockremote/test_mockremote.py
@@ -3,7 +3,7 @@ import copy
from collections import defaultdict
from bunch import Bunch
-from backend.exceptions import MockRemoteError, CoprSignError
+from backend.exceptions import MockRemoteError, CoprSignError, BuilderError
import tempfile
import shutil
@@ -73,6 +73,7 @@ class TestMockRemote(object):
self.mc_builder = patcher.start()
self.mr = MockRemote(self.HOST, self.JOB, callback=self.mr_cb,
macros=self.MACROS, opts=self.OPTS)
+ self.mr.check()
yield
patcher.stop()
@@ -120,6 +121,7 @@ class TestMockRemote(object):
def test_default_callback(self, f_mock_remote, capsys):
mr_2 = MockRemote(self.HOST, self.JOB, macros=self.MACROS, opts=self.OPTS)
+ mr_2.check()
out, err = capsys.readouterr()
assert isinstance(mr_2.callback, DefaultCallBack)
@@ -128,8 +130,9 @@ class TestMockRemote(object):
def test_no_job_chroot(self, f_mock_remote, capsys):
job_2 = copy.deepcopy(self.JOB)
job_2.chroot = None
+ mr_2 = MockRemote(self.HOST, job_2, macros=self.MACROS, opts=self.OPTS)
with pytest.raises(MockRemoteError):
- mr_2 = MockRemote(self.HOST, job_2, macros=self.MACROS, opts=self.OPTS)
+ mr_2.check()
out, err = capsys.readouterr()
@@ -231,8 +234,7 @@ class TestMockRemote(object):
self.mr.mark_dir_with_build_id = MagicMock()
build_details = MagicMock()
- self.mr.builder.build.return_value = (True, STDOUT, STDERR, build_details)
- self.mr.builder.download.return_value = (True, STDOUT, STDERR)
+ self.mr.builder.build.return_value = (build_details, STDOUT)
result = self.mr.build_pkg_and_process_results()
@@ -251,8 +253,8 @@ class TestMockRemote(object):
assert os.path.exists(os.path.join(self.DESTDIR_CHROOT, "mockchain.log"))
def test_build_pkg_and_process_results_error_on_download(self, f_mock_remote):
- self.mr.builder.build.return_value = (True, STDOUT, STDERR, {})
- self.mr.builder.download.return_value = (False, STDOUT, STDERR)
+ self.mr.builder.build.return_value = ({}, STDOUT)
+ self.mr.builder.download.side_effect = BuilderError(msg="STDERR")
self.mr.mark_dir_with_build_id = MagicMock()
self.mr.on_success_build = MagicMock()
@@ -264,8 +266,9 @@ class TestMockRemote(object):
assert not os.path.exists(os.path.join(self.DESTDIR_CHROOT, "mockchain.log"))
def test_build_pkg_and_process_results_error_on_build(self, f_mock_remote):
- self.mr.builder.build.return_value = (False, STDOUT, STDERR, {})
- self.mr.builder.download.return_value = (True, STDOUT, STDERR)
+ # self.mr.builder.build.return_value = ({}, STDOUT)
+ self.mr.builder.build.side_effect = BuilderError(msg="STDERR")
+ # self.mr.builder.download.return_value = BuilderError(msg="STDERR")
self.mr.mark_dir_with_build_id = MagicMock()
self.mr.on_success_build = MagicMock()
9 years, 3 months