Sign In
Sign Up
Sign In
Sign Up
Manage this list
×
Keyboard Shortcuts
Thread View
j
: Next unread message
k
: Previous unread message
j a
: Jump to all threads
j l
: Jump to MailingList overview
2024
June
May
April
March
February
January
2023
December
November
October
September
August
July
June
May
April
March
February
January
2022
December
November
October
September
August
July
June
May
April
March
February
January
2021
December
November
October
September
August
July
June
May
April
March
February
January
2020
December
November
October
September
August
July
June
May
April
March
February
January
2019
December
November
October
September
August
July
June
May
April
March
February
January
2018
December
November
October
September
August
July
June
May
April
March
February
January
2017
December
November
October
September
August
July
June
May
April
March
February
January
2016
December
November
October
September
August
July
June
May
April
March
February
January
2015
December
November
October
September
August
July
June
May
April
March
February
January
2014
December
November
October
September
August
July
June
May
April
March
February
January
2013
December
November
October
September
List overview
Download
copr-commits
November 2014
----- 2024 -----
June 2024
May 2024
April 2024
March 2024
February 2024
January 2024
----- 2023 -----
December 2023
November 2023
October 2023
September 2023
August 2023
July 2023
June 2023
May 2023
April 2023
March 2023
February 2023
January 2023
----- 2022 -----
December 2022
November 2022
October 2022
September 2022
August 2022
July 2022
June 2022
May 2022
April 2022
March 2022
February 2022
January 2022
----- 2021 -----
December 2021
November 2021
October 2021
September 2021
August 2021
July 2021
June 2021
May 2021
April 2021
March 2021
February 2021
January 2021
----- 2020 -----
December 2020
November 2020
October 2020
September 2020
August 2020
July 2020
June 2020
May 2020
April 2020
March 2020
February 2020
January 2020
----- 2019 -----
December 2019
November 2019
October 2019
September 2019
August 2019
July 2019
June 2019
May 2019
April 2019
March 2019
February 2019
January 2019
----- 2018 -----
December 2018
November 2018
October 2018
September 2018
August 2018
July 2018
June 2018
May 2018
April 2018
March 2018
February 2018
January 2018
----- 2017 -----
December 2017
November 2017
October 2017
September 2017
August 2017
July 2017
June 2017
May 2017
April 2017
March 2017
February 2017
January 2017
----- 2016 -----
December 2016
November 2016
October 2016
September 2016
August 2016
July 2016
June 2016
May 2016
April 2016
March 2016
February 2016
January 2016
----- 2015 -----
December 2015
November 2015
October 2015
September 2015
August 2015
July 2015
June 2015
May 2015
April 2015
March 2015
February 2015
January 2015
----- 2014 -----
December 2014
November 2014
October 2014
September 2014
August 2014
July 2014
June 2014
May 2014
April 2014
March 2014
February 2014
January 2014
----- 2013 -----
December 2013
November 2013
October 2013
September 2013
copr-commits@lists.fedorahosted.org
2 participants
61 discussions
Start a n
N
ew thread
[copr] master: [frontend] fixed paramater validation for API hanlde `create_new_copr` (919e6e7)
by vgologuz@fedoraproject.org
24 Nov '14
24 Nov '14
Repository :
http://git.fedorahosted.org/cgit/copr.git
On branch : master >--------------------------------------------------------------- commit 919e6e78fe7fac339748456378f1ad0c96831a25 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Mon Nov 24 15:34:29 2014 +0100 [frontend] fixed paramater validation for API hanlde `create_new_copr` >--------------------------------------------------------------- frontend/coprs_frontend/coprs/forms.py | 5 +- frontend/coprs_frontend/coprs/logic/users_logic.py | 10 ++ .../coprs/views/api_ns/api_general.py | 5 +- frontend/coprs_frontend/coprs/views/misc.py | 10 +- frontend/coprs_frontend/tests/test_helpers.py | 1 - .../test_views/test_api_ns/test_api_general.py | 100 ++++++++++++++++++++ frontend/run_tests.sh | 6 + 7 files changed, 129 insertions(+), 8 deletions(-) diff --git a/frontend/coprs_frontend/coprs/forms.py b/frontend/coprs_frontend/coprs/forms.py index 3a69f1f..538ff92 100644 --- a/frontend/coprs_frontend/coprs/forms.py +++ b/frontend/coprs_frontend/coprs/forms.py @@ -125,7 +125,10 @@ class CoprFormFactory(object): validators=[UrlListValidator()], filters=[StringListFilter()]) - auto_createrepo = wtforms.BooleanField(default=True) + # FIXME: false_values used for API, json API shouldn't not be + # validated in the same way as plain POST requests + auto_createrepo = wtforms.BooleanField( + default=True, false_values=["", "false", False]) @property def selected_chroots(self): diff --git a/frontend/coprs_frontend/coprs/logic/users_logic.py b/frontend/coprs_frontend/coprs/logic/users_logic.py index 4c2638b..3d494e8 100644 --- a/frontend/coprs_frontend/coprs/logic/users_logic.py +++ b/frontend/coprs_frontend/coprs/logic/users_logic.py @@ -1,9 +1,19 @@ from coprs import exceptions +from coprs.models import User + class UsersLogic(object): @classmethod + def get(cls, username): + return User.query.filter(User.username == username) + + @classmethod + def get_by_api_login(cls, login): + return User.query.filter(User.api_login == login) + + @classmethod def raise_if_cant_update_copr(cls, user, copr, message): """ Raise InsufficientRightsException if given user cant update diff --git a/frontend/coprs_frontend/coprs/views/api_ns/api_general.py b/frontend/coprs_frontend/coprs/views/api_ns/api_general.py index 0ca9780..cae3495 100644 --- a/frontend/coprs_frontend/coprs/views/api_ns/api_general.py +++ b/frontend/coprs_frontend/coprs/views/api_ns/api_general.py @@ -80,6 +80,9 @@ def api_new_copr(username): elif form.validate_on_submit(): infos = [] + auto_createrepo = form.auto_createrepo.data + if "auto_createrepo" not in flask.request.data: + auto_createrepo = True try: copr = coprs_logic.CoprsLogic.add( name=form.name.data.strip(), @@ -89,7 +92,7 @@ def api_new_copr(username): description=form.description.data, instructions=form.instructions.data, check_for_duplicates=True, - auto_createrepo=form.auto_createrepo.data + auto_createrepo=auto_createrepo ) infos.append("New project was successfully created.") diff --git a/frontend/coprs_frontend/coprs/views/misc.py b/frontend/coprs_frontend/coprs/views/misc.py index f244b47..7d6138a 100644 --- a/frontend/coprs_frontend/coprs/views/misc.py +++ b/frontend/coprs_frontend/coprs/views/misc.py @@ -10,6 +10,7 @@ from coprs import db from coprs import helpers from coprs import models from coprs import oid +from coprs.logic.users_logic import UsersLogic def fed_openidize_name(name): @@ -189,16 +190,15 @@ def api_login_required(f): @functools.wraps(f) def decorated_function(*args, **kwargs): token = None - username = None + apt_login = None if "Authorization" in flask.request.headers: base64string = flask.request.headers["Authorization"] base64string = base64string.split()[1].strip() userstring = base64.b64decode(base64string) - (username, token) = userstring.split(":") + (apt_login, token) = userstring.split(":") token_auth = False - if token and username: - user = models.User.query.filter( - models.User.api_login == username).first() + if token and apt_login: + user = UsersLogic.get_by_api_login(apt_login).first() if (user and user.api_token == token and user.api_token_expiration >= datetime.date.today()): diff --git a/frontend/coprs_frontend/tests/test_helpers.py b/frontend/coprs_frontend/tests/test_helpers.py index 4e41748..65ffe9e 100644 --- a/frontend/coprs_frontend/tests/test_helpers.py +++ b/frontend/coprs_frontend/tests/test_helpers.py @@ -80,7 +80,6 @@ class TestHelpers(CoprsTestCase): https_url = "
https://example.com/repo
" orig = app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] - print("Orig: {}".format(orig)) try: app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] = "https" assert fix_protocol_for_backend(https_url) == https_url diff --git a/frontend/coprs_frontend/tests/test_views/test_api_ns/test_api_general.py b/frontend/coprs_frontend/tests/test_views/test_api_ns/test_api_general.py new file mode 100644 index 0000000..1829acc --- /dev/null +++ b/frontend/coprs_frontend/tests/test_views/test_api_ns/test_api_general.py @@ -0,0 +1,100 @@ +import base64 +import json + +import pytest +import sqlalchemy + +from coprs.logic.users_logic import UsersLogic +from coprs.logic.coprs_logic import CoprsLogic +from tests.coprs_test_case import CoprsTestCase, TransactionDecorator + + +class TestCreateCopr(CoprsTestCase): + copr_name = "copr_api_1" + instructions = "1234" + description = "567890" + repos = "
http://example.com/repo
" # TODO: better response on on http:// entries + initial_pkgs = "
http://example.com/pkg
" + + def post_api_with_auth(self, url, content): + userstring = "{}:{}".format(self.u1.api_login, self.u1.api_token) + base64string_user = base64.b64encode(userstring) + base64string = "Basic " + base64string_user + + return self.tc.post( + url, + content_type="application/json", + data=content, + headers={ + "Authorization": base64string + } + ) + + @TransactionDecorator("u1") + def test_api_create_copr_ok_minimal(self, f_users, f_mock_chroots, f_db): + self.db.session.add_all([self.u1, self.mc1]) + self.tc.post("/api/new/") + + content = { + "name": self.copr_name, + self.mc1.name: "y", + # "repos": repos, + # "initial_pkgs": initial_pkgs, + # "description": description, + # "instructions": instructions + } + content_encoded = json.dumps(content) + + with pytest.raises(sqlalchemy.orm.exc.NoResultFound): + CoprsLogic.get(None, self.u1.name, self.copr_name).one() + + r = self.post_api_with_auth( + "/api/coprs/{}/new/".format(self.u1.name), + content_encoded + ) + response = json.loads(r.data) + assert "New project was successfully created" in response["message"] + + copr = self.models.Copr.query.filter(self.models.Copr.name == self.copr_name).one() + assert copr.name == self.copr_name + assert [self.mc1.name] == [c.name for c in copr.active_chroots] + assert copr.repos == '' + assert copr.owner.id == self.u1.id + assert copr.auto_createrepo + + + @TransactionDecorator("u1") + def test_api_create_copr_ok_all(self, f_users, f_mock_chroots, f_db): + self.db.session.add_all([self.u1, self.mc1]) + self.tc.post("/api/new/") + + content = { + "name": self.copr_name, + self.mc1.name: "y", + "repos": self.repos, + "initial_pkgs": self.initial_pkgs, + "description": self.description, + "instructions": self.instructions, + "auto_createrepo": False + } + content_encoded = json.dumps(content) + + with pytest.raises(sqlalchemy.orm.exc.NoResultFound): + CoprsLogic.get(None, self.u1.name, self.copr_name).one() + + r = self.post_api_with_auth( + "/api/coprs/{}/new/".format(self.u1.name), + content_encoded + ) + response = json.loads(r.data) + assert "New project was successfully created" in response["message"] + + copr = self.models.Copr.query.filter(self.models.Copr.name == self.copr_name).one() + assert copr.name == self.copr_name + assert [self.mc1.name] == [c.name for c in copr.active_chroots] + assert copr.repos == self.repos + assert copr.owner.id == self.u1.id + assert copr.description == self.description + assert copr.instructions == self.instructions + + assert not copr.auto_createrepo diff --git a/frontend/run_tests.sh b/frontend/run_tests.sh new file mode 100755 index 0000000..41e390e --- /dev/null +++ b/frontend/run_tests.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +cd coprs_frontend +COPR_CONFIG="$(pwd)/config/copr_unit_test.conf" python -m pytest tests \ + --cov-report term-missing --cov coprs $@ +
1
0
0
0
[copr] master: [backend] debugging (0f3d486)
by vgologuz@fedoraproject.org
24 Nov '14
24 Nov '14
Repository :
http://git.fedorahosted.org/cgit/copr.git
On branch : master >--------------------------------------------------------------- commit 0f3d48662ecde28b7d893bcadc4f5ddf6224eb14 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Fri Nov 21 11:56:27 2014 +0100 [backend] debugging >--------------------------------------------------------------- backend/backend/mockremote/__init__.py | 4 +++- backend/backend/mockremote/builder.py | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/backend/backend/mockremote/__init__.py b/backend/backend/mockremote/__init__.py index bce9895..421e13f 100755 --- a/backend/backend/mockremote/__init__.py +++ b/backend/backend/mockremote/__init__.py @@ -148,7 +148,7 @@ class MockRemote(object): self.failed = [] self.finished = [] - # self.callback.log("self dict: {}".format(self.__dict__)) + self.callback.log("self dict: {}".format(self.__dict__)) @property def chroot_dir(self): @@ -285,11 +285,13 @@ class MockRemote(object): # building self.callback.start_build(self.pkg) b_status, b_out, b_err, build_details = self.builder.build(self.pkg) + self.callback.log("builder.build output: {}".format((b_status, b_out, b_err, build_details))) self.callback.end_build(self.pkg) # downloading self.callback.start_download(self.pkg) d_success, d_out, d_err = self.builder.download(self.pkg, self.chroot_dir) + self.callback.log("builder.download output {}".format((d_success, d_out, d_err))) if not d_success: raise MockRemoteError( "Failure to download {0}: {1}".format(self.pkg, d_out + d_err)) diff --git a/backend/backend/mockremote/builder.py b/backend/backend/mockremote/builder.py index 228e75f..ef261fe 100644 --- a/backend/backend/mockremote/builder.py +++ b/backend/backend/mockremote/builder.py @@ -37,7 +37,7 @@ class Builder(object): self.hostname, self.username, self.timeout) self.root_conn = _create_ans_conn(self.hostname, "root", self.timeout) - # self.callback.log("Created builder: {}".format(self.__dict__)) + self.callback.log("Created builder: {}".format(self.__dict__)) # Before use: check out the host - make sure it can build/be contacted/etc # self.check() @@ -239,7 +239,6 @@ class Builder(object): # return success/failure,stdout,stderr of build command # returns success_bool, out, err - success = False build_details = {} self.modify_base_buildroot() @@ -266,12 +265,13 @@ class Builder(object): return_on_error=["stdout", "stderr"]) if is_err: - return (success, err_results.get("stdout", ""), + return (False, err_results.get("stdout", ""), err_results.get("stderr", ""), build_details) # we know the command ended successfully but not if the pkg built # successfully err, is_err, out = self.check_build_success(pkg, results) + success = False if not is_err: success = True self.collect_built_packages(build_details, pkg)
1
0
0
0
[copr] tag 'copr-cli-1.40-1' created
by vgologuz@fedoraproject.org
21 Nov '14
21 Nov '14
Repository :
http://git.fedorahosted.org/cgit/copr.git
New tag : copr-cli-1.40-1 Referencing: 88aef4ad7aefcca446b6825d7fbc97925e402974
1
0
0
0
[copr] master: Automatic commit of package [copr-cli] release [1.40-1]. (47d08b5)
by vgologuz@fedoraproject.org
21 Nov '14
21 Nov '14
Repository :
http://git.fedorahosted.org/cgit/copr.git
On branch : master >--------------------------------------------------------------- commit 47d08b51e939d8419c675a178fe9abda72bad95a Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Fri Nov 21 11:29:43 2014 +0100 Automatic commit of package [copr-cli] release [1.40-1]. >--------------------------------------------------------------- cli/copr-cli.spec | 7 ++++++- rel-eng/packages/copr-cli | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/cli/copr-cli.spec b/cli/copr-cli.spec index a872e3a..af7f641 100644 --- a/cli/copr-cli.spec +++ b/cli/copr-cli.spec @@ -4,7 +4,7 @@ %endif Name: copr-cli -Version: 1.39 +Version: 1.40 Release: 1%{?dist} Summary: Command line interface for COPR @@ -90,6 +90,11 @@ install -p man/copr.1 %{buildroot}/%{_mandir}/man1/ %endif %changelog +* Fri Nov 21 2014 Valentin Gologuzov <vgologuz(a)redhat.com> 1.40-1 +- updated to use newer version of python-copr +- minor changes in commands output +- print debug logs only when user provides "--debug" option + * Mon Oct 20 2014 Miroslav Suchý <msuchy(a)redhat.com> 1.39-1 - add man page for copr(1) - [cli] [RHBZ: #1149889] RFE: download command in copr-cli diff --git a/rel-eng/packages/copr-cli b/rel-eng/packages/copr-cli index 03bded4..5bd8686 100644 --- a/rel-eng/packages/copr-cli +++ b/rel-eng/packages/copr-cli @@ -1 +1 @@ -1.39-1 cli/ +1.40-1 cli/
1
0
0
0
[copr] tag 'copr-backend-1.47-1' created
by vgologuz@fedoraproject.org
20 Nov '14
20 Nov '14
Repository :
http://git.fedorahosted.org/cgit/copr.git
New tag : copr-backend-1.47-1 Referencing: dcfe6dfd0a676597c17527312d69804a1ede3505
1
0
0
0
[copr] master: Automatic commit of package [copr-backend] release [1.47-1]. (f8ace3d)
by vgologuz@fedoraproject.org
20 Nov '14
20 Nov '14
Repository :
http://git.fedorahosted.org/cgit/copr.git
On branch : master >--------------------------------------------------------------- commit f8ace3d3d6053f3024312c01df36fd6e975aa669 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Thu Nov 20 16:50:54 2014 +0100 Automatic commit of package [copr-backend] release [1.47-1]. >--------------------------------------------------------------- backend/copr-backend.spec | 8 +++++++- rel-eng/packages/copr-backend | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/backend/copr-backend.spec b/backend/copr-backend.spec index 15f3c2a..4ad39fa 100644 --- a/backend/copr-backend.spec +++ b/backend/copr-backend.spec @@ -3,7 +3,7 @@ %endif Name: copr-backend -Version: 1.46 +Version: 1.47 Release: 1%{?dist} Summary: Backend for Copr @@ -180,6 +180,12 @@ useradd -r -g copr -G lighttpd -s /bin/bash -c "COPR user" copr %exclude %{_pkgdocdir}/playbooks %changelog +* Thu Nov 20 2014 Valentin Gologuzov <vgologuz(a)redhat.com> 1.47-1 +- refactored mockremote, added explicit BuildJob class +- allow to spawn builder in advance +- copr-prune-repo respects auto_createrepo option +- bugfix: repeated config reads produced constantly growing lists + * Fri Oct 24 2014 Valentin Gologuzov <vgologuz(a)redhat.com> 1.46-1 - [backend] added handling of new action type: "createrepo" - [backend] added dependency on `python-copr` diff --git a/rel-eng/packages/copr-backend b/rel-eng/packages/copr-backend index faaeb1c..8274b5b 100644 --- a/rel-eng/packages/copr-backend +++ b/rel-eng/packages/copr-backend @@ -1 +1 @@ -1.46-1 backend/ +1.47-1 backend/
1
0
0
0
[copr] master: [backend] unittest backend.mockremote.Mockremote (a3c58d4)
by vgologuz@fedoraproject.org
20 Nov '14
20 Nov '14
Repository :
http://git.fedorahosted.org/cgit/copr.git
On branch : master >--------------------------------------------------------------- commit a3c58d4e475b118f4bf5b2250a96261d153c10eb Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Wed Nov 19 16:00:03 2014 +0100 [backend] unittest backend.mockremote.Mockremote >--------------------------------------------------------------- backend/backend/constants.py | 2 +- backend/backend/dispatcher.py | 16 +- backend/backend/helpers.py | 7 +- backend/backend/job.py | 2 + backend/backend/mockremote/__init__.py | 95 +++++---- backend/backend/mockremote/builder.py | 3 +- backend/backend/mockremote/callback.py | 2 +- backend/run/copr_mockremote.py | 8 +- backend/tests/mockremote/test_builder.py | 1 - backend/tests/mockremote/test_mockremote.py | 317 +++++++++++++++++++++++++++ 10 files changed, 391 insertions(+), 62 deletions(-) diff --git a/backend/backend/constants.py b/backend/backend/constants.py index a20667a..dcb5330 100644 --- a/backend/backend/constants.py +++ b/backend/backend/constants.py @@ -8,7 +8,7 @@ DEF_REMOTE_BASEDIR = "/var/tmp" DEF_TIMEOUT = 3600 DEF_REPOS = [] DEF_CHROOT = None -DEF_USER = "mockbuilder" +DEF_BUILD_USER = "mockbuilder" DEF_DESTDIR = os.getcwd() DEF_MACROS = {} DEF_BUILDROOT_PKGS = "" diff --git a/backend/backend/dispatcher.py b/backend/backend/dispatcher.py index d653f23..1dc4614 100644 --- a/backend/backend/dispatcher.py +++ b/backend/backend/dispatcher.py @@ -571,16 +571,17 @@ class Worker(multiprocessing.Process): mr = MockRemote( builder_host=ip, job=job, - cont=True, - recurse=True, repos=chroot_repos, macros=macros, lock=self.lock, - do_sign=self.opts.do_sign, + callback=CliLogCallBack( quiet=True, logfn=chrootlogfile), - front_url=self.opts.frontend_base_url, - results_base_url=self.opts.results_baseurl + + opts=self.opts, + # do_sign=self.opts.do_sign, + # front_url=self.opts.frontend_base_url, + # results_base_url=self.opts.results_baseurl ) build_details = mr.build_pkgs() @@ -594,11 +595,6 @@ class Worker(multiprocessing.Process): # record and break self.callback.log("{0} - {1}".format(ip, e)) status = 0 # failure - # else: - # # we can"t really trace back if we just fail normally - # # check if any pkgs didn"t build - # if mr.failed: - # status = 0 # failure self.callback.log( "Finished build: id={0} builder={1}" diff --git a/backend/backend/helpers.py b/backend/backend/helpers.py index c5b3958..116bb54 100644 --- a/backend/backend/helpers.py +++ b/backend/backend/helpers.py @@ -12,6 +12,8 @@ import sys import time from bunch import Bunch +import datetime +from backend.constants import DEF_BUILD_USER from copr.client import CoprClient from .exceptions import CoprBackendError @@ -103,6 +105,9 @@ class BackendConfigReader(object): opts.do_sign = _get_conf( cp, "backend", "do_sign", False, mode="bool") + opts.build_user = _get_conf( + cp, "backend", "build_user", DEF_BUILD_USER) + opts.build_groups_count = _get_conf( cp, "backend", "build_groups", 1, mode="int") @@ -171,7 +176,7 @@ def get_auto_createrepo_status(front_url, username, projectname): def log(lf, msg, quiet=None): if lf: - now = time.time() + now = datetime.datetime.utcnow().isoformat() try: with open(lf, "a") as lfh: fcntl.flock(lfh, fcntl.LOCK_EX) diff --git a/backend/backend/job.py b/backend/backend/job.py index b89349d..1f5b1b1 100644 --- a/backend/backend/job.py +++ b/backend/backend/job.py @@ -39,6 +39,8 @@ class BuildJob(object): self.status = None self.chroot = None + self.buildroot_pkgs = None + # TODO: validate update data for key, val in task_data.items(): key = str(key) diff --git a/backend/backend/mockremote/__init__.py b/backend/backend/mockremote/__init__.py index 5828a1f..bce9895 100755 --- a/backend/backend/mockremote/__init__.py +++ b/backend/backend/mockremote/__init__.py @@ -32,9 +32,10 @@ from __future__ import absolute_import import fcntl import urllib import os +from bunch import Bunch from ..constants import DEF_REMOTE_BASEDIR, DEF_TIMEOUT, DEF_REPOS, \ - DEF_USER, DEF_MACROS + DEF_BUILD_USER, DEF_MACROS from ..exceptions import MockRemoteError @@ -48,7 +49,7 @@ from .callback import DefaultCallBack def get_target_dir(chroot_dir, pkg_name): source_basename = os.path.basename(pkg_name).replace(".src.rpm", "") - return os.path.join(chroot_dir, source_basename) + return os.path.normpath(os.path.join(chroot_dir, source_basename)) class MockRemote(object): @@ -56,16 +57,16 @@ class MockRemote(object): # mock remote now do too much things # idea: send events according to the build progress to handler - def __init__(self, builder_host=None, user=DEF_USER, job=None, - cont=False, recurse=False, repos=None, callback=None, - remote_basedir=DEF_REMOTE_BASEDIR, remote_tempdir=None, - macros=None, lock=None, do_sign=False, - front_url=None, results_base_url=None): + def __init__(self, builder_host=None, job=None, + repos=None, + callback=None, + macros=None, + opts=None, + lock=None): """ - :param builder_host: builder hostname or ip - :param user: user to run as/connect as on builder systems + :param backend.job.BuildJob job: Job object with the following attributes:: :ivar timeout: ssh timeout :ivar destdir: target directory to put built packages @@ -77,58 +78,67 @@ class MockRemote(object): :ivar pkg: pkg to build - :param cont: if a pkg fails to build, continue to the next one - :param bool recurse: if more than one pkg and it fails to build, - try to build the rest and come back to it :param repos: additional repositories for mock :param backend.mockremote.callback.DefaultCallBack callback: object with hooks for notifications about build progress - :param remote_basedir: basedir on builder - :param remote_tempdir: tempdir on builder + :param macros: { "copr_username": ..., "copr_projectname": ..., "vendor": ...} :param multiprocessing.Lock lock: instance of Lock shared between Copr backend process - :param bool do_sign: enable package signing, require configured - signer host and correct /etc/sign.conf - - :param str front_url: url to the copr frontend - + :param DefaultCallback callback: build progress handler + + :param Bunch opts: builder options, used keys:: + :ivar build_user: user to run as/connect as on builder systems + :ivar do_sign: enable package signing, require configured + signer host and correct /etc/sign.conf + :ivar frontend_base_url: url to the copr frontend + :ivar results_base_url: base url for the built results + :ivar remote_basedir: basedir on builder + :ivar remote_tempdir: tempdir on builder + + # Removed: + # :param cont: if a pkg fails to build, continue to the next one-- + # :param bool recurse: if more than one pkg and it fails to build, + # try to build the rest and come back to it """ + self.opts = Bunch( + do_sign=False, + frontend_base_url=None, + results_base_url=u"", + build_user=DEF_BUILD_USER, + remote_basedir=DEF_REMOTE_BASEDIR, + remote_tempdir=None, + ) + if opts: + self.opts.update(opts) self.max_retry_count = 2 # TODO: add config option - if repos is None: - repos = DEF_REPOS - if macros is None: - macros = DEF_MACROS self.job = job - self.repos = repos + self.repos = repos or DEF_REPOS # TODO: remove or re-implement # self.cont = cont # unused since we build only one pkg at time # self.recurse = recurse self.callback = callback - self.remote_basedir = remote_basedir - self.remote_tempdir = remote_tempdir - self.macros = macros + self.macros = macros or DEF_MACROS self.lock = lock - self.do_sign = do_sign - self.front_url = front_url - self.results_base_url = results_base_url or u'' if not self.callback: self.callback = DefaultCallBack() self.callback.log("Setting up builder: {0}".format(builder_host)) - self.builder = Builder(hostname=builder_host, username=user, chroot=self.job.chroot, + self.builder = Builder(hostname=builder_host, + username=self.opts.build_user, + chroot=self.job.chroot, timeout=self.job.timeout or DEF_TIMEOUT, buildroot_pkgs=self.job.buildroot_pkgs, callback=self.callback, - remote_basedir=self.remote_basedir, - remote_tempdir=self.remote_tempdir, + remote_basedir=self.opts.remote_basedir, + remote_tempdir=self.opts.remote_tempdir, macros=self.macros, repos=self.repos) self.builder.check() @@ -138,8 +148,7 @@ class MockRemote(object): self.failed = [] self.finished = [] - #self.built_packages = [] - self.callback.log("self dict: {}".format(self.__dict__)) + # self.callback.log("self dict: {}".format(self.__dict__)) @property def chroot_dir(self): @@ -201,7 +210,7 @@ class MockRemote(object): get_target_dir(self.chroot_dir, self.pkg), callback=self.callback) except Exception as e: - self.callback.log( + self.callback.error( "failed to sign packages " "built from `{}` with error: \n" "{}".format(self.pkg, e) @@ -225,16 +234,16 @@ class MockRemote(object): r_log.close() def do_createrepo(self): - base_url = "/".join([self.results_base_url, self.job.project_owner, + base_url = "/".join([self.opts.results_base_url, self.job.project_owner, self.job.project_name, self.job.chroot]) self.callback.log("Createrepo:: owner: {}; project: {}; " "front url: {}; path: {}; base_url: {}" .format(self.job.project_owner, self.job.project_name, - self.front_url, self.chroot_dir, base_url)) + self.opts.frontend_base_url, self.chroot_dir, base_url)) _, _, err = createrepo( path=self.chroot_dir, - front_url=self.front_url, + front_url=self.opts.frontend_base_url, base_url=base_url, username=self.job.project_owner, projectname=self.job.project_name, @@ -251,7 +260,7 @@ class MockRemote(object): def on_success_build(self): self.callback.log("Success building {0}".format(os.path.basename(self.pkg))) - if self.do_sign: + if self.opts.do_sign: self.sign_built_packages() # self.built_packages.append(self.pkg) @@ -271,6 +280,7 @@ class MockRemote(object): def build_pkg_and_process_results(self): self.prepare_build_dir() + self.mark_dir_with_build_id() # building self.callback.start_build(self.pkg) @@ -279,8 +289,8 @@ class MockRemote(object): # downloading self.callback.start_download(self.pkg) - d_ret, d_out, d_err = self.builder.download(self.pkg, self.chroot_dir) - if not d_ret: + d_success, d_out, d_err = self.builder.download(self.pkg, self.chroot_dir) + if not d_success: raise MockRemoteError( "Failure to download {0}: {1}".format(self.pkg, d_out + d_err)) @@ -288,7 +298,6 @@ class MockRemote(object): self.log_to_file_safe(os.path.join(self.chroot_dir, "mockchain.log"), ["\n\n{0}\n\n".format(self.pkg), b_out], [b_err]) - self.mark_dir_with_build_id() if b_status: self.on_success_build() return build_details diff --git a/backend/backend/mockremote/builder.py b/backend/backend/mockremote/builder.py index 642a01b..228e75f 100644 --- a/backend/backend/mockremote/builder.py +++ b/backend/backend/mockremote/builder.py @@ -33,11 +33,12 @@ class Builder(object): self._remote_tempdir = remote_tempdir self._remote_basedir = remote_basedir # if we're at this point we've connected and done stuff on the host - # TODO: maybe do this lazy ? self.conn = _create_ans_conn( self.hostname, self.username, self.timeout) self.root_conn = _create_ans_conn(self.hostname, "root", self.timeout) + # self.callback.log("Created builder: {}".format(self.__dict__)) + # Before use: check out the host - make sure it can build/be contacted/etc # self.check() diff --git a/backend/backend/mockremote/callback.py b/backend/backend/mockremote/callback.py index 1fdfc5f..0dfbfc9 100644 --- a/backend/backend/mockremote/callback.py +++ b/backend/backend/mockremote/callback.py @@ -30,7 +30,7 @@ class DefaultCallBack(object): class CliLogCallBack(DefaultCallBack): def __init__(self, **kwargs): - DefaultCallBack.__init__(self, **kwargs) + super(CliLogCallBack, self).__init__(**kwargs) def start_build(self, pkg): msg = "Start build: {0}".format(pkg) diff --git a/backend/run/copr_mockremote.py b/backend/run/copr_mockremote.py index ebbcda6..e4c4d1f 100644 --- a/backend/run/copr_mockremote.py +++ b/backend/run/copr_mockremote.py @@ -7,7 +7,7 @@ import sys import os from backend.exceptions import MockRemoteError -from backend.mockremote import MockRemote, DEF_CHROOT, DEF_USER, DEF_REPOS, DEF_TIMEOUT, DEF_DESTDIR +from backend.mockremote import MockRemote, DEF_CHROOT, DEF_BUILD_USER, DEF_REPOS, DEF_TIMEOUT, DEF_DESTDIR from backend.helpers import SortedOptParser from backend.mockremote.callback import CliLogCallBack @@ -45,7 +45,7 @@ def parse_args(args): " defaults to not logging") parser.add_option("-b", "--builder", dest="builder", default=None, help="builder to use") - parser.add_option("-u", dest="user", default=DEF_USER, + parser.add_option("-u", dest="user", default=DEF_BUILD_USER, help="user to run as/connect as on builder systems") parser.add_option("-t", "--timeout", dest="timeout", type="int", default=DEF_TIMEOUT, @@ -129,8 +129,8 @@ def main(args): job=job, builder_host=opts.builder, user=opts.user, - cont=opts.cont, - recurse=opts.recurse, + # cont=opts.cont, + # recurse=opts.recurse, repos=opts.repos, do_sign=opts.do_sign, callback=callback, diff --git a/backend/tests/mockremote/test_builder.py b/backend/tests/mockremote/test_builder.py index c4da674..519c022 100644 --- a/backend/tests/mockremote/test_builder.py +++ b/backend/tests/mockremote/test_builder.py @@ -34,7 +34,6 @@ def noop(*args, **kwargs): pass - def print_all(*args, **kwargs): pprint(args) pprint(kwargs) diff --git a/backend/tests/mockremote/test_mockremote.py b/backend/tests/mockremote/test_mockremote.py index e69de29..b526d27 100644 --- a/backend/tests/mockremote/test_mockremote.py +++ b/backend/tests/mockremote/test_mockremote.py @@ -0,0 +1,317 @@ +# coding: utf-8 +import copy + +from collections import defaultdict +from pprint import pprint +from bunch import Bunch +from backend.exceptions import BuilderError, BuilderTimeOutError, MockRemoteError, CoprSignError + +import tempfile +import shutil +import os + +import six + +if six.PY3: + from unittest import mock + from unittest.mock import patch, MagicMock +else: + import mock + from mock import patch, MagicMock + +import pytest + +import backend.mockremote.builder as builder_module +from backend.mockremote.builder import Builder +from backend.mockremote import MockRemote, get_target_dir +from backend.mockremote.callback import DefaultCallBack +from backend.job import BuildJob + + +def test_get_target_dir(): + for ch_dir, pkg_name, expected in [ + ("/tmp/", "copr-backend-1.46-1.git.35.8ba18d1.fc20.src.rpm", + "/tmp/copr-backend-1.46-1.git.35.8ba18d1.fc20"), + ("/tmp/foo/../bar", "copr-backend-1.46-1.git.35.8ba18d1.fc20.src.rpm", + "/tmp/bar/copr-backend-1.46-1.git.35.8ba18d1.fc20"), + ]: + assert get_target_dir(ch_dir, pkg_name) == expected + +STDOUT = "stdout" +STDERR = "stderr" +COPR_OWNER = "copr_owner" +COPR_NAME = "copr_name" +COPR_VENDOR = "vendor" +# DESTDIR = "/dev/null" +# CHROOT = + +class CallbackTest(DefaultCallBack): + def __init__(self, log_dict, **kwargs): + """ + :param log_dict: defaultdict(list) + :return: + """ + super(CallbackTest, self).__init__(**kwargs) + self.log_dict = log_dict + + def __getattribute__(self, item): + if item in ["log", "error", "start_build", "end_build", + "start_download", "end_download"]: + def fn(msg): + self.log_dict[item].append(msg) + return fn + else: + return super(CallbackTest, self).__getattribute__(item) + + +class TestMockRemote(object): + + + + def get_callback(self): + self._cb_log = defaultdict(list) + return CallbackTest(log_dict=self._cb_log) + + @pytest.yield_fixture + def f_mock_remote(self): + self.mr_cb = self.get_callback() + patcher = mock.patch("backend.mockremote.Builder") + self.mc_builder = patcher.start() + + + + + self.mr = MockRemote(self.HOST, self.JOB, callback=self.mr_cb, + macros=self.MACROS, opts=self.OPTS) + + yield + patcher.stop() + + def setup_method(self, method): + self.test_root_path = tempfile.mkdtemp() + self.CHROOT = "fedora-20_i386" + self.DESTDIR = os.path.join(self.test_root_path, COPR_OWNER, COPR_NAME) + self.DESTDIR_CHROOT = os.path.join(self.DESTDIR, self.CHROOT) + self.FRONT_URL = "
htt://front.example.com
" + self.BASE_URL = "
http://example.com/results
" + + self.PKG_NAME = "foobar" + self.PKG_VERSION = "1.2.3" + + + self.HOST = "127.0.0.1" + self.SRC_PKG_URL = "
http://example.com/{}-{}.src.rpm
".format(self.PKG_NAME, self.PKG_VERSION) + self.JOB = BuildJob({ + "project_owner": COPR_OWNER, + "project_name": COPR_NAME, + "pkgs": self.SRC_PKG_URL, + "repos": "", + "build_id": 12345, + "chroot": self.CHROOT, + }, Bunch({ + "timeout": 1800, + "destdir": self.test_root_path, + "results_baseurl": "/tmp/", + })) + self.MACROS = { + "copr_username": COPR_OWNER, + "copr_projectname": COPR_NAME, + "vendor": COPR_VENDOR + } + self.OPTS = Bunch({ + "do_sign": False, + "results_base_url": self.BASE_URL, + "frontend_base_url": self.FRONT_URL, + }) + + def teardown_method(self, method): + shutil.rmtree(self.test_root_path) + + def test_dummy(self, f_mock_remote): + pass + + def test_default_callback(self, f_mock_remote, capsys): + mr_2 = MockRemote(self.HOST, self.JOB, macros=self.MACROS, opts=self.OPTS) + out, err = capsys.readouterr() + + assert isinstance(mr_2.callback, DefaultCallBack) + assert out + + def test_no_job_chroot(self, f_mock_remote, capsys): + job_2 = copy.deepcopy(self.JOB) + job_2.chroot = None + with pytest.raises(MockRemoteError): + mr_2 = MockRemote(self.HOST, job_2, macros=self.MACROS, opts=self.OPTS) + + out, err = capsys.readouterr() + + @mock.patch("backend.mockremote.get_pubkey") + def test_add_pubkey(self, mc_get_pubkey, f_mock_remote): + self.mr.add_pubkey() + assert mc_get_pubkey.called + expected_path = os.path.join(self.DESTDIR, self.CHROOT, "pubkey.gpg") + assert mc_get_pubkey.call_args == mock.call( + COPR_OWNER, COPR_NAME, expected_path) + + @mock.patch("backend.mockremote.get_pubkey") + def test_add_pubkey_on_exception(self, mc_get_pubkey, f_mock_remote): + mc_get_pubkey.side_effect = CoprSignError("foobar") + self.mr.add_pubkey() + assert self._cb_log["error"][-1] + assert "failed to retrieve" in self._cb_log["error"][-1] + + @mock.patch("backend.mockremote.sign_rpms_in_dir") + def test_sign_built_packages(self, mc_sign_rpms_in_dir, f_mock_remote): + self.mr.sign_built_packages() + assert mc_sign_rpms_in_dir.called + + @mock.patch("backend.mockremote.sign_rpms_in_dir") + def test_sign_built_packages_exception(self, mc_sign_rpms_in_dir, f_mock_remote): + mc_sign_rpms_in_dir.side_effect = IOError() + self.mr.sign_built_packages() + assert self._cb_log["error"][-1] + assert "failed to sign packages" in self._cb_log["error"][-1] + + @mock.patch("backend.mockremote.sign_rpms_in_dir") + def test_sign_built_packages_exception_reraise(self, mc_sign_rpms_in_dir, f_mock_remote): + mc_sign_rpms_in_dir.side_effect = MockRemoteError("foobar") + with pytest.raises(MockRemoteError): + self.mr.sign_built_packages() + + @mock.patch("backend.mockremote.createrepo") + def test_do_createrepo(self, mc_createrepo, f_mock_remote): + mc_createrepo.return_value = ("", "", "") + self.mr.do_createrepo() + assert mc_createrepo.called + expected_call = mock.call( + path=os.path.join(self.DESTDIR, self.CHROOT), + front_url=self.FRONT_URL, + base_url=u"/".join([self.BASE_URL, COPR_OWNER, COPR_NAME, self.CHROOT]), + username=COPR_OWNER, + projectname=COPR_NAME, + lock=None, + ) + assert mc_createrepo.call_args == expected_call + + @mock.patch("backend.mockremote.createrepo") + def test_do_createrepo_on_error(self, mc_createrepo, f_mock_remote): + err_msg = "error occured" + mc_createrepo.return_value = ("", "", err_msg) + self.mr.do_createrepo() + assert len(self._cb_log["error"]) > 0 + assert "Error making local repo" in self._cb_log["error"][0] + assert err_msg in self._cb_log["error"][1] + + def test_on_success_build(self, f_mock_remote): + self.mr.sign_built_packages = MagicMock() + self.mr.do_createrepo = MagicMock() + + self.mr.opts.do_sign = False + self.mr.on_success_build() + + assert not self.mr.sign_built_packages.called + assert self.mr.do_createrepo.called + + self.mr.do_createrepo.reset() + + self.mr.opts.do_sign = True + self.mr.on_success_build() + + assert self.mr.sign_built_packages.called + assert self.mr.do_createrepo.called + + def test_prepare_build_dir_erase_fail_file(self, f_mock_remote): + target_dir = os.path.join( + self.DESTDIR, self.CHROOT, + "{}-{}".format(self.PKG_NAME, self.PKG_VERSION)) + os.makedirs(target_dir) + fail_path = os.path.join(target_dir, "fail") + with open(fail_path, "w") as handle: + handle.write("1") + assert os.path.exists(fail_path) + + self.mr.prepare_build_dir() + + assert not os.path.exists(fail_path) + + def test_prepare_build_dir_creates_dirs(self, f_mock_remote): + self.mr.prepare_build_dir() + assert os.path.exists(self.DESTDIR_CHROOT) + + def test_build_pkg_and_process_results(self, f_mock_remote): + self.mr.on_success_build = MagicMock() + self.mr.mark_dir_with_build_id = MagicMock() + + build_details = MagicMock() + self.mr.builder.build.return_value = (True, STDOUT, STDERR, build_details) + self.mr.builder.download.return_value = (True, STDOUT, STDERR) + + result = self.mr.build_pkg_and_process_results() + + assert id(result) == id(build_details) + + assert self.mr.builder.build.called + assert self.mr.builder.build.call_args == mock.call(self.SRC_PKG_URL) + + assert self.mr.builder.download.called + assert self.mr.builder.download.call_args == \ + mock.call(self.SRC_PKG_URL, self.DESTDIR_CHROOT) + + assert self.mr.mark_dir_with_build_id.called + assert self.mr.on_success_build.called + + assert os.path.exists(os.path.join(self.DESTDIR_CHROOT, "mockchain.log")) + + def test_build_pkg_and_process_results_error_on_download(self, f_mock_remote): + self.mr.builder.build.return_value = (True, STDOUT, STDERR, {}) + self.mr.builder.download.return_value = (False, STDOUT, STDERR) + + self.mr.mark_dir_with_build_id = MagicMock() + self.mr.on_success_build = MagicMock() + with pytest.raises(MockRemoteError): + self.mr.build_pkg_and_process_results() + + assert not self.mr.on_success_build.called + assert self.mr.mark_dir_with_build_id.called + assert not os.path.exists(os.path.join(self.DESTDIR_CHROOT, "mockchain.log")) + + def test_build_pkg_and_process_results_error_on_build(self, f_mock_remote): + self.mr.builder.build.return_value = (False, STDOUT, STDERR, {}) + self.mr.builder.download.return_value = (True, STDOUT, STDERR) + + self.mr.mark_dir_with_build_id = MagicMock() + self.mr.on_success_build = MagicMock() + with pytest.raises(MockRemoteError): + self.mr.build_pkg_and_process_results() + + assert not self.mr.on_success_build.called + assert self.mr.mark_dir_with_build_id.called + + assert os.path.exists(os.path.join(self.DESTDIR_CHROOT, "mockchain.log")) + + def test_mark_dir_with_build_id(self, f_mock_remote): + # TODO: create real test + self.mr.mark_dir_with_build_id() + + def test_build_pkgs_ignore_error_once(self, f_mock_remote): + err_msg = "error message" + build_details = MagicMock() + + self.mr.build_pkg_and_process_results = MagicMock() + self.mr.build_pkg_and_process_results.side_effect = [ + MockRemoteError(err_msg), build_details] + + result = self.mr.build_pkgs() + assert result == build_details + assert err_msg in self._cb_log["error"][-1] + + def test_build_pkgs_exceed_retry(self, f_mock_remote): + err_msg = "error message" + + self.mr.build_pkg_and_process_results = MagicMock() + self.mr.build_pkg_and_process_results.side_effect = MockRemoteError(err_msg) + + with pytest.raises(MockRemoteError): + self.mr.build_pkgs() + +
1
0
0
0
[copr] master: [backend] unittest for backend.mockremote.builder (ddd5404)
by vgologuz@fedoraproject.org
20 Nov '14
20 Nov '14
Repository :
http://git.fedorahosted.org/cgit/copr.git
On branch : master >--------------------------------------------------------------- commit ddd5404b3f2746a78e5f654037a41925d7bbb470 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Tue Nov 18 16:11:28 2014 +0100 [backend] unittest for backend.mockremote.builder >--------------------------------------------------------------- backend/backend/mockremote/__init__.py | 1 + backend/backend/mockremote/builder.py | 41 +- backend/tests/mockremote/test_builder.py | 753 +++++++++++++++++++++ backend/tests/{ => mockremote}/test_mockremote.py | 0 4 files changed, 774 insertions(+), 21 deletions(-) diff --git a/backend/backend/mockremote/__init__.py b/backend/backend/mockremote/__init__.py index 4c60b25..5828a1f 100755 --- a/backend/backend/mockremote/__init__.py +++ b/backend/backend/mockremote/__init__.py @@ -130,6 +130,7 @@ class MockRemote(object): remote_basedir=self.remote_basedir, remote_tempdir=self.remote_tempdir, macros=self.macros, repos=self.repos) + self.builder.check() if not self.job.chroot: raise MockRemoteError("No chroot specified!") diff --git a/backend/backend/mockremote/builder.py b/backend/backend/mockremote/builder.py index fcdd0ee..642a01b 100644 --- a/backend/backend/mockremote/builder.py +++ b/backend/backend/mockremote/builder.py @@ -1,9 +1,10 @@ import os import pipes import socket -import subprocess +from subprocess import Popen, PIPE import time -import ansible.runner + +from ansible.runner import Runner from ..exceptions import BuilderError, BuilderTimeOutError @@ -26,20 +27,19 @@ class Builder(object): self.macros = macros or {} # rename macros to mock_ext_options self.callback = callback - if buildroot_pkgs is None: - self.buildroot_pkgs = "" - else: - self.buildroot_pkgs = buildroot_pkgs + self.buildroot_pkgs = buildroot_pkgs or "" self.checked = False self._remote_tempdir = remote_tempdir self._remote_basedir = remote_basedir # if we're at this point we've connected and done stuff on the host + # TODO: maybe do this lazy ? self.conn = _create_ans_conn( self.hostname, self.username, self.timeout) self.root_conn = _create_ans_conn(self.hostname, "root", self.timeout) - # check out the host - make sure it can build/be contacted/etc - self.check() + + # Before use: check out the host - make sure it can build/be contacted/etc + # self.check() @property def remote_build_dir(self): @@ -56,6 +56,7 @@ class Builder(object): results = self._run_ansible(create_tmpdir_cmd) tempdir = None + # TODO: use check_for_ans_error for _, resdict in results["contacted"].items(): tempdir = resdict["stdout"] @@ -149,7 +150,7 @@ class Builder(object): build_details["built_packages"] = list(results["contacted"].values())[0][u"stdout"] self.callback.log("Packages:\n{}".format(build_details["built_packages"])) - def check_build_success(self, is_err, pkg, results): + def check_build_success(self, pkg, results): myresults = get_ans_results(results, self.hostname) out = myresults.get("stdout", "") err = myresults.get("stderr", "") @@ -184,7 +185,7 @@ class Builder(object): def get_package_version(self, pkg): self.callback.log("Getting package information: version") - results = self._run_ansible("rpm -qp --qf \"%{{VERSION}}\n\" {}".format(pkg)) + results = self._run_ansible("rpm -qp --qf \"%{{VERSION}}\" {}".format(pkg)) if "contacted" in results: # TODO: do more sane return list(results["contacted"].values())[0][u"stdout"] @@ -269,7 +270,7 @@ class Builder(object): # we know the command ended successfully but not if the pkg built # successfully - err, is_err, out = self.check_build_success(is_err, pkg, results) + err, is_err, out = self.check_build_success(pkg, results) if not is_err: success = True self.collect_built_packages(build_details, pkg) @@ -280,7 +281,6 @@ class Builder(object): # download the pkg to destdir using rsync + ssh # return success/failure, stdout, stderr - success = False rpd = self._get_remote_pkg_dir(pkg) # make spaces work w/our rsync command below :( destdir = "'" + destdir.replace("'", "'\\''") + "'" @@ -290,8 +290,7 @@ class Builder(object): command = "{0} -avH -e {1} {2} {3}/".format( rsync, ssh_opts, remote_src, destdir) - cmd = subprocess.Popen(command, shell=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + cmd = Popen(command, shell=True, stdout=PIPE, stderr=PIPE) # rsync results into opts.destdir out, err = cmd.communicate() @@ -345,7 +344,7 @@ class Builder(object): if is_err: if "rc" in err_results: errors.append( - "Warning: {0} lacks mockchain on chroot {1}".format( + "Warning: {0} lacks mockchain binary or mock config for chroot {1}".format( self.hostname, self.chroot)) else: errors.append(err_results["msg"]) @@ -358,12 +357,12 @@ class Builder(object): def _create_ans_conn(hostname, username, timeout): - ans_conn = ansible.runner.Runner(remote_user=username, - host_list=hostname + ",", - pattern=hostname, - forks=1, - transport="ssh", - timeout=timeout) + ans_conn = Runner(remote_user=username, + host_list=hostname + ",", + pattern=hostname, + forks=1, + transport="ssh", + timeout=timeout) return ans_conn diff --git a/backend/tests/mockremote/test_builder.py b/backend/tests/mockremote/test_builder.py new file mode 100644 index 0000000..c4da674 --- /dev/null +++ b/backend/tests/mockremote/test_builder.py @@ -0,0 +1,753 @@ +# coding: utf-8 +import copy + +from collections import defaultdict +from pprint import pprint +from backend.exceptions import BuilderError, BuilderTimeOutError + +import tempfile +import shutil +import os + +import six + +if six.PY3: + from unittest import mock + from unittest.mock import patch, MagickMock +else: + import mock + from mock import patch, MagicMock + +import pytest + +import backend.mockremote.builder as builder_module +from backend.mockremote.builder import Builder + +# @pytest.yield_fixture +# def mc_ansible_runner(): +# patcher = mock.patch("backend.mockremote.builder.Runner") +# yield patcher.start() +# patcher.stop() + + +def noop(*args, **kwargs): + pass + + + +def print_all(*args, **kwargs): + pprint(args) + pprint(kwargs) + + +class TestBuilder(object): + BUILDER_BUILDROOT_PKGS = [] + BUILDER_CHROOT = "fedora-20-i386" + BUILDER_TIMEOUT = 1024 + BUILDER_HOSTNAME = "
example.com
" + BUILDER_USER = "copr_builder" + BUILDER_REMOTE_BASEDIR = "/tmp/copr-backend-test" + BUILDER_REMOTE_TMPDIR = "/tmp/copr-backend-test-tmp" + BUILDER_PKG_BASE = "foovar-2.41.f21" + BUILDER_PKG = "
http://example.com/foovar-2.41.f21.src.rpm
" + + STDOUT = "stdout" + STDERR = "stderr" + + def get_test_builder(self): + builder = Builder( + hostname=self.BUILDER_HOSTNAME, + username=self.BUILDER_USER, + timeout=self.BUILDER_TIMEOUT, + chroot=self.BUILDER_CHROOT, + buildroot_pkgs=self.BUILDER_BUILDROOT_PKGS, + remote_basedir=self.BUILDER_REMOTE_BASEDIR, + remote_tempdir=self.BUILDER_REMOTE_TMPDIR, + callback=self.mc_callback, + ) + builder.checked = True + return builder + + def setup_method(self, method): + self.mc_ansible_runner_patcher = mock.patch("backend.mockremote.builder.Runner") + self.mc_ansible_runner = self.mc_ansible_runner_patcher.start() + self.mc_ansible_runner.side_effect = lambda **kwargs: mock.MagicMock(**kwargs) + + self.test_root_path = tempfile.mkdtemp() + #print("Temp dir: {}".format(self.test_root_path)) + + self.mc_callback = mock.MagicMock() + self._cb_log = [] + self.mc_callback.log.side_effect = lambda msg: self._cb_log.append(msg) + + self.stage = 0 + self.stage_ctx = defaultdict(dict) + + @property + def buildcmd(self): + return self.gen_mockchain_command(self.BUILDER_PKG) + # + def teardown_method(self, method): + self.mc_ansible_runner_patcher.stop() + # remote tmp dir + + if os.path.exists(self.test_root_path): + shutil.rmtree(self.test_root_path) + + def test_constructor(self): + assert not self.mc_ansible_runner.called + builder = self.get_test_builder() + assert self.mc_ansible_runner.called + + assert builder.conn.remote_user == self.BUILDER_USER + assert builder.root_conn.remote_user == "root" + + def test_get_remote_pkg_dir(self): + builder = self.get_test_builder() + for pkg_name, pkg_fixed in [ + ("copr-backend-1.46-1.git.33.5733924.fc20.src.rpm", "copr-backend-1.46-1.git.33.5733924.fc20"), + ("/tmp/kf5-baloo-5.1.0.1-1.fc20.src.rpm", "kf5-baloo-5.1.0.1-1.fc20") + ]: + expected = "/".join([self.BUILDER_REMOTE_TMPDIR, "build", "results", + self.BUILDER_CHROOT, pkg_fixed]) + assert builder._get_remote_pkg_dir(pkg_name) == expected + + def test_run_ansible(self): + builder = self.get_test_builder() + ans_cmd = "foo bar" + + for conn, as_root in [(builder.conn, False), (builder.root_conn, True)]: + for module_name in [None, "foo", "copy"]: + run_count = conn.run.call_count + builder._run_ansible(ans_cmd, as_root=as_root) + assert conn.run.call_count == run_count + 1 + assert conn.module_args == ans_cmd + assert conn.module_name == module_name or "shell" + + def test_check_for_ans_answer(self): + """ + Silly test. Ansible api has almost no documentation, + so we can only cover some return patterns :( + + """ + tested_func = builder_module.check_for_ans_error + + cases = [ + { + "args": [ + { + "dark": {}, + "contacted": {} + }, self.BUILDER_HOSTNAME + ], + "kwargs": {}, + "expected_return": (False, {}), + "expected_exception": None + }, + { + "args": [ + { + "dark": {self.BUILDER_HOSTNAME: ""}, + "contacted": {} + }, self.BUILDER_HOSTNAME + ], + "kwargs": {}, + "expected_return": ( + True, + {"msg": "Error: Could not contact/connect to {}.".format(self.BUILDER_HOSTNAME)}), + "expected_exception": None + }, + { + "args": [ + { + "dark": {}, + "contacted": {self.BUILDER_HOSTNAME: { + "rc": 0, + "stdout": "stdout", + "stderr": "stderr", + "stdother": "stdother", + }} + }, self.BUILDER_HOSTNAME + ], + "kwargs": {}, + "expected_return": (False, {"rc": 0}), + "expected_exception": None + }, + { + "args": [ + { + "dark": {}, + "contacted": {self.BUILDER_HOSTNAME: { + "rc": 1, + "stdout": "stdout", + "stderr": "stderr", + "stdother": "stdother", + }} + }, self.BUILDER_HOSTNAME + ], + "kwargs": {}, + "expected_return": ( + True, { + "rc": 1, + "msg": "rc 1 not in success_codes", + "stdout": "stdout", + "stderr": "stderr", + + }), + "expected_exception": None + }, + { + "args": [ + { + "dark": {}, + "contacted": {self.BUILDER_HOSTNAME: { + "rc": 1, + "stdout": "stdout", + "stderr": "stderr", + "stdother": "stdother", + + }} + }, self.BUILDER_HOSTNAME + ], + "kwargs": {"success_codes": [0, 1]}, + "expected_return": (False, {"rc": 1}), + "expected_exception": None + }, + { + "args": [ + { + "dark": {}, + "contacted": {self.BUILDER_HOSTNAME: { + "rc": 2, + "stdout": "stdout", + "stderr": "stderr", + "stdother": "stdother", + }} + }, self.BUILDER_HOSTNAME + ], + "kwargs": {"err_codes": [2, 3], "return_on_error": ["stdother"]}, + "expected_return": (True, {"rc": 2, "msg": "rc 2 matched err_codes", + "stdother": "stdother"}), + "expected_exception": None + }, + { + "args": [ + { + "dark": {}, + "contacted": {self.BUILDER_HOSTNAME: { + "failed": True, + "stdout": "stdout", + "stderr": "stderr", + "stdother": "stdother", + }} + }, self.BUILDER_HOSTNAME + ], + "kwargs": {}, + "expected_return": (True, {"msg": "results included failed as true", + "stdout": "stdout", "stderr": "stderr"}), + "expected_exception": None + } + ] + for case in cases: + if case["expected_exception"]: + with pytest.raises(case["expected_exception"]): + tested_func(*case["args"], **case["kwargs"]) + else: + result = tested_func(*case["args"], **case["kwargs"]) + assert result == case["expected_return"] + + def test_get_ans_results(self): + result_obj = "RESULT_STRING" + results = {"dark": {self.BUILDER_HOSTNAME: result_obj}, "contacted": {}} + assert result_obj == builder_module.get_ans_results(results, self.BUILDER_HOSTNAME) + + results = {"contacted": {self.BUILDER_HOSTNAME: result_obj}, "dark": {}} + assert result_obj == builder_module.get_ans_results(results, self.BUILDER_HOSTNAME) + + results = {"contacted": {self.BUILDER_HOSTNAME: "wrong_obj"}, + "dark": {self.BUILDER_HOSTNAME: result_obj}} + assert result_obj == builder_module.get_ans_results(results, self.BUILDER_HOSTNAME) + + results = {"contacted": {}, "dark": {}} + assert {} == builder_module.get_ans_results(results, self.BUILDER_HOSTNAME) + + def test_check_not_repeated(self): + builder = self.get_test_builder() + assert builder.check() == (True, []) + assert not builder.conn.called + assert not builder.root_conn.called + + def test_check_hostname_check(self): + builder = self.get_test_builder() + for name in ["*", "256.0.0.1"]: + with pytest.raises(BuilderError): + builder.checked = False + builder.hostname = name + builder.check() + + def test_check_missing_required_binaries(self): + builder = self.get_test_builder() + + def ans_run(): + self.stage_ctx[self.stage]["conn"] = copy.deepcopy(builder.conn) + ret_map = { + 0: {"contacted": {self.BUILDER_HOSTNAME: {"rc": 1, }}}, + 1: {"contacted": {self.BUILDER_HOSTNAME: {"rc": 0, }}} + } + self.stage += 1 + return ret_map[self.stage - 1] + + builder.conn.run.side_effect = ans_run + with pytest.raises(BuilderError) as err: + builder.checked = False + builder.check() + + # import ipdb; ipdb.set_trace() + assert "/bin/rpm -q mock rsync" in self.stage_ctx[0]["conn"].module_args + + assert "does not have mock or rsync installed" in err.value.msg + + def test_check_missing_mockchain_or_mock_config(self): + builder = self.get_test_builder() + + def ans_run(): + self.stage_ctx[self.stage]["conn"] = copy.deepcopy(builder.conn) + ret_map = { + 0: {"contacted": {self.BUILDER_HOSTNAME: {"rc": 0, }}}, + 1: {"contacted": {self.BUILDER_HOSTNAME: {"rc": 1, }}} + } + self.stage += 1 + return ret_map[self.stage - 1] + + builder.conn.run.side_effect = ans_run + with pytest.raises(BuilderError) as err: + builder.checked = False + builder.check() + #print(builder.conn.module_args) + #assert "usr/bin/test -f " in builder.conn.module_args + + assert "/usr/bin/test -f /usr/bin/mockchain" in self.stage_ctx[1]["conn"].module_args + assert "/usr/bin/test -f /etc/mock/{}.cfg".format(self.BUILDER_CHROOT) in \ + self.stage_ctx[1]["conn"].module_args + + assert "lacks mockchain binary or mock config for chroot " in \ + err.value.msg + + def test_check_missing_alt_err_messages(self): + builder = self.get_test_builder() + + def ans_run(): + self.stage_ctx[self.stage]["conn"] = copy.deepcopy(builder.conn) + ret_map = { + 0: {"contacted": {self.BUILDER_HOSTNAME: {"failed": "fatal_1", }}}, + 1: {"contacted": {self.BUILDER_HOSTNAME: {"failed": "fatal_2", }}} + } + self.stage += 1 + return ret_map[self.stage - 1] + + builder.conn.run.side_effect = ans_run + with pytest.raises(BuilderError) as err: + builder.checked = False + builder.check() + #print(builder.conn.module_args) + #assert "usr/bin/test -f " in builder.conn.module_args + + assert "/usr/bin/test -f /usr/bin/mockchain" in self.stage_ctx[1]["conn"].module_args + assert "/usr/bin/test -f /etc/mock/{}.cfg".format(self.BUILDER_CHROOT) in \ + self.stage_ctx[1]["conn"].module_args + + assert "results included failed as true" in err.value.msg + + def test_check_set_checked(self): + builder = self.get_test_builder() + + builder.conn.run.return_value = {"contacted": {self.BUILDER_HOSTNAME: {"rc": 0, }}} + builder.checked = False + builder.check() + assert builder.checked + + def test_tempdir_nop_when_provided(self): + builder = self.get_test_builder() + assert builder.tempdir == self.BUILDER_REMOTE_TMPDIR + assert not builder.conn.run.called + + def test_tempdir_failed_to_create(self): + builder = self.get_test_builder() + builder._remote_tempdir = None + + builder.conn.run.return_value = {"contacted": { + self.BUILDER_HOSTNAME: {"failed": "fatal_1", "stdout": None}}} + + with pytest.raises(BuilderError) as err: + x = builder.tempdir + + def test_tempdir_correct_creation(self): + builder = self.get_test_builder() + builder._remote_tempdir = None + + new_tmp_dir = "/tmp/new/" + + def ans_run(): + self.stage_ctx[self.stage]["conn"] = copy.deepcopy(builder.conn) + ret_map = { + 0: {"contacted": { + self.BUILDER_HOSTNAME: {"rc": 0, "stdout": new_tmp_dir}}}, + 1: {"contacted": { + self.BUILDER_HOSTNAME: {"rc": 0}}}, + } + self.stage += 1 + return ret_map[self.stage - 1] + + builder.conn.run.side_effect = ans_run + x = builder.tempdir + assert x == new_tmp_dir + assert "/bin/mktemp -d {0}".format(self.BUILDER_REMOTE_BASEDIR) in \ + self.stage_ctx[0]["conn"].module_args + + assert "/bin/chmod 755 {}".format(new_tmp_dir) in \ + self.stage_ctx[1]["conn"].module_args + + def test_tempdir_setter(self): + builder = self.get_test_builder() + builder._remote_tempdir = None + new_tmp_dir = "/tmp/new/" + builder.tempdir = new_tmp_dir + assert builder.tempdir == new_tmp_dir + + def test_modify_base_buildroot_malicious_vars(self): + builder = self.get_test_builder() + + for bad_pkg in [ + "../'HOME-example.src.pkg", + # i'm assuming that this also should cause error + #"~HOM/E-example.src.pkg; rm -rf", + #"../%HOME-example.src.pkg", + #"../%HOME-example.src.pkg" + + ]: + with pytest.raises(BuilderError) as err: + builder.buildroot_pkgs = bad_pkg + builder.modify_base_buildroot() + + def test_modify_base_buildroot_on_error(self, ): + builder = self.get_test_builder() + br_pkgs = "foo bar" + builder.buildroot_pkgs = br_pkgs + builder.root_conn.run.return_value = { + "contacted": {self.BUILDER_HOSTNAME: {"rc": 1, "stdout": None}}, + "dark": {} + } + + builder.modify_base_buildroot() + + assert "putting {} into minimal buildroot of fedora-20-i386".format(br_pkgs) in self._cb_log[0] + + assert not builder.conn.run.called + assert builder.root_conn.run.called + + expected = ( + "dest=/etc/mock/{}.cfg " + "line=\"config_opts['chroot_setup_cmd'] = 'install @buildsys-build {}'\" " + "regexp=\"^.*chroot_setup_cmd.*$\"" + ).format(self.BUILDER_CHROOT, br_pkgs) + assert builder.root_conn.module_args == expected + + assert len(self._cb_log) == 3 + assert "Error: " in self._cb_log[1] + + def test_modify_base_buildroot(self, ): + builder = self.get_test_builder() + br_pkgs = "foo bar" + builder.buildroot_pkgs = br_pkgs + builder.root_conn.run.return_value = { + "contacted": {self.BUILDER_HOSTNAME: {"rc": 0, "stdout": None}}, + "dark": {} + } + + builder.modify_base_buildroot() + + assert "putting {} into minimal buildroot of fedora-20-i386".format(br_pkgs) in self._cb_log[0] + + assert not builder.conn.run.called + assert builder.root_conn.run.called + + expected = ( + "dest=/etc/mock/{}.cfg " + "line=\"config_opts['chroot_setup_cmd'] = 'install @buildsys-build {}'\" " + "regexp=\"^.*chroot_setup_cmd.*$\"" + ).format(self.BUILDER_CHROOT, br_pkgs) + assert builder.root_conn.module_args == expected + + assert len(self._cb_log) == 1 + + def test_collect_build_packages(self): + builder = self.get_test_builder() + + + bd = {} + stdout = "stdout" + + builder.conn.run.return_value = { + "contacted": {self.BUILDER_HOSTNAME: {"rc": 0, "stdout": stdout}}, + "dark": {} + } + builder.collect_built_packages(bd, self.BUILDER_PKG) + expected = ( + "cd {} && " + "for f in `ls *.rpm |grep -v \"src.rpm$\"`; do" + " rpm -qp --qf \"%{{NAME}} %{{VERSION}}\n\" $f; " + "done".format(builder._get_remote_pkg_dir(self.BUILDER_PKG_BASE)) + ) + assert builder.conn.module_args == expected + assert len(self._cb_log) == 2 + + @mock.patch("backend.mockremote.builder.check_for_ans_error") + def test_check_build_success(self, mc_check_for_ans_errror): + builder = self.get_test_builder() + + mc_check_for_ans_errror.return_value = "is_err", "err_results" + + results = builder.check_build_success(self.BUILDER_PKG, { + "contacted": {self.BUILDER_HOSTNAME: {"rc": 0, + "stdout": "stdout", + "stderr": "stderr"}}, + "dark": {} + }) + + assert ("stderr", "is_err", "stdout") == results + + expected_ans_args = ( + "/usr/bin/test -f " + "/tmp/copr-backend-test-tmp/build/results/" + "{}/{}/success" + ).format(self.BUILDER_CHROOT, self.BUILDER_PKG_BASE) + assert expected_ans_args == builder.conn.module_args + + def test_check_if_pkg_local_or_http_local(self): + pkg_path = os.path.join(self.test_root_path, "{}.src.rpm".format(self.BUILDER_PKG_BASE)) + with open(pkg_path, "w") as handle: + handle.write("1") + + builder = self.get_test_builder() + dest = builder.check_if_pkg_local_or_http(pkg_path) + dest_expected = os.path.join(self.BUILDER_REMOTE_TMPDIR, "{}.src.rpm".format(self.BUILDER_PKG_BASE)) + assert dest == dest_expected + assert builder.conn.module_args == "src={} dest={}".format(pkg_path, dest_expected) + assert builder.conn.module_name == "copy" + + def test_check_if_pkg_local_or_http_remote(self): + builder = self.get_test_builder() + dest = builder.check_if_pkg_local_or_http(self.BUILDER_PKG) + + assert dest == self.BUILDER_PKG + + def test_get_package_version(self): + # TODO: pathc run_ansible with local cmd execution and put a real .src.rpm + builder = self.get_test_builder() + + builder.conn.run.return_value = { + "contacted": {self.BUILDER_HOSTNAME: {"rc": 0, + "stdout": "stdout", + "stderr": "stderr"}}, + "dark": {} + } + + assert "stdout" == builder.get_package_version(self.BUILDER_PKG_BASE + ".src.rpm") + assert builder.conn.module_args == "rpm -qp --qf \"%{{VERSION}}\" {}.src.rpm".format(self.BUILDER_PKG_BASE) + + def test_get_package_version_not_contacted(self): + # TODO: pathc run_ansible with local cmd execution and put a real .src.rpm + builder = self.get_test_builder() + + builder.conn.run.return_value = {} + + assert builder.get_package_version(self.BUILDER_PKG_BASE + ".src.rpm") is None + assert builder.conn.module_args == "rpm -qp --qf \"%{{VERSION}}\" {}.src.rpm".format(self.BUILDER_PKG_BASE) + + def test_get_mockchain_command(self): + builder = self.get_test_builder() + + builder.repos = [ + "
http://example.com/rhel7
", + "
http://example.com/fedora-20
; rm -rf", + "
http://example.com/fedora-$releasever
", + "
http://example.com/fedora-rawhide
", + ] + builder.macros = { + "foo": "bar", + "foo; rm -rf": "bar", + "foo2": "bar; rm -rf" + } + result_cmd = builder.gen_mockchain_command(self.BUILDER_PKG) + expected = ( + "/usr/bin/mockchain -r fedora-20-i386 -l /tmp/copr-backend-test-tmp/build/" + " -a
http://example.com/rhel7
-a '
http://example.com/fedora-20
; rm -rf' " + "-a '
http://example.com/fedora-$releasever
' -a
http://example.com/fedora-rawhide
" + "-m '--define=foo bar' -m '--define=foo; rm -rf bar' -m '--define=foo2 bar; rm -rf'" + "
http://example.com/foovar-2.41.f21.src.rpm
") + assert result_cmd == expected + + builder.chroot = "fedora-rawhide" + builder.repos = [ + "
http://example.com/rhel7
", + "
http://example.com/fedora-20
; rm -rf", + "
http://example.com/fedora-$releasever
", + "
http://example.com/fedora-rawhide
", + ] + builder.macros = { + "foo": "bar", + "foo; rm -rf": "bar", + "foo2": "bar; rm -rf" + } + result_cmd = builder.gen_mockchain_command(self.BUILDER_PKG) + expected = ( + "/usr/bin/mockchain -r fedora-rawhide -l /tmp/copr-backend-test-tmp/build/" + " -a
http://example.com/rhel7
-a '
http://example.com/fedora-20
; rm -rf' " + "-a
http://example.com/fedora-rawhide
-a
http://example.com/fedora-rawhide
" + "-m '--define=foo bar' -m '--define=foo; rm -rf bar' -m '--define=foo2 bar; rm -rf'" + "
http://example.com/foovar-2.41.f21.src.rpm
") + assert result_cmd == expected + + @mock.patch("backend.mockremote.builder.time") + def test_run_command_and_wait_timeout(self, mc_time): + build_cmd = "foo bar" + builder = self.get_test_builder() + + mc_poller = mock.MagicMock() + mc_poller.poll.return_value = {"contacted": {}, "dark": {}} + builder.conn.run_async.return_value = None, mc_poller + builder.timeout = 50 + + mc_time.return_value = None + + with pytest.raises(BuilderTimeOutError): + builder.run_command_and_wait(build_cmd) + + @mock.patch("backend.mockremote.builder.time") + def test_run_command_and_wait(self, mc_time): + build_cmd = "foo bar" + builder = self.get_test_builder() + + mc_poller = mock.MagicMock() + builder.conn.run_async.return_value = None, mc_poller + builder.timeout = 100 + + expected_result = {"contacted": {self.BUILDER_HOSTNAME: True}, "dark": {}} + + def poll(): + if self.stage < 2: + return {"contacted": {}, "dark": {}} + else: + return expected_result + + mc_poller.poll.side_effect = poll + + def incr_stage(*args, **kwargs): + self.stage += 1 + + mc_time.sleep.side_effect = incr_stage + builder.run_command_and_wait(build_cmd) + + @mock.patch("backend.mockremote.builder.Popen") + def test_download(self, mc_popen): + builder = self.get_test_builder() + + for ret_code, expected_success in [(0, True), (1, False), (23, False)]: + mc_cmd = mock.MagicMock() + mc_cmd.communicate.return_value = self.STDOUT, self.STDERR + mc_cmd.returncode = ret_code + mc_popen.return_value = mc_cmd + success, stdout, stderr = builder.download(self.BUILDER_PKG, self.BUILDER_REMOTE_BASEDIR) + + expected = ( + "/usr/bin/rsync -avH -e 'ssh -o PasswordAuthentication=no -o StrictHostKeyChecking=no'" + " copr_builder@example.com:/tmp/copr-backend-test-tmp/build/results/fedora-20-i386/foovar-2.41.f21 " + "'/tmp/copr-backend-test'/") + + assert mc_popen.call_args[0][0] == expected + assert expected_success == success + assert stdout == self.STDOUT + assert stderr == self.STDERR + + def test_build(self): + builder = self.get_test_builder() + builder.modify_base_buildroot = MagicMock() + builder.check_if_pkg_local_or_http = MagicMock() + builder.check_if_pkg_local_or_http.return_value = self.BUILDER_PKG + + builder.get_package_version = MagicMock() + builder.get_package_version.return_value = None + + builder.run_command_and_wait = MagicMock() + successful_wait_result = { + "contacted": {self.BUILDER_HOSTNAME: { + "rc": 0, "stdout": self.STDOUT, "stderr": self.STDERR + }}, + "dark": {} + } + builder.run_command_and_wait.return_value = successful_wait_result + + builder.check_build_success = MagicMock() + builder.check_build_success.return_value = (self.STDERR, False, self.STDOUT) + + builder.collect_built_packages = MagicMock() + + success, stdout, stderr, build_details = builder.build(self.BUILDER_PKG) + assert success + assert stdout == self.STDOUT + assert stderr == self.STDERR + assert builder.modify_base_buildroot.called + assert builder.check_if_pkg_local_or_http.called + assert builder.run_command_and_wait.called + assert builder.check_build_success.called + assert builder.collect_built_packages + assert build_details == {} + + # test providing version + builder.get_package_version.return_value = "srpm_version" + success, stdout, stderr, build_details = builder.build(self.BUILDER_PKG) + + assert "pkg_version" in build_details + assert build_details["pkg_version"] == "srpm_version" + + # test timeout handle + builder.run_command_and_wait.side_effect = BuilderTimeOutError("msg") + success, stdout, stderr, build_details = builder.build(self.BUILDER_PKG) + assert not success + assert stderr == "Timeout expired" + assert build_details["pkg_version"] == "srpm_version" + + # remove timeout + builder.run_command_and_wait.side_effect = None + success, stdout, stderr, build_details = builder.build(self.BUILDER_PKG) + assert success + + # error inside wait result + unsuccessful_wait_result = { + "contacted": {self.BUILDER_HOSTNAME: { + "rc": 1, "stdout": self.STDOUT, "stderr": self.STDERR + }}, + "dark": {} + } + builder.run_command_and_wait.return_value = unsuccessful_wait_result + success, stdout, stderr, build_details = builder.build(self.BUILDER_PKG) + assert not success + + # make wait result successful again + builder.run_command_and_wait.return_value = successful_wait_result + # error during build check + builder.check_build_success.return_value = (self.STDERR, True, self.STDOUT) + assert not success + assert stdout == self.STDOUT + assert stderr == self.STDERR + + # revert to successful check build + builder.check_build_success.return_value = (self.STDERR, False, self.STDOUT) + # check update build details + def upd(bd, pkg): + bd["foo"] = "bar" + + builder.collect_built_packages.side_effect = upd + success, stdout, stderr, build_details = builder.build(self.BUILDER_PKG) + assert success + assert build_details["foo"] == "bar" diff --git a/backend/tests/test_mockremote.py b/backend/tests/mockremote/test_mockremote.py similarity index 100% rename from backend/tests/test_mockremote.py rename to backend/tests/mockremote/test_mockremote.py
1
0
0
0
[copr] master: [backend] refactored mockremote (0c4fab3)
by vgologuz@fedoraproject.org
20 Nov '14
20 Nov '14
Repository :
http://git.fedorahosted.org/cgit/copr.git
On branch : master >--------------------------------------------------------------- commit 0c4fab395a30f37dbcf319ff80bf1de89c78fa83 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Fri Nov 7 14:55:11 2014 +0100 [backend] refactored mockremote >--------------------------------------------------------------- Diff suppressed because of size. To see it, use: git diff --patch-with-stat --no-color --find-copies-harder --ignore-space-at-eol ^0c4fab395a30f37dbcf319ff80bf1de89c78fa83~1 0c4fab395a30f37dbcf319ff80bf1de89c78fa83
1
0
0
0
[copr] tag 'python-copr-1.55-1' created
by vgologuz@fedoraproject.org
20 Nov '14
20 Nov '14
Repository :
http://git.fedorahosted.org/cgit/copr.git
New tag : python-copr-1.55-1 Referencing: 04e0f02a2f88ada4d3cf94514340e6c36c127cfe
1
0
0
0
← Newer
1
2
3
4
5
6
7
Older →
Jump to page:
1
2
3
4
5
6
7
Results per page:
10
25
50
100
200