Repository :
http://git.fedorahosted.org/cgit/copr.git
On branch : master
---------------------------------------------------------------
commit e539a10a7a34d3f4ff983704470d30306a132d07
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Tue Mar 24 13:42:09 2015 +0100
[backend] safer copr_prune_results script, unittests
---------------------------------------------------------------
backend/backend/createrepo.py | 2 +
backend/run/copr_prune_results.py | 28 ++--
backend/tests/_resources/foox.tar.gz | Bin 17135362 -> 6549985 bytes
backend/tests/run/test_copr_prune_results.py | 168 ++++++++++++++++++++++++--
4 files changed, 171 insertions(+), 27 deletions(-)
diff --git a/backend/backend/createrepo.py b/backend/backend/createrepo.py
index 092fa6c..c20656f 100644
--- a/backend/backend/createrepo.py
+++ b/backend/backend/createrepo.py
@@ -47,6 +47,8 @@ def createrepo_unsafe(path, lock=None, dest_dir=None, base_url=None):
cmd = Popen(comm, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = cmd.communicate()
+ # TODO: rewrite with raise CreateRepoException on returncode != 0
+ # and return only stdout on success
return cmd.returncode, out, err
diff --git a/backend/run/copr_prune_results.py b/backend/run/copr_prune_results.py
index 9f77f81..d9445fd 100644
--- a/backend/run/copr_prune_results.py
+++ b/backend/run/copr_prune_results.py
@@ -58,20 +58,19 @@ class Pruner(object):
log.info("Removing failed build: {}".format(build_path))
shutil.rmtree(build_path)
- def prune_obsolete_success_builds(self, chroot_path, username, projectname):
+ def prune_obsolete_success_builds(self, chroot_path):
"""
Uses bash script which invokes repoquery to find obsolete build_dirs
:param chroot_path: path to the chroot directory
- :return:
"""
# import ipdb; ipdb.set_trace()
cmd = map(str, [self.prune_sh_script, chroot_path, self.days])
handle = Popen(cmd, stdout=PIPE, stderr=PIPE)
stdout, stderr = handle.communicate()
if handle.returncode != 0:
- log.error("Failed to prune old builds for copr {}/{}\n STDOUT: \n{}\n
STDERR: \n{}\n"
- .format(username, projectname, stdout.decode(), stderr.decode()))
+ log.error("Failed to prune old builds at: {} \n STDOUT: \n{}\n STDERR:
\n{}\n"
+ .format(chroot_path, stdout.decode(), stderr.decode()))
return
# log.debug("find obsolete returned:\n {}\n stderr: \n
{}".format(stdout, stderr))
for line in stdout.split("\n"):
@@ -121,31 +120,32 @@ class Pruner(object):
# run prune project sh
for sub_dir_name in os.listdir(project_path):
- if sub_dir_name == "repodata":
- continue
chroot_path = os.path.join(project_path, sub_dir_name)
if not os.path.isdir(chroot_path):
continue
try:
self.prune_failed_builds(chroot_path)
- self.prune_obsolete_success_builds(chroot_path, username, projectname)
+ self.prune_obsolete_success_builds(chroot_path)
except Exception as err:
log.exception(err)
- log.error("Error during prune copr {}/{}".format(username,
projectname))
+ log.error("Error during prune copr {}/{}:{}".format(username,
projectname, sub_dir_name))
- log.debug("Prune done for {}/{}".format(username, projectname))
+ log.debug("Prune done for {}/{}:{}".format(username, projectname,
sub_dir_name))
# run createrepo
try:
- retcode, stdout, stderr = createrepo_unsafe(project_path)
+ retcode, stdout, stderr = createrepo_unsafe(chroot_path)
if retcode != 0:
log.error(
- "Createrepo for copr {}/{}\n STDOUT: \n{}\n STDERR:
\n{}\n"
- .format(username, projectname, stdout.decode(),
stderr.decode()))
+ "Failed to createrepo for copr {}/{}:{}\n STDOUT: \n{}\n
STDERR: \n{}\n"
+ .format(username, projectname, sub_dir_name, stdout.decode(),
stderr.decode()))
+ else:
+ log.info("Createrepo done for copr {}/{}:{}"
+ .format(username, projectname, sub_dir_name))
except Exception as exception:
- log.exception("Createrepo for copr {}/{} failed with error:
{}"
- .format(username, projectname, exception))
+ log.exception("Createrepo for copr {}/{}:{} failed with error:
{}"
+ .format(username, projectname, sub_dir_name, exception))
log.info("Prune finished for copr {}/{}".format(username,
projectname))
diff --git a/backend/tests/_resources/foox.tar.gz b/backend/tests/_resources/foox.tar.gz
index aaebe8f..7b3c975 100644
Binary files a/backend/tests/_resources/foox.tar.gz and
b/backend/tests/_resources/foox.tar.gz differ
diff --git a/backend/tests/run/test_copr_prune_results.py
b/backend/tests/run/test_copr_prune_results.py
index 074e9dc..25c4b80 100644
--- a/backend/tests/run/test_copr_prune_results.py
+++ b/backend/tests/run/test_copr_prune_results.py
@@ -8,6 +8,7 @@ import tempfile
import time
from bunch import Bunch
from subprocess import Popen, PIPE
+from copr.client.exceptions import CoprException
import pytest
@@ -30,8 +31,28 @@ def mc_popen():
with mock.patch("{}.Popen".format(MODULE_REF)) as handle:
yield handle
+(a)pytest.yield_fixture
+def mc_bcr():
+ with mock.patch("{}.BackendConfigReader".format(MODULE_REF)) as handle:
+ yield handle
+
+(a)pytest.yield_fixture
+def mc_cru():
+ with mock.patch("{}.createrepo_unsafe".format(MODULE_REF)) as handle:
+ yield handle
+
+(a)pytest.yield_fixture
+def mc_gacs():
+ with mock.patch("{}.get_auto_createrepo_status".format(MODULE_REF)) as
handle:
+ yield handle
+
+(a)pytest.yield_fixture
+def mc_pruner():
+ with mock.patch("{}.Pruner".format(MODULE_REF)) as handle:
+ yield handle
from copr_prune_results import Pruner
+from copr_prune_results import main as prune_main
def assert_same_dirs(a, b):
@@ -42,6 +63,7 @@ def assert_same_dirs(a, b):
raise AssertionError("Directories {}, {} aren't equal, diff out: \n {}
\n {}"
.format(a, b, stdout, stderr))
+
class TestPruneResults(object):
def setup_method(self, method):
@@ -54,22 +76,24 @@ class TestPruneResults(object):
self.pkg_2 = "hello-2.8-1.fc20"
self.pkg_2_obsolete = "hello-1.0.fc20"
self.prj = "foox"
- self.chroots = ["fedora-20-i386", "fedora-20-x86_64",
- "fedora-21-i386", "fedora-21-x86_64"]
+ self.chroots = ["fedora-20-i386", "fedora-20-x86_64"]
self.opts = Bunch(
days=14,
find_obsolete_script=os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir,
- "run", "copr_find_obsolete_builds.sh"))
+ "run",
"copr_find_obsolete_builds.sh")),
+
+ frontend_base_url="http://example.com",
+ destdir=self.tmp_dir_name
)
self.log_file_path = os.path.join(self.tmp_dir_name, "_unittest.log")
logging.basicConfig(
- # filename=self.log_file_path,
- stream=sys.stdout,
+ filename=self.log_file_path,
+ # stream=sys.stdout,
format='[%(asctime)s][%(levelname)6s]: %(message)s',
level=logging.DEBUG
)
@@ -161,10 +185,7 @@ class TestPruneResults(object):
os.utime(os.path.join(self.tmp_dir_name, self.prj, self.chroots[0],
self.pkg_1_obsolete, "success"),
(time.time(), time.time()))
- self.pruner.prune_obsolete_success_builds(
- os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]),
- self.username, self.coprname
- )
+ self.pruner.prune_obsolete_success_builds(os.path.join(self.tmp_dir_name,
self.prj, self.chroots[0]))
assert_same_dirs(
os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]),
@@ -178,13 +199,134 @@ class TestPruneResults(object):
mc_handle.communicate.return_value = ("foo", "bar")
# doesn't touch FS if `find_obsolete_build` produce error return code
- self.pruner.prune_obsolete_success_builds(
- os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]),
- self.username, self.coprname
- )
+ self.pruner.prune_obsolete_success_builds(os.path.join(self.tmp_dir_name,
self.prj, self.chroots[0]))
assert mc_popen.called
assert_same_dirs(
os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]),
os.path.join(self.expect_dir_name, self.prj, self.chroots[0]),
)
+
+ def test_prune_project_ok(self, test_pruner, mc_cru, mc_gacs):
+ self.pruner.prune_failed_builds = MagicMock()
+ self.pruner.prune_obsolete_success_builds = MagicMock()
+ mc_cru.return_value = (0, "", "")
+
+ self.pruner.prune_project(os.path.join(self.tmp_dir_name, self.prj),
+ self.username, self.coprname)
+
+ expected_path_set = set([
+ os.path.join(self.tmp_dir_name, self.prj, chroot)
+ for chroot in self.chroots
+ ])
+ assert set([
+ call[0][0] for call in
+ self.pruner.prune_failed_builds.call_args_list
+ ]) == expected_path_set
+ assert set([
+ call[0][0] for call in
+ self.pruner.prune_obsolete_success_builds.call_args_list
+ ]) == expected_path_set
+ assert set([
+ call[0][0] for call in
+ mc_cru.call_args_list
+ ]) == expected_path_set
+
+
+ def test_prune_project_handle_gacs_error(self, test_pruner, mc_cru, mc_gacs):
+ self.pruner.prune_failed_builds = MagicMock()
+ self.pruner.prune_obsolete_success_builds = MagicMock()
+
+ mc_gacs.side_effect = CoprException()
+
+ self.pruner.prune_project(os.path.join(self.tmp_dir_name, self.prj),
+ self.username, self.coprname)
+
+ assert not self.pruner.prune_failed_builds.called
+ assert not self.pruner.prune_obsolete_success_builds.called
+
+ def test_prune_project_handle_errors(self, test_pruner, mc_cru, mc_gacs):
+ self.pruner.prune_failed_builds = MagicMock()
+ self.pruner.prune_obsolete_success_builds = MagicMock()
+ mc_gacs.return_value = True
+
+
+ # 0. createrepo_unsafe failure
+ mc_cru.return_value = (1, "stdout", "stderr")
+
+ self.pruner.prune_project(os.path.join(self.tmp_dir_name, self.prj),
+ self.username, self.coprname)
+
+ assert self.pruner.prune_failed_builds.called
+ assert self.pruner.prune_obsolete_success_builds.called
+
+ self.pruner.prune_failed_builds.reset_mock()
+ self.pruner.prune_obsolete_success_builds.reset_mock()
+
+ mc_cru.return_value = (0, "", "")
+ # 1. createrepo_unsafe raises error
+ mc_cru.side_effect = IOError()
+
+ self.pruner.prune_project(os.path.join(self.tmp_dir_name, self.prj),
+ self.username, self.coprname)
+
+ assert self.pruner.prune_failed_builds.called
+ assert self.pruner.prune_obsolete_success_builds.called
+
+ self.pruner.prune_failed_builds.reset_mock()
+ self.pruner.prune_obsolete_success_builds.reset_mock()
+
+ mc_cru.side_effect = None
+ # 2. prune_obsolete_success_builds raises error
+
+ self.pruner.prune_obsolete_success_builds.side_effect = IOError()
+
+ self.pruner.prune_project(os.path.join(self.tmp_dir_name, self.prj),
+ self.username, self.coprname)
+
+ assert mc_cru.called
+
+ self.pruner.prune_failed_builds.reset_mock()
+ self.pruner.prune_obsolete_success_builds.reset_mock()
+ mc_cru.reset_mock()
+
+ # 3. prune_failed_builds raises error
+
+ self.pruner.prune_obsolete_success_builds.side_effect = None
+ self.pruner.prune_failed_builds.side_effect = IOError()
+
+ self.pruner.prune_project(os.path.join(self.tmp_dir_name, self.prj),
+ self.username, self.coprname)
+
+ assert mc_cru.called
+
+
+ def test_prune_project_skip_when_acr_disabled(self, test_pruner, mc_cru, mc_gacs):
+ self.pruner.prune_failed_builds = MagicMock()
+ self.pruner.prune_obsolete_success_builds = MagicMock()
+
+ mc_gacs.return_value = False
+
+ self.pruner.prune_project(os.path.join(self.tmp_dir_name, self.prj),
+ self.username, self.coprname)
+
+ assert not self.pruner.prune_failed_builds.called
+ assert not self.pruner.prune_obsolete_success_builds.called
+
+ def test_run(self, test_pruner):
+ self.pruner.prune_project = MagicMock()
+
+ self.pruner.run()
+
+
+ def test_main(self, mc_pruner, mc_bcr):
+ prune_main()
+ assert mc_pruner.called
+ assert mc_pruner.return_value.run.called
+ assert mc_bcr.called
+ assert mc_bcr.call_args[0][0] == "/etc/copr/copr-be.conf"
+
+ os.environ["BACKEND_CONFIG"] = "foobar"
+ prune_main()
+ assert mc_bcr.call_args[0][0] == "foobar"
+