Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
commit 60ccf4c6e115309614dbe4652f0659497fb6eb21 Author: Valentin Gologuzov vgologuz@redhat.com Date: Mon Oct 20 11:54:27 2014 +0200
[backend] unittest for Action and minor refactoring
backend/backend/actions.py | 48 +++-- backend/copr-be.py | 5 +- cli/run_tests.sh => backend/run_test.sh | 2 +- backend/tests/test_action.py | 448 +++++++++++++++++++++++++++++++ 4 files changed, 484 insertions(+), 19 deletions(-)
diff --git a/backend/backend/actions.py b/backend/backend/actions.py index a16f894..edcbd78 100644 --- a/backend/backend/actions.py +++ b/backend/backend/actions.py @@ -1,20 +1,23 @@ -from bunch import Bunch -from callback import FrontendCallback import os.path import shutil import time
+from bunch import Bunch + from mockremote import createrepo
class Action(object): + """ Object to send data back to fronted + + :param multiprocessing.Queue events: ? collects events for logging
- """ Object to send data back to fronted """ + """
- def __init__(self, opts, events, action, lock): + def __init__(self, events, action, lock, frontend_callback, destdir): super(Action, self).__init__() - self.frontend_callback = FrontendCallback(opts) - self.destdir = opts.destdir + self.frontend_callback = frontend_callback + self.destdir = destdir self.data = action self.events = events self.lock = lock @@ -27,7 +30,7 @@ class Action(object): result = Bunch() result.id = self.data["id"]
- if self.data["action_type"] == 0: # delete + if self.data["action_type"] == ActionType.DELETE: if self.data["object_type"] == "copr": self.event("Action delete copr") project = self.data["old_value"] @@ -42,7 +45,7 @@ class Action(object): self.event("Action delete build") project = self.data["old_value"] packages = [os.path.basename(x).replace(".src.rpm", "") for x in \ - self.data["data"].split()] + self.data["data"].split()]
path = os.path.join(self.destdir, project)
@@ -65,8 +68,9 @@ class Action(object): # files as well - that would be wrong. for pkg in packages: if self.data["object_type"] == "build-succeeded" or \ - (self.data["object_type"] == "build-failed" and - os.path.exists(os.path.join(path, chroot, pkg, "fail"))): + (self.data["object_type"] == "build-failed" and + os.path.exists(os.path.join(path, chroot, pkg, "fail")) + ): pkg_path = os.path.join(path, chroot, pkg) if os.path.isdir(pkg_path): self.event("Removing build {0}".format(pkg_path)) @@ -95,12 +99,12 @@ class Action(object): result.job_ended_on = time.time() result.result = 1 # success
- elif self.data["action_type"] == 1: # rename + elif self.data["action_type"] == ActionType.RENAME: self.event("Action rename") - old_path = os.path.normpath( - self.destdir + '/', self.data["old_value"]) - new_path = os.path.normpath( - self.destdir + '/', self.data["new_value"]) + old_path = os.path.normpath(os.path.join( + self.destdir, self.data["old_value"])) + new_path = os.path.normpath(os.path.join( + self.destdir, self.data["new_value"]))
if os.path.exists(old_path): if not os.path.exists(new_path): @@ -113,8 +117,20 @@ class Action(object): result.result = 1 # success result.job_ended_on = time.time()
- elif self.data["action_type"] == 2: # legal-flag + elif self.data["action_type"] == ActionType.LEGAL_FLAG: self.event("Action legal-flag: ignoring")
if "result" in result: self.frontend_callback.update({"actions": [result]}) + + +class ActionType(object): + DELETE = 0 + RENAME = 1 + LEGAL_FLAG = 2 + + +class ActionResult(object): + WAITING = 0 + SUCCESS = 1 + FAILURE = 2 diff --git a/backend/copr-be.py b/backend/copr-be.py index abc5301..c213770 100755 --- a/backend/copr-be.py +++ b/backend/copr-be.py @@ -22,7 +22,7 @@ import setproctitle import signal import sys import time - +from callback import FrontendCallback
def _get_conf(cp, section, option, default, mode=None): """ @@ -111,7 +111,8 @@ class CoprJobGrab(multiprocessing.Process): len(r_json["actions"])))
for action in r_json["actions"]: - ao = Action(self.opts, self.events, action, self.lock) + ao = Action(self.events, action, self.lock, destdir=self.opts.destdir, + frontend_callback=FrontendCallback(self.opts)) ao.run()
def run(self): diff --git a/cli/run_tests.sh b/backend/run_test.sh similarity index 52% copy from cli/run_tests.sh copy to backend/run_test.sh index 326c3c4..03aa25a 100755 --- a/cli/run_tests.sh +++ b/backend/run_test.sh @@ -1,4 +1,4 @@ #!/bin/sh
-PYTHONPATH=./copr_cli:$PYTHONPATH python -B -m pytest -s --cov-report term-missing --cov ./copr_cli/ tests $@ +PYTHONPATH=.:$PYTHONPATH python -B -m pytest -s --cov-report term-missing --cov ./backend/actions.py ./tests/ $@ #PYTHONPATH=../python/:./copr_cli:$PYTHONPATH python3 -B -m pytest --cov-report term-missing --cov ./copr_cli/ $@ diff --git a/backend/tests/test_action.py b/backend/tests/test_action.py new file mode 100644 index 0000000..934765d --- /dev/null +++ b/backend/tests/test_action.py @@ -0,0 +1,448 @@ +import os +import argparse +from collections import defaultdict +import json +from pprint import pprint +from _pytest.capture import capsys +import pytest +import tempfile +import shutil + +import six +import time + +if six.PY3: + from unittest import mock + from unittest.mock import MagicMock +else: + import mock + from mock import MagicMock + +import logging + +logging.basicConfig( + level=logging.INFO, + format='[%(asctime)s] {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s', + datefmt='%H:%M:%S' +) + +log = logging.getLogger() +log.info("Logger initiated") + +from backend.actions import Action, ActionType, ActionResult + +import multiprocessing +if six.PY3: + import queue + from queue import Empty as EmptyQueue +else: + import Queue as queue + from Queue import Empty as EmptyQueue + +@mock.patch("backend.actions.time") +class TestAction(object): + + def setup_method(self, method): + self.test_q = queue.Queue() + self.tmp_dir_name = None + + self.test_time = time.time() + + self.pkgs = ["foo.src.rpm", "bar.src.rpm"] + self.pkgs_stripped = ["foo", "bar"] + + def teardown_method(self, method): + self.rm_tmp_dir() + + def rm_tmp_dir(self): + if self.tmp_dir_name: + shutil.rmtree(self.tmp_dir_name) + self.tmp_dir_name = None + + def make_temp_dir(self): + root_tmp_dir = tempfile.gettempdir() + subdir = "test_action_{}".format(time.time()) + self.tmp_dir_name = os.path.join(root_tmp_dir, subdir) + #print(self.tmp_dir_name) + os.mkdir(self.tmp_dir_name) + os.mkdir(os.path.join(self.tmp_dir_name, "old_dir")) + + self.test_content = "time: {}\n".format(self.test_time) + + + + return self.tmp_dir_name + + def test_action_event(self, mc_time): + test_action = Action(events=self.test_q, action={}, lock=None, + frontend_callback=None, destdir=None) + with pytest.raises(EmptyQueue): + test_action.events.get_nowait() + + test_string = "Foo Bar" + + mc_time.time.return_value = self.test_time + test_action.event(test_string) + result_dict = test_action.events.get() + assert result_dict["when"] == self.test_time + assert result_dict["who"] == "action" + assert result_dict["what"] == test_string + + def test_action_run_legal_flag(self, mc_time): + mc_time.time.return_value = self.test_time + mc_front_cb = MagicMock() + test_action = Action( + action={ + "action_type": ActionType.LEGAL_FLAG, + "id": 1 + }, + events=self.test_q, lock=None, + frontend_callback=mc_front_cb, + destdir=None + ) + test_action.run() + assert not mc_front_cb.called + + result_dict = self.test_q.get() + assert result_dict["when"] == self.test_time + assert result_dict["who"] == "action" + assert result_dict["what"] == "Action legal-flag: ignoring" + with pytest.raises(EmptyQueue): + test_action.events.get_nowait() + + def test_action_run_rename(self, mc_time): + + mc_time.time.return_value = self.test_time + mc_front_cb = MagicMock() + + tmp_dir = self.make_temp_dir() + with open(os.path.join(self.tmp_dir_name, "old_dir", "foobar.txt"), "w") as handle: + handle.write(self.test_content) + + test_action = Action( + action={ + "action_type": ActionType.RENAME, + "id": 1, + "old_value": "old_dir", + "new_value": "new_dir" + }, + events=self.test_q, lock=None, + frontend_callback=mc_front_cb, + destdir=tmp_dir + ) + test_action.run() + result_dict = mc_front_cb.update.call_args[0][0]["actions"][0] + + assert result_dict["id"] == 1 + assert result_dict["result"] == ActionResult.SUCCESS + assert result_dict["job_ended_on"] == self.test_time + + assert not os.path.exists(os.path.join(tmp_dir, "old_dir")) + assert os.path.exists(os.path.join(tmp_dir, "new_dir")) + assert os.path.exists(os.path.join(tmp_dir, "new_dir", "foobar.txt")) + with open(os.path.join(tmp_dir, "new_dir", "foobar.txt")) as handle: + assert handle.read() == self.test_content + + def test_action_run_rename_success_on_empty_src(self, mc_time): + mc_time.time.return_value = self.test_time + mc_front_cb = MagicMock() + + tmp_dir = self.make_temp_dir() + + test_action = Action( + action={ + "action_type": ActionType.RENAME, + "id": 1, + "old_value": "old_dir", + "new_value": "new_dir" + }, + events=self.test_q, lock=None, + frontend_callback=mc_front_cb, + destdir=os.path.join(tmp_dir, "dir-not-exists") + ) + test_action.run() + result_dict = mc_front_cb.update.call_args[0][0]["actions"][0] + + assert result_dict["id"] == 1 + assert result_dict["result"] == ActionResult.SUCCESS + assert result_dict["job_ended_on"] == self.test_time + + assert os.path.exists(os.path.join(tmp_dir, "old_dir")) + + def test_action_run_rename_failure_on_destination_exists(self, mc_time): + mc_time.time.return_value = self.test_time + mc_front_cb = MagicMock() + + tmp_dir = self.make_temp_dir() + with open(os.path.join(self.tmp_dir_name, "old_dir", "foobar.txt"), "w") as handle: + handle.write(self.test_content) + os.mkdir(os.path.join(tmp_dir, "new_dir")) + + test_action = Action( + action={ + "action_type": ActionType.RENAME, + "id": 1, + "old_value": "old_dir", + "new_value": "new_dir" + }, + events=self.test_q, lock=None, + frontend_callback=mc_front_cb, + destdir=tmp_dir + ) + test_action.run() + result_dict = mc_front_cb.update.call_args[0][0]["actions"][0] + + assert result_dict["id"] == 1 + assert result_dict["result"] == ActionResult.FAILURE + assert result_dict["message"] == "Destination directory already exist." + assert result_dict["job_ended_on"] == self.test_time + + assert os.path.exists(os.path.join(tmp_dir, "old_dir")) + assert os.path.exists(os.path.join(tmp_dir, "new_dir")) + assert not os.path.exists(os.path.join(tmp_dir, "new_dir", "foobar.txt")) + + def test_action_run_delete_copr(self, mc_time): + mc_time.time.return_value = self.test_time + mc_front_cb = MagicMock() + + tmp_dir = self.make_temp_dir() + + test_action = Action( + action={ + "action_type": ActionType.DELETE, + "object_type": "copr", + "id": 6, + "old_value": "old_dir", + }, + events=self.test_q, lock=None, + frontend_callback=mc_front_cb, + destdir=os.path.join(tmp_dir, "dir-not-exists") + ) + test_action.run() + + result_dict = mc_front_cb.update.call_args[0][0]["actions"][0] + assert result_dict["id"] == 6 + assert result_dict["result"] == ActionResult.SUCCESS + assert result_dict["job_ended_on"] == self.test_time + + event_dict = self.test_q.get_nowait() + + assert event_dict["what"] == "Action delete copr" + assert event_dict["who"] == "action" + assert event_dict["when"] == self.test_time + with pytest.raises(EmptyQueue): + self.test_q.get_nowait() + + assert os.path.exists(os.path.join(tmp_dir, "old_dir")) + + def test_action_run_delete_copr_remove_folders(self, mc_time): + mc_time.time.return_value = self.test_time + mc_front_cb = MagicMock() + + tmp_dir = self.make_temp_dir() + + test_action = Action( + action={ + "action_type": ActionType.DELETE, + "object_type": "copr", + "id": 6, + "old_value": "old_dir", + }, + events=self.test_q, lock=None, + frontend_callback=mc_front_cb, + destdir=tmp_dir + ) + test_action.run() + + result_dict = mc_front_cb.update.call_args[0][0]["actions"][0] + assert result_dict["id"] == 6 + assert result_dict["result"] == ActionResult.SUCCESS + assert result_dict["job_ended_on"] == self.test_time + + event_dict = self.test_q.get_nowait() + + assert event_dict["what"] == "Action delete copr" + assert event_dict["who"] == "action" + assert event_dict["when"] == self.test_time + + event_dict_2 = self.test_q.get_nowait() + + assert "Removing copr" in event_dict_2["what"] + assert event_dict["who"] == "action" + assert event_dict["when"] == self.test_time + + assert not os.path.exists(os.path.join(tmp_dir, "old_dir")) + + def test_delete_no_chroot_dirs(self, mc_time): + mc_time.time.return_value = self.test_time + mc_front_cb = MagicMock() + + tmp_dir = self.make_temp_dir() + + for obj_type in ["build-succeeded", "build-skipped", "build-failed"]: + test_action = Action( + action={ + "action_type": ActionType.DELETE, + "object_type": obj_type, + "id": 7, + "old_value": "not-existing-project", + "data": " ".join(self.pkgs), + }, + events=self.test_q, lock=None, + frontend_callback=mc_front_cb, + destdir=tmp_dir + ) + with mock.patch("backend.actions.shutil") as mc_shutil: + test_action.run() + assert not mc_shutil.rmtree.called + + ev_1 = self.test_q.get_nowait() + assert "Action delete build" == ev_1["what"] + assert ev_1["who"] == "action" + + ev_2 = self.test_q.get_nowait() + assert "Packages to delete" in ev_2["what"] + assert " ".join(self.pkgs_stripped) in ev_2["what"] + assert ev_2["who"] == "action" + + ev_3 = self.test_q.get_nowait() + assert "Copr path" in ev_3["what"] + assert ev_3["who"] == "action" + + with pytest.raises(EmptyQueue): + self.test_q.get_nowait() + + @mock.patch("backend.actions.createrepo") + def test_delete_build_succeeded(self, mc_createrepo, mc_time): + mc_time.time.return_value = self.test_time + mc_front_cb = MagicMock() + + mc_createrepo.return_value = (0, "", "") + #mc_createrepo.side_effect = IOError() + + tmp_dir = self.make_temp_dir() + + chroot_1_dir = os.path.join(tmp_dir, "old_dir", "fedora20") + os.mkdir(chroot_1_dir) + foo_pkg_dir = os.path.join(chroot_1_dir, "foo") + os.mkdir(foo_pkg_dir) + chroot_2_dir = os.path.join(tmp_dir, "old_dir", "epel7") + os.mkdir(chroot_2_dir) + + with open(os.path.join(chroot_1_dir, "foo", "foo.src.rpm"), "w") as fh: + fh.write("foo\n") + + log_path = os.path.join(chroot_1_dir, "build-42.log") + with open(log_path, "w") as fh: + fh.write(self.test_content) + + test_action = Action( + action={ + "action_type": ActionType.DELETE, + "object_type": "build-succeeded", + "id": 7, + "old_value": "old_dir", + "data": " ".join(self.pkgs), + "object_id": 42 + }, + events=self.test_q, lock=None, + frontend_callback=mc_front_cb, + destdir=tmp_dir + ) + + assert os.path.exists(foo_pkg_dir) + test_action.run() + assert not os.path.exists(foo_pkg_dir) + assert not os.path.exists(log_path) + + assert_what_from_queue(self.test_q, msg_list=[ + "Action delete build", + "Packages to delete " + " ".join(self.pkgs_stripped), + "Copr path", + ]) + + def assert_epel7(): + assert_what_from_queue(self.test_q, msg_list=[ + "Package foo dir not found in chroot epel7", + "Package bar dir not found in chroot epel7", + ]) + + + def assert_fedora20(): + assert_what_from_queue(self.test_q, msg_list=[ + "Removing build ", + "Running createrepo", + "Package bar dir not found in chroot fedora20", + "Running createrepo", + + ]) + + ev = self.test_q.get_nowait() + assert ev["who"] == "action" + assert "In chroot epel7" in ev["what"] or "In chroot fedora20" in ev["what"] + + if "In chroot epel7" in ev["what"]: + assert_epel7() + assert_what_from_queue(self.test_q, msg_list=["In chroot fedora20"]) + assert_fedora20() + else: + assert_fedora20() + assert_what_from_queue(self.test_q, msg_list=["In chroot epel7"]) + assert_epel7() + + assert_what_from_queue(self.test_q, msg_list=["Removing log"]) + + with pytest.raises(EmptyQueue): + self.test_q.get_nowait() + + @mock.patch("backend.actions.createrepo") + def test_delete_build_succeeded_createrepo_error(self, mc_createrepo, mc_time): + mc_time.time.return_value = self.test_time + mc_front_cb = MagicMock() + + mc_createrepo.return_value = (1, "", "Create repo failed") + #mc_createrepo.side_effect = IOError() + + tmp_dir = self.make_temp_dir() + + chroot_1_dir = os.path.join(tmp_dir, "old_dir", "fedora20") + os.mkdir(chroot_1_dir) + foo_pkg_dir = os.path.join(chroot_1_dir, "foo") + os.mkdir(foo_pkg_dir) + chroot_2_dir = os.path.join(tmp_dir, "old_dir", "epel7") + os.mkdir(chroot_2_dir) + + with open(os.path.join(chroot_1_dir, "foo", "foo.src.rpm"), "w") as fh: + fh.write("foo\n") + + test_action = Action( + action={ + "action_type": ActionType.DELETE, + "object_type": "build-succeeded", + "id": 7, + "old_value": "old_dir", + "data": " ".join(self.pkgs), + "object_id": 42 + }, + events=self.test_q, lock=None, + frontend_callback=mc_front_cb, + destdir=tmp_dir + ) + + test_action.run() + error_event_recorded = False + + while not self.test_q.empty(): + ev = self.test_q.get_nowait() + if "Error making local repo" in ev["what"]: + error_event_recorded = True + + assert error_event_recorded + + + +def assert_what_from_queue(q, msg_list, who="action"): + for msg in msg_list: + ev = q.get_nowait() + assert ev["who"] == who + assert msg in ev["what"]