Repository :
http://git.fedorahosted.org/cgit/copr.git
On branch : master
---------------------------------------------------------------
commit 60ccf4c6e115309614dbe4652f0659497fb6eb21
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Mon Oct 20 11:54:27 2014 +0200
[backend] unittest for Action and minor refactoring
---------------------------------------------------------------
backend/backend/actions.py | 48 +++--
backend/copr-be.py | 5 +-
cli/run_tests.sh => backend/run_test.sh | 2 +-
backend/tests/test_action.py | 448 +++++++++++++++++++++++++++++++
4 files changed, 484 insertions(+), 19 deletions(-)
diff --git a/backend/backend/actions.py b/backend/backend/actions.py
index a16f894..edcbd78 100644
--- a/backend/backend/actions.py
+++ b/backend/backend/actions.py
@@ -1,20 +1,23 @@
-from bunch import Bunch
-from callback import FrontendCallback
import os.path
import shutil
import time
+from bunch import Bunch
+
from mockremote import createrepo
class Action(object):
+ """ Object to send data back to fronted
+
+ :param multiprocessing.Queue events: ? collects events for logging
- """ Object to send data back to fronted """
+ """
- def __init__(self, opts, events, action, lock):
+ def __init__(self, events, action, lock, frontend_callback, destdir):
super(Action, self).__init__()
- self.frontend_callback = FrontendCallback(opts)
- self.destdir = opts.destdir
+ self.frontend_callback = frontend_callback
+ self.destdir = destdir
self.data = action
self.events = events
self.lock = lock
@@ -27,7 +30,7 @@ class Action(object):
result = Bunch()
result.id = self.data["id"]
- if self.data["action_type"] == 0: # delete
+ if self.data["action_type"] == ActionType.DELETE:
if self.data["object_type"] == "copr":
self.event("Action delete copr")
project = self.data["old_value"]
@@ -42,7 +45,7 @@ class Action(object):
self.event("Action delete build")
project = self.data["old_value"]
packages = [os.path.basename(x).replace(".src.rpm",
"") for x in \
- self.data["data"].split()]
+ self.data["data"].split()]
path = os.path.join(self.destdir, project)
@@ -65,8 +68,9 @@ class Action(object):
# files as well - that would be wrong.
for pkg in packages:
if self.data["object_type"] ==
"build-succeeded" or \
- (self.data["object_type"] ==
"build-failed" and
- os.path.exists(os.path.join(path, chroot, pkg,
"fail"))):
+ (self.data["object_type"] ==
"build-failed" and
+ os.path.exists(os.path.join(path, chroot, pkg,
"fail"))
+ ):
pkg_path = os.path.join(path, chroot, pkg)
if os.path.isdir(pkg_path):
self.event("Removing build
{0}".format(pkg_path))
@@ -95,12 +99,12 @@ class Action(object):
result.job_ended_on = time.time()
result.result = 1 # success
- elif self.data["action_type"] == 1: # rename
+ elif self.data["action_type"] == ActionType.RENAME:
self.event("Action rename")
- old_path = os.path.normpath(
- self.destdir + '/', self.data["old_value"])
- new_path = os.path.normpath(
- self.destdir + '/', self.data["new_value"])
+ old_path = os.path.normpath(os.path.join(
+ self.destdir, self.data["old_value"]))
+ new_path = os.path.normpath(os.path.join(
+ self.destdir, self.data["new_value"]))
if os.path.exists(old_path):
if not os.path.exists(new_path):
@@ -113,8 +117,20 @@ class Action(object):
result.result = 1 # success
result.job_ended_on = time.time()
- elif self.data["action_type"] == 2: # legal-flag
+ elif self.data["action_type"] == ActionType.LEGAL_FLAG:
self.event("Action legal-flag: ignoring")
if "result" in result:
self.frontend_callback.update({"actions": [result]})
+
+
+class ActionType(object):
+ DELETE = 0
+ RENAME = 1
+ LEGAL_FLAG = 2
+
+
+class ActionResult(object):
+ WAITING = 0
+ SUCCESS = 1
+ FAILURE = 2
diff --git a/backend/copr-be.py b/backend/copr-be.py
index abc5301..c213770 100755
--- a/backend/copr-be.py
+++ b/backend/copr-be.py
@@ -22,7 +22,7 @@ import setproctitle
import signal
import sys
import time
-
+from callback import FrontendCallback
def _get_conf(cp, section, option, default, mode=None):
"""
@@ -111,7 +111,8 @@ class CoprJobGrab(multiprocessing.Process):
len(r_json["actions"])))
for action in r_json["actions"]:
- ao = Action(self.opts, self.events, action, self.lock)
+ ao = Action(self.events, action, self.lock, destdir=self.opts.destdir,
+ frontend_callback=FrontendCallback(self.opts))
ao.run()
def run(self):
diff --git a/cli/run_tests.sh b/backend/run_test.sh
similarity index 52%
copy from cli/run_tests.sh
copy to backend/run_test.sh
index 326c3c4..03aa25a 100755
--- a/cli/run_tests.sh
+++ b/backend/run_test.sh
@@ -1,4 +1,4 @@
#!/bin/sh
-PYTHONPATH=./copr_cli:$PYTHONPATH python -B -m pytest -s --cov-report term-missing --cov
./copr_cli/ tests $@
+PYTHONPATH=.:$PYTHONPATH python -B -m pytest -s --cov-report term-missing --cov
./backend/actions.py ./tests/ $@
#PYTHONPATH=../python/:./copr_cli:$PYTHONPATH python3 -B -m pytest --cov-report
term-missing --cov ./copr_cli/ $@
diff --git a/backend/tests/test_action.py b/backend/tests/test_action.py
new file mode 100644
index 0000000..934765d
--- /dev/null
+++ b/backend/tests/test_action.py
@@ -0,0 +1,448 @@
+import os
+import argparse
+from collections import defaultdict
+import json
+from pprint import pprint
+from _pytest.capture import capsys
+import pytest
+import tempfile
+import shutil
+
+import six
+import time
+
+if six.PY3:
+ from unittest import mock
+ from unittest.mock import MagicMock
+else:
+ import mock
+ from mock import MagicMock
+
+import logging
+
+logging.basicConfig(
+ level=logging.INFO,
+ format='[%(asctime)s] {%(pathname)s:%(lineno)d} %(levelname)s -
%(message)s',
+ datefmt='%H:%M:%S'
+)
+
+log = logging.getLogger()
+log.info("Logger initiated")
+
+from backend.actions import Action, ActionType, ActionResult
+
+import multiprocessing
+if six.PY3:
+ import queue
+ from queue import Empty as EmptyQueue
+else:
+ import Queue as queue
+ from Queue import Empty as EmptyQueue
+
+(a)mock.patch("backend.actions.time")
+class TestAction(object):
+
+ def setup_method(self, method):
+ self.test_q = queue.Queue()
+ self.tmp_dir_name = None
+
+ self.test_time = time.time()
+
+ self.pkgs = ["foo.src.rpm", "bar.src.rpm"]
+ self.pkgs_stripped = ["foo", "bar"]
+
+ def teardown_method(self, method):
+ self.rm_tmp_dir()
+
+ def rm_tmp_dir(self):
+ if self.tmp_dir_name:
+ shutil.rmtree(self.tmp_dir_name)
+ self.tmp_dir_name = None
+
+ def make_temp_dir(self):
+ root_tmp_dir = tempfile.gettempdir()
+ subdir = "test_action_{}".format(time.time())
+ self.tmp_dir_name = os.path.join(root_tmp_dir, subdir)
+ #print(self.tmp_dir_name)
+ os.mkdir(self.tmp_dir_name)
+ os.mkdir(os.path.join(self.tmp_dir_name, "old_dir"))
+
+ self.test_content = "time: {}\n".format(self.test_time)
+
+
+
+ return self.tmp_dir_name
+
+ def test_action_event(self, mc_time):
+ test_action = Action(events=self.test_q, action={}, lock=None,
+ frontend_callback=None, destdir=None)
+ with pytest.raises(EmptyQueue):
+ test_action.events.get_nowait()
+
+ test_string = "Foo Bar"
+
+ mc_time.time.return_value = self.test_time
+ test_action.event(test_string)
+ result_dict = test_action.events.get()
+ assert result_dict["when"] == self.test_time
+ assert result_dict["who"] == "action"
+ assert result_dict["what"] == test_string
+
+ def test_action_run_legal_flag(self, mc_time):
+ mc_time.time.return_value = self.test_time
+ mc_front_cb = MagicMock()
+ test_action = Action(
+ action={
+ "action_type": ActionType.LEGAL_FLAG,
+ "id": 1
+ },
+ events=self.test_q, lock=None,
+ frontend_callback=mc_front_cb,
+ destdir=None
+ )
+ test_action.run()
+ assert not mc_front_cb.called
+
+ result_dict = self.test_q.get()
+ assert result_dict["when"] == self.test_time
+ assert result_dict["who"] == "action"
+ assert result_dict["what"] == "Action legal-flag: ignoring"
+ with pytest.raises(EmptyQueue):
+ test_action.events.get_nowait()
+
+ def test_action_run_rename(self, mc_time):
+
+ mc_time.time.return_value = self.test_time
+ mc_front_cb = MagicMock()
+
+ tmp_dir = self.make_temp_dir()
+ with open(os.path.join(self.tmp_dir_name, "old_dir",
"foobar.txt"), "w") as handle:
+ handle.write(self.test_content)
+
+ test_action = Action(
+ action={
+ "action_type": ActionType.RENAME,
+ "id": 1,
+ "old_value": "old_dir",
+ "new_value": "new_dir"
+ },
+ events=self.test_q, lock=None,
+ frontend_callback=mc_front_cb,
+ destdir=tmp_dir
+ )
+ test_action.run()
+ result_dict = mc_front_cb.update.call_args[0][0]["actions"][0]
+
+ assert result_dict["id"] == 1
+ assert result_dict["result"] == ActionResult.SUCCESS
+ assert result_dict["job_ended_on"] == self.test_time
+
+ assert not os.path.exists(os.path.join(tmp_dir, "old_dir"))
+ assert os.path.exists(os.path.join(tmp_dir, "new_dir"))
+ assert os.path.exists(os.path.join(tmp_dir, "new_dir",
"foobar.txt"))
+ with open(os.path.join(tmp_dir, "new_dir", "foobar.txt")) as
handle:
+ assert handle.read() == self.test_content
+
+ def test_action_run_rename_success_on_empty_src(self, mc_time):
+ mc_time.time.return_value = self.test_time
+ mc_front_cb = MagicMock()
+
+ tmp_dir = self.make_temp_dir()
+
+ test_action = Action(
+ action={
+ "action_type": ActionType.RENAME,
+ "id": 1,
+ "old_value": "old_dir",
+ "new_value": "new_dir"
+ },
+ events=self.test_q, lock=None,
+ frontend_callback=mc_front_cb,
+ destdir=os.path.join(tmp_dir, "dir-not-exists")
+ )
+ test_action.run()
+ result_dict = mc_front_cb.update.call_args[0][0]["actions"][0]
+
+ assert result_dict["id"] == 1
+ assert result_dict["result"] == ActionResult.SUCCESS
+ assert result_dict["job_ended_on"] == self.test_time
+
+ assert os.path.exists(os.path.join(tmp_dir, "old_dir"))
+
+ def test_action_run_rename_failure_on_destination_exists(self, mc_time):
+ mc_time.time.return_value = self.test_time
+ mc_front_cb = MagicMock()
+
+ tmp_dir = self.make_temp_dir()
+ with open(os.path.join(self.tmp_dir_name, "old_dir",
"foobar.txt"), "w") as handle:
+ handle.write(self.test_content)
+ os.mkdir(os.path.join(tmp_dir, "new_dir"))
+
+ test_action = Action(
+ action={
+ "action_type": ActionType.RENAME,
+ "id": 1,
+ "old_value": "old_dir",
+ "new_value": "new_dir"
+ },
+ events=self.test_q, lock=None,
+ frontend_callback=mc_front_cb,
+ destdir=tmp_dir
+ )
+ test_action.run()
+ result_dict = mc_front_cb.update.call_args[0][0]["actions"][0]
+
+ assert result_dict["id"] == 1
+ assert result_dict["result"] == ActionResult.FAILURE
+ assert result_dict["message"] == "Destination directory already
exist."
+ assert result_dict["job_ended_on"] == self.test_time
+
+ assert os.path.exists(os.path.join(tmp_dir, "old_dir"))
+ assert os.path.exists(os.path.join(tmp_dir, "new_dir"))
+ assert not os.path.exists(os.path.join(tmp_dir, "new_dir",
"foobar.txt"))
+
+ def test_action_run_delete_copr(self, mc_time):
+ mc_time.time.return_value = self.test_time
+ mc_front_cb = MagicMock()
+
+ tmp_dir = self.make_temp_dir()
+
+ test_action = Action(
+ action={
+ "action_type": ActionType.DELETE,
+ "object_type": "copr",
+ "id": 6,
+ "old_value": "old_dir",
+ },
+ events=self.test_q, lock=None,
+ frontend_callback=mc_front_cb,
+ destdir=os.path.join(tmp_dir, "dir-not-exists")
+ )
+ test_action.run()
+
+ result_dict = mc_front_cb.update.call_args[0][0]["actions"][0]
+ assert result_dict["id"] == 6
+ assert result_dict["result"] == ActionResult.SUCCESS
+ assert result_dict["job_ended_on"] == self.test_time
+
+ event_dict = self.test_q.get_nowait()
+
+ assert event_dict["what"] == "Action delete copr"
+ assert event_dict["who"] == "action"
+ assert event_dict["when"] == self.test_time
+ with pytest.raises(EmptyQueue):
+ self.test_q.get_nowait()
+
+ assert os.path.exists(os.path.join(tmp_dir, "old_dir"))
+
+ def test_action_run_delete_copr_remove_folders(self, mc_time):
+ mc_time.time.return_value = self.test_time
+ mc_front_cb = MagicMock()
+
+ tmp_dir = self.make_temp_dir()
+
+ test_action = Action(
+ action={
+ "action_type": ActionType.DELETE,
+ "object_type": "copr",
+ "id": 6,
+ "old_value": "old_dir",
+ },
+ events=self.test_q, lock=None,
+ frontend_callback=mc_front_cb,
+ destdir=tmp_dir
+ )
+ test_action.run()
+
+ result_dict = mc_front_cb.update.call_args[0][0]["actions"][0]
+ assert result_dict["id"] == 6
+ assert result_dict["result"] == ActionResult.SUCCESS
+ assert result_dict["job_ended_on"] == self.test_time
+
+ event_dict = self.test_q.get_nowait()
+
+ assert event_dict["what"] == "Action delete copr"
+ assert event_dict["who"] == "action"
+ assert event_dict["when"] == self.test_time
+
+ event_dict_2 = self.test_q.get_nowait()
+
+ assert "Removing copr" in event_dict_2["what"]
+ assert event_dict["who"] == "action"
+ assert event_dict["when"] == self.test_time
+
+ assert not os.path.exists(os.path.join(tmp_dir, "old_dir"))
+
+ def test_delete_no_chroot_dirs(self, mc_time):
+ mc_time.time.return_value = self.test_time
+ mc_front_cb = MagicMock()
+
+ tmp_dir = self.make_temp_dir()
+
+ for obj_type in ["build-succeeded", "build-skipped",
"build-failed"]:
+ test_action = Action(
+ action={
+ "action_type": ActionType.DELETE,
+ "object_type": obj_type,
+ "id": 7,
+ "old_value": "not-existing-project",
+ "data": " ".join(self.pkgs),
+ },
+ events=self.test_q, lock=None,
+ frontend_callback=mc_front_cb,
+ destdir=tmp_dir
+ )
+ with mock.patch("backend.actions.shutil") as mc_shutil:
+ test_action.run()
+ assert not mc_shutil.rmtree.called
+
+ ev_1 = self.test_q.get_nowait()
+ assert "Action delete build" == ev_1["what"]
+ assert ev_1["who"] == "action"
+
+ ev_2 = self.test_q.get_nowait()
+ assert "Packages to delete" in ev_2["what"]
+ assert " ".join(self.pkgs_stripped) in ev_2["what"]
+ assert ev_2["who"] == "action"
+
+ ev_3 = self.test_q.get_nowait()
+ assert "Copr path" in ev_3["what"]
+ assert ev_3["who"] == "action"
+
+ with pytest.raises(EmptyQueue):
+ self.test_q.get_nowait()
+
+ @mock.patch("backend.actions.createrepo")
+ def test_delete_build_succeeded(self, mc_createrepo, mc_time):
+ mc_time.time.return_value = self.test_time
+ mc_front_cb = MagicMock()
+
+ mc_createrepo.return_value = (0, "", "")
+ #mc_createrepo.side_effect = IOError()
+
+ tmp_dir = self.make_temp_dir()
+
+ chroot_1_dir = os.path.join(tmp_dir, "old_dir", "fedora20")
+ os.mkdir(chroot_1_dir)
+ foo_pkg_dir = os.path.join(chroot_1_dir, "foo")
+ os.mkdir(foo_pkg_dir)
+ chroot_2_dir = os.path.join(tmp_dir, "old_dir", "epel7")
+ os.mkdir(chroot_2_dir)
+
+ with open(os.path.join(chroot_1_dir, "foo", "foo.src.rpm"),
"w") as fh:
+ fh.write("foo\n")
+
+ log_path = os.path.join(chroot_1_dir, "build-42.log")
+ with open(log_path, "w") as fh:
+ fh.write(self.test_content)
+
+ test_action = Action(
+ action={
+ "action_type": ActionType.DELETE,
+ "object_type": "build-succeeded",
+ "id": 7,
+ "old_value": "old_dir",
+ "data": " ".join(self.pkgs),
+ "object_id": 42
+ },
+ events=self.test_q, lock=None,
+ frontend_callback=mc_front_cb,
+ destdir=tmp_dir
+ )
+
+ assert os.path.exists(foo_pkg_dir)
+ test_action.run()
+ assert not os.path.exists(foo_pkg_dir)
+ assert not os.path.exists(log_path)
+
+ assert_what_from_queue(self.test_q, msg_list=[
+ "Action delete build",
+ "Packages to delete " + " ".join(self.pkgs_stripped),
+ "Copr path",
+ ])
+
+ def assert_epel7():
+ assert_what_from_queue(self.test_q, msg_list=[
+ "Package foo dir not found in chroot epel7",
+ "Package bar dir not found in chroot epel7",
+ ])
+
+
+ def assert_fedora20():
+ assert_what_from_queue(self.test_q, msg_list=[
+ "Removing build ",
+ "Running createrepo",
+ "Package bar dir not found in chroot fedora20",
+ "Running createrepo",
+
+ ])
+
+ ev = self.test_q.get_nowait()
+ assert ev["who"] == "action"
+ assert "In chroot epel7" in ev["what"] or "In chroot
fedora20" in ev["what"]
+
+ if "In chroot epel7" in ev["what"]:
+ assert_epel7()
+ assert_what_from_queue(self.test_q, msg_list=["In chroot
fedora20"])
+ assert_fedora20()
+ else:
+ assert_fedora20()
+ assert_what_from_queue(self.test_q, msg_list=["In chroot epel7"])
+ assert_epel7()
+
+ assert_what_from_queue(self.test_q, msg_list=["Removing log"])
+
+ with pytest.raises(EmptyQueue):
+ self.test_q.get_nowait()
+
+ @mock.patch("backend.actions.createrepo")
+ def test_delete_build_succeeded_createrepo_error(self, mc_createrepo, mc_time):
+ mc_time.time.return_value = self.test_time
+ mc_front_cb = MagicMock()
+
+ mc_createrepo.return_value = (1, "", "Create repo failed")
+ #mc_createrepo.side_effect = IOError()
+
+ tmp_dir = self.make_temp_dir()
+
+ chroot_1_dir = os.path.join(tmp_dir, "old_dir", "fedora20")
+ os.mkdir(chroot_1_dir)
+ foo_pkg_dir = os.path.join(chroot_1_dir, "foo")
+ os.mkdir(foo_pkg_dir)
+ chroot_2_dir = os.path.join(tmp_dir, "old_dir", "epel7")
+ os.mkdir(chroot_2_dir)
+
+ with open(os.path.join(chroot_1_dir, "foo", "foo.src.rpm"),
"w") as fh:
+ fh.write("foo\n")
+
+ test_action = Action(
+ action={
+ "action_type": ActionType.DELETE,
+ "object_type": "build-succeeded",
+ "id": 7,
+ "old_value": "old_dir",
+ "data": " ".join(self.pkgs),
+ "object_id": 42
+ },
+ events=self.test_q, lock=None,
+ frontend_callback=mc_front_cb,
+ destdir=tmp_dir
+ )
+
+ test_action.run()
+ error_event_recorded = False
+
+ while not self.test_q.empty():
+ ev = self.test_q.get_nowait()
+ if "Error making local repo" in ev["what"]:
+ error_event_recorded = True
+
+ assert error_event_recorded
+
+
+
+def assert_what_from_queue(q, msg_list, who="action"):
+ for msg in msg_list:
+ ev = q.get_nowait()
+ assert ev["who"] == who
+ assert msg in ev["what"]