Repository :
http://git.fedorahosted.org/cgit/copr.git
On branch : master
---------------------------------------------------------------
commit f6ec0a9cdf210a342d5786c0187fee1786a28529
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Fri Jul 31 19:44:06 2015 +0200
[dist-git] more test coverage
---------------------------------------------------------------
backend/copr-backend.spec | 4 +-
dist-git/copr-dist-git.spec | 20 ++-
dist-git/dist_git/dist_git_importer.py | 29 ++--
dist-git/dist_git/helpers.py | 7 +-
dist-git/tests/test_dist_git_importer.py | 236 +++++++++++++++++++++++++++++-
5 files changed, 267 insertions(+), 29 deletions(-)
diff --git a/backend/copr-backend.spec b/backend/copr-backend.spec
index 62519f7..7f14487 100644
--- a/backend/copr-backend.spec
+++ b/backend/copr-backend.spec
@@ -185,8 +185,8 @@ cp -a conf/logstash/copr_backend.conf
%{buildroot}%{_pkgdocdir}/examples/%{_sysc
redis-server --port 7777 &> /dev/null &
-#PYTHONPATH=backend:run:$PYTHONPATH python -B -m pytest \
-# -s -v --cov-report term-missing --cov ./backend --cov ./run ./tests/
+PYTHONPATH=backend:run:$PYTHONPATH python -B -m pytest \
+ -s -v --cov-report term-missing --cov ./backend --cov ./run ./tests/
kill %1
diff --git a/dist-git/copr-dist-git.spec b/dist-git/copr-dist-git.spec
index d117676..652e3c0 100644
--- a/dist-git/copr-dist-git.spec
+++ b/dist-git/copr-dist-git.spec
@@ -14,13 +14,17 @@ Source0: %{name}-%{version}.tar.gz
BuildArch: noarch
-BuildRequires: systemd
-
-Requires: systemd
-Requires: dist-git
-Requires: python-bunch
-Requires: python-requests
-Requires: pyrpkg
+BuildRequires: systemd
+BuildRequires: dist-git
+BuildRequires: python-bunch
+BuildRequires: python-requests
+BuildRequires: pyrpkg
+
+Requires: systemd
+Requires: dist-git
+Requires: python-bunch
+Requires: python-requests
+Requires: pyrpkg
%description
COPR is lightweight build system. It allows you to create new project in WebUI
@@ -55,6 +59,8 @@ cp -a conf/logrotate
%{buildroot}%{_sysconfdir}/logrotate.d/copr-dist-git
touch %{buildroot}%{_var}/log/copr-dist-git/main.log
%check
+PYTHONPATH=.:$PYTHONPATH python -B -m pytest \
+ -v --cov-report term-missing --cov ./dist_git ./tests/
%post
# change context to be readable by cgit
diff --git a/dist-git/dist_git/dist_git_importer.py
b/dist-git/dist_git/dist_git_importer.py
index f548863..88df6a6 100755
--- a/dist-git/dist_git/dist_git_importer.py
+++ b/dist-git/dist_git/dist_git_importer.py
@@ -109,6 +109,7 @@ class ImportTask(object):
class DistGitImporter(object):
def __init__(self, opts):
+ self.is_running = False
self.opts = opts
self.get_url =
"{}/backend/importing/".format(self.opts.frontend_base_url)
@@ -136,29 +137,25 @@ class DistGitImporter(object):
@staticmethod
def fetch_srpm(task, fetched_srpm_path):
log.debug("download the package")
-
try:
urlretrieve(task.package_url, fetched_srpm_path)
except IOError:
raise PackageDownloadException()
- return fetched_srpm_path
- @staticmethod
- def my_upload(repo_dir, reponame, filename, filehash):
+ def my_upload(self, repo_dir, reponame, filename, filehash):
"""
This is a replacement function for uploading sources.
Rpkg uses upload.cgi for uploading which doesn't make sense
on the local machine.
"""
- lookaside = "/var/lib/dist-git/cache/lookaside/pkgs/"
source = os.path.join(repo_dir, filename)
- destination = os.path.join(lookaside, reponame, filename, filehash, filename)
+ destination = os.path.join(self.opts.lookaside_location, reponame,
+ filename, filehash, filename)
if not os.path.exists(destination):
os.makedirs(os.path.dirname(destination))
shutil.copyfile(source, destination)
- @classmethod
- def git_import_srpm(cls, task, filepath):
+ def git_import_srpm(self, task, filepath):
"""
Imports a source rpm file into local dist git.
Repository name is in the Copr Style: user/project/package
@@ -195,7 +192,7 @@ class DistGitImporter(object):
# rpkg calls upload.cgi script on the dist git server
# here, I just copy the source files manually with custom function
# I also add one parameter "repo_dir" to that function with this
hack
- commands.lookasidecache.upload = types.MethodType(cls.my_upload, repo_dir)
+ commands.lookasidecache.upload = types.MethodType(self.my_upload, repo_dir)
log.debug("clone the pkg repository into tmp directory")
commands.clone(module, tmp, task.branch)
@@ -219,10 +216,12 @@ class DistGitImporter(object):
except rpkgError:
log.exception("error during commit and push, ignored")
- git_hash = commands.commithash
+ return commands.commithash
+ except Exception:
+ log.exception("Unexpected")
+ raise PackageImportException()
finally:
shutil.rmtree(tmp)
- return git_hash
@staticmethod
def pkg_name_evr(srpm_path):
@@ -310,7 +309,8 @@ class DistGitImporter(object):
def run(self):
log.info("DistGitImported initialized")
- while True:
+ self.is_running = True
+ while self.is_running:
mb_task = self.try_to_obtain_new_task()
if mb_task is None:
time.sleep(self.opts.sleep_time)
@@ -347,10 +347,7 @@ def main():
log.info("Using configuration: \n"
"{}".format(opts))
importer = DistGitImporter(opts)
- try:
- importer.run()
- except KeyboardInterrupt:
- return
+ importer.run()
if __name__ == "__main__":
diff --git a/dist-git/dist_git/helpers.py b/dist-git/dist_git/helpers.py
index eb90f4a..3631840 100644
--- a/dist-git/dist_git/helpers.py
+++ b/dist-git/dist_git/helpers.py
@@ -77,6 +77,11 @@ class DistGitConfigReader(object):
)
opts.cgit_pkg_list_location = _get_conf(
- cp, "cgit_pkg_list_location", "log_dir",
"/var/lib/copr-dist-git/cgit_pkg_list"
+ cp, "dist-git", "cgit_pkg_list_location",
"/var/lib/copr-dist-git/cgit_pkg_list"
)
+
+ opts.lookaside_location = _get_conf(
+ cp, "dist-git", "lookaside_location",
"/var/lib/dist-git/cache/lookaside/pkgs/"
+ )
+
return opts
diff --git a/dist-git/tests/test_dist_git_importer.py
b/dist-git/tests/test_dist_git_importer.py
index 4cc54b4..7c003d9 100644
--- a/dist-git/tests/test_dist_git_importer.py
+++ b/dist-git/tests/test_dist_git_importer.py
@@ -9,6 +9,7 @@ import tempfile
import shutil
import time
from bunch import Bunch
+from pyrpkg import rpkgError
import pytest
import six
@@ -20,14 +21,35 @@ else:
import mock
from mock import MagicMock
-from dist_git.dist_git_importer import DistGitImporter, SourceType, ImportTask
+from dist_git.dist_git_importer import DistGitImporter, SourceType, ImportTask,
PackageDownloadException, \
+ PackageImportException, PackageQueryException, main
MODULE_REF = 'dist_git.dist_git_importer'
@pytest.yield_fixture
+def mc_dgcr():
+ with mock.patch("{}.DistGitConfigReader".format(MODULE_REF)) as handle:
+ yield handle
+
+(a)pytest.yield_fixture
+def mc_dgi():
+ with mock.patch("{}.DistGitImporter".format(MODULE_REF)) as handle:
+ yield handle
+
+(a)pytest.yield_fixture
+def mc_time():
+ with mock.patch("{}.time".format(MODULE_REF)) as handle:
+ yield handle
+
+(a)pytest.yield_fixture
def mc_popen():
- with mock.patch(MODULE_REF) as handle:
+ with mock.patch("{}.Popen".format(MODULE_REF)) as handle:
+ yield handle
+
+(a)pytest.yield_fixture
+def mc_call():
+ with mock.patch("{}.call".format(MODULE_REF)) as handle:
yield handle
@@ -42,8 +64,18 @@ def mc_post():
with mock.patch("{}.post".format(MODULE_REF)) as handle:
yield handle
+(a)pytest.yield_fixture
+def mc_urlretrieve():
+ with mock.patch("{}.urlretrieve".format(MODULE_REF)) as handle:
+ yield handle
-if False:
+(a)pytest.yield_fixture
+def mc_pyrpkg_commands():
+ with mock.patch("{}.Commands".format(MODULE_REF)) as handle:
+ yield handle
+
+
+if True:
logging.basicConfig(
level=logging.DEBUG,
format='[%(asctime)s][%(levelname)s][%(name)s][%(module)s:%(lineno)d]
%(message)s',
@@ -53,17 +85,27 @@ if False:
class TestDistGitImporter(object):
def setup_method(self, method):
+ self.tmp_dir_name = self.make_temp_dir()
+ self.lookaside_location = os.path.join(self.tmp_dir_name, "lookaside")
self.opts = Bunch({
"frontend_base_url": "http://front",
"frontend_auth": "secure_password",
+ "lookaside_location": self.lookaside_location,
+
+ "cgit_pkg_list_location": self.tmp_dir_name,
+ "sleep_time": 10,
+ "log_dir": self.tmp_dir_name
})
self.dgi = DistGitImporter(self.opts)
self.USER_NAME = "foo"
self.PROJECT_NAME = "bar"
+ self.PACKAGE_NAME = "bar_app"
+ self.PACKAGE_VERSION = "2:0.01-1.fc20"
self.BRANCH = "f22"
+ self.FILE_HASH = "1234abc"
self.task_data_1 = {
"task_id": 123,
"user": self.USER_NAME,
@@ -86,6 +128,24 @@ class TestDistGitImporter(object):
self.task_1 = ImportTask.from_dict(self.task_data_1, self.opts)
self.task_2 = ImportTask.from_dict(self.task_data_2, self.opts)
+ self.fetched_srpm_path = "/tmp/none/"
+
+
+ def teardown_method(self, method):
+ self.rm_tmp_dir()
+
+ def rm_tmp_dir(self):
+ if self.tmp_dir_name:
+ shutil.rmtree(self.tmp_dir_name)
+ self.tmp_dir_name = None
+
+ def make_temp_dir(self):
+ root_tmp_dir = tempfile.gettempdir()
+ subdir = "test_{}_{}".format(MODULE_REF, time.time())
+ self.tmp_dir_name = os.path.join(root_tmp_dir, subdir)
+ os.mkdir(self.tmp_dir_name)
+ return self.tmp_dir_name
+
def test_try_to_obtain_new_task_empty(self, mc_get):
mc_get.return_value.json.return_value = {"builds": []}
assert self.dgi.try_to_obtain_new_task() is None
@@ -116,3 +176,173 @@ class TestDistGitImporter(object):
task_data["source_type"] = 999999
mc_get.return_value.json.return_value = {"builds": [task_data]}
assert self.dgi.try_to_obtain_new_task() is None
+
+ def test_fetch_srpm_on(self, mc_urlretrieve):
+ self.dgi.fetch_srpm(self.task_1, self.fetched_srpm_path)
+ assert mc_urlretrieve.called
+
+ mc_urlretrieve.side_effect = IOError
+ with pytest.raises(PackageDownloadException):
+ self.dgi.fetch_srpm(self.task_1, self.fetched_srpm_path)
+
+ def test_my_upload(self):
+ filename = "source"
+ source_path = os.path.join(self.tmp_dir_name, filename)
+ with open(source_path, "w") as handle:
+ handle.write("1")
+
+ reponame = self.PROJECT_NAME
+ target = "/".join([
+ self.lookaside_location, reponame, filename, self.FILE_HASH, filename
+ ])
+ assert not os.path.exists(target)
+ self.dgi.my_upload(self.tmp_dir_name, reponame, filename, self.FILE_HASH)
+ assert os.path.exists(target)
+
+ def test_git_import_srpm(self, mc_pyrpkg_commands):
+ # stupid test, just for the coverage
+ mc_cmd = MagicMock()
+ mc_pyrpkg_commands.return_value = mc_cmd
+ mc_cmd.commithash = self.FILE_HASH
+
+ filename = "source"
+ source_path = os.path.join(self.tmp_dir_name, filename)
+
+ self.task_1.package_name = self.PACKAGE_NAME
+ assert self.dgi.git_import_srpm(self.task_1, source_path) == self.FILE_HASH
+
+ # check exception handling
+ for err in [IOError, OSError, ValueError]:
+ mc_cmd.import_srpm.side_effect = err
+ with pytest.raises(PackageImportException):
+ self.dgi.git_import_srpm(self.task_1, source_path)
+ mc_cmd.import_srpm.side_effect = None
+
+ mc_cmd.push.side_effect = rpkgError
+ assert self.dgi.git_import_srpm(self.task_1, source_path) == self.FILE_HASH
+
+ for err in [IOError, OSError, ValueError]:
+ mc_pyrpkg_commands.side_effect = err
+ with pytest.raises(PackageImportException):
+ self.dgi.git_import_srpm(self.task_1, source_path)
+
+ def test_pkg_name_evr(self, mc_popen):
+ mc_comm = MagicMock()
+ mc_popen.return_value.communicate = mc_comm
+
+ test_plan = [
+ (("(none)", "0.1", "1.fc20"),
"0.1-1.fc20"),
+ (("2", "0.1", "1.fc20"),
"2:0.1-1.fc20")
+ ]
+ for (e, v, r), expected in test_plan:
+ mc_comm.return_value = ("foo {} {} {}".format(e, v, r), None)
+ assert self.dgi.pkg_name_evr("/dev/null") == ("foo",
expected)
+
+ def test_pkg_name_evr_error_handling(self, mc_popen):
+ mc_comm = MagicMock()
+ mc_popen.return_value.communicate = mc_comm
+
+ test_plan = [
+ (("(none)", "0.1", "1.fc20"),
"0.1-1.fc20"),
+ (("2", "0.1", "1.fc20"),
"2:0.1-1.fc20")
+ ]
+
+ mc_popen.side_effect = OSError
+ with pytest.raises(PackageQueryException):
+ self.dgi.pkg_name_evr("/dev/null")
+
+ mc_popen.side_effect = None
+ for (e, v, r), expected in test_plan:
+ mc_comm.return_value = ("foo {} {} {}".format(e, v, r), "err
msg")
+ with pytest.raises(PackageQueryException):
+ self.dgi.pkg_name_evr("/dev/null")
+
+ for (e, v, r), expected in test_plan:
+ mc_comm.return_value = ("", None)
+ with pytest.raises(PackageQueryException):
+ self.dgi.pkg_name_evr("/dev/null")
+
+ def test_before_git_import(self, mc_call):
+ # dummy test, just for coverage
+ self.dgi.before_git_import(self.task_1)
+ assert mc_call.called
+
+ def test_after_git_import(self, mc_call):
+ # dummy test, just for coverage
+ self.dgi.after_git_import()
+ assert mc_call.called
+
+ def test_past_back(self, mc_post):
+ dd = {"foo": "bar"}
+ self.dgi.post_back(dd)
+ assert mc_post.called
+
+ def test_past_back_safe(self, mc_post):
+ dd = {"foo": "bar"}
+ self.dgi.post_back_safe(dd)
+ assert mc_post.called
+ mc_post.reset_mock()
+ assert not mc_post.called
+
+ mc_post.side_effect = IOError
+ self.dgi.post_back_safe(dd)
+ assert mc_post.called
+
+ def test_do_import(self):
+ internal_methods = [
+ "fetch_srpm",
+ "pkg_name_evr",
+ "before_git_import",
+ "git_import_srpm",
+ "after_git_import",
+ "post_back"
+ ]
+ mc_methods = {name: MagicMock() for name in internal_methods}
+ for name, mc in mc_methods.items():
+ setattr(self.dgi, name, mc)
+
+ mc_methods["pkg_name_evr"].return_value = self.PACKAGE_NAME,
self.PACKAGE_VERSION
+ mc_methods["git_import_srpm"].return_value = self.FILE_HASH
+
+ self.dgi.do_import(self.task_1)
+ assert self.task_1.package_name == self.PACKAGE_NAME
+ assert self.task_1.package_version == self.PACKAGE_VERSION
+ assert self.task_1.git_hash == self.FILE_HASH
+
+ for name in internal_methods:
+ mc = mc_methods[name]
+ for err in (IOError, PackageImportException, PackageDownloadException,
PackageQueryException):
+ mc.side_effect = err
+ self.dgi.do_import(self.task_1)
+
+ mc.side_effect = None
+
+ def test_run(self, mc_time):
+ self.dgi.try_to_obtain_new_task = MagicMock()
+ self.dgi.do_import = MagicMock()
+
+ def stop_run(*args, **kwargs):
+ self.dgi.is_running = False
+
+ mc_time.sleep.side_effect = stop_run
+
+ self.dgi.try_to_obtain_new_task.return_value = None
+ self.dgi.run()
+ assert not self.dgi.do_import.called
+
+ self.dgi.try_to_obtain_new_task.return_value = self.task_1
+ self.dgi.do_import.side_effect = stop_run
+ self.dgi.run()
+ assert self.dgi.do_import.call_args == mock.call(self.task_1)
+
+ def test_main(self, mc_dgi, mc_dgcr):
+ # dummy test, just for coverage
+ mc_dgcr.return_value.read.return_value = self.opts
+ main()
+
+ assert mc_dgi.called
+
+ mc_dgcr.return_value.read.side_effect = IOError()
+ with pytest.raises(SystemExit):
+ main()
+