[copr] master: [dist-git] copied systemd unit file from ansible (e225321)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit e22532174f0586eaccb46f8e484442f1844f7e26
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Fri Jul 31 19:56:57 2015 +0200
[dist-git] copied systemd unit file from ansible
>---------------------------------------------------------------
dist-git/copr-dist-git.service | 13 +++++++++++++
1 files changed, 13 insertions(+), 0 deletions(-)
diff --git a/dist-git/copr-dist-git.service b/dist-git/copr-dist-git.service
index e69de29..7bce43b 100644
--- a/dist-git/copr-dist-git.service
+++ b/dist-git/copr-dist-git.service
@@ -0,0 +1,13 @@
+[Unit]
+Description=copr aux service to import srpm into dist-git
+Requires=dist-git.socket
+After=dist-git.socket
+
+
+[Service]
+User=copr-service
+Group=copr-service
+ExecStart=/usr/bin/python2.7 /usr/share/copr/dist_git/dist_git_importer.py
+
+[Install]
+WantedBy=multi-user.target
8 years, 10 months
[copr] master: [dist-git] more test coverage (f6ec0a9)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit f6ec0a9cdf210a342d5786c0187fee1786a28529
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Fri Jul 31 19:44:06 2015 +0200
[dist-git] more test coverage
>---------------------------------------------------------------
backend/copr-backend.spec | 4 +-
dist-git/copr-dist-git.spec | 20 ++-
dist-git/dist_git/dist_git_importer.py | 29 ++--
dist-git/dist_git/helpers.py | 7 +-
dist-git/tests/test_dist_git_importer.py | 236 +++++++++++++++++++++++++++++-
5 files changed, 267 insertions(+), 29 deletions(-)
diff --git a/backend/copr-backend.spec b/backend/copr-backend.spec
index 62519f7..7f14487 100644
--- a/backend/copr-backend.spec
+++ b/backend/copr-backend.spec
@@ -185,8 +185,8 @@ cp -a conf/logstash/copr_backend.conf %{buildroot}%{_pkgdocdir}/examples/%{_sysc
redis-server --port 7777 &> /dev/null &
-#PYTHONPATH=backend:run:$PYTHONPATH python -B -m pytest \
-# -s -v --cov-report term-missing --cov ./backend --cov ./run ./tests/
+PYTHONPATH=backend:run:$PYTHONPATH python -B -m pytest \
+ -s -v --cov-report term-missing --cov ./backend --cov ./run ./tests/
kill %1
diff --git a/dist-git/copr-dist-git.spec b/dist-git/copr-dist-git.spec
index d117676..652e3c0 100644
--- a/dist-git/copr-dist-git.spec
+++ b/dist-git/copr-dist-git.spec
@@ -14,13 +14,17 @@ Source0: %{name}-%{version}.tar.gz
BuildArch: noarch
-BuildRequires: systemd
-
-Requires: systemd
-Requires: dist-git
-Requires: python-bunch
-Requires: python-requests
-Requires: pyrpkg
+BuildRequires: systemd
+BuildRequires: dist-git
+BuildRequires: python-bunch
+BuildRequires: python-requests
+BuildRequires: pyrpkg
+
+Requires: systemd
+Requires: dist-git
+Requires: python-bunch
+Requires: python-requests
+Requires: pyrpkg
%description
COPR is lightweight build system. It allows you to create new project in WebUI
@@ -55,6 +59,8 @@ cp -a conf/logrotate %{buildroot}%{_sysconfdir}/logrotate.d/copr-dist-git
touch %{buildroot}%{_var}/log/copr-dist-git/main.log
%check
+PYTHONPATH=.:$PYTHONPATH python -B -m pytest \
+ -v --cov-report term-missing --cov ./dist_git ./tests/
%post
# change context to be readable by cgit
diff --git a/dist-git/dist_git/dist_git_importer.py b/dist-git/dist_git/dist_git_importer.py
index f548863..88df6a6 100755
--- a/dist-git/dist_git/dist_git_importer.py
+++ b/dist-git/dist_git/dist_git_importer.py
@@ -109,6 +109,7 @@ class ImportTask(object):
class DistGitImporter(object):
def __init__(self, opts):
+ self.is_running = False
self.opts = opts
self.get_url = "{}/backend/importing/".format(self.opts.frontend_base_url)
@@ -136,29 +137,25 @@ class DistGitImporter(object):
@staticmethod
def fetch_srpm(task, fetched_srpm_path):
log.debug("download the package")
-
try:
urlretrieve(task.package_url, fetched_srpm_path)
except IOError:
raise PackageDownloadException()
- return fetched_srpm_path
- @staticmethod
- def my_upload(repo_dir, reponame, filename, filehash):
+ def my_upload(self, repo_dir, reponame, filename, filehash):
"""
This is a replacement function for uploading sources.
Rpkg uses upload.cgi for uploading which doesn't make sense
on the local machine.
"""
- lookaside = "/var/lib/dist-git/cache/lookaside/pkgs/"
source = os.path.join(repo_dir, filename)
- destination = os.path.join(lookaside, reponame, filename, filehash, filename)
+ destination = os.path.join(self.opts.lookaside_location, reponame,
+ filename, filehash, filename)
if not os.path.exists(destination):
os.makedirs(os.path.dirname(destination))
shutil.copyfile(source, destination)
- @classmethod
- def git_import_srpm(cls, task, filepath):
+ def git_import_srpm(self, task, filepath):
"""
Imports a source rpm file into local dist git.
Repository name is in the Copr Style: user/project/package
@@ -195,7 +192,7 @@ class DistGitImporter(object):
# rpkg calls upload.cgi script on the dist git server
# here, I just copy the source files manually with custom function
# I also add one parameter "repo_dir" to that function with this hack
- commands.lookasidecache.upload = types.MethodType(cls.my_upload, repo_dir)
+ commands.lookasidecache.upload = types.MethodType(self.my_upload, repo_dir)
log.debug("clone the pkg repository into tmp directory")
commands.clone(module, tmp, task.branch)
@@ -219,10 +216,12 @@ class DistGitImporter(object):
except rpkgError:
log.exception("error during commit and push, ignored")
- git_hash = commands.commithash
+ return commands.commithash
+ except Exception:
+ log.exception("Unexpected")
+ raise PackageImportException()
finally:
shutil.rmtree(tmp)
- return git_hash
@staticmethod
def pkg_name_evr(srpm_path):
@@ -310,7 +309,8 @@ class DistGitImporter(object):
def run(self):
log.info("DistGitImported initialized")
- while True:
+ self.is_running = True
+ while self.is_running:
mb_task = self.try_to_obtain_new_task()
if mb_task is None:
time.sleep(self.opts.sleep_time)
@@ -347,10 +347,7 @@ def main():
log.info("Using configuration: \n"
"{}".format(opts))
importer = DistGitImporter(opts)
- try:
- importer.run()
- except KeyboardInterrupt:
- return
+ importer.run()
if __name__ == "__main__":
diff --git a/dist-git/dist_git/helpers.py b/dist-git/dist_git/helpers.py
index eb90f4a..3631840 100644
--- a/dist-git/dist_git/helpers.py
+++ b/dist-git/dist_git/helpers.py
@@ -77,6 +77,11 @@ class DistGitConfigReader(object):
)
opts.cgit_pkg_list_location = _get_conf(
- cp, "cgit_pkg_list_location", "log_dir", "/var/lib/copr-dist-git/cgit_pkg_list"
+ cp, "dist-git", "cgit_pkg_list_location", "/var/lib/copr-dist-git/cgit_pkg_list"
)
+
+ opts.lookaside_location = _get_conf(
+ cp, "dist-git", "lookaside_location", "/var/lib/dist-git/cache/lookaside/pkgs/"
+ )
+
return opts
diff --git a/dist-git/tests/test_dist_git_importer.py b/dist-git/tests/test_dist_git_importer.py
index 4cc54b4..7c003d9 100644
--- a/dist-git/tests/test_dist_git_importer.py
+++ b/dist-git/tests/test_dist_git_importer.py
@@ -9,6 +9,7 @@ import tempfile
import shutil
import time
from bunch import Bunch
+from pyrpkg import rpkgError
import pytest
import six
@@ -20,14 +21,35 @@ else:
import mock
from mock import MagicMock
-from dist_git.dist_git_importer import DistGitImporter, SourceType, ImportTask
+from dist_git.dist_git_importer import DistGitImporter, SourceType, ImportTask, PackageDownloadException, \
+ PackageImportException, PackageQueryException, main
MODULE_REF = 'dist_git.dist_git_importer'
@pytest.yield_fixture
+def mc_dgcr():
+ with mock.patch("{}.DistGitConfigReader".format(MODULE_REF)) as handle:
+ yield handle
+
+(a)pytest.yield_fixture
+def mc_dgi():
+ with mock.patch("{}.DistGitImporter".format(MODULE_REF)) as handle:
+ yield handle
+
+(a)pytest.yield_fixture
+def mc_time():
+ with mock.patch("{}.time".format(MODULE_REF)) as handle:
+ yield handle
+
+(a)pytest.yield_fixture
def mc_popen():
- with mock.patch(MODULE_REF) as handle:
+ with mock.patch("{}.Popen".format(MODULE_REF)) as handle:
+ yield handle
+
+(a)pytest.yield_fixture
+def mc_call():
+ with mock.patch("{}.call".format(MODULE_REF)) as handle:
yield handle
@@ -42,8 +64,18 @@ def mc_post():
with mock.patch("{}.post".format(MODULE_REF)) as handle:
yield handle
+(a)pytest.yield_fixture
+def mc_urlretrieve():
+ with mock.patch("{}.urlretrieve".format(MODULE_REF)) as handle:
+ yield handle
-if False:
+(a)pytest.yield_fixture
+def mc_pyrpkg_commands():
+ with mock.patch("{}.Commands".format(MODULE_REF)) as handle:
+ yield handle
+
+
+if True:
logging.basicConfig(
level=logging.DEBUG,
format='[%(asctime)s][%(levelname)s][%(name)s][%(module)s:%(lineno)d] %(message)s',
@@ -53,17 +85,27 @@ if False:
class TestDistGitImporter(object):
def setup_method(self, method):
+ self.tmp_dir_name = self.make_temp_dir()
+ self.lookaside_location = os.path.join(self.tmp_dir_name, "lookaside")
self.opts = Bunch({
"frontend_base_url": "http://front",
"frontend_auth": "secure_password",
+ "lookaside_location": self.lookaside_location,
+
+ "cgit_pkg_list_location": self.tmp_dir_name,
+ "sleep_time": 10,
+ "log_dir": self.tmp_dir_name
})
self.dgi = DistGitImporter(self.opts)
self.USER_NAME = "foo"
self.PROJECT_NAME = "bar"
+ self.PACKAGE_NAME = "bar_app"
+ self.PACKAGE_VERSION = "2:0.01-1.fc20"
self.BRANCH = "f22"
+ self.FILE_HASH = "1234abc"
self.task_data_1 = {
"task_id": 123,
"user": self.USER_NAME,
@@ -86,6 +128,24 @@ class TestDistGitImporter(object):
self.task_1 = ImportTask.from_dict(self.task_data_1, self.opts)
self.task_2 = ImportTask.from_dict(self.task_data_2, self.opts)
+ self.fetched_srpm_path = "/tmp/none/"
+
+
+ def teardown_method(self, method):
+ self.rm_tmp_dir()
+
+ def rm_tmp_dir(self):
+ if self.tmp_dir_name:
+ shutil.rmtree(self.tmp_dir_name)
+ self.tmp_dir_name = None
+
+ def make_temp_dir(self):
+ root_tmp_dir = tempfile.gettempdir()
+ subdir = "test_{}_{}".format(MODULE_REF, time.time())
+ self.tmp_dir_name = os.path.join(root_tmp_dir, subdir)
+ os.mkdir(self.tmp_dir_name)
+ return self.tmp_dir_name
+
def test_try_to_obtain_new_task_empty(self, mc_get):
mc_get.return_value.json.return_value = {"builds": []}
assert self.dgi.try_to_obtain_new_task() is None
@@ -116,3 +176,173 @@ class TestDistGitImporter(object):
task_data["source_type"] = 999999
mc_get.return_value.json.return_value = {"builds": [task_data]}
assert self.dgi.try_to_obtain_new_task() is None
+
+ def test_fetch_srpm_on(self, mc_urlretrieve):
+ self.dgi.fetch_srpm(self.task_1, self.fetched_srpm_path)
+ assert mc_urlretrieve.called
+
+ mc_urlretrieve.side_effect = IOError
+ with pytest.raises(PackageDownloadException):
+ self.dgi.fetch_srpm(self.task_1, self.fetched_srpm_path)
+
+ def test_my_upload(self):
+ filename = "source"
+ source_path = os.path.join(self.tmp_dir_name, filename)
+ with open(source_path, "w") as handle:
+ handle.write("1")
+
+ reponame = self.PROJECT_NAME
+ target = "/".join([
+ self.lookaside_location, reponame, filename, self.FILE_HASH, filename
+ ])
+ assert not os.path.exists(target)
+ self.dgi.my_upload(self.tmp_dir_name, reponame, filename, self.FILE_HASH)
+ assert os.path.exists(target)
+
+ def test_git_import_srpm(self, mc_pyrpkg_commands):
+ # stupid test, just for the coverage
+ mc_cmd = MagicMock()
+ mc_pyrpkg_commands.return_value = mc_cmd
+ mc_cmd.commithash = self.FILE_HASH
+
+ filename = "source"
+ source_path = os.path.join(self.tmp_dir_name, filename)
+
+ self.task_1.package_name = self.PACKAGE_NAME
+ assert self.dgi.git_import_srpm(self.task_1, source_path) == self.FILE_HASH
+
+ # check exception handling
+ for err in [IOError, OSError, ValueError]:
+ mc_cmd.import_srpm.side_effect = err
+ with pytest.raises(PackageImportException):
+ self.dgi.git_import_srpm(self.task_1, source_path)
+ mc_cmd.import_srpm.side_effect = None
+
+ mc_cmd.push.side_effect = rpkgError
+ assert self.dgi.git_import_srpm(self.task_1, source_path) == self.FILE_HASH
+
+ for err in [IOError, OSError, ValueError]:
+ mc_pyrpkg_commands.side_effect = err
+ with pytest.raises(PackageImportException):
+ self.dgi.git_import_srpm(self.task_1, source_path)
+
+ def test_pkg_name_evr(self, mc_popen):
+ mc_comm = MagicMock()
+ mc_popen.return_value.communicate = mc_comm
+
+ test_plan = [
+ (("(none)", "0.1", "1.fc20"), "0.1-1.fc20"),
+ (("2", "0.1", "1.fc20"), "2:0.1-1.fc20")
+ ]
+ for (e, v, r), expected in test_plan:
+ mc_comm.return_value = ("foo {} {} {}".format(e, v, r), None)
+ assert self.dgi.pkg_name_evr("/dev/null") == ("foo", expected)
+
+ def test_pkg_name_evr_error_handling(self, mc_popen):
+ mc_comm = MagicMock()
+ mc_popen.return_value.communicate = mc_comm
+
+ test_plan = [
+ (("(none)", "0.1", "1.fc20"), "0.1-1.fc20"),
+ (("2", "0.1", "1.fc20"), "2:0.1-1.fc20")
+ ]
+
+ mc_popen.side_effect = OSError
+ with pytest.raises(PackageQueryException):
+ self.dgi.pkg_name_evr("/dev/null")
+
+ mc_popen.side_effect = None
+ for (e, v, r), expected in test_plan:
+ mc_comm.return_value = ("foo {} {} {}".format(e, v, r), "err msg")
+ with pytest.raises(PackageQueryException):
+ self.dgi.pkg_name_evr("/dev/null")
+
+ for (e, v, r), expected in test_plan:
+ mc_comm.return_value = ("", None)
+ with pytest.raises(PackageQueryException):
+ self.dgi.pkg_name_evr("/dev/null")
+
+ def test_before_git_import(self, mc_call):
+ # dummy test, just for coverage
+ self.dgi.before_git_import(self.task_1)
+ assert mc_call.called
+
+ def test_after_git_import(self, mc_call):
+ # dummy test, just for coverage
+ self.dgi.after_git_import()
+ assert mc_call.called
+
+ def test_past_back(self, mc_post):
+ dd = {"foo": "bar"}
+ self.dgi.post_back(dd)
+ assert mc_post.called
+
+ def test_past_back_safe(self, mc_post):
+ dd = {"foo": "bar"}
+ self.dgi.post_back_safe(dd)
+ assert mc_post.called
+ mc_post.reset_mock()
+ assert not mc_post.called
+
+ mc_post.side_effect = IOError
+ self.dgi.post_back_safe(dd)
+ assert mc_post.called
+
+ def test_do_import(self):
+ internal_methods = [
+ "fetch_srpm",
+ "pkg_name_evr",
+ "before_git_import",
+ "git_import_srpm",
+ "after_git_import",
+ "post_back"
+ ]
+ mc_methods = {name: MagicMock() for name in internal_methods}
+ for name, mc in mc_methods.items():
+ setattr(self.dgi, name, mc)
+
+ mc_methods["pkg_name_evr"].return_value = self.PACKAGE_NAME, self.PACKAGE_VERSION
+ mc_methods["git_import_srpm"].return_value = self.FILE_HASH
+
+ self.dgi.do_import(self.task_1)
+ assert self.task_1.package_name == self.PACKAGE_NAME
+ assert self.task_1.package_version == self.PACKAGE_VERSION
+ assert self.task_1.git_hash == self.FILE_HASH
+
+ for name in internal_methods:
+ mc = mc_methods[name]
+ for err in (IOError, PackageImportException, PackageDownloadException, PackageQueryException):
+ mc.side_effect = err
+ self.dgi.do_import(self.task_1)
+
+ mc.side_effect = None
+
+ def test_run(self, mc_time):
+ self.dgi.try_to_obtain_new_task = MagicMock()
+ self.dgi.do_import = MagicMock()
+
+ def stop_run(*args, **kwargs):
+ self.dgi.is_running = False
+
+ mc_time.sleep.side_effect = stop_run
+
+ self.dgi.try_to_obtain_new_task.return_value = None
+ self.dgi.run()
+ assert not self.dgi.do_import.called
+
+ self.dgi.try_to_obtain_new_task.return_value = self.task_1
+ self.dgi.do_import.side_effect = stop_run
+ self.dgi.run()
+ assert self.dgi.do_import.call_args == mock.call(self.task_1)
+
+ def test_main(self, mc_dgi, mc_dgcr):
+ # dummy test, just for coverage
+ mc_dgcr.return_value.read.return_value = self.opts
+ main()
+
+ assert mc_dgi.called
+
+ mc_dgcr.return_value.read.side_effect = IOError()
+ with pytest.raises(SystemExit):
+ main()
+
8 years, 10 months
[copr] master: [dist-git] added initial tests; renamed folder with sources to use underscore instead of dash (dd0ab58)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit dd0ab580430c28b95cbf1b4dc3474c557bc6b7c3
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Fri Jul 31 17:17:25 2015 +0200
[dist-git] added initial tests; renamed folder with sources to use underscore instead of dash
>---------------------------------------------------------------
dist-git/copr-dist-git.spec | 4 +-
dist-git/{dist-git => dist_git}/__init__.py | 0
.../{dist-git => dist_git}/dist_git_importer.py | 74 +++++-------
dist-git/{dist-git => dist_git}/helpers.py | 0
dist-git/run_tests.sh | 1 +
dist-git/tests/test_dist_git_importer.py | 118 ++++++++++++++++++++
6 files changed, 151 insertions(+), 46 deletions(-)
diff --git a/dist-git/copr-dist-git.spec b/dist-git/copr-dist-git.spec
index 09aace1..d117676 100644
--- a/dist-git/copr-dist-git.spec
+++ b/dist-git/copr-dist-git.spec
@@ -38,14 +38,14 @@ This package contains Copr services for Dist Git server
%install
-install -d %{buildroot}%{_datadir}/copr/dist-git
+install -d %{buildroot}%{_datadir}/copr/dist_git
install -d %{buildroot}%{_sysconfdir}/copr
install -d %{buildroot}%{_sysconfdir}/logrotate.d/
install -d %{buildroot}%{_unitdir}
install -d %{buildroot}%{_var}/log/copr-dist-git
install -d %{buildroot}%{_sharedstatedir}/copr-dist-git
-cp -a dist-git/* %{buildroot}%{_datadir}/copr/dist-git
+cp -a dist_git/* %{buildroot}%{_datadir}/copr/dist_git
cp -a conf/copr-dist-git.conf.example %{buildroot}%{_sysconfdir}/copr/copr-dist-git.conf
cp -a copr-dist-git.service %{buildroot}%{_unitdir}/
diff --git a/dist-git/dist-git/__init__.py b/dist-git/dist_git/__init__.py
similarity index 100%
rename from dist-git/dist-git/__init__.py
rename to dist-git/dist_git/__init__.py
diff --git a/dist-git/dist-git/dist_git_importer.py b/dist-git/dist_git/dist_git_importer.py
similarity index 86%
rename from dist-git/dist-git/dist_git_importer.py
rename to dist-git/dist_git/dist_git_importer.py
index 058ae64..f548863 100755
--- a/dist-git/dist-git/dist_git_importer.py
+++ b/dist-git/dist_git/dist_git_importer.py
@@ -4,14 +4,13 @@ import os
import json
import time
import types
-import urllib
+from urllib import urlretrieve
import shutil
import tempfile
import logging
-import subprocess
+from subprocess import PIPE, Popen
-from requests import get
-from requests import post
+from requests import get, post
# pyrpkg uses os.getlogin(). It requires tty which is unavailable when we run this script as a daemon
# very dirty solution for now
@@ -30,17 +29,6 @@ from helpers import DistGitConfigReader
log = logging.getLogger(__name__)
-# Example usage:
-#
-# user = "asamalik"
-# project = "project-for-dist-git"
-# pkg_name = "devtoolset-3"
-# branch = "f20"
-# filepath = "/tmp/rh-php56-php-5.6.5-5.el7.src.rpm"
-#
-# git_hash = import_srpm(user, project, pkg_name, branch, filepath)
-
-
class PackageImportException(Exception):
pass
@@ -53,22 +41,6 @@ class PackageQueryException(Exception):
pass
-def _my_upload(repo_dir, reponame, filename, filehash):
- """
- This is a replacement function for uploading sources.
- Rpkg uses upload.cgi for uploading which doesn't make sense
- on the local machine.
- """
- lookaside = "/var/lib/dist-git/cache/lookaside/pkgs/"
- source = os.path.join(repo_dir, filename)
- destination = os.path.join(lookaside, reponame, filename, filehash, filename)
- if not os.path.exists(destination):
- os.makedirs(os.path.dirname(destination))
- shutil.copyfile(source, destination)
-
-
-
-
class SourceType:
SRPM_LINK = 1
SRPM_UPLOAD = 2
@@ -166,13 +138,27 @@ class DistGitImporter(object):
log.debug("download the package")
try:
- urllib.urlretrieve(task.package_url, fetched_srpm_path)
+ urlretrieve(task.package_url, fetched_srpm_path)
except IOError:
raise PackageDownloadException()
return fetched_srpm_path
@staticmethod
- def git_import_srpm(task, filepath):
+ def my_upload(repo_dir, reponame, filename, filehash):
+ """
+ This is a replacement function for uploading sources.
+ Rpkg uses upload.cgi for uploading which doesn't make sense
+ on the local machine.
+ """
+ lookaside = "/var/lib/dist-git/cache/lookaside/pkgs/"
+ source = os.path.join(repo_dir, filename)
+ destination = os.path.join(lookaside, reponame, filename, filehash, filename)
+ if not os.path.exists(destination):
+ os.makedirs(os.path.dirname(destination))
+ shutil.copyfile(source, destination)
+
+ @classmethod
+ def git_import_srpm(cls, task, filepath):
"""
Imports a source rpm file into local dist git.
Repository name is in the Copr Style: user/project/package
@@ -180,12 +166,12 @@ class DistGitImporter(object):
:type task: ImportTask
"""
- log.debug("importing it and delete the srpm")
+ log.debug("importing srpm into the dist-git")
# I need to use git via SSH because of gitolite as it manages
# permissions with it's hook that relies on gitolite console
# which is a default shell on SSH
- git_bas_eurl = "ssh://copr-dist-git@localhost/%(module)s"
+ git_base_url = "ssh://copr-dist-git@localhost/%(module)s"
tmp = tempfile.mkdtemp()
try:
repo_dir = os.path.join(tmp, task.package_name)
@@ -195,7 +181,7 @@ class DistGitImporter(object):
lookaside="",
lookasidehash="md5",
lookaside_cgi="",
- gitbaseurl=git_bas_eurl,
+ gitbaseurl=git_base_url,
anongiturl="",
branchre="",
kojiconfig="",
@@ -209,7 +195,7 @@ class DistGitImporter(object):
# rpkg calls upload.cgi script on the dist git server
# here, I just copy the source files manually with custom function
# I also add one parameter "repo_dir" to that function with this hack
- commands.lookasidecache.upload = types.MethodType(_my_upload, repo_dir)
+ commands.lookasidecache.upload = types.MethodType(cls.my_upload, repo_dir)
log.debug("clone the pkg repository into tmp directory")
commands.clone(module, tmp, task.branch)
@@ -231,22 +217,22 @@ class DistGitImporter(object):
commands.push()
log.debug("commit and push done")
except rpkgError:
- pass
+ log.exception("error during commit and push, ignored")
+
git_hash = commands.commithash
finally:
shutil.rmtree(tmp)
return git_hash
@staticmethod
- def pkg_name_evr(pkg):
+ def pkg_name_evr(srpm_path):
"""
Queries a package for its name and evr (epoch:version-release)
"""
log.debug("Verifying packagage, getting name and version.")
- cmd = ['rpm', '-qp', '--nosignature', '--qf', '%{NAME} %{EPOCH} %{VERSION} %{RELEASE}', pkg]
+ cmd = ['rpm', '-qp', '--nosignature', '--qf', '%{NAME} %{EPOCH} %{VERSION} %{RELEASE}', srpm_path]
try:
- proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
+ proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
output, error = proc.communicate()
except OSError as e:
raise PackageQueryException(e)
@@ -280,14 +266,14 @@ class DistGitImporter(object):
"""
Could raise error related to networkd connection
"""
- post(self.upload_url, auth=self.auth, data=json.dumps(data_dict), headers=self.headers)
+ return post(self.upload_url, auth=self.auth, data=json.dumps(data_dict), headers=self.headers)
def post_back_safe(self, data_dict):
"""
Ignores any error
"""
try:
- self.post_back(data_dict)
+ return self.post_back(data_dict)
except Exception:
log.exception("Failed to post back to frontend : {}".format(data_dict))
diff --git a/dist-git/dist-git/helpers.py b/dist-git/dist_git/helpers.py
similarity index 100%
rename from dist-git/dist-git/helpers.py
rename to dist-git/dist_git/helpers.py
diff --git a/dist-git/run_tests.sh b/dist-git/run_tests.sh
new file mode 100755
index 0000000..83bc02f
--- /dev/null
+++ b/dist-git/run_tests.sh
@@ -0,0 +1 @@
+PYTHONPATH=./src: python2 -B -m pytest --cov-report term-missing --cov ./dist_git tests $@
diff --git a/dist-git/tests/test_dist_git_importer.py b/dist-git/tests/test_dist_git_importer.py
new file mode 100644
index 0000000..4cc54b4
--- /dev/null
+++ b/dist-git/tests/test_dist_git_importer.py
@@ -0,0 +1,118 @@
+# coding: utf-8
+import json
+import logging
+
+import os
+import copy
+import tarfile
+import tempfile
+import shutil
+import time
+from bunch import Bunch
+import pytest
+
+import six
+
+if six.PY3:
+ from unittest import mock
+ from unittest.mock import MagicMock
+else:
+ import mock
+ from mock import MagicMock
+
+from dist_git.dist_git_importer import DistGitImporter, SourceType, ImportTask
+
+MODULE_REF = 'dist_git.dist_git_importer'
+
+
+(a)pytest.yield_fixture
+def mc_popen():
+ with mock.patch(MODULE_REF) as handle:
+ yield handle
+
+
+(a)pytest.yield_fixture
+def mc_get():
+ with mock.patch("{}.get".format(MODULE_REF)) as handle:
+ yield handle
+
+
+(a)pytest.yield_fixture
+def mc_post():
+ with mock.patch("{}.post".format(MODULE_REF)) as handle:
+ yield handle
+
+
+if False:
+ logging.basicConfig(
+ level=logging.DEBUG,
+ format='[%(asctime)s][%(levelname)s][%(name)s][%(module)s:%(lineno)d] %(message)s',
+ datefmt='%H:%M:%S'
+ )
+
+
+class TestDistGitImporter(object):
+ def setup_method(self, method):
+ self.opts = Bunch({
+ "frontend_base_url": "http://front",
+ "frontend_auth": "secure_password",
+
+ })
+
+ self.dgi = DistGitImporter(self.opts)
+
+ self.USER_NAME = "foo"
+ self.PROJECT_NAME = "bar"
+ self.BRANCH = "f22"
+ self.task_data_1 = {
+ "task_id": 123,
+ "user": self.USER_NAME,
+ "project": self.PROJECT_NAME,
+
+ "branch": self.BRANCH,
+ "source_type": SourceType.SRPM_LINK,
+ "source_json": json.dumps({"url": "http://example.com/pkg.src.rpm"})
+ }
+ self.task_data_2 = {
+ "task_id": 124,
+ "user": self.USER_NAME,
+ "project": self.PROJECT_NAME,
+
+ "branch": self.BRANCH,
+ "source_type": SourceType.SRPM_UPLOAD,
+ "source_json": json.dumps({"tmp": "tmp_2", "pkg": "pkg_2.src.rpm"})
+ }
+
+ self.task_1 = ImportTask.from_dict(self.task_data_1, self.opts)
+ self.task_2 = ImportTask.from_dict(self.task_data_2, self.opts)
+
+ def test_try_to_obtain_new_task_empty(self, mc_get):
+ mc_get.return_value.json.return_value = {"builds": []}
+ assert self.dgi.try_to_obtain_new_task() is None
+
+ def test_try_to_obtain_handle_error(self, mc_get):
+ for err in [IOError, OSError, ValueError]:
+ mc_get.side_effect = err
+ assert self.dgi.try_to_obtain_new_task() is None
+
+ def test_try_to_obtain_ok(self, mc_get):
+ mc_get.return_value.json.return_value = {"builds": [self.task_data_1, self.task_data_2]}
+ task = self.dgi.try_to_obtain_new_task()
+ assert task.task_id == self.task_data_1["task_id"]
+ assert task.user == self.USER_NAME
+ assert task.branch == self.BRANCH
+ assert task.package_url == "http://example.com/pkg.src.rpm"
+
+ def test_try_to_obtain_ok_2(self, mc_get):
+ mc_get.return_value.json.return_value = {"builds": [self.task_data_2, self.task_data_1]}
+ task = self.dgi.try_to_obtain_new_task()
+ assert task.task_id == self.task_data_2["task_id"]
+ assert task.user == self.USER_NAME
+ assert task.branch == self.BRANCH
+ assert task.package_url == "http://front/tmp/tmp_2/pkg_2.src.rpm"
+
+ def test_try_to_obtain_new_task_unknown_source_type(self, mc_get):
+ task_data = copy.deepcopy(self.task_data_1)
+ task_data["source_type"] = 999999
+ mc_get.return_value.json.return_value = {"builds": [task_data]}
+ assert self.dgi.try_to_obtain_new_task() is None
8 years, 10 months
[copr] master: Do not use email in contact automatically (790a1ad)
by asamalik@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 790a1adecc4be0eadd51ac62819529c067431e39
Author: Adam Samalik <asamalik(a)redhat.com>
Date: Fri Jul 31 09:56:59 2015 +0200
Do not use email in contact automatically
This is a new function that might get unnoticed. In that case,
user's email would be exposed to public without their knowing.
So let's make it opt-in instead.
>---------------------------------------------------------------
frontend/coprs_frontend/coprs/forms.py | 1 -
1 files changed, 0 insertions(+), 1 deletions(-)
diff --git a/frontend/coprs_frontend/coprs/forms.py b/frontend/coprs_frontend/coprs/forms.py
index 7a50562..9cbd29c 100644
--- a/frontend/coprs_frontend/coprs/forms.py
+++ b/frontend/coprs_frontend/coprs/forms.py
@@ -167,7 +167,6 @@ class CoprFormFactory(object):
contact = wtforms.StringField(
"Contact",
- default=flask.g.user.mail,
validators=[
wtforms.validators.Optional(),
EmailOrURL()])
8 years, 10 months
[copr] master: update copr form info (c3c8b62)
by asamalik@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit c3c8b62379545a0032ac54de9f714174ee515bf0
Author: Adam Samalik <asamalik(a)redhat.com>
Date: Fri Jul 31 09:42:41 2015 +0200
update copr form info
>---------------------------------------------------------------
.../coprs_frontend/coprs/templates/_helpers.html | 13 ++++++++++++-
.../coprs/templates/coprs/_coprs_forms.html | 20 ++++++--------------
2 files changed, 18 insertions(+), 15 deletions(-)
diff --git a/frontend/coprs_frontend/coprs/templates/_helpers.html b/frontend/coprs_frontend/coprs/templates/_helpers.html
index d420411..f8573fd 100644
--- a/frontend/coprs_frontend/coprs/templates/_helpers.html
+++ b/frontend/coprs_frontend/coprs/templates/_helpers.html
@@ -1,4 +1,4 @@
-{% macro render_field(field, label=None, class='') %}
+{% macro render_field(field, label=None, class='', info=None) %}
{% if not kwargs['hidden'] %}
{% if field.errors %}
<div class="form-group has-error">
@@ -10,11 +10,22 @@
</label>
<div class="col-sm-10">
{{ field(class="form-control", **kwargs)|safe }}
+ <ul class="list-unstyled">
+ {% if info %}
+ <li class="help-block">
+ <small class="text-muted pficon pficon-info">
+ {{info|safe}}
+ </small>
+ </li>
+ {% endif %}
{% if field.errors %}
{% for error in field.errors %}
+ <li class="help-block">
<span class="help-block">{{ error }}</span>
+ </li>
{% endfor %}
{% endif %}
+ </ul>
</div>
</div>
{% else %}
diff --git a/frontend/coprs_frontend/coprs/templates/coprs/_coprs_forms.html b/frontend/coprs_frontend/coprs/templates/coprs/_coprs_forms.html
index 974bc31..56f393c 100644
--- a/frontend/coprs_frontend/coprs/templates/coprs/_coprs_forms.html
+++ b/frontend/coprs_frontend/coprs/templates/coprs/_coprs_forms.html
@@ -22,21 +22,13 @@
{{ render_field(form.name, hidden = True) }}
{{ render_field(form.name, label='Project Name', disabled = True) }}
{% endif %}
- {{ render_field(form.description, rows=5, cols=50, placeholder='Optional - describe your project briefly.') }}
- {{ render_field(form.instructions, rows=5, cols=50, placeholder='Optional - describe how your project can be installed. Where users can report bugs and issues. Link to wiki is good as well.') }}
- {{ render_field(form.homepage, label='Homepage', placeholder='Optional - project homepage') }}
- {{ render_field(form.contact, label='Contact', placeholder='Optional - email address or contact url') }}
+ {{ render_field(form.description, rows=5, cols=50, placeholder='Optional - describe your project briefly.',
+ info='You can use <a href="http://daringfireball.net/projects/markdown/syntax">markdown syntax</a>, inline HTMLis forbidden.') }}
+ {{ render_field(form.instructions, rows=5, cols=50, placeholder='Optional - describe how your project can be installed. Where users can report bugs and issues. Link to wiki is good as well.', info='You can use <a href="http://daringfireball.net/projects/markdown/syntax">markdown syntax</a>, inline HTMLis forbidden.') }}
- <div class="form-group">
- <label class="col-sm-2 control-label" for="textInput-markup">
- </label>
- <div class="col-sm-10">
- <small class="text-muted pficon pficon-info">
- You can use <a href="http://daringfireball.net/projects/markdown/syntax">markdown syntax</a>, inline HTML is forbidden.
- </small>
- </div>
- </div>
+ {{ render_field(form.homepage, label='Homepage', placeholder='Optional - project homepage') }}
+ {{ render_field(form.contact, label='Contact', placeholder='Optional - email address or contact url', info="Use e-mail address or a link to your issue tracker. This information will be shown publicly.") }}
{% if form._mock_chroots_error %}
<div class="form-group has-error">
@@ -59,7 +51,7 @@
</div>
{% endfor %}{% endfor %}
{% if form._mock_chroots_error %}
- <span class=" help-block"> {{ form._mock_chroots_error }}</span>
+ <span class="help-block pficon pficon-warning-triangle-o"> {{ form._mock_chroots_error }}</span>
{% endif %}
</div>
</div>
8 years, 10 months
[copr] master: [dist-git] refactoring importer (3b504f7)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 3b504f78423aced5d1c9cade99e5384771ecb281
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Thu Jul 30 17:46:38 2015 +0200
[dist-git] refactoring importer
>---------------------------------------------------------------
dist-git/dist-git/dist_git_importer.py | 408 ++++++++++++++++++--------------
dist-git/dist-git/helpers.py | 4 +
2 files changed, 233 insertions(+), 179 deletions(-)
diff --git a/dist-git/dist-git/dist_git_importer.py b/dist-git/dist-git/dist_git_importer.py
index a39528b..058ae64 100755
--- a/dist-git/dist-git/dist_git_importer.py
+++ b/dist-git/dist-git/dist_git_importer.py
@@ -67,98 +67,75 @@ def _my_upload(repo_dir, reponame, filename, filehash):
shutil.copyfile(source, destination)
-def import_srpm(user, project, pkg, branch, filepath):
- """
- Imports a source rpm file into local dist git.
- Repository name is in the Copr Style: user/project/package
- filepath is a srpm file locally downloaded somewhere
- """
- # I need to use git via SSH because of gitolite as it manages
- # permissions with it's hook that relies on gitolite console
- # which is a default shell on SSH
- gitbaseurl = "ssh://copr-dist-git@localhost/%(module)s"
- tmp = tempfile.mkdtemp()
- try:
- repo_dir = os.path.join(tmp, pkg)
- log.debug("repo_dir: {}".format(repo_dir))
- # use rpkg for importing the source rpm
- commands = Commands(path=repo_dir,
- lookaside="",
- lookasidehash="md5",
- lookaside_cgi="",
- gitbaseurl=gitbaseurl,
- anongiturl="",
- branchre="",
- kojiconfig="",
- build_client="")
-
- # rpkg gets module_name as a basename of git url
- # we use module_name as "username/projectname/packagename"
- # basename is not working here - so I'm setting it manually
- module = "{}/{}/{}".format(user, project, pkg)
- commands.module_name = module
- # rpkg calls upload.cgi script on the dist git server
- # here, I just copy the source files manually with custom function
- # I also add one parameter "repo_dir" to that function with this hack
- commands.lookasidecache.upload = types.MethodType(_my_upload, repo_dir)
-
- log.debug("clone the pkg repository into tmp directory")
- commands.clone(module, tmp, branch)
-
- log.debug("import the source rpm into git and save filenames of sources")
- try:
- uploadfiles = commands.import_srpm(filepath)
- except Exception:
- log.exception("Failed to import the source rpm: {}".format(filepath))
- raise PackageImportException
- log.info("save the source files into lookaside cache")
- commands.upload(uploadfiles, replace=True)
- log.debug("git push")
- message = "Import of {}".format(os.path.basename(filepath))
- try:
- commands.commit(message)
- commands.push()
- log.debug("commit and push done")
- except rpkgError:
- pass
- git_hash = commands.commithash
- finally:
- shutil.rmtree(tmp)
- return git_hash
-
-
-def pkg_name_evr(pkg):
- """
- Queries a package for its name and evr (epoch:version-release)
- """
- log.debug("Verifying packagage, getting name and version.")
- cmd = ['rpm', '-qp', '--nosignature', '--qf', '%{NAME} %{EPOCH} %{VERSION} %{RELEASE}', pkg]
- try:
- proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
- output, error = proc.communicate()
- except OSError as e:
- raise PackageQueryException(e)
- if error:
- raise PackageQueryException('Error querying srpm: %s' % error)
+class SourceType:
+ SRPM_LINK = 1
+ SRPM_UPLOAD = 2
- try:
- name, epoch, version, release = output.split(" ")
- except ValueError as e:
- raise PackageQueryException(e)
- # Epoch is an integer or '(none)' if not set
- if epoch.isdigit():
- evr = "{}:{}-{}".format(epoch, version, release)
- else:
- evr = "{}-{}".format(version, release)
+class ImportTask(object):
+ def __init__(self):
+
+ self.task_id = None
+ self.user = None
+ self.project = None
+ self.branch = None
+
+ self.source_type = None
+ self.source_json = None
+ self.source_data = None
+
+ self.package_name = None
+ self.package_version = None
+ self.git_hash = None
+
+ self.package_url = None
+
+ @property
+ def reponame(self):
+ if any(x is None for x in [self.user, self.project, self.package_name]):
+ return None
+ else:
+ return "{}/{}/{}".format(self.user, self.project, self.package_name)
+
+ @staticmethod
+ def from_dict(dict_data, opts):
+ task = ImportTask()
- return name, evr
+ task.task_id = dict_data["task_id"]
+ task.user = dict_data["user"]
+ task.project = dict_data["project"]
+ task.branch = dict_data["branch"]
+ task.source_type = dict_data["source_type"]
+ task.source_json = dict_data["source_json"]
+ task.source_data = json.loads(dict_data["source_json"])
-class DistGitImporter():
+ if task.source_type == SourceType.SRPM_LINK:
+ task.package_url = json.loads(task.source_json)["url"]
+
+ elif task.source_type == SourceType.SRPM_UPLOAD:
+ json_tmp = task.source_data["tmp"]
+ json_pkg = task.source_data["pkg"]
+ task.package_url = "{}/tmp/{}/{}".format(opts.frontend_base_url, json_tmp, json_pkg)
+
+ else:
+ raise PackageImportException("Got unknown source type: {}".format(task.source_type))
+
+ return task
+
+ def get_dict_for_frontend(self):
+ return {
+ "task_id": self.task_id,
+ "pkg_name": self.package_name,
+ "pkg_version": self.package_version,
+ "repo_name": self.reponame,
+ "git_hash": self.git_hash
+ }
+
+
+class DistGitImporter(object):
def __init__(self, opts):
self.opts = opts
@@ -167,104 +144,137 @@ class DistGitImporter():
self.auth = ("user", self.opts.frontend_auth)
self.headers = {"content-type": "application/json"}
- def run(self):
+ self.tmp_root = None
- tmp = tempfile.mkdtemp()
- log.info("DistGitImported initialized")
+ def try_to_obtain_new_task(self):
+ log.debug("1. Try to get task data")
try:
- while True:
- log.info("1. Try to get task data")
- try:
- # get the data
- r = get(self.get_url)
-
- # take the first task
- builds_list = r.json()["builds"]
- if len(builds_list) == 0:
- log.debug("No new tasks to process")
- time.sleep(30)
- continue
- task = builds_list[0]
-
- # extract data from it
- task_id = task["task_id"]
- user = task["user"]
- project = task["project"]
- #package = task["package"]
- branch = task["branch"]
- source_type = task["source_type"]
- source_json = task["source_json"]
-
- if source_type == 1: # SRPM link
- package_url = json.loads(source_json)["url"]
- elif source_type == 2: # SRPM upload
- json_tmp = json.loads(source_json)["tmp"]
- json_pkg = json.loads(source_json)["pkg"]
- package_url = "{}/tmp/{}/{}".format(
- self.opts.frontend_base_url, json_tmp, json_pkg)
- else:
- log.error("Got unknown source type: {}".format(source_type))
- raise Exception("Got unknown source type: {}".format(source_type))
-
- except KeyboardInterrupt:
- sys.exit(0)
-
- except Exception:
- log.exception("Failed acquire new packages for import")
- time.sleep(30)
- continue
-
- log.info("2. Importing the package: {}".format(package_url))
- try:
- log.debug("download the package")
- fetched_srpm_path = os.path.join(tmp, os.path.basename(package_url))
- try:
- urllib.urlretrieve(package_url, fetched_srpm_path)
- except IOError:
- raise PackageDownloadException
-
- # todo: check that obtained file is a REAL srpm
- # todo query package name & version and ise real name instead of task["package"]
- # if fetched file is not a proper srpm set error state
- name, version = pkg_name_evr(fetched_srpm_path)
-
- reponame = "{}/{}/{}".format(user, project, name)
- log.debug("make sure repos exist: {}".format(reponame))
- call(["/usr/share/dist-git/git_package.sh", reponame])
- call(["/usr/share/dist-git/git_branch.sh", branch, reponame])
-
- log.debug("import it and delete the srpm")
- git_hash = import_srpm(user, project, name, branch, fetched_srpm_path)
-
- log.debug("send a response - success")
-
- # refresh cgit
- call(["/usr/share/dist-git/cgit_pkg_list.sh", self.opts.cgit_pkg_list_location])
-
- # send a response - success
- data = {"task_id": task_id,
- "pkg_name": name,
- "pkg_version": version,
- "repo_name": reponame,
- "git_hash": git_hash}
- self.post_back(data)
-
- except (PackageImportException, PackageDownloadException, PackageQueryException):
- log.info("send a response - failure during import of: {}".format(package_url))
- self.post_back_safe({"task_id": task_id, "error": "error"})
-
- except Exception:
- log.exception("Unexpected error during package import")
- self.post_back_safe({"task_id": task_id, "error": "error"})
-
- finally:
- try:
- os.remove(fetched_srpm_path)
- except Exception:
- log.exception("Failed to remove fetched srpm")
+ # get the data
+ r = get(self.get_url)
+ # take the first task
+ builds_list = r.json()["builds"]
+ if len(builds_list) == 0:
+ log.debug("No new tasks to process")
+ return
+ return ImportTask.from_dict(builds_list[0], self.opts)
+ except Exception:
+ log.exception("Failed acquire new packages for import")
+ return
+
+ @staticmethod
+ def fetch_srpm(task, fetched_srpm_path):
+ log.debug("download the package")
+
+ try:
+ urllib.urlretrieve(task.package_url, fetched_srpm_path)
+ except IOError:
+ raise PackageDownloadException()
+ return fetched_srpm_path
+
+ @staticmethod
+ def git_import_srpm(task, filepath):
+ """
+ Imports a source rpm file into local dist git.
+ Repository name is in the Copr Style: user/project/package
+ filepath is a srpm file locally downloaded somewhere
+
+ :type task: ImportTask
+ """
+ log.debug("importing it and delete the srpm")
+ # I need to use git via SSH because of gitolite as it manages
+ # permissions with it's hook that relies on gitolite console
+ # which is a default shell on SSH
+ git_bas_eurl = "ssh://copr-dist-git@localhost/%(module)s"
+ tmp = tempfile.mkdtemp()
+ try:
+ repo_dir = os.path.join(tmp, task.package_name)
+ log.debug("repo_dir: {}".format(repo_dir))
+ # use rpkg for importing the source rpm
+ commands = Commands(path=repo_dir,
+ lookaside="",
+ lookasidehash="md5",
+ lookaside_cgi="",
+ gitbaseurl=git_bas_eurl,
+ anongiturl="",
+ branchre="",
+ kojiconfig="",
+ build_client="")
+
+ # rpkg gets module_name as a basename of git url
+ # we use module_name as "username/projectname/package_name"
+ # basename is not working here - so I'm setting it manually
+ module = "{}/{}/{}".format(task.user, task.project, task.package_name)
+ commands.module_name = module
+ # rpkg calls upload.cgi script on the dist git server
+ # here, I just copy the source files manually with custom function
+ # I also add one parameter "repo_dir" to that function with this hack
+ commands.lookasidecache.upload = types.MethodType(_my_upload, repo_dir)
+
+ log.debug("clone the pkg repository into tmp directory")
+ commands.clone(module, tmp, task.branch)
+
+ log.debug("import the source rpm into git and save filenames of sources")
+ try:
+ upload_files = commands.import_srpm(filepath)
+ except Exception:
+ log.exception("Failed to import the source rpm: {}".format(filepath))
+ raise PackageImportException()
+
+ log.info("save the source files into lookaside cache")
+ commands.upload(upload_files, replace=True)
+
+ log.debug("git push")
+ message = "Import of {}".format(os.path.basename(filepath))
+ try:
+ commands.commit(message)
+ commands.push()
+ log.debug("commit and push done")
+ except rpkgError:
+ pass
+ git_hash = commands.commithash
finally:
shutil.rmtree(tmp)
+ return git_hash
+
+ @staticmethod
+ def pkg_name_evr(pkg):
+ """
+ Queries a package for its name and evr (epoch:version-release)
+ """
+ log.debug("Verifying packagage, getting name and version.")
+ cmd = ['rpm', '-qp', '--nosignature', '--qf', '%{NAME} %{EPOCH} %{VERSION} %{RELEASE}', pkg]
+ try:
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ output, error = proc.communicate()
+ except OSError as e:
+ raise PackageQueryException(e)
+ if error:
+ raise PackageQueryException('Error querying srpm: %s' % error)
+
+ try:
+ name, epoch, version, release = output.split(" ")
+ except ValueError as e:
+ raise PackageQueryException(e)
+
+ # Epoch is an integer or '(none)' if not set
+ if epoch.isdigit():
+ evr = "{}:{}-{}".format(epoch, version, release)
+ else:
+ evr = "{}-{}".format(version, release)
+
+ return name, evr
+
+ def after_git_import(self):
+ log.debug("refreshing cgit listing")
+ call(["/usr/share/dist-git/cgit_pkg_list.sh", self.opts.cgit_pkg_list_location])
+
+ @staticmethod
+ def before_git_import(task):
+ log.debug("make sure repos exist: {}".format(task.reponame))
+ call(["/usr/share/dist-git/git_package.sh", task.reponame])
+ call(["/usr/share/dist-git/git_branch.sh", task.branch, task.reponame])
def post_back(self, data_dict):
"""
@@ -281,6 +291,46 @@ class DistGitImporter():
except Exception:
log.exception("Failed to post back to frontend : {}".format(data_dict))
+ def do_import(self, task):
+ """
+ :type task: ImportTask
+ """
+ log.debug("2. Importing the package: {}".format(task.package_url))
+ tmp_root = tempfile.mkdtemp()
+ fetched_srpm_path = os.path.join(tmp_root, "package.src.rpm")
+
+ try:
+ self.fetch_srpm(task, fetched_srpm_path)
+ task.package_name, task.package_version = self.pkg_name_evr(fetched_srpm_path)
+
+ self.before_git_import(task)
+ task.git_hash = self.git_import_srpm(task, fetched_srpm_path)
+ self.after_git_import()
+
+ log.debug("sending a response - success")
+ self.post_back(task.get_dict_for_frontend())
+
+ except (PackageImportException, PackageDownloadException, PackageQueryException):
+ log.info("send a response - failure during import of: {}".format(task.package_url))
+ self.post_back_safe({"task_id": task.task_id, "error": "error"})
+
+ except Exception:
+ log.exception("Unexpected error during package import")
+ self.post_back_safe({"task_id": task.task_id, "error": "error"})
+
+ finally:
+ shutil.rmtree(tmp_root, ignore_errors=True)
+
+ def run(self):
+ log.info("DistGitImported initialized")
+
+ while True:
+ mb_task = self.try_to_obtain_new_task()
+ if mb_task is None:
+ time.sleep(self.opts.sleep_time)
+ else:
+ self.do_import(mb_task)
+
def main():
config_file = None
@@ -291,7 +341,7 @@ def main():
config_reader = DistGitConfigReader(config_file)
try:
opts = config_reader.read()
- except Exception as e:
+ except Exception:
print("Failed to read config file, used file location: `{}`"
.format(config_file))
# sys.exit(1)
diff --git a/dist-git/dist-git/helpers.py b/dist-git/dist-git/helpers.py
index bb7233d..eb90f4a 100644
--- a/dist-git/dist-git/helpers.py
+++ b/dist-git/dist-git/helpers.py
@@ -72,6 +72,10 @@ class DistGitConfigReader(object):
cp, "dist-git", "log_dir", "/var/log/copr-dist-git"
)
+ opts.sleep_time = _get_conf(
+ cp, "dist-git", "sleep_time", 15, mode="int"
+ )
+
opts.cgit_pkg_list_location = _get_conf(
cp, "cgit_pkg_list_location", "log_dir", "/var/lib/copr-dist-git/cgit_pkg_list"
)
8 years, 10 months
[copr] master: [frontend] add importing queue (838d31d)
by asamalik@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 838d31deee083bd7de11cf166e53ffd89245912b
Author: Adam Samalik <asamalik(a)redhat.com>
Date: Thu Jul 30 16:55:22 2015 +0200
[frontend] add importing queue
>---------------------------------------------------------------
.../coprs_frontend/coprs/templates/coprs/show.html | 4 ++++
.../coprs_frontend/coprs/templates/status.html | 3 +++
.../status/{waiting.html => importing.html} | 10 +++-------
.../coprs/views/coprs_ns/coprs_general.py | 6 ++++++
.../coprs/views/status_ns/status_general.py | 9 +++++++++
5 files changed, 25 insertions(+), 7 deletions(-)
diff --git a/frontend/coprs_frontend/coprs/templates/coprs/show.html b/frontend/coprs_frontend/coprs/templates/coprs/show.html
index fc8596e..350173a 100644
--- a/frontend/coprs_frontend/coprs/templates/coprs/show.html
+++ b/frontend/coprs_frontend/coprs/templates/coprs/show.html
@@ -61,6 +61,10 @@
<h3 class="panel-title"> Task Queue </h3>
</div>
<div class="list-group">
+ <a href="{{url_for('status_ns.importing')}}" class="list-group-item">
+ <span class="badge">{{importing_tasks}}</span>
+ Importing
+ </a>
<a href="{{url_for('status_ns.waiting')}}" class="list-group-item">
<span class="badge">{{waiting_tasks}}</span>
Waiting
diff --git a/frontend/coprs_frontend/coprs/templates/status.html b/frontend/coprs_frontend/coprs/templates/status.html
index cabb9f1..c477e4b 100644
--- a/frontend/coprs_frontend/coprs/templates/status.html
+++ b/frontend/coprs_frontend/coprs/templates/status.html
@@ -14,6 +14,9 @@
</ol>
<h1> Task queue </h1>
<ul class="nav nav-tabs">
+ <li class="{% block importing_selected %}{% endblock %}">
+ <a href="{{ url_for('status_ns.importing') }}">Importing</a>
+ </li>
<li class="{% block waiting_selected %}{% endblock %}">
<a href="{{ url_for('status_ns.waiting') }}">Waiting</a>
</li>
diff --git a/frontend/coprs_frontend/coprs/templates/status/waiting.html b/frontend/coprs_frontend/coprs/templates/status/importing.html
similarity index 84%
copy from frontend/coprs_frontend/coprs/templates/status/waiting.html
copy to frontend/coprs_frontend/coprs/templates/status/importing.html
index e1d34e5..c69c2bc 100644
--- a/frontend/coprs_frontend/coprs/templates/status/waiting.html
+++ b/frontend/coprs_frontend/coprs/templates/status/importing.html
@@ -1,8 +1,8 @@
{% extends "status.html" %}
-{% block waiting_selected %}active{% endblock %}
+{% block importing_selected %}active{% endblock %}
{%block status_breadcrumb %}
<li class="active">
- Waiting
+ Importing
</li>
{%endblock%}
{% block status_body %}
@@ -13,7 +13,7 @@
{%else%}
{{number}} tasks
{%endif%}
- are waiting
+ are being imported
</h2>
<table class="table table-striped table-bordered">
@@ -30,11 +30,7 @@
{% for task in tasks %}
<tr>
<td>
- {% if task.state == "starting" %}
- <span class="build-starting">starting</span>
- {% else %}
{{ task.build.submitted_on | time_ago }}
- {% endif %}
</td>
<td><a href="{{ url_for('coprs_ns.copr_detail', username = task.build.copr.owner.name, coprname = task.build.copr.name) }}">{{task.build.copr.owner.name}}/{{task.build.copr.name}}</a></td>
<td><a href="{{ url_for('coprs_ns.copr_build', username = task.build.copr.owner.name, coprname = task.build.copr.name, build_id = task.build_id) }}">{{task.build_id}}</a></td>
diff --git a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
index 8d8e906..26dcf5e 100644
--- a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
+++ b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
@@ -44,12 +44,15 @@ def coprs_show(page=1):
waiting_tasks = len(list(builds_logic.BuildsLogic.get_build_task_queue()))
running_tasks = len(list(builds_logic.BuildsLogic
.get_build_tasks(helpers.StatusEnum("running"))))
+ importing_tasks = len(list(builds_logic.BuildsLogic
+ .get_build_tasks(helpers.StatusEnum("importing"))))
return flask.render_template("coprs/show/all.html",
coprs=coprs,
paginator=paginator,
waiting_tasks=waiting_tasks,
running_tasks=running_tasks,
+ importing_tasks=importing_tasks,
users_builds=users_builds)
@@ -73,6 +76,8 @@ def coprs_by_owner(username=None, page=1):
waiting_tasks = len(list(builds_logic.BuildsLogic.get_build_task_queue()))
running_tasks = len(list(builds_logic.BuildsLogic
.get_build_tasks(helpers.StatusEnum("running"))))
+ importing_tasks = len(list(builds_logic.BuildsLogic
+ .get_build_tasks(helpers.StatusEnum("importing"))))
return flask.render_template("coprs/show/user.html",
user=user,
@@ -80,6 +85,7 @@ def coprs_by_owner(username=None, page=1):
paginator=paginator,
waiting_tasks=waiting_tasks,
running_tasks=running_tasks,
+ importing_tasks=importing_tasks,
users_builds=users_builds)
diff --git a/frontend/coprs_frontend/coprs/views/status_ns/status_general.py b/frontend/coprs_frontend/coprs/views/status_ns/status_general.py
index add0359..58301d0 100644
--- a/frontend/coprs_frontend/coprs/views/status_ns/status_general.py
+++ b/frontend/coprs_frontend/coprs/views/status_ns/status_general.py
@@ -21,3 +21,12 @@ def running():
return flask.render_template("status/running.html",
number=len(list(tasks)),
tasks=tasks)
+
+
+(a)status_ns.route("/importing/")
+def importing():
+ tasks = builds_logic.BuildsLogic.get_build_tasks(
+ helpers.StatusEnum("importing")).limit(200)
+ return flask.render_template("status/importing.html",
+ number=len(list(tasks)),
+ tasks=tasks)
8 years, 10 months
[copr] master: [dist-git] mark build as failed for any error during import (6024b6e)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 6024b6eeb80c0ec924d11c4409f779ada525f5b4
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Thu Jul 30 14:57:37 2015 +0200
[dist-git] mark build as failed for any error during import
>---------------------------------------------------------------
dist-git/dist-git/dist_git_importer.py | 39 +++++++++++++++++++++----------
1 files changed, 26 insertions(+), 13 deletions(-)
diff --git a/dist-git/dist-git/dist_git_importer.py b/dist-git/dist-git/dist_git_importer.py
index 2503f79..a39528b 100755
--- a/dist-git/dist-git/dist_git_importer.py
+++ b/dist-git/dist-git/dist_git_importer.py
@@ -162,11 +162,12 @@ class DistGitImporter():
def __init__(self, opts):
self.opts = opts
+ self.get_url = "{}/backend/importing/".format(self.opts.frontend_base_url)
+ self.upload_url = "{}/backend/import-completed/".format(self.opts.frontend_base_url)
+ self.auth = ("user", self.opts.frontend_auth)
+ self.headers = {"content-type": "application/json"}
+
def run(self):
- get_url = "{}/backend/importing/".format(self.opts.frontend_base_url)
- upload_url = "{}/backend/import-completed/".format(self.opts.frontend_base_url)
- auth = ("user", self.opts.frontend_auth)
- headers = {"content-type": "application/json"}
tmp = tempfile.mkdtemp()
log.info("DistGitImported initialized")
@@ -175,7 +176,7 @@ class DistGitImporter():
log.info("1. Try to get task data")
try:
# get the data
- r = get(get_url)
+ r = get(self.get_url)
# take the first task
builds_list = r.json()["builds"]
@@ -246,18 +247,15 @@ class DistGitImporter():
"pkg_version": version,
"repo_name": reponame,
"git_hash": git_hash}
- post(upload_url, auth=auth, data=json.dumps(data), headers=headers)
+ self.post_back(data)
- except (PackageImportException, PackageDownloadException):
+ except (PackageImportException, PackageDownloadException, PackageQueryException):
log.info("send a response - failure during import of: {}".format(package_url))
- data = {"task_id": task_id, "error": "error"}
- try:
- post(upload_url, auth=auth, data=json.dumps(data), headers=headers)
- except Exception:
- log.exception("Failed to inform frontend about failed import: {}".format(data))
+ self.post_back_safe({"task_id": task_id, "error": "error"})
except Exception:
- log.exception("error during package import")
+ log.exception("Unexpected error during package import")
+ self.post_back_safe({"task_id": task_id, "error": "error"})
finally:
try:
@@ -268,6 +266,21 @@ class DistGitImporter():
finally:
shutil.rmtree(tmp)
+ def post_back(self, data_dict):
+ """
+ Could raise error related to networkd connection
+ """
+ post(self.upload_url, auth=self.auth, data=json.dumps(data_dict), headers=self.headers)
+
+ def post_back_safe(self, data_dict):
+ """
+ Ignores any error
+ """
+ try:
+ self.post_back(data_dict)
+ except Exception:
+ log.exception("Failed to post back to frontend : {}".format(data_dict))
+
def main():
config_file = None
8 years, 10 months
[copr] master: [backend] fix BuildJob.results_dir; eliminated MockRemote.pkg_dest_path (b0063eb)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit b0063eb1bd0aabaaf2112bd8f9034c2f1f0ac64b
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Thu Jul 30 12:38:59 2015 +0200
[backend] fix BuildJob.results_dir; eliminated MockRemote.pkg_dest_path
>---------------------------------------------------------------
backend/backend/daemons/dispatcher.py | 2 ++
backend/backend/job.py | 6 +++---
backend/backend/mockremote/__init__.py | 11 +++--------
backend/tests/mockremote/test_mockremote.py | 8 ++++----
4 files changed, 12 insertions(+), 15 deletions(-)
diff --git a/backend/backend/daemons/dispatcher.py b/backend/backend/daemons/dispatcher.py
index 58076c7..70953e8 100644
--- a/backend/backend/daemons/dispatcher.py
+++ b/backend/backend/daemons/dispatcher.py
@@ -342,6 +342,8 @@ class Worker(multiprocessing.Process):
def copy_mock_logs(self, job):
if not os.path.isdir(job.results_dir):
+ self.log.info("Job results dir doesn't exists, couldn't copy main log; path: {}"
+ .format(job.results_dir))
return
log_names = [(job.chroot_log_name, "mockchain.log.gz"),
diff --git a/backend/backend/job.py b/backend/backend/job.py
index 3a62678..a30c468 100644
--- a/backend/backend/job.py
+++ b/backend/backend/job.py
@@ -87,7 +87,7 @@ class BuildJob(object):
@property
def results_dir(self):
- return os.path.join(self.chroot_dir, self.package_name)
+ return os.path.join(self.chroot_dir, self.target_dir_name)
@property
def target_dir_name(self):
@@ -153,5 +153,5 @@ class BuildJob(object):
return str(self.__unicode__())
def __unicode__(self):
- return u"BuildJob<id: {build_id}, owner: {project_owner}, " \
- u"project: {project_name}, git branch: {git_branch}, git_hash: {git_hash} >".format(**self.__dict__)
+ return u"BuildJob<id: {build_id}, owner: {project_owner}, project: {project_name}, " \
+ u"git branch: {git_branch}, git_hash: {git_hash}, status: {status} >".format(**self.__dict__)
diff --git a/backend/backend/mockremote/__init__.py b/backend/backend/mockremote/__init__.py
index 0b2e0a4..c96190d 100755
--- a/backend/backend/mockremote/__init__.py
+++ b/backend/backend/mockremote/__init__.py
@@ -172,11 +172,6 @@ class MockRemote(object):
def pkg(self):
return self.job.pkg
- @property
- def pkg_dest_path(self):
- return os.path.normpath("{}/{}/{}".format(
- self.job.destdir, self.job.chroot, self.job.target_dir_name))
-
def add_pubkey(self):
"""
Adds pubkey.gpg with public key to ``chroot_dir``
@@ -266,7 +261,7 @@ class MockRemote(object):
self.do_createrepo()
def prepare_build_dir(self):
- p_path = self.pkg_dest_path
+ p_path = self.job.results_dir
# if it's marked as fail, nuke the failure and try to rebuild it
if os.path.exists(os.path.join(p_path, "fail")):
os.unlink(os.path.join(p_path, "fail"))
@@ -316,7 +311,7 @@ class MockRemote(object):
raise MockRemoteError("Error occurred during build {}: {}"
.format(self.job, error))
finally:
- self.builder.download(self.pkg_dest_path)
+ self.builder.download(self.job.results_dir)
# self.add_log_symlinks() # todo: add config option, need this for nginx
self.log.info("End Build: {0}".format(self.job))
@@ -329,7 +324,7 @@ class MockRemote(object):
into the directory with downloaded files.
"""
- info_file_path = os.path.join(self.pkg_dest_path, "build.info")
+ info_file_path = os.path.join(self.job.results_dir, "build.info")
self.log.info("marking build dir with build_id, ")
try:
with open(info_file_path, 'w') as info_file:
diff --git a/backend/tests/mockremote/test_mockremote.py b/backend/tests/mockremote/test_mockremote.py
index d1b17a8..d9986b8 100644
--- a/backend/tests/mockremote/test_mockremote.py
+++ b/backend/tests/mockremote/test_mockremote.py
@@ -176,7 +176,7 @@ class TestMockRemote(object):
assert self.mr.do_createrepo.called
def test_prepare_build_dir_erase_fail_file(self, f_mock_remote):
- target_dir = self.mr.pkg_dest_path
+ target_dir = self.mr.job.results_dir
os.makedirs(target_dir)
fail_path = os.path.join(target_dir, "fail")
with open(fail_path, "w") as handle:
@@ -187,7 +187,7 @@ class TestMockRemote(object):
assert os.path.exists(fail_path) is False
def test_prepare_build_dir_erase_success_file(self, f_mock_remote):
- target_dir = self.mr.pkg_dest_path
+ target_dir = self.mr.job.results_dir
os.makedirs(target_dir)
fail_path = os.path.join(target_dir, "success")
with open(fail_path, "w") as handle:
@@ -200,7 +200,7 @@ class TestMockRemote(object):
def test_prepare_build_dir_creates_dirs(self, f_mock_remote):
self.mr.prepare_build_dir()
- assert os.path.exists(self.mr.pkg_dest_path)
+ assert os.path.exists(self.mr.job.results_dir)
def test_build_pkg_and_process_results(self, f_mock_remote):
self.mr.on_success_build = MagicMock()
@@ -246,7 +246,7 @@ class TestMockRemote(object):
def test_mark_dir_with_build_id(self, f_mock_remote):
# TODO: create real test
- target_dir = self.mr.pkg_dest_path
+ target_dir = self.mr.job.results_dir
os.makedirs(target_dir)
info_file_path = os.path.join(target_dir, "build.info")
8 years, 10 months
[copr] master: [dist-git] don't break on the post failure to frontend (b91dafc)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit b91dafc383c89687a2983981eebe9f9d27517e47
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Thu Jul 30 12:18:37 2015 +0200
[dist-git] don't break on the post failure to frontend
>---------------------------------------------------------------
dist-git/dist-git/dist_git_importer.py | 5 ++++-
1 files changed, 4 insertions(+), 1 deletions(-)
diff --git a/dist-git/dist-git/dist_git_importer.py b/dist-git/dist-git/dist_git_importer.py
index ef75d4d..2503f79 100755
--- a/dist-git/dist-git/dist_git_importer.py
+++ b/dist-git/dist-git/dist_git_importer.py
@@ -251,7 +251,10 @@ class DistGitImporter():
except (PackageImportException, PackageDownloadException):
log.info("send a response - failure during import of: {}".format(package_url))
data = {"task_id": task_id, "error": "error"}
- post(upload_url, auth=auth, data=json.dumps(data), headers=headers)
+ try:
+ post(upload_url, auth=auth, data=json.dumps(data), headers=headers)
+ except Exception:
+ log.exception("Failed to inform frontend about failed import: {}".format(data))
except Exception:
log.exception("error during package import")
8 years, 10 months