[copr] master: Automatic commit of package [copr-frontend] release [1.68-1]. (ade9303)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit ade93032fe77a9156550ba1cb96851e0510ec0c2
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Tue Sep 15 15:51:49 2015 +0200
Automatic commit of package [copr-frontend] release [1.68-1].
>---------------------------------------------------------------
frontend/copr-frontend.spec | 6 +++++-
rel-eng/packages/copr-frontend | 2 +-
2 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/frontend/copr-frontend.spec b/frontend/copr-frontend.spec
index acb7692..6669c8e 100644
--- a/frontend/copr-frontend.spec
+++ b/frontend/copr-frontend.spec
@@ -5,7 +5,7 @@
%endif
Name: copr-frontend
-Version: 1.67
+Version: 1.68
Release: 1%{?dist}
Summary: Frontend for Copr
@@ -200,6 +200,10 @@ service logstash condrestart
#%doc documentation/python-doc
%changelog
+* Tue Sep 15 2015 Valentin Gologuzov <vgologuz(a)redhat.com> 1.68-1
+- fix tests to create tmp directory for srpm upload
+- don't depend on python-copr
+
* Tue Sep 15 2015 Valentin Gologuzov <vgologuz(a)redhat.com> 1.67-1
- new REST-like api
- fixed resubmitting build
diff --git a/rel-eng/packages/copr-frontend b/rel-eng/packages/copr-frontend
index 4dc53b1..0e35442 100644
--- a/rel-eng/packages/copr-frontend
+++ b/rel-eng/packages/copr-frontend
@@ -1 +1 @@
-1.67-1 frontend/
+1.68-1 frontend/
8 years, 9 months
[copr] master: [frontend] fix tests to create tmp directory for srpm upload (08a05c3)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 08a05c37f12210296e88a47b68867624e590ba09
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Tue Sep 15 15:14:42 2015 +0200
[frontend] fix tests to create tmp directory for srpm upload
>---------------------------------------------------------------
frontend/coprs_frontend/config/copr_unit_test.conf | 2 +-
frontend/coprs_frontend/tests/coprs_test_case.py | 12 ++++++++----
2 files changed, 9 insertions(+), 5 deletions(-)
diff --git a/frontend/coprs_frontend/config/copr_unit_test.conf b/frontend/coprs_frontend/config/copr_unit_test.conf
index 51a6e7f..97f7973 100644
--- a/frontend/coprs_frontend/config/copr_unit_test.conf
+++ b/frontend/coprs_frontend/config/copr_unit_test.conf
@@ -57,4 +57,4 @@ PUBLIC_COPR_HOSTNAME = "localhost"
LOG_FILENAME="copr_frontend.log"
# a place for storing srpms until they get uploaded
-SRPM_STORAGE_DIR = "/var/lib/copr/data/srpm_storage"
+SRPM_STORAGE_DIR = os.path.join(LOCAL_TMP_DIR, "srpm_storage")
diff --git a/frontend/coprs_frontend/tests/coprs_test_case.py b/frontend/coprs_frontend/tests/coprs_test_case.py
index 6da78d1..35a670d 100644
--- a/frontend/coprs_frontend/tests/coprs_test_case.py
+++ b/frontend/coprs_frontend/tests/coprs_test_case.py
@@ -32,10 +32,14 @@ class CoprsTestCase(object):
@classmethod
def setup_class(cls):
config = coprs.app.config
- if "LOCAL_TMP_DIR" in config:
- path = os.path.abspath(config["LOCAL_TMP_DIR"])
- if not os.path.exists(path):
- os.makedirs(path)
+ for key in [
+ "LOCAL_TMP_DIR",
+ "SRPM_STORAGE_DIR",
+ ]:
+ if key in config:
+ path = os.path.abspath(config[key])
+ if not os.path.exists(path):
+ os.makedirs(path)
@classmethod
def teardown_class(cls):
8 years, 9 months
[copr] master: [frontend] don't depend on python-copr (69a6041)
by frostyx@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 69a60410c3957b77f8d18426e9c4747c70485d12
Author: Jakub Kadlčík <jkadlcik(a)redhat.com>
Date: Tue Sep 15 13:40:26 2015 +0200
[frontend] don't depend on python-copr
>---------------------------------------------------------------
.../coprs_frontend/run/generate_repo_packages.py | 7 +++----
1 files changed, 3 insertions(+), 4 deletions(-)
diff --git a/frontend/coprs_frontend/run/generate_repo_packages.py b/frontend/coprs_frontend/run/generate_repo_packages.py
index 6b6fa59..ab1289b 100644
--- a/frontend/coprs_frontend/run/generate_repo_packages.py
+++ b/frontend/coprs_frontend/run/generate_repo_packages.py
@@ -23,7 +23,6 @@ sys.path.append(os.path.dirname(here))
from coprs import app
from coprs.logic.coprs_logic import CoprsLogic
-from copr.client.exceptions import CoprException, CoprRequestException
# ----------------------------------------------------------------------------------------------------------------------
@@ -80,7 +79,7 @@ class RepoRpmBuilder(object):
url = urlparse.urljoin(FRONTEND_URL, api)
r = requests.get(url)
if r.status_code != 200:
- raise CoprRequestException("Can't get {}".format(url))
+ raise RuntimeError("Can't get {}".format(url))
return r.content
def generate_repo_package(self):
@@ -109,7 +108,7 @@ class RepoRpmBuilder(object):
out, err = process.communicate()
if process.returncode != 0:
- raise CoprException("Failed rpmbuild for: {}\n{}".format(self.repo_name, err))
+ raise RuntimeError("Failed rpmbuild for: {}\n{}".format(self.repo_name, err))
# ----------------------------------------------------------------------------------------------------------------------
@@ -157,7 +156,7 @@ def main():
builder.generate_repo_package()
print("Created RPM package for: {}".format(builder.repo_name))
- except CoprException as e:
+ except RuntimeError as e:
print(e.message, file=sys.stderr)
8 years, 9 months
[copr] master: [python] fix missing urllib.parse on el7 (b9ee654)
by frostyx@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit b9ee654d02ef2dc25642b872a79f893b8dcfc62c
Author: Jakub Kadlčík <jkadlcik(a)redhat.com>
Date: Tue Sep 15 13:20:20 2015 +0200
[python] fix missing urllib.parse on el7
>---------------------------------------------------------------
python/copr/client/client.py | 7 ++++++-
1 files changed, 6 insertions(+), 1 deletions(-)
diff --git a/python/copr/client/client.py b/python/copr/client/client.py
index c9614a3..16b366d 100644
--- a/python/copr/client/client.py
+++ b/python/copr/client/client.py
@@ -14,9 +14,14 @@ import requests
import six
from six.moves import configparser
-from six.moves.urllib.parse import urlparse
from requests_toolbelt.multipart.encoder import MultipartEncoder
+# urlparse from six is not available on el7
+# because it requires at least python-six-1.4.1
+if sys.version_info.major == 2:
+ from urlparse import urlparse
+else:
+ from urllib.parse import urlparse
if sys.version_info < (2, 7):
class NullHandler(logging.Handler):
8 years, 9 months
[copr] master: Automatic commit of package [copr-frontend] release [1.67-1]. (df30065)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit df3006594dd683dfc0aeb415dd7c158493e33b36
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Tue Sep 15 13:29:04 2015 +0200
Automatic commit of package [copr-frontend] release [1.67-1].
>---------------------------------------------------------------
frontend/copr-frontend.spec | 22 +++++++++++++++++++++-
rel-eng/packages/copr-frontend | 2 +-
2 files changed, 22 insertions(+), 2 deletions(-)
diff --git a/frontend/copr-frontend.spec b/frontend/copr-frontend.spec
index fd80c27..acb7692 100644
--- a/frontend/copr-frontend.spec
+++ b/frontend/copr-frontend.spec
@@ -5,7 +5,7 @@
%endif
Name: copr-frontend
-Version: 1.66
+Version: 1.67
Release: 1%{?dist}
Summary: Frontend for Copr
@@ -200,6 +200,26 @@ service logstash condrestart
#%doc documentation/python-doc
%changelog
+* Tue Sep 15 2015 Valentin Gologuzov <vgologuz(a)redhat.com> 1.67-1
+- new REST-like api
+- fixed resubmitting build
+- validate uploaded file to .src.rpm extension
+- warn user if he use rpmfusion repository for building.
+- give user hint how to give permission to somebody else
+- [frontend][old API] backported `result_dir_url` of BuildChroot to the build
+ details: new field `results_by_chroot`
+- [frontend] [dist-git] provide build failure details
+- fix missing copr names in yum_repos (RhBug:1258943) URL build.results may or
+ may not end with slash, so when not, the urljoin cuts off it's last part
+ (which is copr name)
+- UI updates: 403, 404 errors, notification texts, footer, alerts are dismissable
+- comps.xml support
+- unify printing of form errors (RhBug:1252559)
+- sort chroots alphabetically (RhBug:1253588)
+- add command generate_repo_packages for manage.py
+- build deletion fix (taiga #32)
+- change columns in status (taiga #28)
+
* Tue Aug 11 2015 Miroslav Suchý <msuchy(a)redhat.com> 1.66-1
- correctly join url fragments
- create json for each package (RhBug:1252432)
diff --git a/rel-eng/packages/copr-frontend b/rel-eng/packages/copr-frontend
index 95ed504..4dc53b1 100644
--- a/rel-eng/packages/copr-frontend
+++ b/rel-eng/packages/copr-frontend
@@ -1 +1 @@
-1.66-1 frontend/
+1.67-1 frontend/
8 years, 9 months
[copr] master: [python][frontend] use requests-toolbelt to stream SRPM files (RhBug:1261125) (61817cd)
by frostyx@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 61817cdf03f0bca8f6eadbfae389f41bbfda354f
Author: Jakub Kadlčík <jkadlcik(a)redhat.com>
Date: Tue Sep 15 11:38:47 2015 +0200
[python][frontend] use requests-toolbelt to stream SRPM files (RhBug:1261125)
>---------------------------------------------------------------
frontend/coprs_frontend/coprs/forms.py | 4 ++++
python/copr/client/client.py | 18 ++++++++++--------
python/python-copr.spec | 4 ++++
python/requirements.txt | 3 ++-
python/setup.py | 1 +
5 files changed, 21 insertions(+), 9 deletions(-)
diff --git a/frontend/coprs_frontend/coprs/forms.py b/frontend/coprs_frontend/coprs/forms.py
index 766ada3..7ffe436 100644
--- a/frontend/coprs_frontend/coprs/forms.py
+++ b/frontend/coprs_frontend/coprs/forms.py
@@ -283,6 +283,7 @@ class BuildFormFactory(object):
memory_reqs = wtforms.IntegerField(
"Memory requirements",
validators=[
+ wtforms.validators.Optional(),
wtforms.validators.NumberRange(
min=constants.MIN_BUILD_MEMORY,
max=constants.MAX_BUILD_MEMORY)],
@@ -291,6 +292,7 @@ class BuildFormFactory(object):
timeout = wtforms.IntegerField(
"Timeout",
validators=[
+ wtforms.validators.Optional(),
wtforms.validators.NumberRange(
min=constants.MIN_BUILD_TIMEOUT,
max=constants.MAX_BUILD_TIMEOUT)],
@@ -330,6 +332,7 @@ class BuildFormUploadFactory(object):
memory_reqs = wtforms.IntegerField(
"Memory requirements",
validators=[
+ wtforms.validators.Optional(),
wtforms.validators.NumberRange(
min=constants.MIN_BUILD_MEMORY,
max=constants.MAX_BUILD_MEMORY)],
@@ -338,6 +341,7 @@ class BuildFormUploadFactory(object):
timeout = wtforms.IntegerField(
"Timeout",
validators=[
+ wtforms.validators.Optional(),
wtforms.validators.NumberRange(
min=constants.MIN_BUILD_TIMEOUT,
max=constants.MAX_BUILD_TIMEOUT)],
diff --git a/python/copr/client/client.py b/python/copr/client/client.py
index 63f1395..c9614a3 100644
--- a/python/copr/client/client.py
+++ b/python/copr/client/client.py
@@ -15,6 +15,7 @@ import six
from six.moves import configparser
from six.moves.urllib.parse import urlparse
+from requests_toolbelt.multipart.encoder import MultipartEncoder
if sys.version_info < (2, 7):
@@ -137,7 +138,7 @@ class CoprClient(UnicodeMixin):
return CoprClient(**config)
def _fetch(self, url, data=None, projectname=None, username=None,
- method=None, skip_auth=False, on_error_response=None, files=None):
+ method=None, skip_auth=False, on_error_response=None, headers=None):
""" Fetches data from server,
checks response and raises a CoprRequestException with nice error message
or CoprUnknownResponseException in case of some some error. \n
@@ -150,7 +151,7 @@ class CoprClient(UnicodeMixin):
:param username: [optional] use alternative username
:param on_error_response: [optional] function to handle responses
with bad status code
- :param files: [optional] files to upload
+ :param headers: [optional] custom request headers
:return: deserialized response
:rtype: dict
@@ -168,8 +169,8 @@ class CoprClient(UnicodeMixin):
kwargs["auth"] = (self.login, self.token)
if data is not None:
kwargs["data"] = data
- if files is not None:
- kwargs["files"] = files
+ if headers is not None:
+ kwargs["headers"] = headers
if method not in ["get", "post", "head", "delete", "put"]:
raise Exception("Method {0} not allowed".format(method))
@@ -311,12 +312,11 @@ class CoprClient(UnicodeMixin):
"""
if not username:
username = self.username
-
data = {
"memory_reqs": memory,
"timeout": timeout
}
- files = None
+ headers = None
if urlparse(pkgs[0]).scheme != "":
api_endpoint = "new_build"
@@ -324,7 +324,8 @@ class CoprClient(UnicodeMixin):
else:
try:
api_endpoint = "new_build_upload"
- files = {'pkgs': open(pkgs[0], "rb")}
+ f = open(pkgs[0], "rb")
+ data["pkgs"] = (os.path.basename(f.name), f, "application/x-rpm")
except IOError as e:
raise CoprRequestException(e)
@@ -335,7 +336,8 @@ class CoprClient(UnicodeMixin):
for chroot in chroots or []:
data[chroot] = "y"
- data = self._fetch(url, data, method="post", files=files)
+ m = MultipartEncoder(data)
+ data = self._fetch(url, m, method="post", headers={'Content-Type': m.content_type})
response = CoprResponse(
client=self,
diff --git a/python/python-copr.spec b/python/python-copr.spec
index a63366e..c2fff41 100644
--- a/python/python-copr.spec
+++ b/python/python-copr.spec
@@ -28,6 +28,7 @@ BuildRequires: libxslt
BuildRequires: util-linux
BuildRequires: python-setuptools
BuildRequires: python-requests
+BuildRequires: python-requests-toolbelt
BuildRequires: python-six
BuildRequires: python-mock
BuildRequires: pytest
@@ -39,6 +40,7 @@ BuildRequires: python-docutils
Requires: python-setuptools
Requires: python-six
Requires: python-requests
+Requires: python-requests-toolbelt
%description
COPR is lightweight build system. It allows you to create new project in WebUI,
@@ -58,11 +60,13 @@ BuildRequires: python3-setuptools
BuildRequires: python3-pytest
BuildRequires: python3-mock
BuildRequires: python3-requests
+BuildRequires: python3-requests-toolbelt
BuildRequires: python3-six
Requires: python3-setuptools
Requires: python3-six
Requires: python3-requests
+Requires: python3-requests-toolbelt
%description -n python3-copr
COPR is lightweight build system. It allows you to create new project in WebUI,
diff --git a/python/requirements.txt b/python/requirements.txt
index 0e941ee..b8249e4 100644
--- a/python/requirements.txt
+++ b/python/requirements.txt
@@ -4,5 +4,6 @@
pytest
mock
six
-requests
sphinx
+requests
+requests-toolbelt
diff --git a/python/setup.py b/python/setup.py
index 9df02d3..fa3727f 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -13,6 +13,7 @@ from copr.client import __description__, __version__
requires = [
'requests',
+ 'requests-toolbelt',
'six'
]
8 years, 9 months
[copr] master: [backend] fixed tests (db2319f)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit db2319f28d296ac47ccb8a44437f44889499062e
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Mon Sep 14 18:38:46 2015 +0200
[backend] fixed tests
>---------------------------------------------------------------
backend/backend/daemons/backend.py | 4 +---
backend/tests/deamons/test_backend.py | 19 +------------------
backend/tests/deamons/test_job_grab.py | 6 +++---
3 files changed, 5 insertions(+), 24 deletions(-)
diff --git a/backend/backend/daemons/backend.py b/backend/backend/daemons/backend.py
index 5a89f82..8818f71 100644
--- a/backend/backend/daemons/backend.py
+++ b/backend/backend/daemons/backend.py
@@ -6,8 +6,6 @@ from __future__ import division
from __future__ import absolute_import
import grp
-import multiprocessing
-import os
import pwd
import signal
import sys
@@ -29,7 +27,7 @@ from .dispatcher import Worker
class CoprBackend(object):
"""
- Core process - starts/stops/initializes workers and other backend components
+ Core process - starts/stops/initializes workers
:param config_file: path to the backend configuration file
diff --git a/backend/tests/deamons/test_backend.py b/backend/tests/deamons/test_backend.py
index 7d047d4..e0a7efa 100644
--- a/backend/tests/deamons/test_backend.py
+++ b/backend/tests/deamons/test_backend.py
@@ -4,7 +4,7 @@ import shutil
import time
from munch import Munch
-import pwd
+
import pytest
import retask
from retask import ConnectionError
@@ -67,9 +67,6 @@ class TestBackend(object):
self.bc_patcher = mock.patch("backend.daemons.backend.BackendConfigReader")
self.bc = self.bc_patcher.start()
- self.mp_patcher = mock.patch("backend.daemons.backend.multiprocessing")
- self.mc_mp = self.mp_patcher.start()
-
self.config_file = "/dev/null/copr.conf"
self.ext_opts = {}
@@ -179,16 +176,6 @@ class TestBackend(object):
with pytest.raises(CoprBackendError):
self.be.init_task_queues()
- @mock.patch("backend.daemons.backend.CoprJobGrab")
- def test_dummy_init_sub_process(self, mc_jobgrab, init_be):
-
- self.be.init_sub_process()
- assert mc_jobgrab.called
- assert mc_jobgrab.call_args == mock.call(opts=self.be.opts,
- frontend_client=self.be.frontend_client,
- lock=self.be.lock)
- assert mc_jobgrab.return_value.start.called
-
def test_update_conf(self, init_be):
test_obj = MagicMock()
self.bc_obj.read.return_value = test_obj
@@ -285,9 +272,6 @@ class TestBackend(object):
worker_dead.is_alive.return_value = False
self.be.clean_task_queues = MagicMock()
- self.be.init_sub_process = MagicMock()
- self.be.ensure_sub_processes_alive = MagicMock()
- # self.be.init_task_queues = MagicMock()
self.be.update_conf = MagicMock()
self.be.spin_up_workers_by_group = MagicMock()
self.be.frontend_client = MagicMock()
@@ -308,7 +292,6 @@ class TestBackend(object):
mock.call(self.opts.build_groups[1]),
]
assert self.be.update_conf.called
- assert self.be.ensure_sub_processes_alive.called
assert not self.be.is_running
assert not self.be.workers_by_group_id[0]
assert not self.be.workers_by_group_id[1]
diff --git a/backend/tests/deamons/test_job_grab.py b/backend/tests/deamons/test_job_grab.py
index b1cf828..6cbc59f 100644
--- a/backend/tests/deamons/test_job_grab.py
+++ b/backend/tests/deamons/test_job_grab.py
@@ -138,14 +138,14 @@ class TestJobGrab(object):
@pytest.fixture
def init_jg(self, mc_retask_queue, mc_grc):
- self.jg = CoprJobGrab(self.opts, self.frontend_client, self.lock)
+ self.jg = CoprJobGrab(self.opts, self.frontend_client)
self.jg.connect_queues()
self.jg.vm_manager = MagicMock()
def test_connect_queues(self, mc_retask_queue, mc_grc):
mc_rc = MagicMock()
mc_grc.return_value = mc_rc
- self.jg = CoprJobGrab(self.opts, self.frontend_client, self.lock)
+ self.jg = CoprJobGrab(self.opts, self.frontend_client)
assert len(self.jg.task_queues_by_arch) == 0
self.jg.connect_queues()
@@ -161,7 +161,7 @@ class TestJobGrab(object):
def test_listen_to_pubsub(self, mc_retask_queue, mc_grc):
mc_rc = MagicMock()
mc_grc.return_value = mc_rc
- self.jg = CoprJobGrab(self.opts, self.frontend_client, self.lock)
+ self.jg = CoprJobGrab(self.opts, self.frontend_client)
assert not mc_rc.pubsub.called
self.jg.listen_to_pubsub()
8 years, 9 months
[copr] master: [dist-git] cleanup imports (28218b6)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 28218b6c720daa9b8c95112cb35a0125dbecdb14
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Mon Sep 14 18:13:05 2015 +0200
[dist-git] cleanup imports
>---------------------------------------------------------------
dist-git/dist_git/dist_git_importer.py | 14 +-------------
dist-git/dist_git/helpers.py | 1 +
2 files changed, 2 insertions(+), 13 deletions(-)
diff --git a/dist-git/dist_git/dist_git_importer.py b/dist-git/dist_git/dist_git_importer.py
index 4bdcca9..b48fd69 100755
--- a/dist-git/dist_git/dist_git_importer.py
+++ b/dist-git/dist_git/dist_git_importer.py
@@ -3,28 +3,16 @@
import os
import json
import time
-from urllib import urlretrieve
import shutil
import tempfile
import logging
-from subprocess import PIPE, Popen
+from subprocess import PIPE, Popen, call
from requests import get, post
-# pyrpkg uses os.getlogin(). It requires tty which is unavailable when we run this script as a daemon
-# very dirty solution for now
-import pwd
-import sys
from .exceptions import PackageImportException, PackageDownloadException, PackageQueryException
from .srpm_import import do_git_srpm_import
-os.getlogin = lambda: pwd.getpwuid(os.getuid())[0]
-# monkey patch end
-
-from subprocess import call
-
-from helpers import DistGitConfigReader
-
log = logging.getLogger(__name__)
diff --git a/dist-git/dist_git/helpers.py b/dist-git/dist_git/helpers.py
index 3631840..7f3add4 100644
--- a/dist-git/dist_git/helpers.py
+++ b/dist-git/dist_git/helpers.py
@@ -2,6 +2,7 @@ import os
import logging
import ConfigParser
+# todo: replace with munch, check availability in epel
from bunch import Bunch
log = logging.getLogger(__name__)
8 years, 9 months
[copr] master: [backend] split backend daemon: extracting JobGrab (4b9286a)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 4b9286a614078b8656d5f58ff3711e6204528c00
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Fri Sep 11 20:12:47 2015 +0200
[backend] split backend daemon: extracting JobGrab
>---------------------------------------------------------------
backend/backend/daemons/backend.py | 36 +-------------------
backend/backend/daemons/dispatcher.py | 7 +---
backend/backend/daemons/job_grab.py | 6 +--
backend/backend/mockremote/__init__.py | 10 +-----
.../{copr_run_logger.py => copr_run_job_grab.py} | 8 +++--
...nd-log.service => copr-backend-jobgrab.service} | 6 ++--
backend/run/systemd/copr-backend-vmm.service | 1 +
backend/run/systemd/copr-backend.service | 3 +-
8 files changed, 16 insertions(+), 61 deletions(-)
diff --git a/backend/backend/daemons/backend.py b/backend/backend/daemons/backend.py
index 4a0096b..5a89f82 100644
--- a/backend/backend/daemons/backend.py
+++ b/backend/backend/daemons/backend.py
@@ -23,7 +23,6 @@ from backend.frontend import FrontendClient
from ..exceptions import CoprBackendError
from ..helpers import BackendConfigReader, get_redis_logger
-from .job_grab import CoprJobGrab
from .dispatcher import Worker
@@ -53,7 +52,6 @@ class CoprBackend(object):
self.opts = None
self.update_conf()
- self.lock = multiprocessing.Lock()
self.task_queues = {}
self.frontend_client = FrontendClient(self.opts)
@@ -89,31 +87,6 @@ class CoprBackend(object):
self.clean_task_queues()
- def _start_job_grab(self):
- self.log.info("Starting up Job Grabber")
- self._jobgrab = CoprJobGrab(opts=self.opts,
- frontend_client=self.frontend_client,
- lock=self.lock)
- self._jobgrab.start()
-
- def init_sub_process(self):
- """
- - Create job grabber
- """
- self._start_job_grab()
-
- def ensure_sub_processes_alive(self):
- if self.is_running:
- for proc, start_method in [
- (self._jobgrab, self._start_job_grab),
-
- ]:
- if not proc.is_alive():
- self.log.error("Process `{}` died unexpectedly, restarting".format(proc))
- proc.terminate()
- proc.join()
- start_method()
-
def update_conf(self):
"""
Update backend config from config file
@@ -142,8 +115,7 @@ class CoprBackend(object):
opts=self.opts,
frontend_client=self.frontend_client,
worker_num=self.max_worker_num_by_group_id[group_id],
- group_id=group_id,
- lock=self.lock
+ group_id=group_id
)
self.workers_by_group_id[group_id].append(w)
@@ -202,13 +174,9 @@ class CoprBackend(object):
Starts backend process. Control sub process start/stop.
"""
self.update_conf()
-
self.init_task_queues()
- self.init_sub_process()
time.sleep(1)
-
self.log.info("Initial config: {}".format(self.opts))
- self.log.info("Sub processes was started")
try:
self.log.info("Rescheduling old unfinished builds")
@@ -222,8 +190,6 @@ class CoprBackend(object):
# re-read config into opts
self.update_conf()
- self.ensure_sub_processes_alive()
-
for group in self.opts.build_groups:
group_id = group["id"]
diff --git a/backend/backend/daemons/dispatcher.py b/backend/backend/daemons/dispatcher.py
index 3b379b2..023e929 100644
--- a/backend/backend/daemons/dispatcher.py
+++ b/backend/backend/daemons/dispatcher.py
@@ -39,11 +39,10 @@ class Worker(multiprocessing.Process):
:param Munch opts: backend config
:param int worker_num: worker number
:param int group_id: group_id from the set of groups defined in config
- :param lock: (:py:class:`multiprocessing.Lock`) global backend lock
"""
- def __init__(self, opts, frontend_client, worker_num, group_id, lock=None):
+ def __init__(self, opts, frontend_client, worker_num, group_id):
# base class initialization
multiprocessing.Process.__init__(self, name="worker-builder")
@@ -60,7 +59,6 @@ class Worker(multiprocessing.Process):
# event queue for communicating back to dispatcher
self.kill_received = False
- self.lock = lock
self.frontend_client = frontend_client
self.vm_name = None
@@ -299,8 +297,7 @@ class Worker(multiprocessing.Process):
builder_host=self.vm_ip,
job=job,
logger=build_logger,
- opts=self.opts,
- lock=self.lock,
+ opts=self.opts
)
mr.check()
diff --git a/backend/backend/daemons/job_grab.py b/backend/backend/daemons/job_grab.py
index df6e98e..7a15b97 100644
--- a/backend/backend/daemons/job_grab.py
+++ b/backend/backend/daemons/job_grab.py
@@ -29,7 +29,7 @@ from ..frontend import FrontendClient
from ..vm_manage.manager import VmManager
-class CoprJobGrab(Process):
+class CoprJobGrab(object):
"""
Fetch jobs from the Frontend
@@ -44,9 +44,8 @@ class CoprJobGrab(Process):
"""
- def __init__(self, opts, frontend_client, lock):
+ def __init__(self, opts, frontend_client):
# base class initialization
- super(CoprJobGrab, self).__init__(name="jobgrab")
self.opts = opts
self.arch_to_group_id_map = dict()
@@ -58,7 +57,6 @@ class CoprJobGrab(Process):
self.task_queues_by_group = {}
self.added_jobs_dict = dict() # task_id -> task dict
- self.lock = lock # used only for createrepo calls
self.frontend_client = frontend_client
diff --git a/backend/backend/mockremote/__init__.py b/backend/backend/mockremote/__init__.py
index ceffc28..ece50b7 100755
--- a/backend/backend/mockremote/__init__.py
+++ b/backend/backend/mockremote/__init__.py
@@ -80,7 +80,7 @@ class MockRemote(object):
# idea: send events according to the build progress to handler
def __init__(self, builder_host, job, logger,
- repos=None, opts=None, lock=None,):
+ repos=None, opts=None):
"""
:param builder_host: builder hostname or ip
@@ -101,8 +101,6 @@ class MockRemote(object):
:param macros: { "copr_username": ...,
"copr_projectname": ...,
"vendor": ...}
- :param multiprocessing.Lock lock: instance of Lock shared between
- Copr backend process
:param Munch opts: builder options, used keys::
:ivar build_user: user to run as/connect as on builder systems
@@ -132,12 +130,6 @@ class MockRemote(object):
self.log = logger
self.job = job
- # TODO: remove or re-implement
- # self.cont = cont # unused since we build only one pkg at time
- # self.recurse = recurse
-
- self.lock = lock
-
self.log.info("Setting up builder: {0}".format(builder_host))
# TODO: add option "builder_log_level" to backend config
self.log.setLevel(logging.INFO)
diff --git a/backend/run/copr_run_logger.py b/backend/run/copr_run_job_grab.py
old mode 100755
new mode 100644
similarity index 62%
copy from backend/run/copr_run_logger.py
copy to backend/run/copr_run_job_grab.py
index 6b41a74..47a549b
--- a/backend/run/copr_run_logger.py
+++ b/backend/run/copr_run_job_grab.py
@@ -7,13 +7,15 @@ from __future__ import division
from __future__ import absolute_import
from backend.helpers import get_backend_opts
-from backend.daemons.log import RedisLogHandler
+from backend.daemons.job_grab import CoprJobGrab
+from backend.frontend import FrontendClient
def main():
opts = get_backend_opts()
- handler = RedisLogHandler(opts)
- handler.run()
+ fc = FrontendClient(opts)
+ grabber = CoprJobGrab(opts, frontend_client=fc)
+ grabber.run()
if __name__ == "__main__":
diff --git a/backend/run/systemd/copr-backend-log.service b/backend/run/systemd/copr-backend-jobgrab.service
similarity index 59%
copy from backend/run/systemd/copr-backend-log.service
copy to backend/run/systemd/copr-backend-jobgrab.service
index 1577743..e2cef79 100644
--- a/backend/run/systemd/copr-backend-log.service
+++ b/backend/run/systemd/copr-backend-jobgrab.service
@@ -1,14 +1,14 @@
[Unit]
-Description=Copr Backend service, Log Handler component
+Description=Copr Backend service, Job Grabber component
After=syslog.target network.target auditd.service
-Before=copr-backend.service
+After=copr-backend.service
[Service]
Type=simple
Environment="PYTHONPATH=/usr/share/copr/"
User=copr
Group=copr
-ExecStart=/usr/bin/copr_run_logger.py
+ExecStart=/usr/bin/copr_run_job_grab.py
[Install]
WantedBy=multi-user.target
diff --git a/backend/run/systemd/copr-backend-vmm.service b/backend/run/systemd/copr-backend-vmm.service
index 40a3767..33b4aed 100644
--- a/backend/run/systemd/copr-backend-vmm.service
+++ b/backend/run/systemd/copr-backend-vmm.service
@@ -1,6 +1,7 @@
[Unit]
Description=Copr Backend service, Virtual Machine Management component
After=syslog.target network.target auditd.service
+Before=copr-backend.service
[Service]
Type=simple
diff --git a/backend/run/systemd/copr-backend.service b/backend/run/systemd/copr-backend.service
index ce3a4ef..d0a931c 100644
--- a/backend/run/systemd/copr-backend.service
+++ b/backend/run/systemd/copr-backend.service
@@ -1,8 +1,7 @@
[Unit]
Description=Copr Backend service, Workers controller
After=syslog.target network.target auditd.service
-Requires=copr-backend-vmm.service
-Wants=copr-backend-log.service
+Requires=copr-backend-vmm.service copr-backend-jobgrab.service copr-backend-log.service
[Service]
Type=simple
8 years, 9 months