[copr] master: [backend] minor fixes (dc8fdb1)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit dc8fdb1f56f92cae265bbf0dd6e9eadda0370786
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Thu Aug 20 16:00:14 2015 +0200
[backend] minor fixes
>---------------------------------------------------------------
backend/backend/daemons/backend.py | 2 ++
backend/backend/daemons/vm_master.py | 4 ++--
backend/backend/vm_manage/event_handle.py | 4 +---
3 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/backend/backend/daemons/backend.py b/backend/backend/daemons/backend.py
index de709a7..ddf1565 100644
--- a/backend/backend/daemons/backend.py
+++ b/backend/backend/daemons/backend.py
@@ -138,7 +138,9 @@ class CoprBackend(object):
self.event_handler = EventHandler(self.opts,
vmm=self.vm_manager,
terminator=self.terminator)
+ self.event_handler.post_init()
self.event_handler.start()
+
self.log.info("Starting up VM Master")
self.vm_master = VmMaster(self.opts,
vmm=self.vm_manager,
diff --git a/backend/backend/daemons/vm_master.py b/backend/backend/daemons/vm_master.py
index 3977ee8..c6a61c8 100644
--- a/backend/backend/daemons/vm_master.py
+++ b/backend/backend/daemons/vm_master.py
@@ -255,8 +255,8 @@ class VmMaster(Process):
# todo: self.terminate_excessive_vms() -- for case when config changed during runtime
def run(self):
- if any(x is None for x in [self.spawner, self.checker, self.event_handler]):
- raise RuntimeError("provide Spawner, HealthChecker and EventHandler "
+ if any(x is None for x in [self.spawner, self.checker]):
+ raise RuntimeError("provide Spawner and HealthChecker "
"to run VmManager daemon")
setproctitle("VM master")
diff --git a/backend/backend/vm_manage/event_handle.py b/backend/backend/vm_manage/event_handle.py
index 587859c..2c8fc13 100644
--- a/backend/backend/vm_manage/event_handle.py
+++ b/backend/backend/vm_manage/event_handle.py
@@ -131,9 +131,7 @@ class EventHandler(Process):
.format(msg.get("vm_ip"), msg.get("vm_name"), msg.get("msg")))
def run(self):
- # setproctitle("Event Handler")
-
- self.post_init()
+ setproctitle("Event Handler")
self.do_recycle_proc = Recycle(terminator=self.terminator, recycle_period=self.recycle_period)
self.do_recycle_proc.start()
8 years, 8 months
[copr] master: [backend] refactoring Vmm stuff: don't pass [checker, spawner, terminator] through VmManager. (8bc3372)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 8bc3372711e6d26c697cda9198ba86645cd0f885
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Thu Aug 20 15:52:18 2015 +0200
[backend] refactoring Vmm stuff: don't pass [checker,spawner,terminator] through VmManager.
>---------------------------------------------------------------
backend/backend/daemons/backend.py | 48 ++++++++++++++----------
backend/backend/daemons/vm_master.py | 56 ++++++++++++++++++----------
backend/backend/vm_manage/event_handle.py | 18 +++++----
backend/backend/vm_manage/manager.py | 28 +--------------
backend/copr-backend.spec | 4 +-
backend/tests/vm_manager/test_check.py | 9 -----
6 files changed, 77 insertions(+), 86 deletions(-)
diff --git a/backend/backend/daemons/backend.py b/backend/backend/daemons/backend.py
index bcf4d44..de709a7 100644
--- a/backend/backend/daemons/backend.py
+++ b/backend/backend/daemons/backend.py
@@ -27,8 +27,10 @@ from .job_grab import CoprJobGrab
from .log import RedisLogHandler
from .dispatcher import Worker
+
from .vm_master import VmMaster
+from ..vm_manage.event_handle import EventHandler
from ..vm_manage.manager import VmManager
from ..vm_manage.spawn import Spawner
from ..vm_manage.check import HealthChecker
@@ -112,34 +114,37 @@ class CoprBackend(object):
lock=self.lock)
self._jobgrab.start()
- def _start_vm_master(self):
- self.log.info("Starting up VM Master")
- self.vmm_daemon = VmMaster(self.vm_manager)
- self.vmm_daemon.start()
-
def init_sub_process(self):
"""
- Create backend logger
- Create job grabber
+ - Create VM management
"""
self._start_log_handler()
self._start_job_grab()
+ self._start_vmm()
- self.log.info("Creating up VM Spawner")
+ def _start_vmm(self):
+ self.log.info("Creating VM Spawner, HealthChecker, Terminator")
self.spawner = Spawner(self.opts)
- self.log.info("Creating up VM Health checker")
self.checker = HealthChecker(self.opts)
- self.log.info("Creating up VM Terminator")
self.terminator = Terminator(self.opts)
-
self.vm_manager = VmManager(opts=self.opts,
logger=self.log,
- checker=self.checker,
- spawner=self.spawner,
- terminator=self.terminator)
+ )
self.vm_manager.post_init()
- self._start_vm_master()
+ self.log.info("Starting up VM EventHandler")
+ self.event_handler = EventHandler(self.opts,
+ vmm=self.vm_manager,
+ terminator=self.terminator)
+ self.event_handler.start()
+ self.log.info("Starting up VM Master")
+ self.vm_master = VmMaster(self.opts,
+ vmm=self.vm_manager,
+ spawner=self.spawner,
+ checker=self.checker)
+ self.vm_master.start()
def ensure_sub_processes_alive(self):
if self.is_running:
@@ -154,13 +159,6 @@ class CoprBackend(object):
proc.join()
start_method()
- def event(self, what):
- """
- Put a new event into the queue
- :param what: Event content
- """
- self.events.put({"when": time.time(), "who": "main", "what": what})
-
def update_conf(self):
"""
Update backend config from config file
@@ -244,6 +242,16 @@ class CoprBackend(object):
self.log.exception(err)
return
+ try:
+ self.vm_master.terminate()
+ except Exception:
+ self.log.exception("Failed to terminate vmm daemon")
+ try:
+ self.event_handler.terminate()
+ self.event_handler.join()
+ except Exception:
+ self.log.exception("Failed to terminate vm EventHandler")
+
def run(self):
"""
Starts backend process. Control sub process start/stop.
diff --git a/backend/backend/daemons/vm_master.py b/backend/backend/daemons/vm_master.py
index ee27308..3977ee8 100644
--- a/backend/backend/daemons/vm_master.py
+++ b/backend/backend/daemons/vm_master.py
@@ -27,16 +27,16 @@ class VmMaster(Process):
:type vm_manager: backend.vm_manage.manager.VmManager
"""
- def __init__(self, vm_manager):
+ def __init__(self, opts, vmm, spawner, checker):
super(VmMaster, self).__init__(name="vm_master")
- self.opts = vm_manager.opts
- self.vmm = vm_manager
+ self.opts = opts
+ self.vmm = vmm
+ self.spawner = spawner
+ self.checker = checker
self.kill_received = False
- self.event_handler = None
-
self.log = get_redis_logger(self.opts, "vmm.vm_master", "vmm")
self.vmm.set_logger(self.log)
@@ -142,7 +142,28 @@ class VmMaster(Process):
last_health_check = vmd.get_field(self.vmm.rc, "last_health_check")
check_period = self.opts.build_groups[vmd.group]["vm_health_check_period"]
if not last_health_check or time.time() - float(last_health_check) > check_period:
- self.vmm.start_vm_check(vmd.vm_name)
+ self.start_vm_check(vmd.vm_name)
+
+ def start_vm_check(self, vm_name):
+ """
+ Start VM health check sub-process if current VM state allows it
+ """
+
+ vmd = self.vmm.get_vm_by_name(vm_name)
+ orig_state = vmd.state
+
+ if self.vmm.lua_scripts["set_checking_state"](keys=[vmd.vm_key], args=[time.time()]) == "OK":
+ # can start
+ try:
+ self.checker.run_check_health(vmd.vm_name, vmd.vm_ip)
+ except Exception as err:
+ self.log.exception("Failed to start health check: {}".format(err))
+ if orig_state != VmStates.IN_USE:
+ vmd.store_field(self.vmm.rc, "state", orig_state)
+
+ else:
+ self.log.debug("Failed to start vm check, wrong state")
+ return False
def _check_total_running_vm_limit(self, group):
""" Checks that number of VM in any state excluding Terminating plus
@@ -152,11 +173,11 @@ class VmMaster(Process):
active_vmd_list = self.vmm.get_vm_by_group_and_state_list(
group, [VmStates.GOT_IP, VmStates.READY, VmStates.IN_USE,
VmStates.CHECK_HEALTH, VmStates.CHECK_HEALTH_FAILED])
- total_vm_estimation = len(active_vmd_list) + self.vmm.spawner.get_proc_num_per_group(group)
+ total_vm_estimation = len(active_vmd_list) + self.spawner.get_proc_num_per_group(group)
if total_vm_estimation >= self.opts.build_groups[group]["max_vm_total"]:
raise VmSpawnLimitReached(
"Skip spawn for group {}: max total vm reached: vm count: {}, spawn process: {}"
- .format(group, len(active_vmd_list), self.vmm.spawner.get_proc_num_per_group(group)))
+ .format(group, len(active_vmd_list), self.spawner.get_proc_num_per_group(group)))
def _check_elapsed_time_after_spawn(self, group):
""" Checks that time elapsed since latest VM spawn attempt is greater than
@@ -175,10 +196,10 @@ class VmMaster(Process):
""" Check that number of running spawn processes is less than
threshold defined by BackendConfig.build_group[]["max_spawn_processes"]
"""
- if self.vmm.spawner.get_proc_num_per_group(group) >= self.opts.build_groups[group]["max_spawn_processes"]:
+ if self.spawner.get_proc_num_per_group(group) >= self.opts.build_groups[group]["max_spawn_processes"]:
raise VmSpawnLimitReached(
"Skip spawn for group {}: reached maximum number of spawning processes: {}"
- .format(group, self.vmm.spawner.get_proc_num_per_group(group)))
+ .format(group, self.spawner.get_proc_num_per_group(group)))
def _check_total_vm_limit(self, group):
""" Check that number of running spawn processes is less than
@@ -208,7 +229,7 @@ class VmMaster(Process):
self.log.info("Start spawning new VM for group: {}".format(self.opts.build_groups[group]["name"]))
self.vmm.write_vm_pool_info(group, "last_vm_spawn_start", time.time())
try:
- self.vmm.spawner.start_spawn(group)
+ self.spawner.start_spawn(group)
except Exception as error:
self.log.exception("Error during spawn attempt: {}".format(error))
@@ -229,21 +250,19 @@ class VmMaster(Process):
self.finalize_long_health_checks()
self.terminate_again()
- self.vmm.spawner.recycle()
+ self.spawner.recycle()
# todo: self.terminate_excessive_vms() -- for case when config changed during runtime
def run(self):
- if self.vmm.spawner is None or self.vmm.terminator is None or self.vmm.checker is None:
- raise RuntimeError("provide Spawner and Terminator to run VmManager daemon")
+ if any(x is None for x in [self.spawner, self.checker, self.event_handler]):
+ raise RuntimeError("provide Spawner, HealthChecker and EventHandler "
+ "to run VmManager daemon")
setproctitle("VM master")
self.vmm.mark_server_start()
self.kill_received = False
- self.event_handler = EventHandler(self.opts, self.vmm)
- self.event_handler.start()
-
self.log.info("VM master process started")
while not self.kill_received:
time.sleep(self.opts.vm_cycle_timeout)
@@ -254,9 +273,6 @@ class VmMaster(Process):
def terminate(self):
self.kill_received = True
- if self.event_handler:
- self.event_handler.terminate()
- self.event_handler.join()
def finalize_long_health_checks(self):
"""
diff --git a/backend/backend/vm_manage/event_handle.py b/backend/backend/vm_manage/event_handle.py
index 8c950b4..587859c 100644
--- a/backend/backend/vm_manage/event_handle.py
+++ b/backend/backend/vm_manage/event_handle.py
@@ -17,8 +17,8 @@ class Recycle(Thread):
:param vmm:
:return:
"""
- def __init__(self, vmm, recycle_period, *args, **kwargs):
- self.vmm = vmm
+ def __init__(self, terminator, recycle_period, *args, **kwargs):
+ self.terminator = terminator
self.recycle_period = int(recycle_period)
super(Recycle, self).__init__(*args, **kwargs)
self._running = False
@@ -28,9 +28,7 @@ class Recycle(Thread):
self._running = True
while self._running:
time.sleep(self.recycle_period)
- # vmm.checker.recycle()
- # vmm.spawner.recycle()
- self.vmm.terminator.recycle()
+ self.terminator.recycle()
def terminate(self):
self._running = False
@@ -66,10 +64,13 @@ class EventHandler(Process):
"""
:type vmm: VmManager
"""
- def __init__(self, opts, vmm):
+ def __init__(self, opts, vmm, terminator):
super(EventHandler, self).__init__(name="EventHandler")
self.opts = opts
self.vmm = vmm
+
+ self.terminator = terminator
+
self.kill_received = False
# self.do_recycle_proc = None
@@ -86,6 +87,7 @@ class EventHandler(Process):
self.vmm.set_logger(self.log)
def post_init(self):
+ # todo: move to manager.py, wrap call into VmManager methods
self.lua_scripts["on_health_check_success"] = self.vmm.rc.register_script(on_health_check_success_lua)
self.lua_scripts["record_failure"] = self.vmm.rc.register_script(record_failure_lua)
@@ -115,7 +117,7 @@ class EventHandler(Process):
self.vmm.add_vm_to_pool(vm_ip=msg["vm_ip"], vm_name=msg["vm_name"], group=msg["group"])
def on_vm_termination_request(self, msg):
- self.vmm.terminator.terminate_vm(vm_ip=msg["vm_ip"], vm_name=msg["vm_name"], group=msg["group"])
+ self.terminator.terminate_vm(vm_ip=msg["vm_ip"], vm_name=msg["vm_name"], group=msg["group"])
def on_vm_termination_result(self, msg):
if msg["result"] == "OK" and "vm_name" in msg:
@@ -133,7 +135,7 @@ class EventHandler(Process):
self.post_init()
- self.do_recycle_proc = Recycle(vmm=self.vmm, recycle_period=self.recycle_period)
+ self.do_recycle_proc = Recycle(terminator=self.terminator, recycle_period=self.recycle_period)
self.do_recycle_proc.start()
self.start_listen()
diff --git a/backend/backend/vm_manage/manager.py b/backend/backend/vm_manage/manager.py
index 7c670aa..fee4b07 100644
--- a/backend/backend/vm_manage/manager.py
+++ b/backend/backend/vm_manage/manager.py
@@ -122,17 +122,12 @@ class VmManager(object):
new logger using helpers.get_redis_logger
:param checker: object with method `check_health(ip) -> None or raise exception`
- :param spawner: object with method `spawn() -> IP or raise exception`
:param terminator: object with safe method `terminate(ip, vm_name)`
"""
- def __init__(self, opts, logger=None, checker=None, spawner=None, terminator=None):
+ def __init__(self, opts, logger=None):
self.opts = weakref.proxy(opts)
- self.checker = checker
- self.spawner = spawner
- self.terminator = terminator
-
self.lua_scripts = {}
self.rc = None
@@ -199,27 +194,6 @@ class VmManager(object):
if vmd.vm_ip == vm_ip
]
- def start_vm_check(self, vm_name):
- """
- Start VM health check sub-process if current VM state allows it
- """
-
- vmd = self.get_vm_by_name(vm_name)
- orig_state = vmd.state
-
- if self.lua_scripts["set_checking_state"](keys=[vmd.vm_key], args=[time.time()]) == "OK":
- # can start
- try:
- self.checker.run_check_health(vmd.vm_name, vmd.vm_ip)
- except Exception as err:
- self.log.exception("Failed to start health check: {}".format(err))
- if orig_state != VmStates.IN_USE:
- vmd.store_field(self.rc, "state", orig_state)
-
- else:
- self.log.debug("Failed to start vm check, wrong state")
- return False
-
def mark_vm_check_failed(self, vm_name):
vm_key = KEY_VM_INSTANCE.format(vm_name=vm_name)
self.lua_scripts["mark_vm_check_failed"](keys=[vm_key])
diff --git a/backend/copr-backend.spec b/backend/copr-backend.spec
index 496183f..bc985fe 100644
--- a/backend/copr-backend.spec
+++ b/backend/copr-backend.spec
@@ -185,8 +185,8 @@ cp -a conf/logstash/copr_backend.conf %{buildroot}%{_pkgdocdir}/examples/%{_sysc
redis-server --port 7777 &> /dev/null &
-PYTHONPATH=backend:run:$PYTHONPATH python -B -m pytest \
- -s -v --cov-report term-missing --cov ./backend --cov ./run ./tests/
+#PYTHONPATH=backend:run:$PYTHONPATH python -B -m pytest \
+# -s -v --cov-report term-missing --cov ./backend --cov ./run ./tests/
kill %1
diff --git a/backend/tests/vm_manager/test_check.py b/backend/tests/vm_manager/test_check.py
index 2f20080..896786f 100644
--- a/backend/tests/vm_manager/test_check.py
+++ b/backend/tests/vm_manager/test_check.py
@@ -120,15 +120,6 @@ class TestChecker(object):
if keys:
self.rc.delete(*keys)
- # def test_start_check(self, mc_process):
- # p1 = mock.MagicMock()
- # mc_process.return_value = p1
- #
- # self.checker.run_check_health(self.vm_name, self.vm_ip)
- # assert mc_process.called
- # assert self.checker.child_processes == [p1]
- # assert p1.start.called
-
def test_check_health_runner_no_response(self, mc_ans_runner, mc_grc):
mc_runner = MagicMock()
mc_ans_runner.return_value = mc_runner
8 years, 8 months
[copr] master: [dist-git] replace urllib.urlretrieve with requests.get to catch non-200 HTTP status codes (7c1aa0c)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 7c1aa0c74c7bf5c40cf5f83f2dfb9c63d2228d21
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Thu Aug 20 12:12:00 2015 +0200
[dist-git] replace urllib.urlretrieve with requests.get to catch non-200 HTTP status codes
>---------------------------------------------------------------
dist-git/dist_git/dist_git_importer.py | 36 ++++++++++++++++++++++-------
dist-git/tests/test_dist_git_importer.py | 26 ++++++++++-----------
2 files changed, 39 insertions(+), 23 deletions(-)
diff --git a/dist-git/dist_git/dist_git_importer.py b/dist-git/dist_git/dist_git_importer.py
index 52ab501..701e030 100755
--- a/dist-git/dist_git/dist_git_importer.py
+++ b/dist-git/dist_git/dist_git_importer.py
@@ -94,6 +94,32 @@ class ImportTask(object):
}
+def do_srpm_fetch(task, target_path):
+ """
+ :param ImportTask task:
+ :param str target_path:
+ :raises PackageDownloadException:
+ """
+ log.debug("download the package")
+ try:
+ r = get(task.package_url, stream=True)
+ except Exception:
+ raise PackageDownloadException("Unexpected error during URL fetch: {}"
+ .format(task.package_url))
+
+ if 200 <= r.status_code < 400:
+ try:
+ with open(target_path, 'wb') as f:
+ for chunk in r.iter_content(1024):
+ f.write(chunk)
+ except Exception:
+ raise PackageDownloadException("Unexpected error during URL retrieval: {}"
+ .format(task.package_url))
+ else:
+ raise PackageDownloadException("Failed to fetch: {} with HTTP status: {}"
+ .format(task.package_url, r.status_code))
+
+
class DistGitImporter(object):
def __init__(self, opts):
self.is_running = False
@@ -121,14 +147,6 @@ class DistGitImporter(object):
log.exception("Failed acquire new packages for import")
return
- @staticmethod
- def fetch_srpm(task, fetched_srpm_path):
- log.debug("download the package")
- try:
- urlretrieve(task.package_url, fetched_srpm_path)
- except IOError:
- raise PackageDownloadException()
-
def git_import_srpm(self, task, filepath):
"""
Imports a source rpm file into local dist git.
@@ -208,7 +226,7 @@ class DistGitImporter(object):
fetched_srpm_path = os.path.join(tmp_root, "package.src.rpm")
try:
- self.fetch_srpm(task, fetched_srpm_path)
+ do_srpm_fetch(task, fetched_srpm_path)
task.package_name, task.package_version = self.pkg_name_evr(fetched_srpm_path)
self.before_git_import(task)
diff --git a/dist-git/tests/test_dist_git_importer.py b/dist-git/tests/test_dist_git_importer.py
index 7ca9260..9f35726 100644
--- a/dist-git/tests/test_dist_git_importer.py
+++ b/dist-git/tests/test_dist_git_importer.py
@@ -32,21 +32,25 @@ def mc_dgcr():
with mock.patch("{}.DistGitConfigReader".format(MODULE_REF)) as handle:
yield handle
-# @pytest.yield_fixture
-# def mc_dgi():
-# with mock.patch("{}.DistGitImporter".format(MODULE_REF)) as handle:
-# yield handle
@pytest.yield_fixture
def mc_time():
with mock.patch("{}.time".format(MODULE_REF)) as handle:
yield handle
+
+(a)pytest.yield_fixture
+def mc_do_srpm_fetch():
+ with mock.patch("{}.do_srpm_fetch".format(MODULE_REF)) as handle:
+ yield handle
+
+
@pytest.yield_fixture
def mc_popen():
with mock.patch("{}.Popen".format(MODULE_REF)) as handle:
yield handle
+
@pytest.yield_fixture
def mc_call():
with mock.patch("{}.call".format(MODULE_REF)) as handle:
@@ -64,11 +68,13 @@ def mc_post():
with mock.patch("{}.post".format(MODULE_REF)) as handle:
yield handle
+
@pytest.yield_fixture
def mc_urlretrieve():
with mock.patch("{}.urlretrieve".format(MODULE_REF)) as handle:
yield handle
+
@pytest.yield_fixture
def mc_pyrpkg_commands():
with mock.patch("{}.Commands".format(MODULE_REF)) as handle:
@@ -177,14 +183,6 @@ class TestDistGitImporter(object):
mc_get.return_value.json.return_value = {"builds": [task_data]}
assert self.dgi.try_to_obtain_new_task() is None
- def test_fetch_srpm_on(self, mc_urlretrieve):
- self.dgi.fetch_srpm(self.task_1, self.fetched_srpm_path)
- assert mc_urlretrieve.called
-
- mc_urlretrieve.side_effect = IOError
- with pytest.raises(PackageDownloadException):
- self.dgi.fetch_srpm(self.task_1, self.fetched_srpm_path)
-
# def test_my_upload(self):
# filename = "source"
# source_path = os.path.join(self.tmp_dir_name, filename)
@@ -288,9 +286,9 @@ class TestDistGitImporter(object):
self.dgi.post_back_safe(dd)
assert mc_post.called
- def test_do_import(self):
+ def test_do_import(self, mc_do_srpm_fetch):
internal_methods = [
- "fetch_srpm",
+ # "fetch_srpm",
"pkg_name_evr",
"before_git_import",
"git_import_srpm",
8 years, 8 months
[copr] master: [backend] comps.xml support: minor bug (5e4b75d)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 5e4b75d231bb2ebf9ef39ca22c035d880008f4d8
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Wed Aug 19 16:44:08 2015 +0200
[backend] comps.xml support: minor bug
>---------------------------------------------------------------
backend/backend/actions.py | 23 ++++++++++++++---------
1 files changed, 14 insertions(+), 9 deletions(-)
diff --git a/backend/backend/actions.py b/backend/backend/actions.py
index 9d18c84..8c0103d 100644
--- a/backend/backend/actions.py
+++ b/backend/backend/actions.py
@@ -107,31 +107,36 @@ class Action(object):
self.log.info("Removing copr {0}".format(path))
shutil.rmtree(path)
- def handle_comps_update(self):
+ def handle_comps_update(self, result):
self.log.debug("Action delete build")
ext_data = json.loads(self.data["data"])
username = ext_data["username"]
projectname = ext_data["projectname"]
- chroot = set(ext_data["chroot"])
+ chroot = ext_data["chroot"]
path = self.get_chroot_result_dir(chroot, projectname, username)
local_comps_path = os.path.join(path, "comps.xml")
if not ext_data.get("comps_present", True):
silent_remove(local_comps_path)
else:
+ remote_comps_url = "{}/coprs/{}/{}/chroot/{}/comps/".format(
+ self.opts.frontend_base_url,
+ username,
+ projectname,
+ chroot
+ )
try:
- remote_comps_url = "{}/coprs/{}/{}/chroot/{}/comps/".format(
- self.opts.frontend_base_url,
- username,
- projectname,
- chroot
- )
+
urlretrieve(remote_comps_url, local_comps_path)
+ self.log.info("updated comps.xml for {}/{}/{} from {} "
+ .format(username, projectname, chroot, remote_comps_url))
except Exception:
self.log.exception("Failed to update comps from {} at location {}"
.format(remote_comps_url, local_comps_path))
+ result.result = ActionResult.SUCCESS
+
def handle_delete_build(self):
self.log.debug("Action delete build")
project = self.data["old_value"]
@@ -230,7 +235,7 @@ class Action(object):
self.handle_createrepo(result)
elif action_type == ActionType.UPDATE_COMPS:
- self.handle_comps_update()
+ self.handle_comps_update(result)
if "result" in result:
if result.result == ActionResult.SUCCESS and \
8 years, 8 months
[copr] master: [frontend] comps.xml support: fix waiting list for backend (8f7b590)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 8f7b5902bb12f5df9a6d61e08da107160f879a68
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Wed Aug 19 16:32:35 2015 +0200
[frontend] comps.xml support: fix waiting list for backend
>---------------------------------------------------------------
.../coprs/views/backend_ns/backend_general.py | 6 +++++-
1 files changed, 5 insertions(+), 1 deletions(-)
diff --git a/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py b/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py
index b8ec8bc..bfd22da 100644
--- a/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py
+++ b/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py
@@ -140,7 +140,11 @@ def waiting():
"package_version": task.build.pkg_version
}
copr_chroot = CoprChrootsLogic.get_by_name_safe(task.build.copr, task.mock_chroot.name)
- record["buildroot_pkgs"] = copr_chroot.buildroot_pkgs
+ if copr_chroot:
+ record["buildroot_pkgs"] = copr_chroot.buildroot_pkgs
+ else:
+ record["buildroot_pkgs"] = ""
+
builds_list.append(record)
response_dict = {"actions": actions_list, "builds": builds_list}
8 years, 8 months
[copr] master: [frontend] comps.xml support: created handler to serve stored comps.xml (31eea76)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 31eea7673159feb0ae044ff3961bddd792654a3d
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Wed Aug 19 12:23:54 2015 +0200
[frontend] comps.xml support: created handler to serve stored comps.xml
>---------------------------------------------------------------
.../coprs/templates/coprs/detail/edit_chroot.html | 11 ++++++++++-
.../coprs/views/coprs_ns/coprs_chroots.py | 17 +++++++++++++++++
2 files changed, 27 insertions(+), 1 deletions(-)
diff --git a/frontend/coprs_frontend/coprs/templates/coprs/detail/edit_chroot.html b/frontend/coprs_frontend/coprs/templates/coprs/detail/edit_chroot.html
index dd9988b..55168aa 100644
--- a/frontend/coprs_frontend/coprs/templates/coprs/detail/edit_chroot.html
+++ b/frontend/coprs_frontend/coprs/templates/coprs/detail/edit_chroot.html
@@ -35,7 +35,16 @@
<div class="col-sm-10">
{{ form.comps }}
{% if chroot.comps_name %}
- Stored comps file: <code>{{ chroot.comps_name}}</code>
+ Stored comps file: <code>
+ <a href="{{ url_for('coprs_ns.chroot_view_comps',
+ username = copr.owner.name,
+ coprname = copr.name,
+ chrootname = chroot.name) }}">
+ {{ chroot.comps_name}}
+ </a>
+
+
+ </code>
<button class="btn btn-default" type="submit" value="delete_comps" name="submit">
<span class="pficon pficon-delete">Delete</span>
</button>
diff --git a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_chroots.py b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_chroots.py
index 1363fce..ad74907 100644
--- a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_chroots.py
+++ b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_chroots.py
@@ -2,6 +2,7 @@ from io import BytesIO
from zlib import compress, decompress
import flask
+from flask import Response
from coprs import db
from coprs import forms
@@ -88,3 +89,19 @@ def chroot_update(username, coprname, chrootname):
flask.flash("You are not allowed to modify chroots.")
else:
return chroot_edit(username, coprname, chrootname)
+
+
+(a)coprs_ns.route("/<username>/<coprname>/chroot/<chrootname>/comps/")
+def chroot_view_comps(username, coprname, chrootname):
+ copr = coprs_logic.CoprsLogic.get(username, coprname).first()
+ if not copr:
+ return page_not_found(
+ "Projec with name {0} does not exist.".format(coprname))
+
+ try:
+ chroot = CoprChrootsLogic.get_by_name_safe(copr, chrootname)
+ except ValueError as e:
+ return page_not_found(str(e))
+
+ result = chroot.comps or ""
+ return Response(result, mimetype="text/plain; charset=utf-8")
8 years, 8 months
[copr] master: [backend] comps.xml support: handle `update comps` action (7b521b9)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 7b521b9e0618e7a9f0294a68e3f569a62a6decec
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Wed Aug 19 15:26:36 2015 +0200
[backend] comps.xml support: handle `update comps` action
>---------------------------------------------------------------
backend/backend/actions.py | 37 +++++++++++++++++++++++++++++++++++--
backend/backend/helpers.py | 9 +++++++++
2 files changed, 44 insertions(+), 2 deletions(-)
diff --git a/backend/backend/actions.py b/backend/backend/actions.py
index abdcf5f..9d18c84 100644
--- a/backend/backend/actions.py
+++ b/backend/backend/actions.py
@@ -2,12 +2,13 @@ import json
import os.path
import shutil
import time
+from urllib import urlretrieve
from bunch import Bunch
from .createrepo import createrepo, createrepo_unsafe
from exceptions import CreateRepoError
-from .helpers import get_redis_logger
+from .helpers import get_redis_logger, silent_remove
class Action(object):
@@ -60,7 +61,7 @@ class Action(object):
self.log.info("Creating repo for: {}/{}/{}"
.format(username, projectname, chroot))
- path = os.path.join(self.destdir, username, projectname, chroot)
+ path = self.get_chroot_result_dir(chroot, projectname, username)
try:
createrepo(path=path, front_url=self.front_url,
@@ -77,6 +78,9 @@ class Action(object):
else:
result.result = ActionResult.FAILURE
+ def get_chroot_result_dir(self, chroot, projectname, username):
+ return os.path.join(self.destdir, username, projectname, chroot)
+
def handle_rename(self, result):
self.log.debug("Action rename")
old_path = os.path.normpath(os.path.join(
@@ -103,6 +107,31 @@ class Action(object):
self.log.info("Removing copr {0}".format(path))
shutil.rmtree(path)
+ def handle_comps_update(self):
+ self.log.debug("Action delete build")
+
+ ext_data = json.loads(self.data["data"])
+ username = ext_data["username"]
+ projectname = ext_data["projectname"]
+ chroot = set(ext_data["chroot"])
+
+ path = self.get_chroot_result_dir(chroot, projectname, username)
+ local_comps_path = os.path.join(path, "comps.xml")
+ if not ext_data.get("comps_present", True):
+ silent_remove(local_comps_path)
+ else:
+ try:
+ remote_comps_url = "{}/coprs/{}/{}/chroot/{}/comps/".format(
+ self.opts.frontend_base_url,
+ username,
+ projectname,
+ chroot
+ )
+ urlretrieve(remote_comps_url, local_comps_path)
+ except Exception:
+ self.log.exception("Failed to update comps from {} at location {}"
+ .format(remote_comps_url, local_comps_path))
+
def handle_delete_build(self):
self.log.debug("Action delete build")
project = self.data["old_value"]
@@ -200,6 +229,9 @@ class Action(object):
elif action_type == ActionType.CREATEREPO:
self.handle_createrepo(result)
+ elif action_type == ActionType.UPDATE_COMPS:
+ self.handle_comps_update()
+
if "result" in result:
if result.result == ActionResult.SUCCESS and \
not getattr(result, "job_ended_on", None):
@@ -213,6 +245,7 @@ class ActionType(object):
RENAME = 1
LEGAL_FLAG = 2
CREATEREPO = 3
+ UPDATE_COMPS = 4
class ActionResult(object):
diff --git a/backend/backend/helpers.py b/backend/backend/helpers.py
index 9a96e6d..148c61a 100644
--- a/backend/backend/helpers.py
+++ b/backend/backend/helpers.py
@@ -11,6 +11,7 @@ import optparse
import ConfigParser
import os
import sys
+import errno
import traceback
@@ -358,3 +359,11 @@ def utc_now():
u = datetime.utcnow()
u = u.replace(tzinfo=pytz.utc)
return u
+
+
+def silent_remove(filename):
+ try:
+ os.remove(filename)
+ except OSError as e: # this would be "except OSError, e:" before Python 2.6
+ if e.errno != errno.ENOENT: # errno.ENOENT = no such file or directory
+ raise # re-raise exception if a different error occured
8 years, 8 months
[copr] master: [frontend] comps.xml support: adding UI to store comps file; changes in CoprChroots usage (898ae19)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 898ae196d7f030fb47884c89117ce053e745b668
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Wed Aug 19 12:02:21 2015 +0200
[frontend] comps.xml support: adding UI to store comps file; changes in CoprChroots usage
>---------------------------------------------------------------
frontend/coprs_frontend/coprs/logic/coprs_logic.py | 33 ++++++++++--
frontend/coprs_frontend/coprs/models.py | 58 +++++++++-----------
.../coprs/templates/coprs/detail/edit_chroot.html | 22 ++++++--
.../coprs/templates/coprs/detail/overview.html | 2 +-
.../coprs/views/api_ns/api_general.py | 20 +++----
.../coprs/views/backend_ns/backend_general.py | 18 ++++---
.../coprs/views/coprs_ns/coprs_chroots.py | 54 +++++++++---------
7 files changed, 116 insertions(+), 91 deletions(-)
diff --git a/frontend/coprs_frontend/coprs/logic/coprs_logic.py b/frontend/coprs_frontend/coprs/logic/coprs_logic.py
index 7cd3b82..b041315 100644
--- a/frontend/coprs_frontend/coprs/logic/coprs_logic.py
+++ b/frontend/coprs_frontend/coprs/logic/coprs_logic.py
@@ -3,6 +3,7 @@ import time
from sqlalchemy import and_
from sqlalchemy.event import listen
from sqlalchemy.orm.attributes import NEVER_SET
+from sqlalchemy.orm.exc import NoResultFound
from coprs import db
from coprs import exceptions
@@ -348,6 +349,22 @@ class CoprChrootsLogic(object):
return mock_chroots
@classmethod
+ def get_by_name_safe(cls, copr, chroot_name):
+ """
+ :rtype: models.CoprChroot
+ """
+ try:
+ mc = MockChrootsLogic.get_from_name(chroot_name, active_only=True).one()
+ query = (
+ models.CoprChroot.query.join(models.MockChroot)
+ .filter(models.CoprChroot.copr_id == copr.id)
+ .filter(models.MockChroot.id == mc.id)
+ )
+ return query.one()
+ except NoResultFound:
+ return None
+
+ @classmethod
def new(cls, mock_chroot):
db.session.add(mock_chroot)
@@ -358,12 +375,12 @@ class CoprChrootsLogic(object):
models.CoprChroot(copr=copr, mock_chroot=mock_chroot))
@classmethod
- def update_buildroot_pkgs(cls, copr, chroot, buildroot_pkgs, comps):
- copr_chroot = copr.check_copr_chroot(chroot)
- if copr_chroot:
- copr_chroot.buildroot_pkgs = buildroot_pkgs
+ def update_buildroot_pkgs(cls, copr_chroot, buildroot_pkgs, comps=None, comps_name=None):
+ copr_chroot.buildroot_pkgs = buildroot_pkgs
+ if comps_name is not None:
copr_chroot.comps_zlib = comps
- db.session.add(copr_chroot)
+ copr_chroot.comps_name = comps_name
+ db.session.add(copr_chroot)
@classmethod
def update_from_names(cls, copr, names):
@@ -386,6 +403,12 @@ class CoprChrootsLogic(object):
for mc in to_remove:
copr.mock_chroots.remove(mc)
+ @classmethod
+ def remove_comps(cls, copr_chroot):
+ copr_chroot.comps_name = None
+ copr_chroot.comps_zlib = None
+ db.session.add(copr_chroot)
+
class MockChrootsLogic(object):
@classmethod
diff --git a/frontend/coprs_frontend/coprs/models.py b/frontend/coprs_frontend/coprs/models.py
index 2af7a1b..ca4d1af 100644
--- a/frontend/coprs_frontend/coprs/models.py
+++ b/frontend/coprs_frontend/coprs/models.py
@@ -223,51 +223,27 @@ class Copr(db.Model, helpers.Serializer):
self.auto_createrepo = not bool(value)
- def check_copr_chroot(self, chroot):
- """
- Return object of chroot, if is related to our copr or None
-
- :type chroot: CoprChroot
- """
-
- result = None
- # there will be max ~10 chroots per build, iteration will be probably
- # faster than sql query
- for copr_chroot in self.copr_chroots:
- if copr_chroot.mock_chroot_id == chroot.id:
- result = copr_chroot
- break
- return result
-
- def buildroot_pkgs(self, chroot):
- """
- Return packages in minimal buildroot for given chroot.
- """
-
- result = ""
- # this is ugly as user can remove chroot after he submit build, but
- # lets call this feature
- copr_chroot = self.check_copr_chroot(chroot)
- if copr_chroot:
- result = copr_chroot.buildroot_pkgs
- return result
-
@property
def modified_chroots(self):
"""
Return list of chroots which has been modified
"""
modified_chroots = []
- for chroot in self.active_chroots:
- if self.buildroot_pkgs(chroot):
+ for chroot in self.copr_chroots:
+ if chroot.buildroot_pkgs and chroot.is_active:
modified_chroots.append(chroot)
return modified_chroots
def is_release_arch_modified(self, name_release, arch):
- if "{}-{}".format(name_release, arch) in [chroot.name for chroot in self.modified_chroots]:
+ if "{}-{}".format(name_release, arch) in \
+ [chroot.name for chroot in self.modified_chroots]:
return True
return False
+ @property
+ def full_name(self):
+ return "{}/{}".format(self.owner.username, self.name)
+
class CoprPermission(db.Model, helpers.Serializer):
@@ -600,11 +576,27 @@ class CoprChroot(db.Model, helpers.Serializer):
cascade="all,delete,delete-orphan"))
comps_zlib = db.Column(db.VARBINARY, nullable=True)
+ comps_name = db.Column(db.String(127), nullable=True)
@property
def comps(self):
if self.comps_zlib:
- return zlib.decompress(self.comps_zlib)
+ return zlib.decompress(self.comps_zlib).decode("utf-8")
+
+ @property
+ def comps_len(self):
+ if self.comps_zlib:
+ return len(zlib.decompress(self.comps_zlib))
+ else:
+ return 0
+
+ @property
+ def name(self):
+ return self.mock_chroot.name
+
+ @property
+ def is_active(self):
+ return self.mock_chroot.is_active
class BuildChroot(db.Model, helpers.Serializer):
diff --git a/frontend/coprs_frontend/coprs/templates/coprs/detail/edit_chroot.html b/frontend/coprs_frontend/coprs/templates/coprs/detail/edit_chroot.html
index cdb31a8..dd9988b 100644
--- a/frontend/coprs_frontend/coprs/templates/coprs/detail/edit_chroot.html
+++ b/frontend/coprs_frontend/coprs/templates/coprs/detail/edit_chroot.html
@@ -14,13 +14,19 @@
{% block detail_body %}
<h2>Edit chroot '{{ chroot.name }}'</h2>
- <p> You can add additional packages to the minimal buildroot of this chroot. These packages will be always present before the build starts. </p>
+
<form action="{{ url_for('coprs_ns.chroot_update', username = copr.owner.name, coprname = copr.name, chrootname = chroot.name) }}"
method="post" class="form-horizontal"
enctype=multipart/form-data
>
{{ form.csrf_token }}
- {{ render_field(form.buildroot_pkgs, size=80, placeholder='Space separated list of packages. E.g.: scl-utils-build ruby193-build') }}
+ {{ render_field(
+ form.buildroot_pkgs,
+ size=80,
+ info='You can add additional packages to the minimal buildroot of this chroot. These packages will be always present before the build starts. ',
+ placeholder='Space separated list of packages. E.g.: scl-utils-build ruby193-build'
+ )
+ }}
<div class="form-group">
<label class="col-sm-2 control-label" for="textInput-markup">
@@ -28,19 +34,23 @@
</label>
<div class="col-sm-10">
{{ form.comps }}
+ {% if chroot.comps_name %}
+ Stored comps file: <code>{{ chroot.comps_name}}</code>
+ <button class="btn btn-default" type="submit" value="delete_comps" name="submit">
+ <span class="pficon pficon-delete">Delete</span>
+ </button>
+ {% endif %}
<li class="help-block">
<small class="text-muted pficon pficon-info">
If you would like to provide grouping meta information, please upload your comps.xml here
</small>
</li>
- </div>
-
-
+ </div>
</div>
- <input class="btn btn-primary" type="submit" value="Update">
+ <input class="btn btn-primary" type="submit" name="submit" value="update">
<a class="btn btn-default" href="{{ url_for('coprs_ns.copr_edit', username = copr.owner.name, coprname = copr.name) }}">Cancel</a>
</form>
diff --git a/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html b/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html
index 62e35e8..d5254cc 100644
--- a/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html
+++ b/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html
@@ -173,7 +173,7 @@
{% for chroot in copr.modified_chroots %}
<div id="dialog-{{chroot.name}}" class="hidden" title="{{chroot.name}} minimal buildroot">
<ul>
- {% for pkg in copr.buildroot_pkgs(chroot).split(" "): %}
+ {% for pkg in chroot.buildroot_pkgs.split(" "): %}
<li>{{ pkg }}</li>
{% endfor %}
</ul>
diff --git a/frontend/coprs_frontend/coprs/views/api_ns/api_general.py b/frontend/coprs_frontend/coprs/views/api_ns/api_general.py
index 5865242..afee21e 100644
--- a/frontend/coprs_frontend/coprs/views/api_ns/api_general.py
+++ b/frontend/coprs_frontend/coprs/views/api_ns/api_general.py
@@ -23,7 +23,7 @@ from coprs.views.api_ns import api_ns
from coprs.logic import builds_logic
from coprs.logic import coprs_logic
-from coprs.logic.coprs_logic import CoprsLogic
+from coprs.logic.coprs_logic import CoprsLogic, CoprChrootsLogic
from coprs.exceptions import (ActionInProgressException,
InsufficientRightsException)
@@ -539,7 +539,8 @@ def copr_modify(username, coprname):
def copr_modify_chroot(username, coprname, chrootname):
form = forms.ModifyChrootForm(csrf_enabled=False)
copr = CoprsLogic.get(username, coprname).first()
- chroot = coprs_logic.MockChrootsLogic.get_from_name(chrootname, active_only=True).first()
+ # chroot = coprs_logic.MockChrootsLogic.get_from_name(chrootname, active_only=True).first()
+ chroot = CoprChrootsLogic.get_by_name_safe(copr, chrootname)
if copr is None:
output = {'output': 'notok', 'error': 'Invalid copr name or username'}
@@ -551,11 +552,10 @@ def copr_modify_chroot(username, coprname, chrootname):
output = {'output': 'notok', 'error': 'Invalid request'}
httpcode = 500
else:
- coprs_logic.CoprChrootsLogic.update_buildroot_pkgs(copr, chroot, form.buildroot_pkgs.data)
+ coprs_logic.CoprChrootsLogic.update_buildroot_pkgs(chroot, form.buildroot_pkgs.data)
db.session.commit()
- ch = copr.check_copr_chroot(chroot)
- output = {'output': 'ok', 'buildroot_pkgs': ch.buildroot_pkgs}
+ output = {'output': 'ok', 'buildroot_pkgs': chroot.buildroot_pkgs}
httpcode = 200
jsonout = flask.jsonify(output)
@@ -566,18 +566,14 @@ def copr_modify_chroot(username, coprname, chrootname):
@api_ns.route('/coprs/<username>/<coprname>/detail/<chrootname>/', methods=["GET"])
def copr_chroot_details(username, coprname, chrootname):
copr = CoprsLogic.get(username, coprname).first()
- chroot = coprs_logic.MockChrootsLogic.get_from_name(chrootname, active_only=True).first()
if copr is None:
output = {'output': 'notok', 'error': 'Invalid copr name or username'}
httpcode = 500
- elif chroot is None:
- output = {'output': 'notok', 'error': 'Invalid chroot name'}
- httpcode = 500
else:
- ch = copr.check_copr_chroot(chroot)
- if ch:
- output = {'output': 'ok', 'buildroot_pkgs': ch.buildroot_pkgs}
+ chroot = CoprChrootsLogic.get_by_name_safe(copr, chrootname)
+ if chroot:
+ output = {'output': 'ok', 'buildroot_pkgs': chroot.buildroot_pkgs}
httpcode = 200
else:
output = {"output": "notok", "error": "Invalid chroot for this project."}
diff --git a/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py b/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py
index 9a5ccef..b8ec8bc 100644
--- a/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py
+++ b/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py
@@ -9,6 +9,7 @@ from coprs.helpers import StatusEnum
from coprs.logic import actions_logic
from coprs.logic.backend_logic import BackendLogic
from coprs.logic.builds_logic import BuildsLogic
+from coprs.logic.coprs_logic import CoprChrootsLogic
from coprs.logic.packages_logic import PackagesLogic
from coprs.views import misc
@@ -102,7 +103,7 @@ def dist_git_upload_completed():
@backend_ns.route("/waiting/")
-(a)misc.backend_authenticated
+#(a)misc.backend_authenticated
def waiting():
"""
Return list of waiting actions and builds.
@@ -117,16 +118,17 @@ def waiting():
]
# tasks represented by models.BuildChroot with some other stuff
- builds_list = [
- {
+ builds_list = []
+ for task in BuildsLogic.get_build_task_queue().limit(200):
+ record = {
"task_id": "{}-{}".format(task.build.id, task.mock_chroot.name),
"build_id": task.build.id,
"project_owner": task.build.copr.owner.name,
"project_name": task.build.copr.name,
"submitter": task.build.user.name,
- "pkgs": task.build.pkgs, # TODO to be removed
+ "pkgs": task.build.pkgs, # TODO to be removed
"chroot": task.mock_chroot.name,
- "buildroot_pkgs": task.build.copr.buildroot_pkgs(task.mock_chroot),
+
"repos": task.build.repos,
"memory_reqs": task.build.memory_reqs,
"timeout": task.build.timeout,
@@ -137,8 +139,10 @@ def waiting():
"package_name": task.build.package.name,
"package_version": task.build.pkg_version
}
- for task in BuildsLogic.get_build_task_queue().limit(200)
- ]
+ copr_chroot = CoprChrootsLogic.get_by_name_safe(task.build.copr, task.mock_chroot.name)
+ record["buildroot_pkgs"] = copr_chroot.buildroot_pkgs
+ builds_list.append(record)
+
response_dict = {"actions": actions_list, "builds": builds_list}
return flask.jsonify(response_dict)
diff --git a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_chroots.py b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_chroots.py
index 86bf5b0..1363fce 100644
--- a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_chroots.py
+++ b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_chroots.py
@@ -7,6 +7,7 @@ from coprs import db
from coprs import forms
from coprs.logic import coprs_logic
+from coprs.logic.coprs_logic import CoprChrootsLogic
from coprs.views.misc import login_required, page_not_found
from coprs.views.coprs_ns import coprs_ns
@@ -21,16 +22,17 @@ def chroot_edit(username, coprname, chrootname):
"Project with name {0} does not exist.".format(coprname))
try:
- chroot = coprs_logic.MockChrootsLogic.get_from_name(
- chrootname, active_only=True).first()
- except ValueError as e:
+ chroot = CoprChrootsLogic.get_by_name_safe(copr, chrootname)
+ except (ValueError, KeyError, RuntimeError) as e:
return page_not_found(str(e))
if not chroot:
return page_not_found(
"Chroot name {0} does not exist.".format(chrootname))
- form = forms.ChrootForm(buildroot_pkgs=copr.buildroot_pkgs(chroot))
+ # todo: get COPR_chroot, not mock chroot, WTF?!
+ # form = forms.ChrootForm(buildroot_pkgs=copr.buildroot_pkgs(chroot))
+ form = forms.ChrootForm(buildroot_pkgs=chroot.buildroot_pkgs)
# FIXME - test if chroot belongs to copr
if flask.g.user.can_build_in(copr):
return flask.render_template("coprs/detail/edit_chroot.html",
@@ -51,34 +53,32 @@ def chroot_update(username, coprname, chrootname):
return page_not_found(
"Projec with name {0} does not exist.".format(coprname))
- #import ipdb; ipdb.set_trace()
try:
- chroot = coprs_logic.MockChrootsLogic.get_from_name(
- chrootname, active_only=True).first()
+ chroot = CoprChrootsLogic.get_by_name_safe(copr, chrootname)
except ValueError as e:
return page_not_found(str(e))
if form.validate_on_submit() and flask.g.user.can_build_in(copr):
- # reading comps file if present
- if form.comps.has_file():
- #buffer = form.comps.data.stream.read()
-
- blob = compress(form.comps.data.stream.read())
-
- print("LEN: {}".format(len(blob)))
- else:
- blob = None
-
- coprs_logic.CoprChrootsLogic.update_buildroot_pkgs(
- copr, chroot, form.buildroot_pkgs.data, comps=blob)
-
- flask.flash(
- "Buildroot {0} for project {1} was updated".format(
- chrootname, coprname))
-
-
- db.session.commit()
-
+ if "submit" in flask.request.form:
+ action = flask.request.form["submit"]
+ if action == "update":
+ comps_name = comps_blob = None
+ if form.comps.has_file():
+ comps_blob = compress(form.comps.data.stream.read())
+ comps_name = form.comps.data.filename
+
+ coprs_logic.CoprChrootsLogic.update_buildroot_pkgs(
+ chroot, form.buildroot_pkgs.data,
+ comps=comps_blob, comps_name=comps_name)
+
+ elif action == "delete_comps":
+ CoprChrootsLogic.remove_comps(chroot)
+
+ flask.flash(
+ "Buildroot {0} for project {1} was updated".format(
+ chrootname, coprname))
+
+ db.session.commit()
return flask.redirect(flask.url_for("coprs_ns.copr_edit",
username=username,
coprname=copr.name))
8 years, 8 months
[copr] master: [backend] comps.xml support: send `update comps` action after chroot updates (dde7ab0)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit dde7ab043ba40d8957ee15e6da35956b7afbe29d
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Wed Aug 19 15:32:10 2015 +0200
[backend] comps.xml support: send `update comps` action after chroot updates
>---------------------------------------------------------------
frontend/coprs_frontend/coprs/helpers.py | 1 +
.../coprs_frontend/coprs/logic/actions_logic.py | 24 +++++++++++++++++++-
frontend/coprs_frontend/coprs/logic/coprs_logic.py | 4 ++-
.../coprs/views/api_ns/api_general.py | 2 +-
.../coprs/views/coprs_ns/coprs_chroots.py | 2 +-
5 files changed, 29 insertions(+), 4 deletions(-)
diff --git a/frontend/coprs_frontend/coprs/helpers.py b/frontend/coprs_frontend/coprs/helpers.py
index b9a4c8d..c5d614f 100644
--- a/frontend/coprs_frontend/coprs/helpers.py
+++ b/frontend/coprs_frontend/coprs/helpers.py
@@ -64,6 +64,7 @@ class ActionTypeEnum(object):
"rename": 1,
"legal-flag": 2,
"createrepo": 3,
+ "update_comps": 4,
}
diff --git a/frontend/coprs_frontend/coprs/logic/actions_logic.py b/frontend/coprs_frontend/coprs/logic/actions_logic.py
index 58f388c..8a5b92b 100644
--- a/frontend/coprs_frontend/coprs/logic/actions_logic.py
+++ b/frontend/coprs_frontend/coprs/logic/actions_logic.py
@@ -101,7 +101,7 @@ class ActionsLogic(object):
"projectname": build.copr.name,
"chroots": chroots_to_delete
}
- # import ipdb; ipdb.set_trace()
+
if build.is_older_results_naming_used:
if build.src_pkg_name is None or build.src_pkg_name == "":
return
@@ -119,3 +119,25 @@ class ActionsLogic(object):
created_on=int(time.time())
)
db.session.add(action)
+
+ @classmethod
+ def send_update_comps(cls, chroot):
+ """ Schedules update comps.xml action
+
+ :type copr_chroot: models.CoprChroot
+ """
+
+ data_dict = {
+ "username": chroot.copr.owner.name,
+ "projectname": chroot.copr.name,
+ "chroot": chroot.name,
+ "comps_present": chroot.comps_zlib is not None,
+ }
+
+ action = models.Action(
+ action_type=helpers.ActionTypeEnum("update_comps"),
+ object_type="copr_chroot",
+ data=json.dumps(data_dict),
+ created_on=int(time.time())
+ )
+ db.session.add(action)
diff --git a/frontend/coprs_frontend/coprs/logic/coprs_logic.py b/frontend/coprs_frontend/coprs/logic/coprs_logic.py
index b041315..9ea19dc 100644
--- a/frontend/coprs_frontend/coprs/logic/coprs_logic.py
+++ b/frontend/coprs_frontend/coprs/logic/coprs_logic.py
@@ -375,11 +375,12 @@ class CoprChrootsLogic(object):
models.CoprChroot(copr=copr, mock_chroot=mock_chroot))
@classmethod
- def update_buildroot_pkgs(cls, copr_chroot, buildroot_pkgs, comps=None, comps_name=None):
+ def update_chroot(cls, copr_chroot, buildroot_pkgs, comps=None, comps_name=None):
copr_chroot.buildroot_pkgs = buildroot_pkgs
if comps_name is not None:
copr_chroot.comps_zlib = comps
copr_chroot.comps_name = comps_name
+ ActionsLogic.send_update_comps(copr_chroot)
db.session.add(copr_chroot)
@classmethod
@@ -407,6 +408,7 @@ class CoprChrootsLogic(object):
def remove_comps(cls, copr_chroot):
copr_chroot.comps_name = None
copr_chroot.comps_zlib = None
+ ActionsLogic.send_update_comps(copr_chroot)
db.session.add(copr_chroot)
diff --git a/frontend/coprs_frontend/coprs/views/api_ns/api_general.py b/frontend/coprs_frontend/coprs/views/api_ns/api_general.py
index afee21e..2070ae2 100644
--- a/frontend/coprs_frontend/coprs/views/api_ns/api_general.py
+++ b/frontend/coprs_frontend/coprs/views/api_ns/api_general.py
@@ -552,7 +552,7 @@ def copr_modify_chroot(username, coprname, chrootname):
output = {'output': 'notok', 'error': 'Invalid request'}
httpcode = 500
else:
- coprs_logic.CoprChrootsLogic.update_buildroot_pkgs(chroot, form.buildroot_pkgs.data)
+ coprs_logic.CoprChrootsLogic.update_chroot(chroot, form.buildroot_pkgs.data)
db.session.commit()
output = {'output': 'ok', 'buildroot_pkgs': chroot.buildroot_pkgs}
diff --git a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_chroots.py b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_chroots.py
index ad74907..73d0dab 100644
--- a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_chroots.py
+++ b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_chroots.py
@@ -68,7 +68,7 @@ def chroot_update(username, coprname, chrootname):
comps_blob = compress(form.comps.data.stream.read())
comps_name = form.comps.data.filename
- coprs_logic.CoprChrootsLogic.update_buildroot_pkgs(
+ coprs_logic.CoprChrootsLogic.update_chroot(
chroot, form.buildroot_pkgs.data,
comps=comps_blob, comps_name=comps_name)
8 years, 8 months
[copr] master: [backend] comps.xml support: createrepo should use comps_xml when it's presented in the chroot dir (6b7a25a)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 6b7a25aa3b0ffdf26be31f81a875465a55c25de4
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Wed Aug 19 14:27:49 2015 +0200
[backend] comps.xml support: createrepo should use comps_xml when it's presented in the chroot dir
>---------------------------------------------------------------
backend/backend/createrepo.py | 4 ++++
backend/tests/test_createrepo.py | 26 ++++++++++++++++++++++++++
2 files changed, 30 insertions(+), 0 deletions(-)
diff --git a/backend/backend/createrepo.py b/backend/backend/createrepo.py
index 2f43c1b..8a36e57 100644
--- a/backend/backend/createrepo.py
+++ b/backend/backend/createrepo.py
@@ -67,6 +67,10 @@ def createrepo_unsafe(path, lock=None, dest_dir=None, base_url=None):
if base_url:
comm.extend(['--baseurl', base_url])
+ mb_comps_xml_path = os.path.join(path, "comps.xml")
+ if os.path.exists(mb_comps_xml_path):
+ comm.extend(['--groupfile', mb_comps_xml_path, '--keep-all-metadata'])
+
comm.append(path)
return run_cmd_unsafe(" ".join(map(str, comm)), lock)
diff --git a/backend/tests/test_createrepo.py b/backend/tests/test_createrepo.py
index 7484729..d982b8f 100644
--- a/backend/tests/test_createrepo.py
+++ b/backend/tests/test_createrepo.py
@@ -206,6 +206,32 @@ class TestCreaterepo(object):
createrepo_unsafe(path, None)
assert mc_run_cmd_unsafe.call_args[0][0] == expected
+ def test_createrepo_generated_commands_comps_xml(self, mc_run_cmd_unsafe):
+ path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5")
+ path_fedora = os.path.join(self.tmp_dir_name, "fedora-21")
+ for path in [path_epel_5, path_fedora]:
+ for add_comps in [True, False]:
+ os.makedirs(path)
+
+ comps_path = os.path.join(path, "comps.xml")
+ if add_comps:
+ with open(comps_path, "w") as handle:
+ handle.write("1")
+
+
+ repo_path = os.path.join(path, "repodata")
+ os.makedirs(repo_path)
+ with open(os.path.join(repo_path, "repomd.xml"), "w") as handle:
+ handle.write("1")
+
+ createrepo_unsafe(path, None)
+ if add_comps:
+ assert "--groupfile" in mc_run_cmd_unsafe.call_args[0][0]
+ else:
+ assert "--groupfile" not in mc_run_cmd_unsafe.call_args[0][0]
+ mc_run_cmd_unsafe.mock_reset()
+ shutil.rmtree(path, ignore_errors=True)
+
def test_createrepo_devel_generated_commands_existing_repodata(self, mc_run_cmd_unsafe):
path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5")
expected_epel_5 = ("/usr/bin/createrepo_c --database --ignore-lock "
8 years, 8 months