[copr] master: [frontend][api] Bug 1194592 - User is able to submit directory (9e88498)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 9e884983cae3f126b8c5be0da2314143ea5571c8
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Thu Mar 19 21:39:29 2015 +0100
[frontend][api] Bug 1194592 - User is able to submit directory
>---------------------------------------------------------------
frontend/coprs_frontend/coprs/forms.py | 12 ++++++++++--
.../coprs/views/api_ns/api_general.py | 2 +-
.../coprs/views/coprs_ns/coprs_builds.py | 15 ++++-----------
3 files changed, 15 insertions(+), 14 deletions(-)
diff --git a/frontend/coprs_frontend/coprs/forms.py b/frontend/coprs_frontend/coprs/forms.py
index 76a8c21..b91600b 100644
--- a/frontend/coprs_frontend/coprs/forms.py
+++ b/frontend/coprs_frontend/coprs/forms.py
@@ -97,6 +97,13 @@ class StringListFilter(object):
return regex.sub(lambda x: '\n', result)
+class SrpmUrlListFilter(object):
+
+ def __call__(self, value):
+ return "\n".join(pkg for pkg in value.replace("\n", " ").split(" ")
+ if pkg.endswith(".src.rpm"))
+
+
class ValueToPermissionNumberFilter(object):
def __call__(self, value):
@@ -217,9 +224,10 @@ class BuildFormFactory(object):
pkgs = wtforms.TextAreaField(
"Pkgs",
validators=[
- wtforms.validators.Required(),
+ wtforms.validators.DataRequired(),
UrlListValidator()],
- filters=[StringListFilter()])
+ filters=[StringListFilter(),
+ SrpmUrlListFilter()])
memory_reqs = wtforms.IntegerField(
"Memory requirements",
diff --git a/frontend/coprs_frontend/coprs/views/api_ns/api_general.py b/frontend/coprs_frontend/coprs/views/api_ns/api_general.py
index b8697b4..ed0c219 100644
--- a/frontend/coprs_frontend/coprs/views/api_ns/api_general.py
+++ b/frontend/coprs_frontend/coprs/views/api_ns/api_general.py
@@ -273,7 +273,7 @@ def copr_new_build(username, coprname):
elif form.validate_on_submit() and flask.g.user.can_build_in(copr):
# we're checking authorization above for now
# and also creating separate build for each package
- pkgs = form.pkgs.data.replace('\n', ' ').split(" ")
+ pkgs = form.pkgs.data.split("\n")
ids = []
chroots = []
for chroot in copr.active_chroots:
diff --git a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_builds.py b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_builds.py
index fe076f4..0cd4d56 100644
--- a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_builds.py
+++ b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_builds.py
@@ -96,16 +96,7 @@ def copr_new_build(username, coprname):
form = forms.BuildFormFactory.create_form_cls(copr.active_chroots)()
if form.validate_on_submit():
- pkgs = form.pkgs.data.replace("\n", " ").split(" ")
-
- # validate (and skip bad) urls
- bad_urls = []
- for pkg in pkgs:
- if not re.match("^.*\.src\.rpm$", pkg):
- bad_urls.append(pkg)
- flask.flash("Bad url: {0} (skipped)".format(pkg))
- for bad_url in bad_urls:
- pkgs.remove(bad_url)
+ pkgs = form.pkgs.data.split("\n")
if not pkgs:
flask.flash("No builds submitted")
@@ -134,7 +125,9 @@ def copr_new_build(username, coprname):
flask.flash(str(e))
db.session.rollback()
else:
- flask.flash("Build was added")
+ for pkg in pkgs:
+ flask.flash("New build was created: {}".format(pkg))
+
db.session.commit()
return flask.redirect(flask.url_for("coprs_ns.copr_builds",
9 years, 1 month
[copr] master: [frontend] [rhbz:#1188784] RFE: Include a "last build" item on the overview page (ca6d5a0)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit ca6d5a078045894544be53ce62b42b8f655e343f
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Thu Mar 19 16:35:21 2015 +0100
[frontend] [rhbz:#1188784] RFE: Include a "last build" item on the overview page
>---------------------------------------------------------------
frontend/coprs_frontend/coprs/filters.py | 15 +++++--
frontend/coprs_frontend/coprs/static/copr.css | 13 ++++++
.../templates/coprs/detail/_build_states.html | 40 +++++++-------------
.../templates/coprs/detail/_builds_table.html | 2 +-
.../coprs/templates/coprs/detail/overview.html | 20 ++++++++++
.../coprs/views/coprs_ns/coprs_general.py | 17 +++++---
6 files changed, 70 insertions(+), 37 deletions(-)
diff --git a/frontend/coprs_frontend/coprs/filters.py b/frontend/coprs_frontend/coprs/filters.py
index 35ca893..8952776 100644
--- a/frontend/coprs_frontend/coprs/filters.py
+++ b/frontend/coprs_frontend/coprs/filters.py
@@ -126,11 +126,18 @@ def parse_package_name(pkg):
@app.template_filter("build_state_description")
def build_state_decoration(state):
- description = ""
- if state == "skipped":
- description = "This package has already been built previously"
- return description
+ description_map = {
+ "failed": "Build failed. See logs for more details.",
+ "succeeded": "Successfully built.",
+ "canceled": "The build has been cancelled manually.",
+ "running": "Build in progress.",
+ "pending": "Your build is waiting for a builder.",
+ "skipped": "This package has already been built previously.",
+ "starting": "Trying to acquire and configure builder for task."
+ }
+
+ return description_map.get(state, "")
@app.template_filter("fix_url_https_backend")
diff --git a/frontend/coprs_frontend/coprs/static/copr.css b/frontend/coprs_frontend/coprs/static/copr.css
index 71fbdf5..7c7fb75 100644
--- a/frontend/coprs_frontend/coprs/static/copr.css
+++ b/frontend/coprs_frontend/coprs/static/copr.css
@@ -562,3 +562,16 @@ textarea {
-webkit-border-radius: 4px;
}
+
+.latest_build {
+ font-size: 80%;
+ background-color: #ededed;
+ padding-top: 5px;
+ padding-left: 5px;
+ padding-bottom: 5px;
+ margin-bottom: 10px;
+ border-radius: 4px;
+ -moz-border-radius: 4px;
+ -webkit-border-radius: 4px;
+ text-align: right;
+}
diff --git a/frontend/coprs_frontend/coprs/templates/coprs/detail/_build_states.html b/frontend/coprs_frontend/coprs/templates/coprs/detail/_build_states.html
index adbc743..a3214c2 100644
--- a/frontend/coprs_frontend/coprs/templates/coprs/detail/_build_states.html
+++ b/frontend/coprs_frontend/coprs/templates/coprs/detail/_build_states.html
@@ -2,34 +2,22 @@
<dl class="aligned">
<div>
<dt> Possible build states: </dt>
+ {% for state in [
+ "pending",
+ "starting",
+ "running",
+ "succeeded",
+ "canceled",
+ "skipped",
+ "failed",
+ "canceled",
+ ]%}
<dd>
- <span class="build-pending"> Pending </span>
- - Your build is waiting for a builder.
- </dd>
- <dd>
- <span class="build-starting"> Starting </span>
- - A builder is being spawned.
- </dd>
- <dd>
- <span class="build-running"> Running </span>
- - Build in progress.
- </dd>
- <dd>
- <span class="build-succeeded"> Succeeded </span>
- - Succesfully built.
- </dd>
- <dd>
- <span class="build-skipped"> Skipped </span>
- - Package has been skipped because it has been successfully built before.
- </dd>
- <dd>
- <span class="build-failed"> Failed </span>
- - Build failed. The reason could be timeout or the package itself. See logs for more details.
- </dd>
- <dd>
- <span class="build-cancelled"> Cancelled </span>
- - The build has been cancelled manually.
+ <span class="build-{{state}}"> {{state|capitalize}} </span>
+ - {{state|build_state_description}}
</dd>
+
+ {% endfor %}
</div>
</dl>
{% endmacro %}
diff --git a/frontend/coprs_frontend/coprs/templates/coprs/detail/_builds_table.html b/frontend/coprs_frontend/coprs/templates/coprs/detail/_builds_table.html
index 65a2485..d656d06 100644
--- a/frontend/coprs_frontend/coprs/templates/coprs/detail/_builds_table.html
+++ b/frontend/coprs_frontend/coprs/templates/coprs/detail/_builds_table.html
@@ -30,7 +30,7 @@
{% endif %}
</td>
<td> {{ build.started_on|time_ago(build.ended_on) }} </td>
- <td class="build-{{build.state}}" alt="{{build.state|build_state_description}}">{{ build.state }}</td>
+ <td class="build-{{build.state}}" abbr="{{build.state|build_state_description}}">{{ build.state }}</td>
</tr>
{% endfor %}
</table>
diff --git a/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html b/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html
index 010e900..e108870 100644
--- a/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html
+++ b/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html
@@ -5,6 +5,26 @@
{% block overview_selected %}selected{% endblock %}
{% block detail_body %}
+ <br />
+ <div class="latest_build">
+ {% if latest_build %}
+
+ <a href="{{url_for("coprs_ns.copr_build",
+ username = copr.owner.name,
+ coprname = copr.name,
+ build_id = latest_build.id)}}">
+ Latest build
+ #{{ latest_build.id }}:
+
+ </a>
+ <abbr class="build-{{latest_build.state}}" title="{{latest_build.state|build_state_description}}">{{ latest_build.state }}</abbr>,
+ submitted {{ latest_build.submitted_on|time_ago }} ago
+ {% else %}
+ No build submitted yet
+ {% endif %}
+
+ </div>
+
<h2>Description</h2>
<div class="shift-right">{{ copr.description|markdown|default('Description not filled in by author. Very likely personal repository for testing purpose, which you should not use.', true) }}</div>
<h2>Installation Instructions</h2>
diff --git a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
index 2eb7312..0101f20 100644
--- a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
+++ b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
@@ -218,12 +218,17 @@ def copr_detail(username, coprname):
repos_info_list = sorted(repos_info.values(), key=lambda rec: rec["name_release"])
- return flask.render_template("coprs/detail/overview.html",
- copr=copr,
- form=form,
- repo_dl_stat=repo_dl_stat,
- repos_info_list=repos_info_list,
- )
+ builds = builds_logic.BuildsLogic.get_multiple(
+ flask.g.user, copr=copr).limit(1).all()
+
+ return flask.render_template(
+ "coprs/detail/overview.html",
+ copr=copr,
+ form=form,
+ repo_dl_stat=repo_dl_stat,
+ repos_info_list=repos_info_list,
+ latest_build=builds[0] if len(builds) == 1 else None,
+ )
@coprs_ns.route("/<username>/<coprname>/permissions/")
9 years, 1 month
[copr] master: [copr] don't allow acquire VMs that was last checked before server restart. (f5faeb1)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit f5faeb173bf2de29b9167b10e255ff61f45b3b71
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Wed Mar 18 17:40:27 2015 +0100
[copr] don't allow acquire VMs that was last checked before server restart.
>---------------------------------------------------------------
backend/backend/daemons/vm_master.py | 2 +-
backend/backend/vm_manage/__init__.py | 11 +++--
backend/backend/vm_manage/manager.py | 31 +++++++++----
backend/tests/deamons/test_vm_master.py | 3 +
backend/tests/vm_manager/test_manager.py | 73 +++++++++++++++++++++++++++---
5 files changed, 99 insertions(+), 21 deletions(-)
diff --git a/backend/backend/daemons/vm_master.py b/backend/backend/daemons/vm_master.py
index 3ea33f5..0e0029c 100644
--- a/backend/backend/daemons/vm_master.py
+++ b/backend/backend/daemons/vm_master.py
@@ -149,7 +149,7 @@ class VmMaster(Process):
raise RuntimeError("provide Spawner and Terminator to run VmManager daemon")
setproctitle("VM master")
-
+ self.vmm.mark_server_start()
self.kill_received = False
self.event_handler = EventHandler(self.vmm)
diff --git a/backend/backend/vm_manage/__init__.py b/backend/backend/vm_manage/__init__.py
index 8ea5338..556f8d1 100644
--- a/backend/backend/vm_manage/__init__.py
+++ b/backend/backend/vm_manage/__init__.py
@@ -25,17 +25,18 @@ PUBSUB_INTERRUPT_BUILDER = "copr:backend:interrupt_build:pubsub::{}"
PUBSUB_VM_TERMINATION = "copr:backend:vm_termination:pubsub::{vm_name}"
# message should contain string "terminating"
-KEY_VM_GROUPS = "copr:backend:vm_groups:set::"
-# set of available groups,
-# TODO: remove it, use opts.build_groups_count
-
KEY_VM_POOL = "copr:backend:vm_pool:set::{group}"
# set of vm_names of vm available for `group`
KEY_VM_POOL_INFO = "copr:backend:vm_pool_info:hset::{group}"
-# hashset with additional information for `group`, used fields:
+# hset with additional information for `group`, used fields:
# - "last_vm_spawn_start": latest time when VM spawn was initiated for this `group`
+KEY_SERVER_INFO = "copr:backend:server_info:hset::"
+# common shared info about server, not stritly related to VMM, maybe move it to helpers later
+# used fields:
+# "server_start_timestamp" -> unixtime string
+
KEY_VM_INSTANCE = "copr:backend:vm_instance:hset::{vm_name}"
# hset to store VmDescriptor
diff --git a/backend/backend/vm_manage/manager.py b/backend/backend/vm_manage/manager.py
index b4f059f..3f2f9bd 100644
--- a/backend/backend/vm_manage/manager.py
+++ b/backend/backend/vm_manage/manager.py
@@ -14,7 +14,7 @@ from backend.exceptions import VmError, NoVmAvailable
from backend.helpers import get_redis_connection
from .models import VmDescriptor
-from . import VmStates, KEY_VM_INSTANCE, KEY_VM_POOL, EventTopics, PUBSUB_MB
+from . import VmStates, KEY_VM_INSTANCE, KEY_VM_POOL, EventTopics, PUBSUB_MB, KEY_SERVER_INFO
# KEYS[1]: VMD key
# ARGV[1] current timestamp for `last_health_check`
@@ -43,10 +43,16 @@ local old_state = redis.call("HGET", KEYS[1], "state")
if old_state ~= "ready" then
return nil
else
- redis.call("HMSET", KEYS[1], "state", "in_use", "bound_to_user", ARGV[1],
- "used_by_pid", ARGV[2], "in_use_since", ARGV[3],
- "task_id", ARGV[4], "build_id", ARGV[5], "chroot", ARGV[6])
- return "OK"
+ local last_health_check = tonumber(redis.call("HGET", KEYS[1], "last_health_check"))
+ local server_restart_time = tonumber(redis.call("HGET", KEYS[2], "server_start_timestamp"))
+ if last_health_check and server_restart_time and last_health_check > server_restart_time then
+ redis.call("HMSET", KEYS[1], "state", "in_use", "bound_to_user", ARGV[1],
+ "used_by_pid", ARGV[2], "in_use_since", ARGV[3],
+ "task_id", ARGV[4], "build_id", ARGV[5], "chroot", ARGV[6])
+ return "OK"
+ else
+ return nil
+ end
end
"""
@@ -185,19 +191,25 @@ class VmManager(object):
vm_key = KEY_VM_INSTANCE.format(vm_name=vm_name)
self.lua_scripts["mark_vm_check_failed"](keys=[vm_key])
+ def mark_server_start(self):
+ self.rc.hset(KEY_SERVER_INFO, "server_start_timestamp", time.time())
+
def acquire_vm(self, group, username, pid, task_id=None, build_id=None, chroot=None):
"""
Try to acquire VM from pool
+
:param group: builder group id, as defined in config
:type group: int
:param username: build owner username, VMM prefer to reuse an existing VM which was use by the same user
:param pid: builder pid to release VM after build process unhandled death
+
+ :rtype: VmDescriptor
+ :raises: NoVmAvailable when manager couldn't find suitable VM for the given group and user
"""
- # TODO: reject request if user acquired #machines > threshold_vm_per_user
vmd_list = self.get_all_vm_in_group(group)
vm_count_used_by_user = len([
vmd for vmd in vmd_list if
- vmd.bound_to_user==username and vmd.state==VmStates.IN_USE
+ vmd.bound_to_user == username and vmd.state == VmStates.IN_USE
])
if vm_count_used_by_user >= self.opts.build_groups[group]["max_vm_per_user"]:
raise NoVmAvailable("No VM are available, user `{}` already acquired #{} VMs"
@@ -213,8 +225,9 @@ class VmManager(object):
# TODO: record last_copr_backend_startup_time at startup
for vmd in all_vms:
vm_key = KEY_VM_INSTANCE.format(vm_name=vmd.vm_name)
- if self.lua_scripts["acquire_vm"](keys=[vm_key], args=[username, pid, time.time(),
- task_id, build_id, chroot]) == "OK":
+ if self.lua_scripts["acquire_vm"](keys=[vm_key, KEY_SERVER_INFO],
+ args=[username, pid, time.time(),
+ task_id, build_id, chroot]) == "OK":
return vmd
else:
raise NoVmAvailable("No VM are available, please wait in queue. Group: {}".format(group))
diff --git a/backend/tests/deamons/test_vm_master.py b/backend/tests/deamons/test_vm_master.py
index 191ba1d..0a8d8e1 100644
--- a/backend/tests/deamons/test_vm_master.py
+++ b/backend/tests/deamons/test_vm_master.py
@@ -355,6 +355,7 @@ class TestVmMaster(object):
None
]
self.vm_master.do_cycle = types.MethodType(mc_do_cycle, self.vm_master)
+ self.vmm.mark_server_start = MagicMock()
self.stage = 0
def on_sleep(*args, **kwargs):
@@ -371,6 +372,8 @@ class TestVmMaster(object):
assert mc_event_handler.called
assert mc_event_handler.return_value.start.called
+ assert self.vmm.mark_server_start.called
+
err_log = self.queue.get(timeout=1)
assert err_log is not None
assert "Unhandled error:" in err_log["what"]
diff --git a/backend/tests/vm_manager/test_manager.py b/backend/tests/vm_manager/test_manager.py
index 0faff78..1769b21 100644
--- a/backend/tests/vm_manager/test_manager.py
+++ b/backend/tests/vm_manager/test_manager.py
@@ -1,5 +1,6 @@
# coding: utf-8
import json
+import random
import types
import time
@@ -10,7 +11,7 @@ import six
from backend import exceptions
from backend.exceptions import VmError, NoVmAvailable
-from backend.vm_manage import VmStates, KEY_VM_POOL, PUBSUB_VM_TERMINATION, PUBSUB_MB, EventTopics
+from backend.vm_manage import VmStates, KEY_VM_POOL, PUBSUB_VM_TERMINATION, PUBSUB_MB, EventTopics, KEY_SERVER_INFO
from backend.vm_manage.manager import VmManager
from backend.daemons.vm_master import VmMaster
from backend.helpers import get_redis_connection
@@ -191,10 +192,43 @@ class TestManager(object):
self.checker.run_check_health.reset_mock()
assert vmd.get_field(self.rc, "state") == state
- def test_acquire_vm_extra_kwargs(self):
+ def test_acquire_vm_no_vm_after_server_restart(self, mc_time):
vmd = self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, self.group)
vmd.store_field(self.rc, "state", VmStates.READY)
+ # undefined both last_health_check and server_start_timestamp
+ with pytest.raises(NoVmAvailable):
+ self.vmm.acquire_vm(0, self.username, 42)
+
+ # only server start timestamp is defined
+ mc_time.time.return_value = 1
+ self.vmm.mark_server_start()
+ with pytest.raises(NoVmAvailable):
+ self.vmm.acquire_vm(0, self.username, 42)
+
+ # only last_health_check defined
+ self.rc.delete(KEY_SERVER_INFO)
+ vmd.store_field(self.rc, "last_health_check", 0)
+ with pytest.raises(NoVmAvailable):
+ self.vmm.acquire_vm(0, self.username, 42)
+
+ # both defined but last_health_check < server_start_time
+ self.vmm.mark_server_start()
+ with pytest.raises(NoVmAvailable):
+ self.vmm.acquire_vm(0, self.username, 42)
+
+ # and finally last_health_check > server_start_time
+ vmd.store_field(self.rc, "last_health_check", 2)
+ vmd_res = self.vmm.acquire_vm(0, self.username, 42)
+ assert vmd.vm_name == vmd_res.vm_name
+
+ def test_acquire_vm_extra_kwargs(self, mc_time):
+ mc_time.time.return_value = 0
+ self.vmm.mark_server_start()
+ vmd = self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, self.group)
+ vmd.store_field(self.rc, "state", VmStates.READY)
+ vmd.store_field(self.rc, "last_health_check", 2)
+
kwargs = {
"task_id": "20-fedora-20-x86_64",
"build_id": "20",
@@ -204,13 +238,18 @@ class TestManager(object):
for k, v in kwargs.items():
assert vmd_got.get_field(self.rc, k) == v
- def test_acquire_vm(self):
+ def test_acquire_vm(self, mc_time):
+ mc_time.time.return_value = 0
+ self.vmm.mark_server_start()
+
vmd_main = self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, self.group)
vmd_alt = self.vmm.add_vm_to_pool(self.vm_ip, "alternative", self.group)
vmd_main.store_field(self.rc, "state", VmStates.READY)
vmd_alt.store_field(self.rc, "state", VmStates.READY)
vmd_alt.store_field(self.vmm.rc, "bound_to_user", self.username)
+ vmd_main.store_field(self.rc, "last_health_check", 2)
+ vmd_alt.store_field(self.rc, "last_health_check", 2)
vmd_got_first = self.vmm.acquire_vm(group=self.group, username=self.username, pid=self.pid)
assert vmd_got_first.vm_name == "alternative"
@@ -220,12 +259,16 @@ class TestManager(object):
with pytest.raises(NoVmAvailable):
self.vmm.acquire_vm(group=self.group, username=self.username, pid=self.pid)
- def test_acquire_vm_per_user_limit(self):
+ def test_acquire_vm_per_user_limit(self, mc_time):
+ mc_time.time.return_value = 0
+ self.vmm.mark_server_start()
+
max_vm_per_user = self.opts.build_groups[0]["max_vm_per_user"]
acquired_vmd_list = []
for idx in range(max_vm_per_user + 1):
vmd = self.vmm.add_vm_to_pool("127.0.{}.1".format(idx), "vm_{}".format(idx), self.group)
vmd.store_field(self.rc, "state", VmStates.READY)
+ vmd.store_field(self.rc, "last_health_check", 2)
acquired_vmd_list.append(vmd)
for idx in range(max_vm_per_user):
@@ -237,8 +280,12 @@ class TestManager(object):
acquired_vmd_list[-1].store_field(self.rc, "state", VmStates.READY)
self.vmm.acquire_vm(0, self.username, 42)
- def test_acquire_only_ready_state(self):
+ def test_acquire_only_ready_state(self, mc_time):
+ mc_time.time.return_value = 0
+ self.vmm.mark_server_start()
+
vmd_main = self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, self.group)
+ vmd_main.store_field(self.rc, "last_health_check", 2)
for state in [VmStates.IN_USE, VmStates.GOT_IP, VmStates.CHECK_HEALTH,
VmStates.TERMINATING, VmStates.CHECK_HEALTH_FAILED]:
@@ -246,13 +293,19 @@ class TestManager(object):
with pytest.raises(NoVmAvailable):
self.vmm.acquire_vm(group=self.group, username=self.username, pid=self.pid)
- def test_acquire_and_release_vm(self):
+ def test_acquire_and_release_vm(self, mc_time):
+ mc_time.time.return_value = 0
+ self.vmm.mark_server_start()
+
+
vmd_main = self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, self.group)
vmd_alt = self.vmm.add_vm_to_pool(self.vm_ip, "alternative", self.group)
vmd_main.store_field(self.rc, "state", VmStates.READY)
vmd_alt.store_field(self.rc, "state", VmStates.READY)
vmd_alt.store_field(self.vmm.rc, "bound_to_user", self.username)
+ vmd_main.store_field(self.rc, "last_health_check", 2)
+ vmd_alt.store_field(self.rc, "last_health_check", 2)
vmd_got_first = self.vmm.acquire_vm(group=self.group, username=self.username, pid=self.pid)
assert vmd_got_first.vm_name == "alternative"
@@ -394,3 +447,11 @@ class TestManager(object):
r3 = sorted(r3, key=lambda vmd: vmd.vm_name)
assert r3[0].vm_name == "a1"
assert r3[1].vm_name == "a2"
+
+ def test_mark_server_start(self, mc_time):
+ assert self.rc.hget(KEY_SERVER_INFO, "server_start_timestamp") is None
+ for i in range(100):
+ val = 100 * i + 0.12345
+ mc_time.time.return_value = val
+ self.vmm.mark_server_start()
+ assert self.rc.hget(KEY_SERVER_INFO, "server_start_timestamp") == "{}".format(val)
9 years, 1 month
[copr] master: [backend] Added limit to acquire_vm based on VMs used by the same username at the current moment. (13e52b6)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 13e52b6ca285c1dbf67f38276be0f3b5d0acfa82
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Wed Mar 18 16:52:24 2015 +0100
[backend] Added limit to acquire_vm based on VMs used by the same username at the current moment.
>---------------------------------------------------------------
backend/backend/helpers.py | 4 ++--
backend/backend/vm_manage/manager.py | 17 +++++++++++++++++
backend/tests/vm_manager/test_manager.py | 20 +++++++++++++++++++-
3 files changed, 38 insertions(+), 3 deletions(-)
diff --git a/backend/backend/helpers.py b/backend/backend/helpers.py
index 0604992..b12f786 100644
--- a/backend/backend/helpers.py
+++ b/backend/backend/helpers.py
@@ -141,10 +141,10 @@ class BackendConfigReader(object):
"max_vm_total": _get_conf(
cp, "backend", "group{}_max_vm_total".format(group_id),
# default=16, mode="int"),
- default=2, mode="int"),
+ default=8, mode="int"),
"max_vm_per_user": _get_conf(
cp, "backend", "group{}_max_vm_total".format(group_id),
- default=2, mode="int"),
+ default=4, mode="int"),
"max_spawn_processes": _get_conf(
cp, "backend", "group{}_max_spawn_processes".format(group_id),
default=2, mode="int"),
diff --git a/backend/backend/vm_manage/manager.py b/backend/backend/vm_manage/manager.py
index f1466b9..b4f059f 100644
--- a/backend/backend/vm_manage/manager.py
+++ b/backend/backend/vm_manage/manager.py
@@ -132,6 +132,15 @@ class VmManager(object):
return range(self.opts.build_groups_count)
def add_vm_to_pool(self, vm_ip, vm_name, group):
+ """
+ Adds newly spawned VM into the pool of available builders
+
+ :param vm_ip: IP
+ :param vm_name: VM name
+ :param group: builder group
+ :type group: int
+ :rtype: VmDescriptor
+ """
# print("\n ADD VM TO POOL")
if self.rc.sismember(KEY_VM_POOL.format(group=group), vm_name):
raise VmError("Can't add VM `{}` to the pool, such name already used".format(vm_name))
@@ -186,6 +195,14 @@ class VmManager(object):
"""
# TODO: reject request if user acquired #machines > threshold_vm_per_user
vmd_list = self.get_all_vm_in_group(group)
+ vm_count_used_by_user = len([
+ vmd for vmd in vmd_list if
+ vmd.bound_to_user==username and vmd.state==VmStates.IN_USE
+ ])
+ if vm_count_used_by_user >= self.opts.build_groups[group]["max_vm_per_user"]:
+ raise NoVmAvailable("No VM are available, user `{}` already acquired #{} VMs"
+ .format(username, vm_count_used_by_user))
+
ready_vmd_list = [vmd for vmd in vmd_list if vmd.state == VmStates.READY]
# trying to find VM used by this user
dirtied_by_user = [vmd for vmd in ready_vmd_list if vmd.bound_to_user == username]
diff --git a/backend/tests/vm_manager/test_manager.py b/backend/tests/vm_manager/test_manager.py
index 4383c2e..0faff78 100644
--- a/backend/tests/vm_manager/test_manager.py
+++ b/backend/tests/vm_manager/test_manager.py
@@ -56,7 +56,8 @@ class TestManager(object):
build_groups={
0: {
"name": "base",
- "archs": ["i386", "x86_64"]
+ "archs": ["i386", "x86_64"],
+ "max_vm_per_user": 3,
}
},
@@ -219,6 +220,23 @@ class TestManager(object):
with pytest.raises(NoVmAvailable):
self.vmm.acquire_vm(group=self.group, username=self.username, pid=self.pid)
+ def test_acquire_vm_per_user_limit(self):
+ max_vm_per_user = self.opts.build_groups[0]["max_vm_per_user"]
+ acquired_vmd_list = []
+ for idx in range(max_vm_per_user + 1):
+ vmd = self.vmm.add_vm_to_pool("127.0.{}.1".format(idx), "vm_{}".format(idx), self.group)
+ vmd.store_field(self.rc, "state", VmStates.READY)
+ acquired_vmd_list.append(vmd)
+
+ for idx in range(max_vm_per_user):
+ vmd = self.vmm.acquire_vm(0, self.username, idx)
+
+ with pytest.raises(NoVmAvailable):
+ self.vmm.acquire_vm(0, self.username, 42)
+
+ acquired_vmd_list[-1].store_field(self.rc, "state", VmStates.READY)
+ self.vmm.acquire_vm(0, self.username, 42)
+
def test_acquire_only_ready_state(self):
vmd_main = self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, self.group)
9 years, 1 month
[copr] master: New python dependencies (40a2230)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 40a2230fa7013cb6c9dab1689395a080f69fc9dd
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Wed Mar 18 15:35:05 2015 +0100
New python dependencies
>---------------------------------------------------------------
backend/requirements.txt | 1 +
frontend/requirements.txt | 1 +
2 files changed, 2 insertions(+), 0 deletions(-)
diff --git a/backend/requirements.txt b/backend/requirements.txt
index 2b6438f..386fbc3 100644
--- a/backend/requirements.txt
+++ b/backend/requirements.txt
@@ -10,3 +10,4 @@ IPy
sphinx
sphinx-argparse
netaddr
+psutil
diff --git a/frontend/requirements.txt b/frontend/requirements.txt
index 966de85..7952e74 100644
--- a/frontend/requirements.txt
+++ b/frontend/requirements.txt
@@ -14,3 +14,4 @@ pyLibravatar
pydns
flexmock
decorator
+python-dateutil
9 years, 1 month
[copr] master: [unittest] run/shutdown redis in test_suite.sh (a328abc)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit a328abcb4e6cb262bb64522348cac5e9ceb09eb7
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Wed Mar 18 15:30:22 2015 +0100
[unittest] run/shutdown redis in test_suite.sh
>---------------------------------------------------------------
test_suite.sh | 8 ++++++++
1 files changed, 8 insertions(+), 0 deletions(-)
diff --git a/test_suite.sh b/test_suite.sh
index 61112e9..440fa90 100644
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -2,6 +2,10 @@
find . -path '*/__pycache__' | xargs rm -rfv
#pip install --user virtualenv
+REDIS_PORT=7777
+
+redis-server --port $REDIS_PORT &> _redis.log &
+
virtualenv _venv
source _venv/bin/activate
@@ -23,6 +27,8 @@ pip install -r keygen/requirements.txt
mkdir -p _report
+
+
python -m pytest python/copr/test --junitxml=_report/python-copr.junit.xml --cov-report xml --cov python/copr/client $@
mv {,_report/python-copr.}coverage.xml
@@ -48,3 +54,5 @@ pep8 --max-line-length=120 keygen/src keygen/tests > _report/keygen_pep8.txt ||
deactivate
rm -rf _tmp/*
+
+kill %1
9 years, 1 month
[copr] master: [backend] run tmp redis-server for tests (9d5f246)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 9d5f246718f4b5af4319ddc52495e10aca6f6702
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Wed Mar 18 15:26:22 2015 +0100
[backend] run tmp redis-server for tests
>---------------------------------------------------------------
.gitignore | 2 ++
backend/backend/helpers.py | 5 +++++
backend/backend/mockremote/builder.py | 1 +
backend/backend/vm_manage/check.py | 2 +-
backend/backend/vm_manage/manager.py | 2 ++
backend/run_test.sh | 9 ++++++++-
backend/tests/deamons/test_vm_master.py | 1 +
backend/tests/mockremote/test_builder.py | 5 ++++-
backend/tests/vm_manager/test_check.py | 2 ++
backend/tests/vm_manager/test_event_handle.py | 1 +
backend/tests/vm_manager/test_executor.py | 1 +
backend/tests/vm_manager/test_manager.py | 1 +
backend/tests/vm_manager/test_spawn.py | 1 +
backend/tests/vm_manager/test_terminate.py | 2 +-
frontend/run_tests.sh | 6 ++++++
15 files changed, 37 insertions(+), 4 deletions(-)
diff --git a/.gitignore b/.gitignore
index 44baad8..5b2fc8d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,3 +17,5 @@ _tmp/
backend/docs/build
frontend/coprs_frontend/copr_frontend.log
+
+_redis.log
diff --git a/backend/backend/helpers.py b/backend/backend/helpers.py
index 2a155ce..0604992 100644
--- a/backend/backend/helpers.py
+++ b/backend/backend/helpers.py
@@ -246,6 +246,11 @@ def get_redis_connection(opts):
kwargs = {}
if hasattr(opts, "redis_db"):
kwargs["db"] = opts.redis_db
+ if hasattr(opts, "redis_host"):
+ kwargs["host"] = opts.redis_host
+ if hasattr(opts, "redis_port"):
+ kwargs["port"] = opts.redis_port
+
return StrictRedis(**kwargs)
diff --git a/backend/backend/mockremote/builder.py b/backend/backend/mockremote/builder.py
index 4d8a5fc..064c7fb 100644
--- a/backend/backend/mockremote/builder.py
+++ b/backend/backend/mockremote/builder.py
@@ -269,6 +269,7 @@ class Builder(object):
return results
def setup_pubsub_handler(self):
+
self.rc = get_redis_connection(self.opts)
self.ps = self.rc.pubsub(ignore_subscribe_messages=True)
channel_name = PUBSUB_INTERRUPT_BUILDER.format(self.hostname)
diff --git a/backend/backend/vm_manage/check.py b/backend/backend/vm_manage/check.py
index a31dfa3..b383b48 100644
--- a/backend/backend/vm_manage/check.py
+++ b/backend/backend/vm_manage/check.py
@@ -25,7 +25,7 @@ def check_health(opts, events, vm_name, vm_ip):
events.put({"when": time.time(), "who": "checker", "what": msg})
runner_options = dict(
- remote_user="root",
+ remote_user=opts.build_user or "root",
host_list="{},".format(vm_ip),
pattern=vm_ip,
forks=1,
diff --git a/backend/backend/vm_manage/manager.py b/backend/backend/vm_manage/manager.py
index 4359de2..f1466b9 100644
--- a/backend/backend/vm_manage/manager.py
+++ b/backend/backend/vm_manage/manager.py
@@ -192,6 +192,8 @@ class VmManager(object):
clean_list = [vmd for vmd in ready_vmd_list if vmd.bound_to_user is None]
all_vms = list(chain(dirtied_by_user, clean_list))
+ # TODO: reject if last_health_check < last_copr_backend_startup_time
+ # TODO: record last_copr_backend_startup_time at startup
for vmd in all_vms:
vm_key = KEY_VM_INSTANCE.format(vm_name=vmd.vm_name)
if self.lua_scripts["acquire_vm"](keys=[vm_key], args=[username, pid, time.time(),
diff --git a/backend/run_test.sh b/backend/run_test.sh
index 651bc86..a5941b9 100755
--- a/backend/run_test.sh
+++ b/backend/run_test.sh
@@ -1,5 +1,12 @@
-#!/bin/sh
+#!/usr/bin/bash
+
+REDIS_PORT=7777
+
+redis-server --port $REDIS_PORT &> _redis.log &
+
# PYTHONPATH=backend:run:$PYTHONPATH python -B -m pytest -s ./tests/ $@
PYTHONPATH=backend:run:$PYTHONPATH python -B -m pytest -s --cov-report term-missing --cov ./backend --cov ./run ./tests/ $@
# PYTHONPATH=../python/:./copr_cli:$PYTHONPATH python3 -B -m pytest --cov-report term-missing --cov ./copr_cli/ $@
+
+kill %1
diff --git a/backend/tests/deamons/test_vm_master.py b/backend/tests/deamons/test_vm_master.py
index cae1c59..191ba1d 100644
--- a/backend/tests/deamons/test_vm_master.py
+++ b/backend/tests/deamons/test_vm_master.py
@@ -83,6 +83,7 @@ class TestVmMaster(object):
def setup_method(self, method):
self.opts = Bunch(
redis_db=9,
+ redis_port=7777,
ssh=Bunch(
transport="ssh"
),
diff --git a/backend/tests/mockremote/test_builder.py b/backend/tests/mockremote/test_builder.py
index a2c7f89..8a414f9 100644
--- a/backend/tests/mockremote/test_builder.py
+++ b/backend/tests/mockremote/test_builder.py
@@ -77,7 +77,10 @@ class TestBuilder(object):
timeout=BUILDER_TIMEOUT,
remote_basedir=BUILDER_REMOTE_BASEDIR,
remote_tempdir=BUILDER_REMOTE_TMPDIR,
- results_baseurl="http://example.com"
+ results_baseurl="http://example.com",
+
+ redis_db=9,
+ redis_port=7777,
)
def get_test_builder(self):
diff --git a/backend/tests/vm_manager/test_check.py b/backend/tests/vm_manager/test_check.py
index b01bb5c..3ab3b08 100644
--- a/backend/tests/vm_manager/test_check.py
+++ b/backend/tests/vm_manager/test_check.py
@@ -75,6 +75,7 @@ class TestChecker(object):
self.terminate_pb_path = "{}/terminate.yml".format(self.test_root_path)
self.opts = Bunch(
redis_db=9,
+ redis_port=7777,
ssh=Bunch(
transport="ssh"
),
@@ -87,6 +88,7 @@ class TestChecker(object):
}
},
+ build_user="mockbuilder",
fedmsg_enabled=False,
sleeptime=0.1,
do_sign=True,
diff --git a/backend/tests/vm_manager/test_event_handle.py b/backend/tests/vm_manager/test_event_handle.py
index 3c079fd..e78b2b7 100644
--- a/backend/tests/vm_manager/test_event_handle.py
+++ b/backend/tests/vm_manager/test_event_handle.py
@@ -74,6 +74,7 @@ class TestEventHandle(object):
self.terminate_pb_path = "{}/terminate.yml".format(self.test_root_path)
self.opts = Bunch(
redis_db=9,
+ redis_port=7777,
ssh=Bunch(
transport="ssh"
),
diff --git a/backend/tests/vm_manager/test_executor.py b/backend/tests/vm_manager/test_executor.py
index f0b39dc..d61cd50 100644
--- a/backend/tests/vm_manager/test_executor.py
+++ b/backend/tests/vm_manager/test_executor.py
@@ -37,6 +37,7 @@ class TestExecutor(object):
def setup_method(self, method):
self.opts = Bunch(
redis_db=9,
+ redis_port=7777,
ssh=Bunch(
transport="ssh"
),
diff --git a/backend/tests/vm_manager/test_manager.py b/backend/tests/vm_manager/test_manager.py
index 27f4013..4383c2e 100644
--- a/backend/tests/vm_manager/test_manager.py
+++ b/backend/tests/vm_manager/test_manager.py
@@ -48,6 +48,7 @@ class TestManager(object):
def setup_method(self, method):
self.opts = Bunch(
redis_db=9,
+ redis_port=7777,
ssh=Bunch(
transport="ssh"
),
diff --git a/backend/tests/vm_manager/test_spawn.py b/backend/tests/vm_manager/test_spawn.py
index 69d3deb..a0e637a 100644
--- a/backend/tests/vm_manager/test_spawn.py
+++ b/backend/tests/vm_manager/test_spawn.py
@@ -72,6 +72,7 @@ class TestSpawner(object):
self.spawn_pb_path = "{}/spawn.yml".format(self.test_root_path)
self.opts = Bunch(
redis_db=9,
+ redis_port=7777,
ssh=Bunch(
transport="ssh"
),
diff --git a/backend/tests/vm_manager/test_terminate.py b/backend/tests/vm_manager/test_terminate.py
index 6ab9feb..f00d8ac 100644
--- a/backend/tests/vm_manager/test_terminate.py
+++ b/backend/tests/vm_manager/test_terminate.py
@@ -73,7 +73,7 @@ class TestTerminate(object):
self.test_root_path = tempfile.mkdtemp()
self.terminate_pb_path = "{}/terminate.yml".format(self.test_root_path)
self.opts = Bunch(
- redis_db=9,
+ redis_port=7777,
ssh=Bunch(
transport="ssh"
),
diff --git a/frontend/run_tests.sh b/frontend/run_tests.sh
index 22c7ad7..f6e06f0 100755
--- a/frontend/run_tests.sh
+++ b/frontend/run_tests.sh
@@ -1,6 +1,12 @@
#!/bin/sh
+REDIS_PORT = 7777
+
+redis-server --port $REDIS_PORT &> _redis.log &
+
+
cd coprs_frontend
COPR_CONFIG="$(pwd)/config/copr_unit_test.conf" python -m pytest tests -s $@ # \
#--cov-report term-missing --cov coprs $@
+kill %1
9 years, 1 month
[copr] master: Automatic commit of package [copr-frontend] release [1.56-1]. (ba82d93)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit ba82d934a82878464ecdeafe1ae98f68d1599d1a
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Fri Mar 6 13:45:08 2015 +0100
Automatic commit of package [copr-frontend] release [1.56-1].
>---------------------------------------------------------------
frontend/copr-frontend.spec | 5 ++++-
rel-eng/packages/copr-frontend | 2 +-
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/frontend/copr-frontend.spec b/frontend/copr-frontend.spec
index de79bb2..2969dce 100644
--- a/frontend/copr-frontend.spec
+++ b/frontend/copr-frontend.spec
@@ -5,7 +5,7 @@
%endif
Name: copr-frontend
-Version: 1.55
+Version: 1.56
Release: 1%{?dist}
Summary: Frontend for Copr
@@ -192,6 +192,9 @@ service logstash condrestart
%doc documentation/python-doc
%changelog
+* Fri Mar 06 2015 Valentin Gologuzov <vgologuz(a)redhat.com> 1.56-1
+- hotfix:#1199258] Link to Source RPM on build detail page points to a wrong URL
+
* Mon Mar 02 2015 Valentin Gologuzov <vgologuz(a)redhat.com> 1.55-1
- [frontend] fix tests to be runnable without redis-server.
diff --git a/rel-eng/packages/copr-frontend b/rel-eng/packages/copr-frontend
index 48209f6..1a8fb68 100644
--- a/rel-eng/packages/copr-frontend
+++ b/rel-eng/packages/copr-frontend
@@ -1 +1 @@
-1.55-1 frontend/
+1.56-1 frontend/
9 years, 2 months