Repository :
http://git.fedorahosted.org/cgit/copr.git
On branch : master
---------------------------------------------------------------
commit 92f89e6b0610bf26675021d0f49c7412b1a95613
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Fri Apr 3 17:25:23 2015 +0200
[backend] creating documentation ...
---------------------------------------------------------------
backend/backend/vm_manage/manager.py | 52 +++++++++++++++++---
backend/docs/source/Autodoc.rst | 16 ++++--
backend/docs/source/Backend_design.rst | 26 ++++++++++
backend/docs/source/INSTALL.rst | 41 +++++++++++++++
backend/docs/source/README.rst | 2 +
backend/docs/source/Statistics.rst | 9 +++
backend/docs/source/VmManagement.rst | 3 +
backend/docs/source/conf.py | 4 +-
backend/docs/source/index.rst | 7 ++-
backend/docs/source/package/daemons/vm_master.rst | 6 ++
.../docs/source/package/mockremote/callback.rst | 6 --
backend/docs/source/package/vm_manage/manager.rst | 6 ++
12 files changed, 155 insertions(+), 23 deletions(-)
diff --git a/backend/backend/vm_manage/manager.py b/backend/backend/vm_manage/manager.py
index ddf8ca1..c058f2a 100644
--- a/backend/backend/vm_manage/manager.py
+++ b/backend/backend/vm_manage/manager.py
@@ -105,14 +105,17 @@ end
class VmManager(object):
"""
- VM manager, can be used in two modes:
+ VM manager, it is used for two purposes:
- Daemon which control VMs lifecycle, requires params `spawner,terminator`
- Client to acquire and release VM in builder process
:param opts: Global backend configuration
:type opts: Bunch
- :param callback: object with method `log(msg)`
+ :type logger: logging.Logger
+ :param logger: Logger instance to use inside of manager, if None manager would
create
+ new logger using helpers.get_redis_logger
+
:param checker: object with method `check_health(ip) -> None or raise exception`
:param spawner: object with method `spawn() -> IP or raise exception`
:param terminator: object with safe method `terminate(ip, vm_name)`
@@ -131,6 +134,10 @@ class VmManager(object):
self.log = logger or get_redis_logger(self.opts, "vmm.lib",
"vmm")
def post_init(self):
+ """
+ Self configuration. Should be called before usage of some methods.
+ """
+
self.rc = get_redis_connection(self.opts)
self.lua_scripts["set_checking_state"] =
self.rc.register_script(set_checking_state_lua)
self.lua_scripts["acquire_vm"] =
self.rc.register_script(acquire_vm_lua)
@@ -139,18 +146,26 @@ class VmManager(object):
self.lua_scripts["mark_vm_check_failed"] =
self.rc.register_script(mark_vm_check_failed_lua)
def set_logger(self, logger):
+ """
+ :param logger: Logger to be used by manager
+ :type logger: logging.Logger
+ """
self.log = logger
@property
def vm_groups(self):
+ """
+ :return: VM build groups
+ :rtype: list of int
+ """
return range(self.opts.build_groups_count)
def add_vm_to_pool(self, vm_ip, vm_name, group):
"""
Adds newly spawned VM into the pool of available builders
- :param vm_ip: IP
- :param vm_name: VM name
+ :param str vm_ip: IP
+ :param str vm_name: VM name
:param group: builder group
:type group: int
:rtype: VmDescriptor
@@ -169,6 +184,11 @@ class VmManager(object):
return vmd
def lookup_vms_by_ip(self, vm_ip):
+ """
+ :param vm_ip:
+ :return: List of found VMD with the give ip
+ :rtype: list of VmDescriptor
+ """
return [
vmd for vmd in self.get_all_vm()
if vmd.vm_ip == vm_ip
@@ -208,7 +228,7 @@ class VmManager(object):
:param group: builder group id, as defined in config
:type group: int
- :param username: build owner username, VMM prefer to reuse an existing VM which
was use by the same user
+ :param username: build owner username, VMM prefer to reuse an existing VM which
was used by the same user
:param pid: builder pid to release VM after build process unhandled death
:rtype: VmDescriptor
@@ -288,6 +308,8 @@ class VmManager(object):
def remove_vm_from_pool(self, vm_name):
"""
Backend forgets about VM after this method
+
+ :raises VmError: if VM has wrong state
"""
vmd = self.get_vm_by_name(vm_name)
if vmd.get_field(self.rc, "state") != VmStates.TERMINATING:
@@ -299,10 +321,16 @@ class VmManager(object):
self.log.info("removed vm `{}` from pool".format(vm_name))
def get_all_vm_in_group(self, group):
+ """
+ :rtype: list of VmDescriptor
+ """
vm_name_list = self.rc.smembers(KEY_VM_POOL.format(group=group))
return [VmDescriptor.load(self.rc, vm_name) for vm_name in vm_name_list]
def get_all_vm(self):
+ """
+ :rtype: list of VmDescriptor
+ """
vm_name_list = []
for group in self.vm_groups:
vm_name_list.extend(self.rc.smembers(KEY_VM_POOL.format(group=group)))
@@ -315,6 +343,15 @@ class VmManager(object):
return VmDescriptor.load(self.rc, vm_name)
def get_vm_by_group_and_state_list(self, group, state_list):
+ """
+ Select VM-s for the given group and allowed states
+
+ :param group: filter VM-s by the build group. If ``group is None`` select VM-s
from the all groups
+ :param state_list: VM state should be in the ``state_list``.
+
+ :return: Filtered VM-s
+ :rtype: list of VmDescriptor
+ """
states = set(state_list)
if group is None:
vmd_list = self.get_all_vm()
@@ -324,8 +361,9 @@ class VmManager(object):
def info(self):
"""
- Present information about all managed VMs in human readable form.
- :return:
+ Present information about all managed VMs in a human readable form.
+
+ :rtype: str
"""
dt_fields = {
"last_health_check",
diff --git a/backend/docs/source/Autodoc.rst b/backend/docs/source/Autodoc.rst
index 9baa116..339f215 100644
--- a/backend/docs/source/Autodoc.rst
+++ b/backend/docs/source/Autodoc.rst
@@ -21,10 +21,11 @@ backend.daemons.
Backend daemons, started by copr-be.py
.. toctree::
- package/daemons/backend.rst
- package/daemons/dispatcher.rst
- package/daemons/job_grab.rst
- package/daemons/log.rst
+ package/daemons/backend
+ package/daemons/dispatcher
+ package/daemons/job_grab
+ package/daemons/log
+ package/daemons/vm_master
backend.mockremote.
-------------------
@@ -33,6 +34,11 @@ Package dedicated to executing job builds on remote VMs.
.. toctree::
package/mockremote/__init__
package/mockremote/builder
- package/mockremote/callback
+backend.vm_manage.
+------------------
+Package for VM management only.
+
+.. toctree::
+ package/vm_manage/manager
diff --git a/backend/docs/source/Backend_design.rst
b/backend/docs/source/Backend_design.rst
new file mode 100644
index 0000000..a34575b
--- /dev/null
+++ b/backend/docs/source/Backend_design.rst
@@ -0,0 +1,26 @@
+Backend design
+==============
+
+Backend consists of the multiple process and a few utility scripts.
+
+The main process :py:class:`~backend.daemons.backend.CoprBackend` is started by
``/usr/bin/copr_be.py`` script.
+Default backend configuration is stored in ``/etc/copr/copr-be.conf``. Running **redis**
server are required.
+
+**CoprBackend** process starts the following components at the init:
+ - Centralised logging: :py:class:`~backend.daemons.log.RedisLogHandler` listens for
the redis pubsub for log events
+ - :py:class:`~backend.daemons.job_grab.CoprJobGrab` polling pending builds and
actions from the copr frontend.
+ Builds are routed to the appropriate task queue and action are executed by
**CoprJobGrab** itself.
+ - VM management is controlled by :py:class:`~backend.daemons.vm_master.VmMaster`.
+ See :ref:VmManagement: for details about Vm handling.
+
+After spawning aux processes **CoprBackend** dynamically spawns and terminates worker
processes :py:class:`~backend.daemons.dispatcher.Worker`.
+
+Communication with frontend
+---------------------------
+
+Action processing
+-----------------
+
+Build lifecycle
+---------------
+
diff --git a/backend/docs/source/INSTALL.rst b/backend/docs/source/INSTALL.rst
index 622363a..e7c1959 100644
--- a/backend/docs/source/INSTALL.rst
+++ b/backend/docs/source/INSTALL.rst
@@ -1,2 +1,43 @@
INSTALL
=======
+
+1. Obtain rpms:
+
+Get rpm either from upstream copr:
https://copr.fedoraproject.org/coprs/msuchy/copr/ ::
+
+ dnf copr enable msuchy/copr
+ dnf install -y copr-backend copr-selinux
+
+
+
+or checkout git repo and build latest release with tito::
+
+ git clone
https://git.fedorahosted.org/cgit/copr.git
+ yum install -y tito yum-utils
+ yum-builddep backend/copr-backend.spec
+ yum-builddep selinux/copr-selinux.spec
+ yum-builddep python/python-copr.spec
+ cd backend && tito build --rpm && cd ..
+ cd selinux && tito build --rpm && cd ..
+ cd python && tito build --rpm && cd ..
+ yum localinstall <path to the built rpms>
+::
+
+2. Prepare ansible playbooks to spawn and terminate VM builders using your VM provider.
+ You could find some inspiration in ``backend/conf/playbooks/``.
+
+3. Edit ``/etc/copr/copr-be.conf`` to reflect your setup. Look at example
``backend/conf/copr-be.conf.example``.
+
+4. [Optional] Mount /var/lib/copr/results to the dedicated storage. This location is used
to store and serve build results.
+
+5. Enable and start backend processes ::
+
+ systemctl enable copr-backend redis lighttpd
+ systemctl start copr-backend redis lighttpd
+
+
+Side notes
+----------
+It maybe usefull to look how copr-backend is deployed at fedora@infrastructure using
ansible:
+http://infrastructure.fedoraproject.org/infra/ansible/playbooks/groups/copr-backend.yml
+
diff --git a/backend/docs/source/README.rst b/backend/docs/source/README.rst
index 07cff11..a566a44 100644
--- a/backend/docs/source/README.rst
+++ b/backend/docs/source/README.rst
@@ -5,3 +5,5 @@ COPR is lightweight build system. It allows you to create new project in
WebUI,
and submit new builds and COPR will create yum repository from latest builds.
This package contains backend. Copr-backend is responsible for build process, package
signing, manages project repositories.
+
+Currently copr-backend is intended to use at Fedora 20+.
diff --git a/backend/docs/source/Statistics.rst b/backend/docs/source/Statistics.rst
new file mode 100644
index 0000000..795cffc
--- /dev/null
+++ b/backend/docs/source/Statistics.rst
@@ -0,0 +1,9 @@
+Statistics
+==========
+
+Logstash [1] is used to parse web server logs, filter interesting events (i.e. rpm file
downloads) and send json
+to frontend. Logstash config doesn't have a nice option to define variables, so
config should be copied manually after
+the installation and frontend host should be fixed there.
+
+--
+1. -
http://logstash.net/
diff --git a/backend/docs/source/VmManagement.rst b/backend/docs/source/VmManagement.rst
new file mode 100644
index 0000000..5f9582b
--- /dev/null
+++ b/backend/docs/source/VmManagement.rst
@@ -0,0 +1,3 @@
+Vm Management
+=============
+
diff --git a/backend/docs/source/conf.py b/backend/docs/source/conf.py
index 0d00883..aa9e7f6 100644
--- a/backend/docs/source/conf.py
+++ b/backend/docs/source/conf.py
@@ -50,9 +50,9 @@ copyright = u'2014, Copr dev team'
# built documents.
#
# The short X.Y version.
-version = '0.48'
+version = '0.61'
# The full version, including alpha/beta/rc tags.
-release = '0.48'
+release = version
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
diff --git a/backend/docs/source/index.rst b/backend/docs/source/index.rst
index 58df8e7..09a98f8 100644
--- a/backend/docs/source/index.rst
+++ b/backend/docs/source/index.rst
@@ -6,14 +6,15 @@
Welcome to copr-backend's documentation!
========================================
-Contents:
-
.. toctree::
- :maxdepth: 2
+ :maxdepth: 1
README
INSTALL
+ Backend_design
+ VmManagement
Configuration
+ Statistics
Autodoc
run_scripts
diff --git a/backend/docs/source/package/daemons/vm_master.rst
b/backend/docs/source/package/daemons/vm_master.rst
new file mode 100644
index 0000000..1f19e79
--- /dev/null
+++ b/backend/docs/source/package/daemons/vm_master.rst
@@ -0,0 +1,6 @@
+backend.daemons.vm_master
+=========================
+
+.. automodule:: backend.daemons.vm_master
+ :members:
+ :undoc-members:
diff --git a/backend/docs/source/package/mockremote/callback.rst
b/backend/docs/source/package/mockremote/callback.rst
deleted file mode 100644
index 33f271f..0000000
--- a/backend/docs/source/package/mockremote/callback.rst
+++ /dev/null
@@ -1,6 +0,0 @@
-backend.mockremote.callback
-===========================
-
-.. automodule:: backend.mockremote.callback
- :members:
- :undoc-members:
diff --git a/backend/docs/source/package/vm_manage/manager.rst
b/backend/docs/source/package/vm_manage/manager.rst
new file mode 100644
index 0000000..1ca28a0
--- /dev/null
+++ b/backend/docs/source/package/vm_manage/manager.rst
@@ -0,0 +1,6 @@
+backend.vm_manage.manager
+=========================
+
+.. automodule:: backend.vm_manage.manager
+ :members:
+ :undoc-members: