[copr] master: [backend] repaired unittests (f099328)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit f099328be88f67a06342df5ea089c3fa0ffb11b6
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Thu Apr 9 19:37:33 2015 +0200
[backend] repaired unittests
>---------------------------------------------------------------
Diff suppressed because of size. To see it, use:
git diff --patch-with-stat --no-color --find-copies-harder --ignore-space-at-eol ^f099328be88f67a06342df5ea089c3fa0ffb11b6~1 f099328be88f67a06342df5ea089c3fa0ffb11b6
9 years
[copr] master: [backend] updated builder playbooks (3090376)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 3090376c0d41f91ba4ecc8e4aad5eb92d46408f7
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Wed Apr 8 14:12:21 2015 +0200
[backend] updated builder playbooks
>---------------------------------------------------------------
Diff suppressed because of size. To see it, use:
git diff --patch-with-stat --no-color --find-copies-harder --ignore-space-at-eol ^3090376c0d41f91ba4ecc8e4aad5eb92d46408f7~1 3090376c0d41f91ba4ecc8e4aad5eb92d46408f7
9 years
[copr] master: [backend] creating documentation ... (92f89e6)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 92f89e6b0610bf26675021d0f49c7412b1a95613
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Fri Apr 3 17:25:23 2015 +0200
[backend] creating documentation ...
>---------------------------------------------------------------
backend/backend/vm_manage/manager.py | 52 +++++++++++++++++---
backend/docs/source/Autodoc.rst | 16 ++++--
backend/docs/source/Backend_design.rst | 26 ++++++++++
backend/docs/source/INSTALL.rst | 41 +++++++++++++++
backend/docs/source/README.rst | 2 +
backend/docs/source/Statistics.rst | 9 +++
backend/docs/source/VmManagement.rst | 3 +
backend/docs/source/conf.py | 4 +-
backend/docs/source/index.rst | 7 ++-
backend/docs/source/package/daemons/vm_master.rst | 6 ++
.../docs/source/package/mockremote/callback.rst | 6 --
backend/docs/source/package/vm_manage/manager.rst | 6 ++
12 files changed, 155 insertions(+), 23 deletions(-)
diff --git a/backend/backend/vm_manage/manager.py b/backend/backend/vm_manage/manager.py
index ddf8ca1..c058f2a 100644
--- a/backend/backend/vm_manage/manager.py
+++ b/backend/backend/vm_manage/manager.py
@@ -105,14 +105,17 @@ end
class VmManager(object):
"""
- VM manager, can be used in two modes:
+ VM manager, it is used for two purposes:
- Daemon which control VMs lifecycle, requires params `spawner,terminator`
- Client to acquire and release VM in builder process
:param opts: Global backend configuration
:type opts: Bunch
- :param callback: object with method `log(msg)`
+ :type logger: logging.Logger
+ :param logger: Logger instance to use inside of manager, if None manager would create
+ new logger using helpers.get_redis_logger
+
:param checker: object with method `check_health(ip) -> None or raise exception`
:param spawner: object with method `spawn() -> IP or raise exception`
:param terminator: object with safe method `terminate(ip, vm_name)`
@@ -131,6 +134,10 @@ class VmManager(object):
self.log = logger or get_redis_logger(self.opts, "vmm.lib", "vmm")
def post_init(self):
+ """
+ Self configuration. Should be called before usage of some methods.
+ """
+
self.rc = get_redis_connection(self.opts)
self.lua_scripts["set_checking_state"] = self.rc.register_script(set_checking_state_lua)
self.lua_scripts["acquire_vm"] = self.rc.register_script(acquire_vm_lua)
@@ -139,18 +146,26 @@ class VmManager(object):
self.lua_scripts["mark_vm_check_failed"] = self.rc.register_script(mark_vm_check_failed_lua)
def set_logger(self, logger):
+ """
+ :param logger: Logger to be used by manager
+ :type logger: logging.Logger
+ """
self.log = logger
@property
def vm_groups(self):
+ """
+ :return: VM build groups
+ :rtype: list of int
+ """
return range(self.opts.build_groups_count)
def add_vm_to_pool(self, vm_ip, vm_name, group):
"""
Adds newly spawned VM into the pool of available builders
- :param vm_ip: IP
- :param vm_name: VM name
+ :param str vm_ip: IP
+ :param str vm_name: VM name
:param group: builder group
:type group: int
:rtype: VmDescriptor
@@ -169,6 +184,11 @@ class VmManager(object):
return vmd
def lookup_vms_by_ip(self, vm_ip):
+ """
+ :param vm_ip:
+ :return: List of found VMD with the give ip
+ :rtype: list of VmDescriptor
+ """
return [
vmd for vmd in self.get_all_vm()
if vmd.vm_ip == vm_ip
@@ -208,7 +228,7 @@ class VmManager(object):
:param group: builder group id, as defined in config
:type group: int
- :param username: build owner username, VMM prefer to reuse an existing VM which was use by the same user
+ :param username: build owner username, VMM prefer to reuse an existing VM which was used by the same user
:param pid: builder pid to release VM after build process unhandled death
:rtype: VmDescriptor
@@ -288,6 +308,8 @@ class VmManager(object):
def remove_vm_from_pool(self, vm_name):
"""
Backend forgets about VM after this method
+
+ :raises VmError: if VM has wrong state
"""
vmd = self.get_vm_by_name(vm_name)
if vmd.get_field(self.rc, "state") != VmStates.TERMINATING:
@@ -299,10 +321,16 @@ class VmManager(object):
self.log.info("removed vm `{}` from pool".format(vm_name))
def get_all_vm_in_group(self, group):
+ """
+ :rtype: list of VmDescriptor
+ """
vm_name_list = self.rc.smembers(KEY_VM_POOL.format(group=group))
return [VmDescriptor.load(self.rc, vm_name) for vm_name in vm_name_list]
def get_all_vm(self):
+ """
+ :rtype: list of VmDescriptor
+ """
vm_name_list = []
for group in self.vm_groups:
vm_name_list.extend(self.rc.smembers(KEY_VM_POOL.format(group=group)))
@@ -315,6 +343,15 @@ class VmManager(object):
return VmDescriptor.load(self.rc, vm_name)
def get_vm_by_group_and_state_list(self, group, state_list):
+ """
+ Select VM-s for the given group and allowed states
+
+ :param group: filter VM-s by the build group. If ``group is None`` select VM-s from the all groups
+ :param state_list: VM state should be in the ``state_list``.
+
+ :return: Filtered VM-s
+ :rtype: list of VmDescriptor
+ """
states = set(state_list)
if group is None:
vmd_list = self.get_all_vm()
@@ -324,8 +361,9 @@ class VmManager(object):
def info(self):
"""
- Present information about all managed VMs in human readable form.
- :return:
+ Present information about all managed VMs in a human readable form.
+
+ :rtype: str
"""
dt_fields = {
"last_health_check",
diff --git a/backend/docs/source/Autodoc.rst b/backend/docs/source/Autodoc.rst
index 9baa116..339f215 100644
--- a/backend/docs/source/Autodoc.rst
+++ b/backend/docs/source/Autodoc.rst
@@ -21,10 +21,11 @@ backend.daemons.
Backend daemons, started by copr-be.py
.. toctree::
- package/daemons/backend.rst
- package/daemons/dispatcher.rst
- package/daemons/job_grab.rst
- package/daemons/log.rst
+ package/daemons/backend
+ package/daemons/dispatcher
+ package/daemons/job_grab
+ package/daemons/log
+ package/daemons/vm_master
backend.mockremote.
-------------------
@@ -33,6 +34,11 @@ Package dedicated to executing job builds on remote VMs.
.. toctree::
package/mockremote/__init__
package/mockremote/builder
- package/mockremote/callback
+backend.vm_manage.
+------------------
+Package for VM management only.
+
+.. toctree::
+ package/vm_manage/manager
diff --git a/backend/docs/source/Backend_design.rst b/backend/docs/source/Backend_design.rst
new file mode 100644
index 0000000..a34575b
--- /dev/null
+++ b/backend/docs/source/Backend_design.rst
@@ -0,0 +1,26 @@
+Backend design
+==============
+
+Backend consists of the multiple process and a few utility scripts.
+
+The main process :py:class:`~backend.daemons.backend.CoprBackend` is started by ``/usr/bin/copr_be.py`` script.
+Default backend configuration is stored in ``/etc/copr/copr-be.conf``. Running **redis** server are required.
+
+**CoprBackend** process starts the following components at the init:
+ - Centralised logging: :py:class:`~backend.daemons.log.RedisLogHandler` listens for the redis pubsub for log events
+ - :py:class:`~backend.daemons.job_grab.CoprJobGrab` polling pending builds and actions from the copr frontend.
+ Builds are routed to the appropriate task queue and action are executed by **CoprJobGrab** itself.
+ - VM management is controlled by :py:class:`~backend.daemons.vm_master.VmMaster`.
+ See :ref:VmManagement: for details about Vm handling.
+
+After spawning aux processes **CoprBackend** dynamically spawns and terminates worker processes :py:class:`~backend.daemons.dispatcher.Worker`.
+
+Communication with frontend
+---------------------------
+
+Action processing
+-----------------
+
+Build lifecycle
+---------------
+
diff --git a/backend/docs/source/INSTALL.rst b/backend/docs/source/INSTALL.rst
index 622363a..e7c1959 100644
--- a/backend/docs/source/INSTALL.rst
+++ b/backend/docs/source/INSTALL.rst
@@ -1,2 +1,43 @@
INSTALL
=======
+
+1. Obtain rpms:
+
+Get rpm either from upstream copr: https://copr.fedoraproject.org/coprs/msuchy/copr/ ::
+
+ dnf copr enable msuchy/copr
+ dnf install -y copr-backend copr-selinux
+
+
+
+or checkout git repo and build latest release with tito::
+
+ git clone https://git.fedorahosted.org/cgit/copr.git
+ yum install -y tito yum-utils
+ yum-builddep backend/copr-backend.spec
+ yum-builddep selinux/copr-selinux.spec
+ yum-builddep python/python-copr.spec
+ cd backend && tito build --rpm && cd ..
+ cd selinux && tito build --rpm && cd ..
+ cd python && tito build --rpm && cd ..
+ yum localinstall <path to the built rpms>
+::
+
+2. Prepare ansible playbooks to spawn and terminate VM builders using your VM provider.
+ You could find some inspiration in ``backend/conf/playbooks/``.
+
+3. Edit ``/etc/copr/copr-be.conf`` to reflect your setup. Look at example ``backend/conf/copr-be.conf.example``.
+
+4. [Optional] Mount /var/lib/copr/results to the dedicated storage. This location is used to store and serve build results.
+
+5. Enable and start backend processes ::
+
+ systemctl enable copr-backend redis lighttpd
+ systemctl start copr-backend redis lighttpd
+
+
+Side notes
+----------
+It maybe usefull to look how copr-backend is deployed at fedora@infrastructure using ansible:
+http://infrastructure.fedoraproject.org/infra/ansible/playbooks/groups/copr-backend.yml
+
diff --git a/backend/docs/source/README.rst b/backend/docs/source/README.rst
index 07cff11..a566a44 100644
--- a/backend/docs/source/README.rst
+++ b/backend/docs/source/README.rst
@@ -5,3 +5,5 @@ COPR is lightweight build system. It allows you to create new project in WebUI,
and submit new builds and COPR will create yum repository from latest builds.
This package contains backend. Copr-backend is responsible for build process, package signing, manages project repositories.
+
+Currently copr-backend is intended to use at Fedora 20+.
diff --git a/backend/docs/source/Statistics.rst b/backend/docs/source/Statistics.rst
new file mode 100644
index 0000000..795cffc
--- /dev/null
+++ b/backend/docs/source/Statistics.rst
@@ -0,0 +1,9 @@
+Statistics
+==========
+
+Logstash [1] is used to parse web server logs, filter interesting events (i.e. rpm file downloads) and send json
+to frontend. Logstash config doesn't have a nice option to define variables, so config should be copied manually after
+the installation and frontend host should be fixed there.
+
+--
+1. - http://logstash.net/
diff --git a/backend/docs/source/VmManagement.rst b/backend/docs/source/VmManagement.rst
new file mode 100644
index 0000000..5f9582b
--- /dev/null
+++ b/backend/docs/source/VmManagement.rst
@@ -0,0 +1,3 @@
+Vm Management
+=============
+
diff --git a/backend/docs/source/conf.py b/backend/docs/source/conf.py
index 0d00883..aa9e7f6 100644
--- a/backend/docs/source/conf.py
+++ b/backend/docs/source/conf.py
@@ -50,9 +50,9 @@ copyright = u'2014, Copr dev team'
# built documents.
#
# The short X.Y version.
-version = '0.48'
+version = '0.61'
# The full version, including alpha/beta/rc tags.
-release = '0.48'
+release = version
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
diff --git a/backend/docs/source/index.rst b/backend/docs/source/index.rst
index 58df8e7..09a98f8 100644
--- a/backend/docs/source/index.rst
+++ b/backend/docs/source/index.rst
@@ -6,14 +6,15 @@
Welcome to copr-backend's documentation!
========================================
-Contents:
-
.. toctree::
- :maxdepth: 2
+ :maxdepth: 1
README
INSTALL
+ Backend_design
+ VmManagement
Configuration
+ Statistics
Autodoc
run_scripts
diff --git a/backend/docs/source/package/daemons/vm_master.rst b/backend/docs/source/package/daemons/vm_master.rst
new file mode 100644
index 0000000..1f19e79
--- /dev/null
+++ b/backend/docs/source/package/daemons/vm_master.rst
@@ -0,0 +1,6 @@
+backend.daemons.vm_master
+=========================
+
+.. automodule:: backend.daemons.vm_master
+ :members:
+ :undoc-members:
diff --git a/backend/docs/source/package/mockremote/callback.rst b/backend/docs/source/package/mockremote/callback.rst
deleted file mode 100644
index 33f271f..0000000
--- a/backend/docs/source/package/mockremote/callback.rst
+++ /dev/null
@@ -1,6 +0,0 @@
-backend.mockremote.callback
-===========================
-
-.. automodule:: backend.mockremote.callback
- :members:
- :undoc-members:
diff --git a/backend/docs/source/package/vm_manage/manager.rst b/backend/docs/source/package/vm_manage/manager.rst
new file mode 100644
index 0000000..1ca28a0
--- /dev/null
+++ b/backend/docs/source/package/vm_manage/manager.rst
@@ -0,0 +1,6 @@
+backend.vm_manage.manager
+=========================
+
+.. automodule:: backend.vm_manage.manager
+ :members:
+ :undoc-members:
9 years
[copr] master: [backend] updated example spawn playbook (681e8c9)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 681e8c9fe1282912e65691871b03e7a720d94038
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Fri Apr 3 12:47:57 2015 +0200
[backend] updated example spawn playbook
>---------------------------------------------------------------
.../playbooks/files/{ => mock}/site-defaults.cfg | 0
backend/conf/playbooks/provision_builder_tasks.yml | 45 ++++++++++++--------
2 files changed, 27 insertions(+), 18 deletions(-)
diff --git a/backend/conf/playbooks/files/site-defaults.cfg b/backend/conf/playbooks/files/mock/site-defaults.cfg
similarity index 100%
rename from backend/conf/playbooks/files/site-defaults.cfg
rename to backend/conf/playbooks/files/mock/site-defaults.cfg
diff --git a/backend/conf/playbooks/provision_builder_tasks.yml b/backend/conf/playbooks/provision_builder_tasks.yml
index 7183692..8d67114 100644
--- a/backend/conf/playbooks/provision_builder_tasks.yml
+++ b/backend/conf/playbooks/provision_builder_tasks.yml
@@ -14,6 +14,7 @@
- libsemanage-python
- name: make sure newest rpm
+ # todo: replace with dnf after ansible 1.9 is available
yum: name={{ item }} state=latest
with_items:
- rpm
@@ -21,27 +22,35 @@
- ca-certificates
- name: put updated mock configs into /etc/mock
- copy: src=files/{{ item }} dest=/etc/mock
+ copy: src=files/mock/{{ item }} dest=/etc/mock
with_items:
- site-defaults.cfg
-- name: prepare caches
- async: 3600
- command: mock -r {{ item }} --init
- with_items:
- - epel-5-i386
- - epel-5-x86_64
- - epel-6-i386
- - epel-6-x86_64
- - epel-7-x86_64
- - fedora-20-i386
- - fedora-20-x86_64
- - fedora-21-i386
- - fedora-21-x86_64
- - fedora-22-i386
- - fedora-22-x86_64
- - fedora-rawhide-i386
- - fedora-rawhide-x86_64
+# ansible doesn't support simultaneously usage of async and with_* options
+# it's not even planned for implementation, see https://github.com/ansible/ansible/issues/5841
+#- name: prepare caches
+# when: prepare_base_image is defined
+# async: 3600
+# command: mock -r {{ item }} --init
+# with_items:
+# - epel-5-i386
+# - epel-5-x86_64
+# - epel-6-i386
+# - epel-6-x86_64
+# - epel-7-x86_64
+# - fedora-20-i386
+# - fedora-20-x86_64
+# - fedora-21-i386
+# - fedora-21-x86_64
+# - fedora-22-i386
+# - fedora-22-x86_64
+# - fedora-rawhide-i386
+# - fedora-rawhide-x86_64
+
+- name: prepare cache
+ when: prepare_base_image is defined
+ async: 7200
+ shell: "for i in epel-5-i386 epel-5-x86_64 epel-6-i386 epel-6-x86_64 epel-7-x86_64 fedora-20-i386 fedora-20-x86_64 fedora-21-i386 fedora-21-x86_64 fedora-22-i386 fedora-22-x86_64 fedora-rawhide-i386 fedora-rawhide-x86_64; do mock --init -r $i; done"
- name: mockbuilder user
user: name=mockbuilder groups=mock
9 years
[copr] master: [backend] remove unused (c5ea2d4)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit c5ea2d4fa5c1ff08ec9aa16ad3b1c0efb518f95c
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Fri Apr 3 17:25:03 2015 +0200
[backend] remove unused
>---------------------------------------------------------------
backend/backend/daemons/dispatcher.py | 29 -----------------------------
1 files changed, 0 insertions(+), 29 deletions(-)
diff --git a/backend/backend/daemons/dispatcher.py b/backend/backend/daemons/dispatcher.py
index c2e3c8b..5e63e5a 100644
--- a/backend/backend/daemons/dispatcher.py
+++ b/backend/backend/daemons/dispatcher.py
@@ -27,34 +27,6 @@ except ImportError:
fedmsg = None
-class WorkerCallback(object):
- """
- Callback class for worker. Now used only for message logging
-
- :param logfile: path to the log file
- """
-
- def __init__(self, logfile=None):
- self.logfile = logfile
-
- def log(self, msg):
- """
- Safely writes msg to the logfile
-
- :param str msg: message to be logged
- """
- if self.logfile:
- now = time.strftime("%F %T")
- try:
- with open(self.logfile, 'a') as lf:
- fcntl.flock(lf, fcntl.LOCK_EX)
- lf.write(str(now) + ': ' + msg + '\n')
- fcntl.flock(lf, fcntl.LOCK_UN)
- except (IOError, OSError) as e:
- sys.stderr.write("Could not write to logfile {0} - {1}\n"
- .format(self.logfile, str(e)))
-
-
class Worker(multiprocessing.Process):
"""
Worker process dispatches building tasks. Backend spin-up multiple workers, each
@@ -65,7 +37,6 @@ class Worker(multiprocessing.Process):
:param Bunch opts: backend config
:param int worker_num: worker number
:param int group_id: group_id from the set of groups defined in config
- :param callback: callback object to handle internal workers events. Should implement method ``log(msg)``.
:param lock: (:py:class:`multiprocessing.Lock`) global backend lock
"""
9 years
[copr] master: finish leftovers from commit f6d60d0a (7e774b9)
by Miroslav Suchý
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 7e774b9dca31b5bf9c57b7be982eb81136e3a82a
Author: Miroslav Suchý <msuchy(a)redhat.com>
Date: Thu Apr 2 16:18:46 2015 +0200
finish leftovers from commit f6d60d0a
>---------------------------------------------------------------
frontend/coprs_frontend/manage.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/frontend/coprs_frontend/manage.py b/frontend/coprs_frontend/manage.py
index 489c7f0..0dbd3ff 100755
--- a/frontend/coprs_frontend/manage.py
+++ b/frontend/coprs_frontend/manage.py
@@ -237,7 +237,7 @@ class AlterUserCommand(Command):
def run(self, name, **kwargs):
user = models.User.query.filter(
- models.User.openid_name == models.User.openidize_name(name)).first()
+ models.User.username == name).first()
if not user:
print("No user named {0}.".format(name))
return
9 years
[copr] master: be precise on how to populate data (96c1cb6)
by Miroslav Suchý
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 96c1cb6d65a301c050b1a365c1fbb3da8133fb9c
Author: Miroslav Suchý <msuchy(a)redhat.com>
Date: Thu Apr 2 15:52:19 2015 +0200
be precise on how to populate data
>---------------------------------------------------------------
copr-setup.txt | 5 ++++-
1 files changed, 4 insertions(+), 1 deletions(-)
diff --git a/copr-setup.txt b/copr-setup.txt
index fca0cc9..e5b1bfa 100644
--- a/copr-setup.txt
+++ b/copr-setup.txt
@@ -97,7 +97,7 @@ SQLite
This will use sqlite db file in /var/lib/copr/data/copr.db (manage.py will create that file)
- Note that you have to either create db as copr-fe user or you have to run:
chown copr-fe:copr-fe /var/lib/copr/data/copr.db
- after you run "./manage.py create_db"
+ after you run "./manage.py create_db --alembic alembic.ini"
PostgreSQL
~~~~~~~~~~
@@ -127,6 +127,9 @@ and test the connection:
PGPASSWORD=coprpass psql -a -U copr-fe coprdb
PGPASSWORD=coprpass psql -a -h localhost -U copr-fe coprdb
+Then run:
+ "./manage.py create_db --alembic alembic.ini"
+
Upgrade
=======
After upgrade of copr-frontend you should upgrade db schema:
9 years
[copr] master: [backend] [WIP] playbooks for the new cloud (e11d5a5)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit e11d5a5861d0b720093b947e09b3ee57bc80679d
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Wed Apr 1 18:34:28 2015 +0200
[backend] [WIP] playbooks for the new cloud
>---------------------------------------------------------------
.../playbooks/WIP/provision/files/builder.repo | 23 +
.../playbooks/WIP/provision/files/buildsys.pub | 1 +
.../provision/files/mock}/site-defaults.cfg | 0
.../playbooks/WIP/provision/filter_plugins/os.py | 24 +
backend/conf/playbooks/WIP/provision/inventory | 1 +
.../WIP/provision/library/nova_compute.py | 587 ++++++++++++++++++++
.../provision}/provision_builder_tasks.yml | 44 +-
.../playbooks/WIP/provision/spinup_nova_pb.yml | 86 +++
.../playbooks/WIP/provision/spinup_nova_task.yml | 28 +
9 files changed, 777 insertions(+), 17 deletions(-)
diff --git a/backend/conf/playbooks/WIP/provision/files/builder.repo b/backend/conf/playbooks/WIP/provision/files/builder.repo
new file mode 100644
index 0000000..b1d401e
--- /dev/null
+++ b/backend/conf/playbooks/WIP/provision/files/builder.repo
@@ -0,0 +1,23 @@
+[builder-infrastructure]
+name=Builder Packages from Fedora Infrastructure $releasever - $basearch
+baseurl=http://infrastructure.fedoraproject.org/repo/builder-rpms/$releasever/$basearch/
+enabled=1
+gpgcheck=1
+gpgkey=http://infrastructure.fedoraproject.org/repo/RPM-GPG-KEY-INFRASTRUCTURE
+
+[msuchy-Mock]
+name=Copr repo for Mock owned by msuchy
+description=Mock for RHEL6 with patch from https://bugzilla.redhat.com/show_bug.cgi?id=1028438 and https://bugzilla.redhat.com/show_bug.cgi?id=1034805
+baseurl=http://172.16.5.5/results/msuchy/Mock/epel-6-$basearch/
+skip_if_unavailable=True
+gpgcheck=0
+enabled=1
+
+[msuchy-scl-utils]
+name=Copr repo for scl-utils owned by msuchy
+description=scl-utils with patch from https://bugzilla.redhat.com/show_bug.cgi?id=985233
+baseurl=http://172.16.5.5/results/msuchy/scl-utils/epel-6-$basearch/
+skip_if_unavailable=True
+gpgcheck=0
+enabled=1
+
diff --git a/backend/conf/playbooks/WIP/provision/files/buildsys.pub b/backend/conf/playbooks/WIP/provision/files/buildsys.pub
new file mode 100644
index 0000000..10790ee
--- /dev/null
+++ b/backend/conf/playbooks/WIP/provision/files/buildsys.pub
@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCeTO0ddXuhDZYM9HyM0a47aeV2yIVWhTpddrQ7/RAIs99XyrsicQLABzmdMBfiZnP0FnHBF/e+2xEkT8hHJpX6bX81jjvs2bb8KP18Nh8vaXI3QospWrRygpu1tjzqZT0Llh4ZVFscum8TrMw4VWXclzdDw6x7csCBjSttqq8F3iTJtQ9XM9/5tCAAOzGBKJrsGKV1CNIrfUo5CSzY+IUVIr8XJ93IB2ZQVASK34T/49egmrWlNB32fqAbDMC+XNmobgn6gO33Yq5Ly7Dk4kqTUx2TEaqDkZfhsVu0YcwV81bmqsltRvpj6bIXrEoMeav7nbuqKcPLTxWEY/2icePF
diff --git a/backend/conf/playbooks/files/site-defaults.cfg b/backend/conf/playbooks/WIP/provision/files/mock/site-defaults.cfg
similarity index 100%
copy from backend/conf/playbooks/files/site-defaults.cfg
copy to backend/conf/playbooks/WIP/provision/files/mock/site-defaults.cfg
diff --git a/backend/conf/playbooks/WIP/provision/filter_plugins/os.py b/backend/conf/playbooks/WIP/provision/filter_plugins/os.py
new file mode 100644
index 0000000..0d3dab0
--- /dev/null
+++ b/backend/conf/playbooks/WIP/provision/filter_plugins/os.py
@@ -0,0 +1,24 @@
+from novaclient.v1_1.client import Client
+
+def nova_result_to_builder_ip(nova_result, network_name):
+ return nova_result["addresses"][network_name][0]["addr"]
+
+def network_name_to_id(network_name, username, password, tenant_name, auth_url):
+ nt = Client(username, password, tenant_name, auth_url, insecure=True)
+ # import ipdb; ipdb.set_trace()
+ return nt.networks.find(label=network_name).id
+
+
+
+class FilterModule(object):
+ def filters(self):
+ return {
+ "nova_result_to_builder_ip": nova_result_to_builder_ip,
+ # "flavor_id_to_name": flavor_id_to_name,
+ # "flavor_name_to_id": flavor_name_to_id,
+ # "image_id_to_name": image_id_to_name,
+ # "image_name_to_id": image_name_to_id,
+ "network_name_to_id": network_name_to_id,
+ # "network_id_to_name": network_id_to_name,
+ }
+
diff --git a/backend/conf/playbooks/WIP/provision/inventory b/backend/conf/playbooks/WIP/provision/inventory
new file mode 100644
index 0000000..2fbb50c
--- /dev/null
+++ b/backend/conf/playbooks/WIP/provision/inventory
@@ -0,0 +1 @@
+localhost
diff --git a/backend/conf/playbooks/WIP/provision/library/nova_compute.py b/backend/conf/playbooks/WIP/provision/library/nova_compute.py
new file mode 100644
index 0000000..4c1e977
--- /dev/null
+++ b/backend/conf/playbooks/WIP/provision/library/nova_compute.py
@@ -0,0 +1,587 @@
+#!/usr/bin/python
+#coding: utf-8 -*-
+
+# (c) 2013, Benno Joy <benno(a)ansible.com>
+# (c) 2013, John Dewey <john(a)dewey.ws>
+#
+# This module is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software. If not, see <http://www.gnu.org/licenses/>.
+
+import operator
+import os
+
+try:
+ from novaclient.v1_1 import client as nova_client
+ from novaclient.v1_1 import floating_ips
+ from novaclient import exceptions
+ from novaclient import utils
+ import time
+except ImportError:
+ print("failed=True msg='novaclient is required for this module'")
+
+DOCUMENTATION = '''
+---
+module: nova_compute
+version_added: "1.2"
+short_description: Create/Delete VMs from OpenStack
+description:
+ - Create or Remove virtual machines from Openstack.
+options:
+ login_username:
+ description:
+ - login username to authenticate to keystone
+ required: true
+ default: admin
+ login_password:
+ description:
+ - Password of login user
+ required: true
+ default: 'yes'
+ login_tenant_name:
+ description:
+ - The tenant name of the login user
+ required: true
+ default: 'yes'
+ auth_url:
+ description:
+ - The keystone url for authentication
+ required: false
+ default: 'http://127.0.0.1:35357/v2.0/'
+ region_name:
+ description:
+ - Name of the region
+ required: false
+ default: None
+ state:
+ description:
+ - Indicate desired state of the resource
+ choices: ['present', 'absent']
+ default: present
+ name:
+ description:
+ - Name that has to be given to the instance
+ required: true
+ default: None
+ image_id:
+ description:
+ - The id of the base image to boot. Mutually exclusive with image_name
+ required: true
+ default: None
+ image_name:
+ description:
+ - The name of the base image to boot. Mutually exclusive with image_id
+ required: true
+ default: None
+ version_added: "1.8"
+ image_exclude:
+ description:
+ - Text to use to filter image names, for the case, such as HP, where there are multiple image names matching the common identifying portions. image_exclude is a negative match filter - it is text that may not exist in the image name. Defaults to "(deprecated)"
+ version_added: "1.8"
+ flavor_id:
+ description:
+ - The id of the flavor in which the new VM has to be created. Mutually exclusive with flavor_ram
+ required: false
+ default: 1
+ flavor_ram:
+ description:
+ - The minimum amount of ram in MB that the flavor in which the new VM has to be created must have. Mutually exclusive with flavor_id
+ required: false
+ default: 1
+ version_added: "1.8"
+ flavor_include:
+ description:
+ - Text to use to filter flavor names, for the case, such as Rackspace, where there are multiple flavors that have the same ram count. flavor_include is a positive match filter - it must exist in the flavor name.
+ version_added: "1.8"
+ key_name:
+ description:
+ - The key pair name to be used when creating a VM
+ required: false
+ default: None
+ security_groups:
+ description:
+ - The name of the security group to which the VM should be added
+ required: false
+ default: None
+ nics:
+ description:
+ - A list of network id's to which the VM's interface should be attached
+ required: false
+ default: None
+ auto_floating_ip:
+ description:
+ - Should a floating ip be auto created and assigned
+ required: false
+ default: 'yes'
+ version_added: "1.8"
+ floating_ips:
+ description:
+ - list of valid floating IPs that pre-exist to assign to this node
+ required: false
+ default: None
+ version_added: "1.8"
+ floating_ip_pools:
+ description:
+ - list of floating IP pools from which to choose a floating IP
+ required: false
+ default: None
+ version_added: "1.8"
+ availability_zone:
+ description:
+ - Name of the availability zone
+ required: false
+ default: None
+ version_added: "1.8"
+ meta:
+ description:
+ - A list of key value pairs that should be provided as a metadata to the new VM
+ required: false
+ default: None
+ wait:
+ description:
+ - If the module should wait for the VM to be created.
+ required: false
+ default: 'yes'
+ wait_for:
+ description:
+ - The amount of time the module should wait for the VM to get into active state
+ required: false
+ default: 180
+ config_drive:
+ description:
+ - Whether to boot the server with config drive enabled
+ required: false
+ default: 'no'
+ version_added: "1.8"
+ user_data:
+ description:
+ - Opaque blob of data which is made available to the instance
+ required: false
+ default: None
+ version_added: "1.6"
+requirements: ["novaclient"]
+'''
+
+EXAMPLES = '''
+# Creates a new VM and attaches to a network and passes metadata to the instance
+- nova_compute:
+ state: present
+ login_username: admin
+ login_password: admin
+ login_tenant_name: admin
+ name: vm1
+ image_id: 4f905f38-e52a-43d2-b6ec-754a13ffb529
+ key_name: ansible_key
+ wait_for: 200
+ flavor_id: 4
+ nics:
+ - net-id: 34605f38-e52a-25d2-b6ec-754a13ffb723
+ meta:
+ hostname: test1
+ group: uge_master
+
+# Creates a new VM in HP Cloud AE1 region availability zone az2 and automatically assigns a floating IP
+- name: launch a nova instance
+ hosts: localhost
+ tasks:
+ - name: launch an instance
+ nova_compute:
+ state: present
+ login_username: username
+ login_password: Equality7-2521
+ login_tenant_name: username-project1
+ name: vm1
+ auth_url: https://region-b.geo-1.identity.hpcloudsvc.com:35357/v2.0/
+ region_name: region-b.geo-1
+ availability_zone: az2
+ image_id: 9302692b-b787-4b52-a3a6-daebb79cb498
+ key_name: test
+ wait_for: 200
+ flavor_id: 101
+ security_groups: default
+ auto_floating_ip: yes
+
+# Creates a new VM in HP Cloud AE1 region availability zone az2 and assigns a pre-known floating IP
+- name: launch a nova instance
+ hosts: localhost
+ tasks:
+ - name: launch an instance
+ nova_compute:
+ state: present
+ login_username: username
+ login_password: Equality7-2521
+ login_tenant_name: username-project1
+ name: vm1
+ auth_url: https://region-b.geo-1.identity.hpcloudsvc.com:35357/v2.0/
+ region_name: region-b.geo-1
+ availability_zone: az2
+ image_id: 9302692b-b787-4b52-a3a6-daebb79cb498
+ key_name: test
+ wait_for: 200
+ flavor_id: 101
+ floating-ips:
+ - 12.34.56.79
+
+# Creates a new VM with 4G of RAM on Ubuntu Trusty, ignoring deprecated images
+- name: launch a nova instance
+ hosts: localhost
+ tasks:
+ - name: launch an instance
+ nova_compute:
+ name: vm1
+ state: present
+ login_username: username
+ login_password: Equality7-2521
+ login_tenant_name: username-project1
+ auth_url: https://region-b.geo-1.identity.hpcloudsvc.com:35357/v2.0/
+ region_name: region-b.geo-1
+ image_name: Ubuntu Server 14.04
+ image_exclude: deprecated
+ flavor_ram: 4096
+
+# Creates a new VM with 4G of RAM on Ubuntu Trusty on a Rackspace Performance node in DFW
+- name: launch a nova instance
+ hosts: localhost
+ tasks:
+ - name: launch an instance
+ nova_compute:
+ name: vm1
+ state: present
+ login_username: username
+ login_password: Equality7-2521
+ login_tenant_name: username-project1
+ auth_url: https://identity.api.rackspacecloud.com/v2.0/
+ region_name: DFW
+ image_name: Ubuntu 14.04 LTS (Trusty Tahr) (PVHVM)
+ flavor_ram: 4096
+ flavor_include: Performance
+'''
+
+
+
+def _delete_server(module, nova):
+ name = None
+ server_list = None
+ try:
+ server_list = nova.servers.list(True, {'name': module.params['name']})
+ if server_list:
+ server = [x for x in server_list if x.name == module.params['name']]
+ nova.servers.delete(server.pop())
+ except Exception, e:
+ module.fail_json( msg = "Error in deleting vm: %s" % e.message)
+ if module.params['wait'] == 'no':
+ module.exit_json(changed = True, result = "deleted")
+ expire = time.time() + int(module.params['wait_for'])
+ while time.time() < expire:
+ name = nova.servers.list(True, {'name': module.params['name']})
+ if not name:
+ module.exit_json(changed = True, result = "deleted")
+ time.sleep(5)
+ module.fail_json(msg = "Timed out waiting for server to get deleted, please check manually")
+
+
+def _add_floating_ip_from_pool(module, nova, server):
+
+ # instantiate FloatingIPManager object
+ floating_ip_obj = floating_ips.FloatingIPManager(nova)
+
+ # empty dict and list
+ usable_floating_ips = {}
+ pools = []
+
+ # user specified
+ pools = module.params['floating_ip_pools']
+
+ # get the list of all floating IPs. Mileage may
+ # vary according to Nova Compute configuration
+ # per cloud provider
+ all_floating_ips = floating_ip_obj.list()
+
+ # iterate through all pools of IP address. Empty
+ # string means all and is the default value
+ for pool in pools:
+ # temporary list per pool
+ pool_ips = []
+ # loop through all floating IPs
+ for f_ip in all_floating_ips:
+ # if not reserved and the correct pool, add
+ if f_ip.instance_id is None and (f_ip.pool == pool):
+ pool_ips.append(f_ip.ip)
+ # only need one
+ break
+
+ # if the list is empty, add for this pool
+ if not pool_ips:
+ try:
+ new_ip = nova.floating_ips.create(pool)
+ except Exception, e:
+ module.fail_json(msg = "Unable to create floating ip")
+ pool_ips.append(new_ip.ip)
+ # Add to the main list
+ usable_floating_ips[pool] = pool_ips
+
+ # finally, add ip(s) to instance for each pool
+ for pool in usable_floating_ips:
+ for ip in usable_floating_ips[pool]:
+ try:
+ server.add_floating_ip(ip)
+ # We only need to assign one ip - but there is an inherent
+ # race condition and some other cloud operation may have
+ # stolen an available floating ip
+ break
+ except Exception, e:
+ module.fail_json(msg = "Error attaching IP %s to instance %s: %s " % (ip, server.id, e.message))
+
+
+def _add_floating_ip_list(module, server, ips):
+ # add ip(s) to instance
+ for ip in ips:
+ try:
+ server.add_floating_ip(ip)
+ except Exception, e:
+ module.fail_json(msg = "Error attaching IP %s to instance %s: %s " % (ip, server.id, e.message))
+
+
+def _add_auto_floating_ip(module, nova, server):
+
+ try:
+ new_ip = nova.floating_ips.create()
+ except Exception as e:
+ module.fail_json(msg = "Unable to create floating ip: %s" % (e.message))
+
+ try:
+ server.add_floating_ip(new_ip)
+ except Exception as e:
+ # Clean up - we auto-created this ip, and it's not attached
+ # to the server, so the cloud will not know what to do with it
+ server.floating_ips.delete(new_ip)
+ module.fail_json(msg = "Error attaching IP %s to instance %s: %s " % (ip, server.id, e.message))
+
+
+def _add_floating_ip(module, nova, server):
+
+ if module.params['floating_ip_pools']:
+ _add_floating_ip_from_pool(module, nova, server)
+ elif module.params['floating_ips']:
+ _add_floating_ip_list(module, server, module.params['floating_ips'])
+ elif module.params['auto_floating_ip']:
+ _add_auto_floating_ip(module, nova, server)
+ else:
+ return server
+
+ # this may look redundant, but if there is now a
+ # floating IP, then it needs to be obtained from
+ # a recent server object if the above code path exec'd
+ try:
+ server = nova.servers.get(server.id)
+ except Exception, e:
+ module.fail_json(msg = "Error in getting info from instance: %s " % e.message)
+ return server
+
+
+def _get_image_id(module, nova):
+ if module.params['image_name']:
+ for image in nova.images.list():
+ if (module.params['image_name'] in image.name and (
+ not module.params['image_exclude']
+ or module.params['image_exclude'] not in image.name)):
+ return image.id
+ module.fail_json(msg = "Error finding image id from name(%s)" % module.params['image_name'])
+ return module.params['image_id']
+
+
+def _get_flavor_id(module, nova):
+ if module.params['flavor_ram']:
+ for flavor in sorted(nova.flavors.list(), key=operator.attrgetter('ram')):
+ if (flavor.ram >= module.params['flavor_ram'] and
+ (not module.params['flavor_include'] or module.params['flavor_include'] in flavor.name)):
+ return flavor.id
+ module.fail_json(msg = "Error finding flavor with %sMB of RAM" % module.params['flavor_ram'])
+ return module.params['flavor_id']
+
+
+def _create_server(module, nova):
+ image_id = _get_image_id(module, nova)
+ flavor_id = _get_flavor_id(module, nova)
+ bootargs = [module.params['name'], image_id, flavor_id]
+ bootkwargs = {
+ 'nics' : module.params['nics'],
+ 'meta' : module.params['meta'],
+ 'security_groups': module.params['security_groups'].split(','),
+ #userdata is unhyphenated in novaclient, but hyphenated here for consistency with the ec2 module:
+ 'userdata': module.params['user_data'],
+ 'config_drive': module.params['config_drive'],
+ }
+
+ for optional_param in ('region_name', 'key_name', 'availability_zone'):
+ if module.params[optional_param]:
+ bootkwargs[optional_param] = module.params[optional_param]
+ try:
+ server = nova.servers.create(*bootargs, **bootkwargs)
+ server = nova.servers.get(server.id)
+ except Exception, e:
+ module.fail_json( msg = "Error in creating instance: %s " % e.message)
+ if module.params['wait'] == 'yes':
+ expire = time.time() + int(module.params['wait_for'])
+ while time.time() < expire:
+ try:
+ server = nova.servers.get(server.id)
+ except Exception, e:
+ module.fail_json( msg = "Error in getting info from instance: %s" % e.message)
+ if server.status == 'ACTIVE':
+ server = _add_floating_ip(module, nova, server)
+
+ private = openstack_find_nova_addresses(getattr(server, 'addresses'), 'fixed', 'private')
+ public = openstack_find_nova_addresses(getattr(server, 'addresses'), 'floating', 'public')
+
+ # now exit with info
+ module.exit_json(changed = True, id = server.id, private_ip=''.join(private), public_ip=''.join(public), status = server.status, info = server._info)
+
+ if server.status == 'ERROR':
+ module.fail_json(msg = "Error in creating the server, please check logs")
+ time.sleep(2)
+
+ module.fail_json(msg = "Timeout waiting for the server to come up.. Please check manually")
+ if server.status == 'ERROR':
+ module.fail_json(msg = "Error in creating the server.. Please check manually")
+ private = openstack_find_nova_addresses(getattr(server, 'addresses'), 'fixed', 'private')
+ public = openstack_find_nova_addresses(getattr(server, 'addresses'), 'floating', 'public')
+
+ module.exit_json(changed = True, id = info['id'], private_ip=''.join(private), public_ip=''.join(public), status = server.status, info = server._info)
+
+
+def _delete_floating_ip_list(module, nova, server, extra_ips):
+ for ip in extra_ips:
+ nova.servers.remove_floating_ip(server=server.id, address=ip)
+
+
+def _check_floating_ips(module, nova, server):
+ changed = False
+ if module.params['floating_ip_pools'] or module.params['floating_ips'] or module.params['auto_floating_ip']:
+ ips = openstack_find_nova_addresses(server.addresses, 'floating')
+ if not ips:
+ # If we're configured to have a floating but we don't have one,
+ # let's add one
+ server = _add_floating_ip(module, nova, server)
+ changed = True
+ elif module.params['floating_ips']:
+ # we were configured to have specific ips, let's make sure we have
+ # those
+ missing_ips = []
+ for ip in module.params['floating_ips']:
+ if ip not in ips:
+ missing_ips.append(ip)
+ if missing_ips:
+ server = _add_floating_ip_list(module, server, missing_ips)
+ changed = True
+ extra_ips = []
+ for ip in ips:
+ if ip not in module.params['floating_ips']:
+ extra_ips.append(ip)
+ if extra_ips:
+ _delete_floating_ip_list(module, server, extra_ips)
+ changed = True
+ return (changed, server)
+
+
+def _get_server_state(module, nova):
+ server = None
+ try:
+ servers = nova.servers.list(True, {'name': module.params['name']})
+ if servers:
+ # the {'name': module.params['name']} will also return servers
+ # with names that partially match the server name, so we have to
+ # strictly filter here
+ servers = [x for x in servers if x.name == module.params['name']]
+ if servers:
+ server = servers[0]
+ except Exception, e:
+ module.fail_json(msg = "Error in getting the server list: %s" % e.message)
+ if server and module.params['state'] == 'present':
+ if server.status != 'ACTIVE':
+ module.fail_json( msg="The VM is available but not Active. state:" + server.status)
+ (ip_changed, server) = _check_floating_ips(module, nova, server)
+ private = openstack_find_nova_addresses(getattr(server, 'addresses'), 'fixed', 'private')
+ public = openstack_find_nova_addresses(getattr(server, 'addresses'), 'floating', 'public')
+ module.exit_json(changed = ip_changed, id = server.id, public_ip = ''.join(public), private_ip = ''.join(private), info = server._info)
+ if server and module.params['state'] == 'absent':
+ return True
+ if module.params['state'] == 'absent':
+ module.exit_json(changed = False, result = "not present")
+ return True
+
+
+
+def main():
+ argument_spec = openstack_argument_spec()
+ argument_spec.update(dict(
+ name = dict(required=True),
+ image_id = dict(default=None),
+ image_name = dict(default=None),
+ image_exclude = dict(default='(deprecated)'),
+ flavor_id = dict(default=1),
+ flavor_ram = dict(default=None, type='int'),
+ flavor_include = dict(default=None),
+ key_name = dict(default=None),
+ security_groups = dict(default='default'),
+ nics = dict(default=None),
+ meta = dict(default=None),
+ wait = dict(default='yes', choices=['yes', 'no']),
+ wait_for = dict(default=180),
+ state = dict(default='present', choices=['absent', 'present']),
+ user_data = dict(default=None),
+ config_drive = dict(default=False, type='bool'),
+ auto_floating_ip = dict(default=False, type='bool'),
+ floating_ips = dict(default=None),
+ floating_ip_pools = dict(default=None),
+ insecure = dict(default=False, type='bool'),
+ ))
+ module = AnsibleModule(
+ argument_spec=argument_spec,
+ mutually_exclusive=[
+ ['auto_floating_ip','floating_ips'],
+ ['auto_floating_ip','floating_ip_pools'],
+ ['floating_ips','floating_ip_pools'],
+ ['image_id','image_name'],
+ ['flavor_id','flavor_ram'],
+ ],
+ )
+
+ nova = nova_client.Client(module.params['login_username'],
+ module.params['login_password'],
+ module.params['login_tenant_name'],
+ module.params['auth_url'],
+ region_name=module.params['region_name'],
+ insecure=module.params['insecure'],
+ service_type='compute')
+ try:
+ nova.authenticate()
+ except exceptions.Unauthorized, e:
+ module.fail_json(msg = "Invalid OpenStack Nova credentials.: %s" % e.message)
+ except exceptions.AuthorizationFailure, e:
+ module.fail_json(msg = "Unable to authorize user: %s" % e.message)
+
+ if module.params['state'] == 'present':
+ if not module.params['image_id'] and not module.params['image_name']:
+ module.fail_json( msg = "Parameter 'image_id' or `image_name` is required if state == 'present'")
+ else:
+ _get_server_state(module, nova)
+ _create_server(module, nova)
+ if module.params['state'] == 'absent':
+ _get_server_state(module, nova)
+ _delete_server(module, nova)
+
+# this is magic, see lib/ansible/module_common.py
+from ansible.module_utils.basic import *
+from ansible.module_utils.openstack import *
+main()
+
diff --git a/backend/conf/playbooks/provision_builder_tasks.yml b/backend/conf/playbooks/WIP/provision/provision_builder_tasks.yml
similarity index 56%
copy from backend/conf/playbooks/provision_builder_tasks.yml
copy to backend/conf/playbooks/WIP/provision/provision_builder_tasks.yml
index 7183692..02db4c5 100644
--- a/backend/conf/playbooks/provision_builder_tasks.yml
+++ b/backend/conf/playbooks/WIP/provision/provision_builder_tasks.yml
@@ -14,6 +14,7 @@
- libsemanage-python
- name: make sure newest rpm
+ # todo: replace with dnf after ansible 1.9 is available
yum: name={{ item }} state=latest
with_items:
- rpm
@@ -21,27 +22,36 @@
- ca-certificates
- name: put updated mock configs into /etc/mock
- copy: src=files/{{ item }} dest=/etc/mock
+ copy: src=files/mock/{{ item }} dest=/etc/mock
with_items:
- site-defaults.cfg
-- name: prepare caches
+# ansible doesn't support simultaneously usage of async and with_* options
+# it's not even planned for implementation, see https://github.com/ansible/ansible/issues/5841
+#- name: prepare caches
+# when: prepare_base_image is defined
+# async: 3600
+# command: mock -r {{ item }} --init
+# with_items:
+# - epel-5-i386
+# - epel-5-x86_64
+# - epel-6-i386
+# - epel-6-x86_64
+# - epel-7-x86_64
+# - fedora-20-i386
+# - fedora-20-x86_64
+# - fedora-21-i386
+# - fedora-21-x86_64
+# - fedora-22-i386
+# - fedora-22-x86_64
+# - fedora-rawhide-i386
+# - fedora-rawhide-x86_64
+
+
+- name: prepare cache
+ when: prepare_base_image is defined
async: 3600
- command: mock -r {{ item }} --init
- with_items:
- - epel-5-i386
- - epel-5-x86_64
- - epel-6-i386
- - epel-6-x86_64
- - epel-7-x86_64
- - fedora-20-i386
- - fedora-20-x86_64
- - fedora-21-i386
- - fedora-21-x86_64
- - fedora-22-i386
- - fedora-22-x86_64
- - fedora-rawhide-i386
- - fedora-rawhide-x86_64
+ shell: "for i in epel-5-i386 epel-5-x86_64 epel-6-i386 epel-6-x86_64 epel-7-x86_64 fedora-20-i386 fedora-20-x86_64 fedora-21-i386 fedora-21-x86_64 fedora-22-i386 fedora-22-x86_64 fedora-rawhide-i386 fedora-rawhide-x86_64; do mock --init -r $i; done"
- name: mockbuilder user
user: name=mockbuilder groups=mock
diff --git a/backend/conf/playbooks/WIP/provision/spinup_nova_pb.yml b/backend/conf/playbooks/WIP/provision/spinup_nova_pb.yml
new file mode 100644
index 0000000..b1bdde6
--- /dev/null
+++ b/backend/conf/playbooks/WIP/provision/spinup_nova_pb.yml
@@ -0,0 +1,86 @@
+- name: check/create instance
+ hosts: 127.0.0.1
+ #hosts: copr-back-stg
+ gather_facts: False
+
+ vars:
+ OS_AUTH_URL: "https://fed-cloud09.cloud.fedoraproject.org:5000/v2.0"
+ image_id: "e71dd9f6-ec73-44dd-946c-25b09fce332d"
+ # With the addition of Keystone we have standardized on the term **tenant**
+ # # as the entity that owns the resources.
+ OS_TENANT_ID: "566a072fb1694950998ad191fee3833b"
+ OS_TENANT_NAME: "coprdev"
+ #
+ # # In addition to the owning entity (tenant), openstack stores the entity
+ # # performing the action as the **user**.
+ OS_USERNAME: "FILLME"
+ #
+ # # With Keystone you pass the keystone password.
+ # echo "Please enter your OpenStack Password: "
+ # read -sr OS_PASSWORD_INPUT
+ OS_PASSWORD: "FILLME"
+ #
+ # _OS_AUTH_OPTS: "--os-auth-url {{OS_AUTH_URL}} --os-username {{OS_USERNAME}} --os-password {{OS_PASSWORD}} --os-tenant-name {{OS_TENANT_NAME}} --os-tenant-id {{OS_TENANT_ID}} "
+ # VM_NAME_OR_ID: "vm_from_nova_cli_001"
+
+ network_name: "copr-net"
+ cloud_networks:
+ #- net-id: "53fb02fd-6e97-4cfc-b298-a2ff867daa52"
+ - net-id: "6797b4e3-28a0-4a14-9689-e506d0cca90d"
+ #- net-id: "dcb11bd5-47df-43ed-bb2f-5d9c39bfedcd"
+ keypair: buildsys
+ max_spawn_time: 600
+
+ #prepare_base_image: True
+
+ #vars_prompt:
+ # - name: "OS_PASSWORD"
+ # prompt: "Enter OS password"
+ # private: yes
+ tasks:
+ - name: generate builder name
+ # local_action: command echo "Copr_builder_{{ 999999999 | random }}"
+ #ocal_action: command echo "Copr_builder_138954194"
+ local_action: set_fact vm_name="Copr_builder_{{ 999999999 | random }}"
+ #register: vm_name
+ - debug: msg="VM_NAME={{ vm_name }}"
+
+ - include: "spinup_nova_task.yml"
+
+
+ # - set_fact: builder_ip="{{ nova.info|nova_result_to_builder_ip }}"
+
+ - debug: msg="VM_IP={{ builder_ip }}"
+
+ - name: wait for he host to be hot
+ local_action: wait_for host={{ builder_ip }} port=22 delay=1 timeout=600
+
+ #- name: sleep
+ # local_action: command sleep 20
+
+
+
+ - name: wait until ssh is available
+ local_action: shell false; until [ "$?" -eq "0" ]; do sleep 2; ssh -o PasswordAuthentication=no fedora@{{ builder_ip }} 'echo foobar' 2>/dev/null; done
+ async: 600
+ poll: 2
+
+
+ #egister: echo_result
+ #ommand: echo foobar
+ #ntil: echo_result.stdout == "foobar"
+ #etries: 15
+ #elay: 5
+ #gnore_errors: True
+
+
+- name: provision builder
+ hosts: builder_temp_group
+ gather_facts: False
+ sudo: True
+ user: fedora
+
+ tasks:
+ #- debug: msg="hello"
+ # # include: "provision_task.yml"
+ - include: "provision_builder_tasks.yml"
diff --git a/backend/conf/playbooks/WIP/provision/spinup_nova_task.yml b/backend/conf/playbooks/WIP/provision/spinup_nova_task.yml
new file mode 100644
index 0000000..ca7c661
--- /dev/null
+++ b/backend/conf/playbooks/WIP/provision/spinup_nova_task.yml
@@ -0,0 +1,28 @@
+- name: spin/ensure vm with nova_compute
+ local_action:
+ module: nova_compute
+ insecure: True
+ auth_url: "{{OS_AUTH_URL}}"
+ login_username: "{{OS_USERNAME}}"
+ login_password: "{{OS_PASSWORD}}"
+ login_tenant_name: "{{OS_TENANT_NAME}}"
+ name: "{{ vm_name }}"
+ # image_id: 86422ca2-6eeb-435c-87e8-402b3c7c3b7b
+ image_id: "{{ image_id }}"
+ wait_for: "{{ max_spawn_time }}"
+ flavor_id: 2
+ security_groups: ssh-anywhere-coprdev,default,ssh-from-persistent-coprdev #,ssh-internal-persistent
+ key_name: "{{ keypair }}"
+ nics:
+ - net-id: "{{ network_name|network_name_to_id(OS_USERNAME, OS_PASSWORD, OS_TENANT_NAME, OS_AUTH_URL) }}"
+ register: nova
+
+- debug: msg="{{ nova.info }}"
+
+
+- set_fact: builder_ip="{{ nova.info|nova_result_to_builder_ip(network_name) }}"
+
+- name: add it to the special group
+ local_action: add_host hostname={{ builder_ip }} groupname=builder_temp_group
+
+
9 years
[copr] master: [backend] don't provide logstash config directly, add only example to documentation (d972007)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit d97200785495e29c0fd885785d764c51d062c8b0
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Tue Mar 31 16:18:26 2015 +0200
[backend] don't provide logstash config directly, add only example to documentation
>---------------------------------------------------------------
backend/conf/logstash/frontend.hostname | 1 -
backend/copr-backend.spec | 27 ++++++++++++---------------
2 files changed, 12 insertions(+), 16 deletions(-)
diff --git a/backend/conf/logstash/frontend.hostname b/backend/conf/logstash/frontend.hostname
deleted file mode 100644
index 2bcb45c..0000000
--- a/backend/conf/logstash/frontend.hostname
+++ /dev/null
@@ -1 +0,0 @@
-copr-fe-dev.cloud.fedoraproject.org
diff --git a/backend/copr-backend.spec b/backend/copr-backend.spec
index 43aee1e..f43343f 100644
--- a/backend/copr-backend.spec
+++ b/backend/copr-backend.spec
@@ -103,10 +103,11 @@ only.
%build
+# TODO: remove when sphinx docs added
# build documentation
-pushd documentation
-make %{?_smp_mflags} python
-popd
+# pushd documentation
+# make %{?_smp_mflags} python
+# popd
%install
@@ -114,7 +115,6 @@ install -d %{buildroot}%{_sharedstatedir}/copr
install -d %{buildroot}%{_sharedstatedir}/copr/jobs
install -d %{buildroot}%{_sharedstatedir}/copr/public_html/results
install -d %{buildroot}%{_var}/log/copr
-install -d %{buildroot}%{_var}/log/copr/workers/
install -d %{buildroot}%{_pkgdocdir}/lighttpd/
install -d %{buildroot}%{_datadir}/copr/backend
install -d %{buildroot}%{_sysconfdir}/copr
@@ -142,9 +142,6 @@ cp -a conf/tmpfiles.d/* %{buildroot}/%{_tmpfilesdir}
touch %{buildroot}%{_var}/log/copr/copr.log
touch %{buildroot}%{_var}/log/copr/prune_old.log
-for i in `seq 7`; do
- touch %{buildroot}%{_var}/log/copr/workers/worker-$i.log
-done
touch %{buildroot}%{_var}/run/copr-backend/copr-be.pid
install -m 0644 copr-backend.service %{buildroot}/%{_unitdir}/
@@ -152,16 +149,18 @@ install -m 0644 conf/copr.sudoers.d %{buildroot}%{_sysconfdir}/sudoers.d/copr
install -d %{buildroot}%{_sysconfdir}/logstash.d
-cp -a conf/logstash/copr_backend.conf %{buildroot}%{_sysconfdir}/logstash.d/copr_backend.conf
+
install -d %{buildroot}%{_datadir}/logstash/patterns/
cp -a conf/logstash/lighttpd.pattern %{buildroot}%{_datadir}/logstash/patterns/lighttpd.pattern
-cp -a conf/logstash/frontend.hostname %{buildroot}%{_sysconfdir}/copr/
#doc
-cp -a documentation/python-doc %{buildroot}%{_pkgdocdir}/
+# cp -a documentation/python-doc %{buildroot}%{_pkgdocdir}/
cp -a conf/playbooks %{buildroot}%{_pkgdocdir}/
+install -d %{buildroot}%{_pkgdocdir}/examples/%{_sysconfdir}/logstash.d
+cp -a conf/logstash/copr_backend.conf %{buildroot}%{_pkgdocdir}/examples/%{_sysconfdir}/logstash.d/copr_backend.conf
+
%check
# redis-server --port 7777 &> /dev/null &
@@ -196,11 +195,9 @@ useradd -r -g copr -G lighttpd -s /bin/bash -c "COPR user" copr
%dir %attr(0755, copr, copr) %{_sharedstatedir}/copr/public_html/
%dir %attr(0755, copr, copr) %{_sharedstatedir}/copr/public_html/results
%dir %attr(0755, copr, copr) %{_var}/log/copr
-%dir %attr(0755, copr, copr) %{_var}/log/copr/workers
%dir %attr(0755, copr, copr) %{_var}/run/copr-backend
%ghost %{_var}/log/copr/*.log
-%ghost %{_var}/log/copr/workers/worker-*.log
%ghost %{_var}/run/copr-backend/copr-be.pid
%config(noreplace) %{_sysconfdir}/logrotate.d/copr-backend
@@ -209,13 +206,11 @@ useradd -r -g copr -G lighttpd -s /bin/bash -c "COPR user" copr
%doc %{_pkgdocdir}/playbooks
%dir %{_sysconfdir}/copr
%config(noreplace) %attr(0640, root, copr) %{_sysconfdir}/copr/copr-be.conf
-%config(noreplace) %attr(0640, root, copr) %{_sysconfdir}/copr/frontend.hostname
%{_unitdir}/copr-backend.service
%{_tmpfilesdir}/copr-backend.conf
%{_bindir}/*
%config(noreplace) %{_sysconfdir}/cron.daily/copr-backend
-%config(noreplace) %{_sysconfdir}/logstash.d/copr_backend.conf
%{_datadir}/logstash/patterns/lighttpd.pattern
@@ -223,7 +218,9 @@ useradd -r -g copr -G lighttpd -s /bin/bash -c "COPR user" copr
%files doc
%license LICENSE
-%doc %{_pkgdocdir}/python-doc
+%doc
+# %{_pkgdocdir}/python-doc
+%{_pkgdocdir}/
%exclude %{_pkgdocdir}/lighttpd
%exclude %{_pkgdocdir}/playbooks
9 years