Repository :
http://git.fedorahosted.org/cgit/copr.git
On branch : master
---------------------------------------------------------------
commit daaaf9d3adb5161fa78106065637479913f40d07
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Wed Mar 25 15:11:36 2015 +0100
[image_builder] initial release, due to OS bug, we cannot create snapshot after
provision through API, need to do it manually in the WebUI.
---------------------------------------------------------------
backend/conf/playbooks/files/site-defaults.cfg | 156 ++++++++++++++++++++
backend/conf/playbooks/provision_builder_pb.yml | 16 ++
backend/conf/playbooks/provision_builder_tasks.yml | 62 ++++++++
backend/conf/playbooks/spawn_new_tasks.yml | 22 +++
builder_image/requirements.txt | 6 +
builder_image/src/__init__.py | 1 +
builder_image/src/config.py | 34 +++++
builder_image/src/copr_prepare_builder_image.py | 88 +++++++++++
8 files changed, 385 insertions(+), 0 deletions(-)
diff --git a/backend/conf/playbooks/files/site-defaults.cfg
b/backend/conf/playbooks/files/site-defaults.cfg
new file mode 100644
index 0000000..15ebd36
--- /dev/null
+++ b/backend/conf/playbooks/files/site-defaults.cfg
@@ -0,0 +1,156 @@
+# mock defaults
+# vim:tw=0:ts=4:sw=4:et:
+#
+# This config file is for site-specific default values that apply across all
+# configurations. Options specified in this config file can be overridden in
+# the individual mock config files.
+#
+# The site-defaults.cfg delivered by default has NO options set. Only set
+# options here if you want to override the defaults.
+#
+# Entries in this file follow the same format as other mock config files.
+# config_opts['foo'] = bar
+
+#############################################################################
+#
+# Things that we recommend you set in site-defaults.cfg:
+#
+# config_opts['basedir'] = '/var/lib/mock/'
+# config_opts['cache_topdir'] = '/var/cache/mock'
+# Note: the path pointed to by basedir and cache_topdir must be owned
+# by group 'mock' and must have mode: g+rws
+# config_opts['rpmbuild_timeout'] = 0
+# config_opts['use_host_resolv'] = True
+
+# You can configure log format to pull from logging.ini formats of these names:
+# config_opts['build_log_fmt_name'] = "unadorned"
+# config_opts['root_log_fmt_name'] = "detailed"
+# config_opts['state_log_fmt_name'] = "state"
+#
+# mock will normally set up a minimal chroot /dev.
+# If you want to use a pre-configured /dev, disable this and use the bind-mount
+# plugin to mount your special /dev
+# config_opts['internal_dev_setup'] = True
+#
+# internal_setarch defaults to 'True' if the python 'ctypes' package is
+# available. It is in the python std lib on >= python 2.5. On older versions,
+# it is available as an addon. On systems w/o ctypes, it will default to 'False'
+# config_opts['internal_setarch'] = False
+#
+# the cleanup_on_* options allow you to automatically clean and remove the
+# mock build directory, but only take effect if --resultdir is used.
+# config_opts provides fine-grained control. cmdline only has big hammer
+#
+# config_opts['cleanup_on_success'] = 1
+# config_opts['cleanup_on_failure'] = 1
+
+# if you want mock to automatically run createrepo on the rpms in your
+# resultdir.
+# config_opts['createrepo_on_rpms'] = False
+# config_opts['createrepo_command'] = '/usr/bin/createrepo -d -q -x
*.src.rpm'
+
+#############################################################################
+#
+# plugin related. Below are the defaults. Change to suit your site
+# policy. site-defaults.cfg is a good place to do this.
+#
+# NOTE: Some of the caching options can theoretically affect build
+# reproducability. Change with care.
+#
+config_opts['plugin_conf']['package_state_enable'] = False
+config_opts['plugin_conf']['ccache_enable'] = False
+# config_opts['plugin_conf']['ccache_opts']['max_cache_size'] =
'4G'
+# config_opts['plugin_conf']['ccache_opts']['compress'] = None
+# config_opts['plugin_conf']['ccache_opts']['dir'] =
"%(cache_topdir)s/%(root)s/ccache/"
+config_opts['plugin_conf']['yum_cache_enable'] = False
+# config_opts['plugin_conf']['yum_cache_opts']['max_age_days'] =
30
+# config_opts['plugin_conf']['yum_cache_opts']['dir'] =
"%(cache_topdir)s/%(root)s/yum_cache/"
+config_opts['plugin_conf']['root_cache_enable'] = False
+# config_opts['plugin_conf']['root_cache_opts']['max_age_days'] =
15
+# config_opts['plugin_conf']['root_cache_opts']['dir'] =
"%(cache_topdir)s/%(root)s/root_cache/"
+#
config_opts['plugin_conf']['root_cache_opts']['compress_program']
= "pigz"
+# config_opts['plugin_conf']['root_cache_opts']['extension'] =
".gz"
+# config_opts['plugin_conf']['root_cache_opts']['exclude_dirs'] =
["./proc", "./sys", "./dev",
+#
"./tmp/ccache", "./var/cache/yum" ]
+#
+# bind mount plugin is enabled by default but has no configured directories to
+# mount
+config_opts['plugin_conf']['bind_mount_enable'] = False
+#
config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('/host/path',
'/bind/mount/path/in/chroot/' ))
+#
+# config_opts['plugin_conf']['tmpfs_enable'] = False
+# config_opts['plugin_conf']['tmpfs_opts']['required_ram_mb'] =
1024
+# config_opts['plugin_conf']['tmpfs_opts']['max_fs_size'] =
'512m'
+
+#############################################################################
+#
+# environment for chroot
+#
+# config_opts['environment']['TERM'] = 'vt100'
+# config_opts['environment']['SHELL'] = '/bin/bash'
+# config_opts['environment']['HOME'] = '/builddir'
+# config_opts['environment']['HOSTNAME'] = 'mock'
+# config_opts['environment']['PATH'] =
'/usr/bin:/bin:/usr/sbin:/sbin'
+# config_opts['environment']['PROMPT_COMMAND'] = 'echo -n
"<mock-chroot>"'
+# config_opts['environment']['LANG'] =
os.environ.setdefault('LANG', 'en_US.UTF-8')
+# config_opts['environment']['TZ'] = os.environ.setdefault('TZ',
'EST5EDT')
+
+#############################################################################
+#
+# Things that you can change, but we dont recommend it:
+# config_opts['chroothome'] = '/builddir'
+# config_opts['clean'] = True
+
+#############################################################################
+#
+# Things that must be adjusted if SCM integration is used:
+#
+# config_opts['scm'] = True
+# config_opts['scm_opts']['method'] = 'git'
+# config_opts['scm_opts']['cvs_get'] = 'cvs -d /srv/cvs co SCM_BRN
SCM_PKG'
+# config_opts['scm_opts']['git_get'] = 'git clone SCM_BRN
git://localhost/SCM_PKG.git SCM_PKG'
+# config_opts['scm_opts']['svn_get'] = 'svn co
file:///srv/svn/SCM_PKG/SCM_BRN SCM_PKG'
+# config_opts['scm_opts']['spec'] = 'SCM_PKG.spec'
+# config_opts['scm_opts']['ext_src_dir'] = '/dev/null'
+# config_opts['scm_opts']['write_tar'] = True
+# config_opts['scm_opts']['git_timestamps'] = True
+
+# These options are also recognized but usually defined in cmd line
+# with --scm-option package=<pkg> --scm-option branch=<branch>
+# config_opts['scm_opts']['package'] = 'mypkg'
+# config_opts['scm_opts']['branch'] = 'master'
+
+#############################################################################
+#
+# Things that are best suited for individual chroot config files:
+#
+# MUST SET (in individual chroot cfg file):
+# config_opts['root'] = 'name-of-yum-build-dir'
+# config_opts['target_arch'] = 'i386'
+# config_opts['yum.conf'] = ''
+# config_opts['yum_common_opts'] = []
+#
+# CAN SET, defaults usually work ok:
+# config_opts['chroot_setup_cmd'] = 'install buildsys-build'
+# config_opts['log_config_file'] = 'logging.ini'
+# config_opts['more_buildreqs']['srpm_name-version-release'] =
'dependencies'
+# config_opts['macros']['%Add_your_macro_name_here'] = "add macro
value here"
+# config_opts['files']['path/name/no/leading/slash'] = "put file
contents here."
+# config_opts['chrootuid'] = os.getuid()
+
+# If you change chrootgid, you must also change "mock" to the correct group
+# name in this line of the mock PAM config:
+# auth sufficient pam_succeed_if.so user ingroup mock use_uid quiet
+# config_opts['chrootgid'] = grp.getgrnam("mock")[2]
+
+# config_opts['useradd'] = '/usr/sbin/useradd -m -u %(uid)s -g %(gid)s -d
%(home)s -n %(user)s' # Fedora/RedHat
+#
+# Security related
+# config_opts['no_root_shells'] = False
+#
+# Proxy settings (https_proxy, ftp_proxy, and no_proxy can also be set)
+# config_opts['http_proxy'] = 'http://localhost:3128'
+
+config_opts['plugin_conf']['compress_logs_enable'] = True
+config_opts['plugin_conf']['compress_logs_opts'] = {}
+config_opts['plugin_conf']['compress_logs_opts']['command'] =
"/usr/bin/gzip -9"
diff --git a/backend/conf/playbooks/provision_builder_pb.yml
b/backend/conf/playbooks/provision_builder_pb.yml
new file mode 100644
index 0000000..f2f94bf
--- /dev/null
+++ b/backend/conf/playbooks/provision_builder_pb.yml
@@ -0,0 +1,16 @@
+- hosts: localhost
+ gather_facts: False
+ tasks:
+ - local_action: add_host hostname={{ vm_ip }} groupname=builder_temp_group
+
+- hosts: builder_temp_group
+ user: fedora
+ sudo: true
+ gather_facts: True
+ #vars:
+ #- files: files/
+
+ tasks:
+ # - debug: msg="test"
+ - include: "provision_builder_tasks.yml"
+
diff --git a/backend/conf/playbooks/provision_builder_tasks.yml
b/backend/conf/playbooks/provision_builder_tasks.yml
new file mode 100644
index 0000000..7183692
--- /dev/null
+++ b/backend/conf/playbooks/provision_builder_tasks.yml
@@ -0,0 +1,62 @@
+- name: install pkgs
+ yum: state=present pkg={{ item }}
+ with_items:
+ - dnf
+ - mock
+ - mock-lvm
+ - createrepo
+ - yum-utils
+ - pyliblzma
+ - rsync
+ - openssh-clients
+ - rsync
+ - libselinux-python
+ - libsemanage-python
+
+- name: make sure newest rpm
+ yum: name={{ item }} state=latest
+ with_items:
+ - rpm
+ - glib2
+ - ca-certificates
+
+- name: put updated mock configs into /etc/mock
+ copy: src=files/{{ item }} dest=/etc/mock
+ with_items:
+ - site-defaults.cfg
+
+- name: prepare caches
+ async: 3600
+ command: mock -r {{ item }} --init
+ with_items:
+ - epel-5-i386
+ - epel-5-x86_64
+ - epel-6-i386
+ - epel-6-x86_64
+ - epel-7-x86_64
+ - fedora-20-i386
+ - fedora-20-x86_64
+ - fedora-21-i386
+ - fedora-21-x86_64
+ - fedora-22-i386
+ - fedora-22-x86_64
+ - fedora-rawhide-i386
+ - fedora-rawhide-x86_64
+
+- name: mockbuilder user
+ user: name=mockbuilder groups=mock
+
+- name: mockbuilder .ssh
+ file: state=directory path=/home/mockbuilder/.ssh mode=0700 owner=mockbuilder
group=mockbuilder
+
+- name: mockbuilder authorized_keys
+ authorized_key: user=mockbuilder key='{{ lookup('file',
'/home/copr/provision/files/buildsys.pub') }}'
+
+- name: root authorized_keys
+ authorized_key: user=root key='{{ lookup('file',
'/home/copr/provision/files/buildsys.pub') }}'
+
+- lineinfile: dest=/etc/mock/fedora-rawhide-x86_64.cfg
line="config_opts['package_manager'] = 'dnf'" state=absent
+- lineinfile: dest=/etc/mock/fedora-rawhide-i386.cfg
line="config_opts['package_manager'] = 'dnf'" state=absent
+
+- lineinfile: dest=/etc/security/limits.conf line="* soft nofile 10240"
insertafter=EOF
+- lineinfile: dest=/etc/security/limits.conf line="* hard nofile 10240"
insertafter=EOF
diff --git a/backend/conf/playbooks/spawn_new_tasks.yml
b/backend/conf/playbooks/spawn_new_tasks.yml
new file mode 100644
index 0000000..802c0d8
--- /dev/null
+++ b/backend/conf/playbooks/spawn_new_tasks.yml
@@ -0,0 +1,22 @@
+- name: spin UP VM using nova_compute
+ sudo: False
+ local_action:
+ module: nova_compute
+ insecure: True
+ auth_url: "{{os_auth_url}}"
+ login_username: "admin"
+ login_password: "{{ADMIN_PASS}}"
+ login_tenant_name: "{{inventory_tenant}}"
+ name: "{{inventory_instance_name}}"
+ image_id: "{{ image }}"
+ wait_for: 300
+ flavor_id: "{{ flavor_id }}"
+ security_groups: "{{security_group}}"
+ key_name: "{{ keypair }}"
+ nics: "{{ cloud_networks }}"
+ # floating_ip_pools: "{{ floating_ip_pools }}"
+ floating_ips:
+ - "{{public_ip}}"
+ register: nova_result
+
+- debug: msg="{{ nova_result.info }}"
diff --git a/builder_image/requirements.txt b/builder_image/requirements.txt
new file mode 100644
index 0000000..7f4b3d2
--- /dev/null
+++ b/builder_image/requirements.txt
@@ -0,0 +1,6 @@
+ipdb
+ansible
+python-novaclient
+python-glanceclient
+python-keystoneclient
+# neutronclient
diff --git a/builder_image/src/__init__.py b/builder_image/src/__init__.py
new file mode 100644
index 0000000..57d631c
--- /dev/null
+++ b/builder_image/src/__init__.py
@@ -0,0 +1 @@
+# coding: utf-8
diff --git a/builder_image/src/config.py b/builder_image/src/config.py
new file mode 100644
index 0000000..d738fc0
--- /dev/null
+++ b/builder_image/src/config.py
@@ -0,0 +1,34 @@
+# coding: utf-8
+
+
+class Config(object):
+ OS_AUTH_URL = "https://fed-cloud09.cloud.fedoraproject.org:5000/v2.0"
+
+ # With the addition of Keystone we have standardized on the term **tenant**
+ # as the entity that owns the resources.
+ # OS_TENANT_ID = "a3fa9697346046ef9e691a21527923a5"
+ # OS_TENANT_NAME = "copr"
+ OS_TENANT_ID = "566a072fb1694950998ad191fee3833b"
+ OS_TENANT_NAME = "coprdev"
+
+ # In addition to the owning entity (tenant), openstack stores the entity
+ # performing the action as the **user**.
+ OS_USERNAME = "username"
+ OS_PASSWORD = "password"
+
+ # If your configuration has multiple regions, we set that information here.
+ # OS_REGION_NAME is optional and only valid in certain environments.
+ OS_REGION_NAME = "RegionOne"
+
+ flavor_name = "m1.small"
+
+ vm_name = "builder_base_image"
+ image_name = "Fedora-Cloud-Base-20141203-21"
+ key_name = "fas"
+
+ net_name_list = ["coprdev-net", "copr-net",
"external"]
+ security_groups = ["default", "ssh-anywhere-coprdev"]
+
+ provision_pb_path = ""
+
+ floating_ips_pool = "external"
diff --git a/builder_image/src/copr_prepare_builder_image.py
b/builder_image/src/copr_prepare_builder_image.py
new file mode 100644
index 0000000..adf1f2d
--- /dev/null
+++ b/builder_image/src/copr_prepare_builder_image.py
@@ -0,0 +1,88 @@
+# coding: utf-8
+
+
+from ansible import errors, runner
+from glanceclient import Client as GlanceClient
+from keystoneclient import session
+from keystoneclient.auth.identity import v2 as identity
+# from neutronclient.neutron.client import Client as NeutronClient
+from novaclient.v2.client import Client
+import glanceclient.exc
+import json
+import novaclient.exceptions
+
+from config import Config
+
+
+class ImageBuilder(object):
+
+ """
+
+ :type config: Config
+ """
+ def __init__(self, config):
+ self.config = config
+ self.vm_ip = None
+
+ self.nova_result = None
+ self.float_ip = None
+
+ def post_init(self):
+ self.nt = Client(
+ self.config.OS_USERNAME,
+ self.config.OS_PASSWORD,
+ self.config.OS_TENANT_NAME,
+ self.config.OS_AUTH_URL,
+ insecure=True,
+ )
+
+ def spawn_vm(self):
+ if len(self.nt.servers.findall(name=self.config.vm_name)) > 0:
+ raise RuntimeError("Server already exists")
+
+ nics = [
+ {"net-id": net.id}
+ for net in [
+ self.nt.networks.find(human_id=net_name)
+ for net_name in self.config.net_name_list
+ ]]
+ self.nova_result = self.nt.servers.create(
+ name=self.config.vm_name,
+ flavor=self.nt.flavors.find(name=self.config.flavor_name),
+ image=self.nt.images.find(name=self.config.image_name),
+ key_name=self.config.key_name,
+ nics=nics,
+ security_groups=self.config.security_groups,
+ )
+
+ if getattr(self.config, "floating_ips_pool"):
+ self.float_ip =
self.nt.floating_ips.create(pool=self.config.floating_ips_pool)
+ self.nova_result.add_floating_ip(self.float_ip)
+
+ def run_provision(self):
+ pass
+
+ def create_image_from_vm(self, image_name):
+ pass
+
+
+ def terminate_vm(self):
+ if self.nova_result:
+ self.nova_result.delete()
+ if self.float_ip:
+ self.float_ip.delete()
+
+
+def main():
+ ib = ImageBuilder(Config())
+ try:
+ ib.post_init()
+
+
+
+ finally:
+ ib.terminate_vm()
+
+
+if __name__ == "__main__":
+ main()