[copr] master: [frontend] Bug 1206562 - Cannot delete Copr because it incorrectly thinks there are unfinished builds. Solution: `failed` but unfinished (ended_on is null) builds should be rescheduled. (cddbf44)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit cddbf44de7f8f05bf3fcedc58f7d1dde567948d3
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Mon Mar 30 14:23:59 2015 +0200
[frontend] Bug 1206562 - Cannot delete Copr because it incorrectly thinks there are unfinished builds. Solution: `failed` but unfinished (ended_on is null) builds should be rescheduled.
>---------------------------------------------------------------
.../coprs_frontend/coprs/logic/builds_logic.py | 8 +++++---
1 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/frontend/coprs_frontend/coprs/logic/builds_logic.py b/frontend/coprs_frontend/coprs/logic/builds_logic.py
index 2712e48..b088ef9 100644
--- a/frontend/coprs_frontend/coprs/logic/builds_logic.py
+++ b/frontend/coprs_frontend/coprs/logic/builds_logic.py
@@ -59,7 +59,8 @@ class BuildsLogic(object):
models.BuildChroot.status == helpers.StatusEnum("pending"),
models.BuildChroot.status == helpers.StatusEnum("starting"),
and_(
- models.BuildChroot.status == helpers.StatusEnum("running"),
+ models.BuildChroot.status.in_([
+ helpers.StatusEnum("running"), helpers.StatusEnum("failed")]),
models.Build.started_on < int(time.time() - 1.1 * MAX_BUILD_TIMEOUT),
models.Build.ended_on.is_(None)
)
@@ -218,11 +219,12 @@ class BuildsLogic(object):
"""
if not user.can_build_in(build.copr):
raise exceptions.InsufficientRightsException(
- "You are not allowed to delete this build.")
+ "You are not allowed to delete build `{}`.".format(build.id))
if not build.deletable:
+ # from celery.contrib import rdb; rdb.set_trace()
raise exceptions.ActionInProgressException(
- "You can not delete build which is not finished.",
+ "You can not delete build `{}` which is not finished.".format(build.id),
"Unfinished build")
# Only failed, finished, succeeded get here.
9 years
[copr] master: [image_builder] initial release, due to OS bug, we cannot create snapshot after provision through API, need to do it manually in the WebUI. (daaaf9d)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit daaaf9d3adb5161fa78106065637479913f40d07
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Wed Mar 25 15:11:36 2015 +0100
[image_builder] initial release, due to OS bug, we cannot create snapshot after provision through API, need to do it manually in the WebUI.
>---------------------------------------------------------------
backend/conf/playbooks/files/site-defaults.cfg | 156 ++++++++++++++++++++
backend/conf/playbooks/provision_builder_pb.yml | 16 ++
backend/conf/playbooks/provision_builder_tasks.yml | 62 ++++++++
backend/conf/playbooks/spawn_new_tasks.yml | 22 +++
builder_image/requirements.txt | 6 +
builder_image/src/__init__.py | 1 +
builder_image/src/config.py | 34 +++++
builder_image/src/copr_prepare_builder_image.py | 88 +++++++++++
8 files changed, 385 insertions(+), 0 deletions(-)
diff --git a/backend/conf/playbooks/files/site-defaults.cfg b/backend/conf/playbooks/files/site-defaults.cfg
new file mode 100644
index 0000000..15ebd36
--- /dev/null
+++ b/backend/conf/playbooks/files/site-defaults.cfg
@@ -0,0 +1,156 @@
+# mock defaults
+# vim:tw=0:ts=4:sw=4:et:
+#
+# This config file is for site-specific default values that apply across all
+# configurations. Options specified in this config file can be overridden in
+# the individual mock config files.
+#
+# The site-defaults.cfg delivered by default has NO options set. Only set
+# options here if you want to override the defaults.
+#
+# Entries in this file follow the same format as other mock config files.
+# config_opts['foo'] = bar
+
+#############################################################################
+#
+# Things that we recommend you set in site-defaults.cfg:
+#
+# config_opts['basedir'] = '/var/lib/mock/'
+# config_opts['cache_topdir'] = '/var/cache/mock'
+# Note: the path pointed to by basedir and cache_topdir must be owned
+# by group 'mock' and must have mode: g+rws
+# config_opts['rpmbuild_timeout'] = 0
+# config_opts['use_host_resolv'] = True
+
+# You can configure log format to pull from logging.ini formats of these names:
+# config_opts['build_log_fmt_name'] = "unadorned"
+# config_opts['root_log_fmt_name'] = "detailed"
+# config_opts['state_log_fmt_name'] = "state"
+#
+# mock will normally set up a minimal chroot /dev.
+# If you want to use a pre-configured /dev, disable this and use the bind-mount
+# plugin to mount your special /dev
+# config_opts['internal_dev_setup'] = True
+#
+# internal_setarch defaults to 'True' if the python 'ctypes' package is
+# available. It is in the python std lib on >= python 2.5. On older versions,
+# it is available as an addon. On systems w/o ctypes, it will default to 'False'
+# config_opts['internal_setarch'] = False
+#
+# the cleanup_on_* options allow you to automatically clean and remove the
+# mock build directory, but only take effect if --resultdir is used.
+# config_opts provides fine-grained control. cmdline only has big hammer
+#
+# config_opts['cleanup_on_success'] = 1
+# config_opts['cleanup_on_failure'] = 1
+
+# if you want mock to automatically run createrepo on the rpms in your
+# resultdir.
+# config_opts['createrepo_on_rpms'] = False
+# config_opts['createrepo_command'] = '/usr/bin/createrepo -d -q -x *.src.rpm'
+
+#############################################################################
+#
+# plugin related. Below are the defaults. Change to suit your site
+# policy. site-defaults.cfg is a good place to do this.
+#
+# NOTE: Some of the caching options can theoretically affect build
+# reproducability. Change with care.
+#
+config_opts['plugin_conf']['package_state_enable'] = False
+config_opts['plugin_conf']['ccache_enable'] = False
+# config_opts['plugin_conf']['ccache_opts']['max_cache_size'] = '4G'
+# config_opts['plugin_conf']['ccache_opts']['compress'] = None
+# config_opts['plugin_conf']['ccache_opts']['dir'] = "%(cache_topdir)s/%(root)s/ccache/"
+config_opts['plugin_conf']['yum_cache_enable'] = False
+# config_opts['plugin_conf']['yum_cache_opts']['max_age_days'] = 30
+# config_opts['plugin_conf']['yum_cache_opts']['dir'] = "%(cache_topdir)s/%(root)s/yum_cache/"
+config_opts['plugin_conf']['root_cache_enable'] = False
+# config_opts['plugin_conf']['root_cache_opts']['max_age_days'] = 15
+# config_opts['plugin_conf']['root_cache_opts']['dir'] = "%(cache_topdir)s/%(root)s/root_cache/"
+# config_opts['plugin_conf']['root_cache_opts']['compress_program'] = "pigz"
+# config_opts['plugin_conf']['root_cache_opts']['extension'] = ".gz"
+# config_opts['plugin_conf']['root_cache_opts']['exclude_dirs'] = ["./proc", "./sys", "./dev",
+# "./tmp/ccache", "./var/cache/yum" ]
+#
+# bind mount plugin is enabled by default but has no configured directories to
+# mount
+config_opts['plugin_conf']['bind_mount_enable'] = False
+# config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('/host/path', '/bind/mount/path/in/chroot/' ))
+#
+# config_opts['plugin_conf']['tmpfs_enable'] = False
+# config_opts['plugin_conf']['tmpfs_opts']['required_ram_mb'] = 1024
+# config_opts['plugin_conf']['tmpfs_opts']['max_fs_size'] = '512m'
+
+#############################################################################
+#
+# environment for chroot
+#
+# config_opts['environment']['TERM'] = 'vt100'
+# config_opts['environment']['SHELL'] = '/bin/bash'
+# config_opts['environment']['HOME'] = '/builddir'
+# config_opts['environment']['HOSTNAME'] = 'mock'
+# config_opts['environment']['PATH'] = '/usr/bin:/bin:/usr/sbin:/sbin'
+# config_opts['environment']['PROMPT_COMMAND'] = 'echo -n "<mock-chroot>"'
+# config_opts['environment']['LANG'] = os.environ.setdefault('LANG', 'en_US.UTF-8')
+# config_opts['environment']['TZ'] = os.environ.setdefault('TZ', 'EST5EDT')
+
+#############################################################################
+#
+# Things that you can change, but we dont recommend it:
+# config_opts['chroothome'] = '/builddir'
+# config_opts['clean'] = True
+
+#############################################################################
+#
+# Things that must be adjusted if SCM integration is used:
+#
+# config_opts['scm'] = True
+# config_opts['scm_opts']['method'] = 'git'
+# config_opts['scm_opts']['cvs_get'] = 'cvs -d /srv/cvs co SCM_BRN SCM_PKG'
+# config_opts['scm_opts']['git_get'] = 'git clone SCM_BRN git://localhost/SCM_PKG.git SCM_PKG'
+# config_opts['scm_opts']['svn_get'] = 'svn co file:///srv/svn/SCM_PKG/SCM_BRN SCM_PKG'
+# config_opts['scm_opts']['spec'] = 'SCM_PKG.spec'
+# config_opts['scm_opts']['ext_src_dir'] = '/dev/null'
+# config_opts['scm_opts']['write_tar'] = True
+# config_opts['scm_opts']['git_timestamps'] = True
+
+# These options are also recognized but usually defined in cmd line
+# with --scm-option package=<pkg> --scm-option branch=<branch>
+# config_opts['scm_opts']['package'] = 'mypkg'
+# config_opts['scm_opts']['branch'] = 'master'
+
+#############################################################################
+#
+# Things that are best suited for individual chroot config files:
+#
+# MUST SET (in individual chroot cfg file):
+# config_opts['root'] = 'name-of-yum-build-dir'
+# config_opts['target_arch'] = 'i386'
+# config_opts['yum.conf'] = ''
+# config_opts['yum_common_opts'] = []
+#
+# CAN SET, defaults usually work ok:
+# config_opts['chroot_setup_cmd'] = 'install buildsys-build'
+# config_opts['log_config_file'] = 'logging.ini'
+# config_opts['more_buildreqs']['srpm_name-version-release'] = 'dependencies'
+# config_opts['macros']['%Add_your_macro_name_here'] = "add macro value here"
+# config_opts['files']['path/name/no/leading/slash'] = "put file contents here."
+# config_opts['chrootuid'] = os.getuid()
+
+# If you change chrootgid, you must also change "mock" to the correct group
+# name in this line of the mock PAM config:
+# auth sufficient pam_succeed_if.so user ingroup mock use_uid quiet
+# config_opts['chrootgid'] = grp.getgrnam("mock")[2]
+
+# config_opts['useradd'] = '/usr/sbin/useradd -m -u %(uid)s -g %(gid)s -d %(home)s -n %(user)s' # Fedora/RedHat
+#
+# Security related
+# config_opts['no_root_shells'] = False
+#
+# Proxy settings (https_proxy, ftp_proxy, and no_proxy can also be set)
+# config_opts['http_proxy'] = 'http://localhost:3128'
+
+config_opts['plugin_conf']['compress_logs_enable'] = True
+config_opts['plugin_conf']['compress_logs_opts'] = {}
+config_opts['plugin_conf']['compress_logs_opts']['command'] = "/usr/bin/gzip -9"
diff --git a/backend/conf/playbooks/provision_builder_pb.yml b/backend/conf/playbooks/provision_builder_pb.yml
new file mode 100644
index 0000000..f2f94bf
--- /dev/null
+++ b/backend/conf/playbooks/provision_builder_pb.yml
@@ -0,0 +1,16 @@
+- hosts: localhost
+ gather_facts: False
+ tasks:
+ - local_action: add_host hostname={{ vm_ip }} groupname=builder_temp_group
+
+- hosts: builder_temp_group
+ user: fedora
+ sudo: true
+ gather_facts: True
+ #vars:
+ #- files: files/
+
+ tasks:
+ # - debug: msg="test"
+ - include: "provision_builder_tasks.yml"
+
diff --git a/backend/conf/playbooks/provision_builder_tasks.yml b/backend/conf/playbooks/provision_builder_tasks.yml
new file mode 100644
index 0000000..7183692
--- /dev/null
+++ b/backend/conf/playbooks/provision_builder_tasks.yml
@@ -0,0 +1,62 @@
+- name: install pkgs
+ yum: state=present pkg={{ item }}
+ with_items:
+ - dnf
+ - mock
+ - mock-lvm
+ - createrepo
+ - yum-utils
+ - pyliblzma
+ - rsync
+ - openssh-clients
+ - rsync
+ - libselinux-python
+ - libsemanage-python
+
+- name: make sure newest rpm
+ yum: name={{ item }} state=latest
+ with_items:
+ - rpm
+ - glib2
+ - ca-certificates
+
+- name: put updated mock configs into /etc/mock
+ copy: src=files/{{ item }} dest=/etc/mock
+ with_items:
+ - site-defaults.cfg
+
+- name: prepare caches
+ async: 3600
+ command: mock -r {{ item }} --init
+ with_items:
+ - epel-5-i386
+ - epel-5-x86_64
+ - epel-6-i386
+ - epel-6-x86_64
+ - epel-7-x86_64
+ - fedora-20-i386
+ - fedora-20-x86_64
+ - fedora-21-i386
+ - fedora-21-x86_64
+ - fedora-22-i386
+ - fedora-22-x86_64
+ - fedora-rawhide-i386
+ - fedora-rawhide-x86_64
+
+- name: mockbuilder user
+ user: name=mockbuilder groups=mock
+
+- name: mockbuilder .ssh
+ file: state=directory path=/home/mockbuilder/.ssh mode=0700 owner=mockbuilder group=mockbuilder
+
+- name: mockbuilder authorized_keys
+ authorized_key: user=mockbuilder key='{{ lookup('file', '/home/copr/provision/files/buildsys.pub') }}'
+
+- name: root authorized_keys
+ authorized_key: user=root key='{{ lookup('file', '/home/copr/provision/files/buildsys.pub') }}'
+
+- lineinfile: dest=/etc/mock/fedora-rawhide-x86_64.cfg line="config_opts['package_manager'] = 'dnf'" state=absent
+- lineinfile: dest=/etc/mock/fedora-rawhide-i386.cfg line="config_opts['package_manager'] = 'dnf'" state=absent
+
+- lineinfile: dest=/etc/security/limits.conf line="* soft nofile 10240" insertafter=EOF
+- lineinfile: dest=/etc/security/limits.conf line="* hard nofile 10240" insertafter=EOF
diff --git a/backend/conf/playbooks/spawn_new_tasks.yml b/backend/conf/playbooks/spawn_new_tasks.yml
new file mode 100644
index 0000000..802c0d8
--- /dev/null
+++ b/backend/conf/playbooks/spawn_new_tasks.yml
@@ -0,0 +1,22 @@
+- name: spin UP VM using nova_compute
+ sudo: False
+ local_action:
+ module: nova_compute
+ insecure: True
+ auth_url: "{{os_auth_url}}"
+ login_username: "admin"
+ login_password: "{{ADMIN_PASS}}"
+ login_tenant_name: "{{inventory_tenant}}"
+ name: "{{inventory_instance_name}}"
+ image_id: "{{ image }}"
+ wait_for: 300
+ flavor_id: "{{ flavor_id }}"
+ security_groups: "{{security_group}}"
+ key_name: "{{ keypair }}"
+ nics: "{{ cloud_networks }}"
+ # floating_ip_pools: "{{ floating_ip_pools }}"
+ floating_ips:
+ - "{{public_ip}}"
+ register: nova_result
+
+- debug: msg="{{ nova_result.info }}"
diff --git a/builder_image/requirements.txt b/builder_image/requirements.txt
new file mode 100644
index 0000000..7f4b3d2
--- /dev/null
+++ b/builder_image/requirements.txt
@@ -0,0 +1,6 @@
+ipdb
+ansible
+python-novaclient
+python-glanceclient
+python-keystoneclient
+# neutronclient
diff --git a/builder_image/src/__init__.py b/builder_image/src/__init__.py
new file mode 100644
index 0000000..57d631c
--- /dev/null
+++ b/builder_image/src/__init__.py
@@ -0,0 +1 @@
+# coding: utf-8
diff --git a/builder_image/src/config.py b/builder_image/src/config.py
new file mode 100644
index 0000000..d738fc0
--- /dev/null
+++ b/builder_image/src/config.py
@@ -0,0 +1,34 @@
+# coding: utf-8
+
+
+class Config(object):
+ OS_AUTH_URL = "https://fed-cloud09.cloud.fedoraproject.org:5000/v2.0"
+
+ # With the addition of Keystone we have standardized on the term **tenant**
+ # as the entity that owns the resources.
+ # OS_TENANT_ID = "a3fa9697346046ef9e691a21527923a5"
+ # OS_TENANT_NAME = "copr"
+ OS_TENANT_ID = "566a072fb1694950998ad191fee3833b"
+ OS_TENANT_NAME = "coprdev"
+
+ # In addition to the owning entity (tenant), openstack stores the entity
+ # performing the action as the **user**.
+ OS_USERNAME = "username"
+ OS_PASSWORD = "password"
+
+ # If your configuration has multiple regions, we set that information here.
+ # OS_REGION_NAME is optional and only valid in certain environments.
+ OS_REGION_NAME = "RegionOne"
+
+ flavor_name = "m1.small"
+
+ vm_name = "builder_base_image"
+ image_name = "Fedora-Cloud-Base-20141203-21"
+ key_name = "fas"
+
+ net_name_list = ["coprdev-net", "copr-net", "external"]
+ security_groups = ["default", "ssh-anywhere-coprdev"]
+
+ provision_pb_path = ""
+
+ floating_ips_pool = "external"
diff --git a/builder_image/src/copr_prepare_builder_image.py b/builder_image/src/copr_prepare_builder_image.py
new file mode 100644
index 0000000..adf1f2d
--- /dev/null
+++ b/builder_image/src/copr_prepare_builder_image.py
@@ -0,0 +1,88 @@
+# coding: utf-8
+
+
+from ansible import errors, runner
+from glanceclient import Client as GlanceClient
+from keystoneclient import session
+from keystoneclient.auth.identity import v2 as identity
+# from neutronclient.neutron.client import Client as NeutronClient
+from novaclient.v2.client import Client
+import glanceclient.exc
+import json
+import novaclient.exceptions
+
+from config import Config
+
+
+class ImageBuilder(object):
+
+ """
+
+ :type config: Config
+ """
+ def __init__(self, config):
+ self.config = config
+ self.vm_ip = None
+
+ self.nova_result = None
+ self.float_ip = None
+
+ def post_init(self):
+ self.nt = Client(
+ self.config.OS_USERNAME,
+ self.config.OS_PASSWORD,
+ self.config.OS_TENANT_NAME,
+ self.config.OS_AUTH_URL,
+ insecure=True,
+ )
+
+ def spawn_vm(self):
+ if len(self.nt.servers.findall(name=self.config.vm_name)) > 0:
+ raise RuntimeError("Server already exists")
+
+ nics = [
+ {"net-id": net.id}
+ for net in [
+ self.nt.networks.find(human_id=net_name)
+ for net_name in self.config.net_name_list
+ ]]
+ self.nova_result = self.nt.servers.create(
+ name=self.config.vm_name,
+ flavor=self.nt.flavors.find(name=self.config.flavor_name),
+ image=self.nt.images.find(name=self.config.image_name),
+ key_name=self.config.key_name,
+ nics=nics,
+ security_groups=self.config.security_groups,
+ )
+
+ if getattr(self.config, "floating_ips_pool"):
+ self.float_ip = self.nt.floating_ips.create(pool=self.config.floating_ips_pool)
+ self.nova_result.add_floating_ip(self.float_ip)
+
+ def run_provision(self):
+ pass
+
+ def create_image_from_vm(self, image_name):
+ pass
+
+
+ def terminate_vm(self):
+ if self.nova_result:
+ self.nova_result.delete()
+ if self.float_ip:
+ self.float_ip.delete()
+
+
+def main():
+ ib = ImageBuilder(Config())
+ try:
+ ib.post_init()
+
+
+
+ finally:
+ ib.terminate_vm()
+
+
+if __name__ == "__main__":
+ main()
9 years, 1 month
[copr] master: [backend] tests cleanup (bf311c8)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit bf311c8767831f4453256ba07f0a4400b3489716
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Tue Mar 24 16:28:11 2015 +0100
[backend] tests cleanup
>---------------------------------------------------------------
backend/backend/daemons/vm_master.py | 4 +++-
backend/backend/mockremote/builder.py | 6 +++---
backend/backend/vm_manage/spawn.py | 2 +-
backend/conf/copr-be.conf.example | 2 +-
backend/copr-backend.spec | 9 +++++----
backend/run/copr_get_vm_info.py | 3 +++
backend/tests/mockremote/test_builder.py | 21 +++++++++++++++++----
backend/tests/run/test_copr_prune_results.py | 6 ++----
8 files changed, 35 insertions(+), 18 deletions(-)
diff --git a/backend/backend/daemons/vm_master.py b/backend/backend/daemons/vm_master.py
index 0fa49f2..ad1fd70 100644
--- a/backend/backend/daemons/vm_master.py
+++ b/backend/backend/daemons/vm_master.py
@@ -84,6 +84,8 @@ class VmMaster(Process):
self.vmm.start_vm_check(vmd.vm_name)
def try_spawn_one(self, group):
+ # TODO: add setting "max_vm_in_ready_state", when this number reached, do not spawn more VMS, min value = 1
+
max_vm_total = self.opts.build_groups[group]["max_vm_total"]
active_vmd_list = self.vmm.get_vm_by_group_and_state_list(
group, [VmStates.GOT_IP, VmStates.READY, VmStates.IN_USE, VmStates.CHECK_HEALTH])
@@ -92,7 +94,7 @@ class VmMaster(Process):
total_vm_estimation = len(active_vmd_list) + self.vmm.spawner.children_number
if total_vm_estimation >= max_vm_total:
self.log("Skip spawn: max total vm reached for group {}: vm count: {}, spawn process: {}"
- .format(group, len(active_vmd_list), self.opts.build_groups[group]["max_vm_total"]))
+ .format(group, len(active_vmd_list), self.vmm.spawner.children_number))
return
last_vm_spawn_start = self.vmm.rc.hget(KEY_VM_POOL_INFO.format(group=group), "last_vm_spawn_start")
if last_vm_spawn_start:
diff --git a/backend/backend/mockremote/builder.py b/backend/backend/mockremote/builder.py
index 064c7fb..a22f2a3 100644
--- a/backend/backend/mockremote/builder.py
+++ b/backend/backend/mockremote/builder.py
@@ -354,10 +354,10 @@ class Builder(object):
def check(self):
# do check of host
try:
+ # requires name resolve facility
socket.gethostbyname(self.hostname)
- except socket.gaierror:
- raise BuilderError("{0} could not be resolved".format(
- self.hostname))
+ except IOError:
+ raise BuilderError("{0} could not be resolved".format(self.hostname))
try:
# check_for_ans_error(res, self.hostname)
diff --git a/backend/backend/vm_manage/spawn.py b/backend/backend/vm_manage/spawn.py
index 77658be..38a6e64 100644
--- a/backend/backend/vm_manage/spawn.py
+++ b/backend/backend/vm_manage/spawn.py
@@ -143,7 +143,7 @@ class Spawner(Executor):
def start_spawn(self, group):
self.recycle()
- spawn_playbook = None
+ # spawn_playbook = None
try:
spawn_playbook = self.opts.build_groups[group]["spawn_playbook"]
except KeyError:
diff --git a/backend/conf/copr-be.conf.example b/backend/conf/copr-be.conf.example
index 70e78ac..4a34206 100644
--- a/backend/conf/copr-be.conf.example
+++ b/backend/conf/copr-be.conf.example
@@ -41,7 +41,7 @@ group0_archs=i386,x86_64
group0_spawn_playbook=/srv/copr-work/provision/builderpb-PC.yml
group0_terminate_playbook=/srv/copr-work/provision/terminatepb-PC.yml
group0_max_workers=8
-
+group0_max_vm_total=2
# That variables will be send to
# terminate_playbook ansible script.
diff --git a/backend/copr-backend.spec b/backend/copr-backend.spec
index 45e5fff..e6a2b5c 100644
--- a/backend/copr-backend.spec
+++ b/backend/copr-backend.spec
@@ -20,15 +20,14 @@ BuildArch: noarch
BuildRequires: asciidoc
BuildRequires: libxslt
BuildRequires: util-linux
+BuildRequires: yum-utils
BuildRequires: python-setuptools
BuildRequires: python-requests
BuildRequires: python2-devel
BuildRequires: python-copr
BuildRequires: systemd
BuildRequires: redis
-%if 0%{?rhel} < 7 && 0%{?rhel} > 0
-BuildRequires: python-argparse
-%endif
+
#for doc package
BuildRequires: epydoc
BuildRequires: graphviz
@@ -48,6 +47,7 @@ BuildRequires: ansible >= 1.2
BuildRequires: python-IPy
BuildRequires: python-paramiko
BuildRequires: python-psutil
+# BuildRequires: python-ipdb
BuildRequires: wget
Requires: obs-signd
@@ -75,6 +75,7 @@ Requires: fedmsg
Requires: gawk
Requires: crontabs
Requires: python-paramiko
+# Requires: python-ipdb
# Requires: logstash
Requires(post): systemd
@@ -166,7 +167,7 @@ cp -a conf/playbooks %{buildroot}%{_pkgdocdir}/
redis-server --port 7777 &> /dev/null &
PYTHONPATH=backend:run:$PYTHONPATH python -B -m pytest \
- -s --cov-report term-missing --cov ./backend --cov ./run ./tests/
+ -s -v --cov-report term-missing --cov ./backend --cov ./run ./tests/
kill %1
diff --git a/backend/run/copr_get_vm_info.py b/backend/run/copr_get_vm_info.py
index 3d80611..bf9f3e6 100644
--- a/backend/run/copr_get_vm_info.py
+++ b/backend/run/copr_get_vm_info.py
@@ -1,5 +1,8 @@
# coding: utf-8
+import sys
+sys.path.append("/usr/share/copr/")
+
from backend.helpers import BackendConfigReader
from backend.vm_manage.manager import VmManager
diff --git a/backend/tests/mockremote/test_builder.py b/backend/tests/mockremote/test_builder.py
index 8a414f9..f2da9b3 100644
--- a/backend/tests/mockremote/test_builder.py
+++ b/backend/tests/mockremote/test_builder.py
@@ -3,6 +3,7 @@ import copy
from collections import defaultdict
from pprint import pprint
+import socket
from bunch import Bunch
from backend.exceptions import BuilderError, BuilderTimeOutError, AnsibleCallError, AnsibleResponseError, VmError
@@ -35,6 +36,13 @@ from backend.mockremote.builder import Builder
MODULE_REF = "backend.mockremote.builder"
+
+(a)pytest.yield_fixture
+def mc_socket():
+ with mock.patch("{}.socket".format(MODULE_REF)) as handle:
+ yield handle
+
+
def noop(*args, **kwargs):
pass
@@ -301,7 +309,8 @@ class TestBuilder(object):
results = {"contacted": {}, "dark": {}}
assert {} == builder_module.get_ans_results(results, self.BUILDER_HOSTNAME)
- def test_check_hostname_check(self):
+ def test_check_hostname_check(self, mc_socket):
+ mc_socket.gethostbyname.side_effect = socket.gaierror()
builder = self.get_test_builder()
for name in ["*", "256.0.0.1"]:
with pytest.raises(BuilderError):
@@ -309,8 +318,9 @@ class TestBuilder(object):
builder.hostname = name
builder.check()
- def test_check_missing_required_binaries(self):
+ def test_check_missing_required_binaries(self, mc_socket):
builder = self.get_test_builder()
+ self.stage = 0
def ans_run():
self.stage_ctx[self.stage]["conn"] = copy.deepcopy(builder.conn)
@@ -326,11 +336,12 @@ class TestBuilder(object):
builder.check()
# import ipdb; ipdb.set_trace()
+ # pprint(self.stage_ctx)
assert "/bin/rpm -q mock rsync" in self.stage_ctx[0]["conn"].module_args
assert "does not have mock or rsync installed" in err.value.msg
- def test_check_missing_mockchain_or_mock_config(self):
+ def test_check_missing_mockchain_or_mock_config(self, mc_socket):
builder = self.get_test_builder()
def ans_run():
@@ -346,13 +357,14 @@ class TestBuilder(object):
with pytest.raises(BuilderError) as err:
builder.check()
+ # pprint(self.stage_ctx)
assert "/usr/bin/test -f /usr/bin/mockchain" in self.stage_ctx[1]["conn"].module_args
# assert "/usr/bin/test -f /etc/mock/{}.cfg".format(self.BUILDER_CHROOT) in \
# self.stage_ctx[1]["conn"].module_args
assert "missing mockchain binary" in err.value.msg
- def test_check_missing_mock_config(self):
+ def test_check_missing_mock_config(self, mc_socket):
builder = self.get_test_builder()
ret_map = {
@@ -370,6 +382,7 @@ class TestBuilder(object):
with pytest.raises(BuilderError) as err:
builder.check()
+ # pprint(self.stage_ctx)
assert "/usr/bin/test -f /usr/bin/mockchain" in self.stage_ctx[1]["conn"].module_args
assert "/usr/bin/test -f /etc/mock/{}.cfg".format(self.BUILDER_CHROOT) in \
self.stage_ctx[2]["conn"].module_args
diff --git a/backend/tests/run/test_copr_prune_results.py b/backend/tests/run/test_copr_prune_results.py
index 25c4b80..1a4be8c 100644
--- a/backend/tests/run/test_copr_prune_results.py
+++ b/backend/tests/run/test_copr_prune_results.py
@@ -78,7 +78,7 @@ class TestPruneResults(object):
self.prj = "foox"
self.chroots = ["fedora-20-i386", "fedora-20-x86_64"]
self.opts = Bunch(
- days=14,
+ prune_days=14,
find_obsolete_script=os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir,
@@ -113,7 +113,7 @@ class TestPruneResults(object):
def make_temp_dir(self):
root_tmp_dir = tempfile.gettempdir()
- subdir = "test_action_{}".format(self.test_time)
+ subdir = "test_prune_old_{}".format(self.test_time)
self.tmp_dir_name = os.path.join(root_tmp_dir, subdir)
os.mkdir(self.tmp_dir_name)
@@ -300,7 +300,6 @@ class TestPruneResults(object):
assert mc_cru.called
-
def test_prune_project_skip_when_acr_disabled(self, test_pruner, mc_cru, mc_gacs):
self.pruner.prune_failed_builds = MagicMock()
self.pruner.prune_obsolete_success_builds = MagicMock()
@@ -318,7 +317,6 @@ class TestPruneResults(object):
self.pruner.run()
-
def test_main(self, mc_pruner, mc_bcr):
prune_main()
assert mc_pruner.called
9 years, 1 month
[copr] master: [backend] ensure that prune script running under the copr user; simpler `copr_find_obsolete_builds` (513441f)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 513441f93ca10e58cb61d262f0db57f0090171f0
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Tue Mar 24 14:09:34 2015 +0100
[backend] ensure that prune script running under the copr user; simpler `copr_find_obsolete_builds`
>---------------------------------------------------------------
backend/run/copr_find_obsolete_builds.sh | 3 +--
backend/run/copr_prune_results.py | 15 ++++++++++-----
2 files changed, 11 insertions(+), 7 deletions(-)
diff --git a/backend/run/copr_find_obsolete_builds.sh b/backend/run/copr_find_obsolete_builds.sh
index dcd3e34..e46df34 100755
--- a/backend/run/copr_find_obsolete_builds.sh
+++ b/backend/run/copr_find_obsolete_builds.sh
@@ -6,7 +6,7 @@ DAYS=$2
echo "# looking for obsolete files in: " $CHROOT_PATH
if [ -d $CHROOT_PATH ]
then
- pushd $CHROOT_PATH >/dev/null
+ cd $CHROOT_PATH
# query latest (sucessfull) packages
LATEST_PKGS=$(mktemp)
@@ -33,5 +33,4 @@ then
done
rm -f $LATEST_PKGS $ERR_LOG
- popd >/dev/null
fi
diff --git a/backend/run/copr_prune_results.py b/backend/run/copr_prune_results.py
index d9445fd..de96dc3 100644
--- a/backend/run/copr_prune_results.py
+++ b/backend/run/copr_prune_results.py
@@ -11,6 +11,7 @@ import sys
import logging
from subprocess import Popen, PIPE
import time
+import pwd
log = logging.getLogger(__name__)
@@ -156,8 +157,12 @@ def main():
pruner.run()
if __name__ == "__main__":
- logging.basicConfig(
- filename="/var/log/copr/prune_old.log",
- format='[%(asctime)s][%(levelname)6s]: %(message)s',
- level=logging.INFO)
- main()
+ if pwd.getpwuid(os.getuid())[0] != "copr":
+ print("This script should be executed under the `copr` user")
+ sys.exit(1)
+ else:
+ logging.basicConfig(
+ filename="/var/log/copr/prune_old.log",
+ format='[%(asctime)s][%(levelname)6s]: %(message)s',
+ level=logging.INFO)
+ main()
9 years, 1 month
[copr] master: [backend] safer copr_prune_results script, unittests (e539a10)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit e539a10a7a34d3f4ff983704470d30306a132d07
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Tue Mar 24 13:42:09 2015 +0100
[backend] safer copr_prune_results script, unittests
>---------------------------------------------------------------
backend/backend/createrepo.py | 2 +
backend/run/copr_prune_results.py | 28 ++--
backend/tests/_resources/foox.tar.gz | Bin 17135362 -> 6549985 bytes
backend/tests/run/test_copr_prune_results.py | 168 ++++++++++++++++++++++++--
4 files changed, 171 insertions(+), 27 deletions(-)
diff --git a/backend/backend/createrepo.py b/backend/backend/createrepo.py
index 092fa6c..c20656f 100644
--- a/backend/backend/createrepo.py
+++ b/backend/backend/createrepo.py
@@ -47,6 +47,8 @@ def createrepo_unsafe(path, lock=None, dest_dir=None, base_url=None):
cmd = Popen(comm, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = cmd.communicate()
+ # TODO: rewrite with raise CreateRepoException on returncode != 0
+ # and return only stdout on success
return cmd.returncode, out, err
diff --git a/backend/run/copr_prune_results.py b/backend/run/copr_prune_results.py
index 9f77f81..d9445fd 100644
--- a/backend/run/copr_prune_results.py
+++ b/backend/run/copr_prune_results.py
@@ -58,20 +58,19 @@ class Pruner(object):
log.info("Removing failed build: {}".format(build_path))
shutil.rmtree(build_path)
- def prune_obsolete_success_builds(self, chroot_path, username, projectname):
+ def prune_obsolete_success_builds(self, chroot_path):
"""
Uses bash script which invokes repoquery to find obsolete build_dirs
:param chroot_path: path to the chroot directory
- :return:
"""
# import ipdb; ipdb.set_trace()
cmd = map(str, [self.prune_sh_script, chroot_path, self.days])
handle = Popen(cmd, stdout=PIPE, stderr=PIPE)
stdout, stderr = handle.communicate()
if handle.returncode != 0:
- log.error("Failed to prune old builds for copr {}/{}\n STDOUT: \n{}\n STDERR: \n{}\n"
- .format(username, projectname, stdout.decode(), stderr.decode()))
+ log.error("Failed to prune old builds at: {} \n STDOUT: \n{}\n STDERR: \n{}\n"
+ .format(chroot_path, stdout.decode(), stderr.decode()))
return
# log.debug("find obsolete returned:\n {}\n stderr: \n {}".format(stdout, stderr))
for line in stdout.split("\n"):
@@ -121,31 +120,32 @@ class Pruner(object):
# run prune project sh
for sub_dir_name in os.listdir(project_path):
- if sub_dir_name == "repodata":
- continue
chroot_path = os.path.join(project_path, sub_dir_name)
if not os.path.isdir(chroot_path):
continue
try:
self.prune_failed_builds(chroot_path)
- self.prune_obsolete_success_builds(chroot_path, username, projectname)
+ self.prune_obsolete_success_builds(chroot_path)
except Exception as err:
log.exception(err)
- log.error("Error during prune copr {}/{}".format(username, projectname))
+ log.error("Error during prune copr {}/{}:{}".format(username, projectname, sub_dir_name))
- log.debug("Prune done for {}/{}".format(username, projectname))
+ log.debug("Prune done for {}/{}:{}".format(username, projectname, sub_dir_name))
# run createrepo
try:
- retcode, stdout, stderr = createrepo_unsafe(project_path)
+ retcode, stdout, stderr = createrepo_unsafe(chroot_path)
if retcode != 0:
log.error(
- "Createrepo for copr {}/{}\n STDOUT: \n{}\n STDERR: \n{}\n"
- .format(username, projectname, stdout.decode(), stderr.decode()))
+ "Failed to createrepo for copr {}/{}:{}\n STDOUT: \n{}\n STDERR: \n{}\n"
+ .format(username, projectname, sub_dir_name, stdout.decode(), stderr.decode()))
+ else:
+ log.info("Createrepo done for copr {}/{}:{}"
+ .format(username, projectname, sub_dir_name))
except Exception as exception:
- log.exception("Createrepo for copr {}/{} failed with error: {}"
- .format(username, projectname, exception))
+ log.exception("Createrepo for copr {}/{}:{} failed with error: {}"
+ .format(username, projectname, sub_dir_name, exception))
log.info("Prune finished for copr {}/{}".format(username, projectname))
diff --git a/backend/tests/_resources/foox.tar.gz b/backend/tests/_resources/foox.tar.gz
index aaebe8f..7b3c975 100644
Binary files a/backend/tests/_resources/foox.tar.gz and b/backend/tests/_resources/foox.tar.gz differ
diff --git a/backend/tests/run/test_copr_prune_results.py b/backend/tests/run/test_copr_prune_results.py
index 074e9dc..25c4b80 100644
--- a/backend/tests/run/test_copr_prune_results.py
+++ b/backend/tests/run/test_copr_prune_results.py
@@ -8,6 +8,7 @@ import tempfile
import time
from bunch import Bunch
from subprocess import Popen, PIPE
+from copr.client.exceptions import CoprException
import pytest
@@ -30,8 +31,28 @@ def mc_popen():
with mock.patch("{}.Popen".format(MODULE_REF)) as handle:
yield handle
+(a)pytest.yield_fixture
+def mc_bcr():
+ with mock.patch("{}.BackendConfigReader".format(MODULE_REF)) as handle:
+ yield handle
+
+(a)pytest.yield_fixture
+def mc_cru():
+ with mock.patch("{}.createrepo_unsafe".format(MODULE_REF)) as handle:
+ yield handle
+
+(a)pytest.yield_fixture
+def mc_gacs():
+ with mock.patch("{}.get_auto_createrepo_status".format(MODULE_REF)) as handle:
+ yield handle
+
+(a)pytest.yield_fixture
+def mc_pruner():
+ with mock.patch("{}.Pruner".format(MODULE_REF)) as handle:
+ yield handle
from copr_prune_results import Pruner
+from copr_prune_results import main as prune_main
def assert_same_dirs(a, b):
@@ -42,6 +63,7 @@ def assert_same_dirs(a, b):
raise AssertionError("Directories {}, {} aren't equal, diff out: \n {} \n {}"
.format(a, b, stdout, stderr))
+
class TestPruneResults(object):
def setup_method(self, method):
@@ -54,22 +76,24 @@ class TestPruneResults(object):
self.pkg_2 = "hello-2.8-1.fc20"
self.pkg_2_obsolete = "hello-1.0.fc20"
self.prj = "foox"
- self.chroots = ["fedora-20-i386", "fedora-20-x86_64",
- "fedora-21-i386", "fedora-21-x86_64"]
+ self.chroots = ["fedora-20-i386", "fedora-20-x86_64"]
self.opts = Bunch(
days=14,
find_obsolete_script=os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir,
- "run", "copr_find_obsolete_builds.sh"))
+ "run", "copr_find_obsolete_builds.sh")),
+
+ frontend_base_url="http://example.com",
+ destdir=self.tmp_dir_name
)
self.log_file_path = os.path.join(self.tmp_dir_name, "_unittest.log")
logging.basicConfig(
- # filename=self.log_file_path,
- stream=sys.stdout,
+ filename=self.log_file_path,
+ # stream=sys.stdout,
format='[%(asctime)s][%(levelname)6s]: %(message)s',
level=logging.DEBUG
)
@@ -161,10 +185,7 @@ class TestPruneResults(object):
os.utime(os.path.join(self.tmp_dir_name, self.prj, self.chroots[0], self.pkg_1_obsolete, "success"),
(time.time(), time.time()))
- self.pruner.prune_obsolete_success_builds(
- os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]),
- self.username, self.coprname
- )
+ self.pruner.prune_obsolete_success_builds(os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]))
assert_same_dirs(
os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]),
@@ -178,13 +199,134 @@ class TestPruneResults(object):
mc_handle.communicate.return_value = ("foo", "bar")
# doesn't touch FS if `find_obsolete_build` produce error return code
- self.pruner.prune_obsolete_success_builds(
- os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]),
- self.username, self.coprname
- )
+ self.pruner.prune_obsolete_success_builds(os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]))
assert mc_popen.called
assert_same_dirs(
os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]),
os.path.join(self.expect_dir_name, self.prj, self.chroots[0]),
)
+
+ def test_prune_project_ok(self, test_pruner, mc_cru, mc_gacs):
+ self.pruner.prune_failed_builds = MagicMock()
+ self.pruner.prune_obsolete_success_builds = MagicMock()
+ mc_cru.return_value = (0, "", "")
+
+ self.pruner.prune_project(os.path.join(self.tmp_dir_name, self.prj),
+ self.username, self.coprname)
+
+ expected_path_set = set([
+ os.path.join(self.tmp_dir_name, self.prj, chroot)
+ for chroot in self.chroots
+ ])
+ assert set([
+ call[0][0] for call in
+ self.pruner.prune_failed_builds.call_args_list
+ ]) == expected_path_set
+ assert set([
+ call[0][0] for call in
+ self.pruner.prune_obsolete_success_builds.call_args_list
+ ]) == expected_path_set
+ assert set([
+ call[0][0] for call in
+ mc_cru.call_args_list
+ ]) == expected_path_set
+
+
+ def test_prune_project_handle_gacs_error(self, test_pruner, mc_cru, mc_gacs):
+ self.pruner.prune_failed_builds = MagicMock()
+ self.pruner.prune_obsolete_success_builds = MagicMock()
+
+ mc_gacs.side_effect = CoprException()
+
+ self.pruner.prune_project(os.path.join(self.tmp_dir_name, self.prj),
+ self.username, self.coprname)
+
+ assert not self.pruner.prune_failed_builds.called
+ assert not self.pruner.prune_obsolete_success_builds.called
+
+ def test_prune_project_handle_errors(self, test_pruner, mc_cru, mc_gacs):
+ self.pruner.prune_failed_builds = MagicMock()
+ self.pruner.prune_obsolete_success_builds = MagicMock()
+ mc_gacs.return_value = True
+
+
+ # 0. createrepo_unsafe failure
+ mc_cru.return_value = (1, "stdout", "stderr")
+
+ self.pruner.prune_project(os.path.join(self.tmp_dir_name, self.prj),
+ self.username, self.coprname)
+
+ assert self.pruner.prune_failed_builds.called
+ assert self.pruner.prune_obsolete_success_builds.called
+
+ self.pruner.prune_failed_builds.reset_mock()
+ self.pruner.prune_obsolete_success_builds.reset_mock()
+
+ mc_cru.return_value = (0, "", "")
+ # 1. createrepo_unsafe raises error
+ mc_cru.side_effect = IOError()
+
+ self.pruner.prune_project(os.path.join(self.tmp_dir_name, self.prj),
+ self.username, self.coprname)
+
+ assert self.pruner.prune_failed_builds.called
+ assert self.pruner.prune_obsolete_success_builds.called
+
+ self.pruner.prune_failed_builds.reset_mock()
+ self.pruner.prune_obsolete_success_builds.reset_mock()
+
+ mc_cru.side_effect = None
+ # 2. prune_obsolete_success_builds raises error
+
+ self.pruner.prune_obsolete_success_builds.side_effect = IOError()
+
+ self.pruner.prune_project(os.path.join(self.tmp_dir_name, self.prj),
+ self.username, self.coprname)
+
+ assert mc_cru.called
+
+ self.pruner.prune_failed_builds.reset_mock()
+ self.pruner.prune_obsolete_success_builds.reset_mock()
+ mc_cru.reset_mock()
+
+ # 3. prune_failed_builds raises error
+
+ self.pruner.prune_obsolete_success_builds.side_effect = None
+ self.pruner.prune_failed_builds.side_effect = IOError()
+
+ self.pruner.prune_project(os.path.join(self.tmp_dir_name, self.prj),
+ self.username, self.coprname)
+
+ assert mc_cru.called
+
+
+ def test_prune_project_skip_when_acr_disabled(self, test_pruner, mc_cru, mc_gacs):
+ self.pruner.prune_failed_builds = MagicMock()
+ self.pruner.prune_obsolete_success_builds = MagicMock()
+
+ mc_gacs.return_value = False
+
+ self.pruner.prune_project(os.path.join(self.tmp_dir_name, self.prj),
+ self.username, self.coprname)
+
+ assert not self.pruner.prune_failed_builds.called
+ assert not self.pruner.prune_obsolete_success_builds.called
+
+ def test_run(self, test_pruner):
+ self.pruner.prune_project = MagicMock()
+
+ self.pruner.run()
+
+
+ def test_main(self, mc_pruner, mc_bcr):
+ prune_main()
+ assert mc_pruner.called
+ assert mc_pruner.return_value.run.called
+ assert mc_bcr.called
+ assert mc_bcr.call_args[0][0] == "/etc/copr/copr-be.conf"
+
+ os.environ["BACKEND_CONFIG"] = "foobar"
+ prune_main()
+ assert mc_bcr.call_args[0][0] == "foobar"
+
9 years, 1 month
[copr] master: [backend] safer copr_prune_results script, [WIP] (10909c0)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 10909c0dadde9fa30c63e7bc5385df3181974292
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Tue Mar 24 02:00:08 2015 +0100
[backend] safer copr_prune_results script, [WIP]
>---------------------------------------------------------------
backend/conf/crontab/copr-backend | 2 +-
backend/run/copr_find_obsolete_builds.sh | 37 +++++
backend/run/copr_prune_old_builds.sh | 52 -------
backend/run/copr_prune_results.py | 193 +++++++++++++++++---------
backend/tests/_resources/foox.tar.gz | Bin 0 -> 17135362 bytes
backend/tests/run/test_copr_prune_results.py | 190 +++++++++++++++++++++++++
6 files changed, 355 insertions(+), 119 deletions(-)
diff --git a/backend/conf/crontab/copr-backend b/backend/conf/crontab/copr-backend
index 68ba1a4..de9f250 100644
--- a/backend/conf/crontab/copr-backend
+++ b/backend/conf/crontab/copr-backend
@@ -1,3 +1,3 @@
#!/usr/bin/sh
-runuser -c "PYTHONUNBUFFERED=1 python /usr/bin/copr_prune_results.py" - copr >> /var/log/copr/prune_old.log 2>&1
+runuser -c "PYTHONUNBUFFERED=1 python /usr/bin/copr_prune_results.py" - copr
diff --git a/backend/run/copr_find_obsolete_builds.sh b/backend/run/copr_find_obsolete_builds.sh
new file mode 100755
index 0000000..dcd3e34
--- /dev/null
+++ b/backend/run/copr_find_obsolete_builds.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+CHROOT_PATH=$1
+DAYS=$2
+
+echo "# looking for obsolete files in: " $CHROOT_PATH
+if [ -d $CHROOT_PATH ]
+then
+ pushd $CHROOT_PATH >/dev/null
+
+ # query latest (sucessfull) packages
+ LATEST_PKGS=$(mktemp)
+ ERR_LOG=$(mktemp)
+ # "yum clean metadata" does not work on this custom repos
+ rm -rf $(find /var/tmp -name query &> /dev/null) &> /dev/null
+ rm -rf /var/tmp/createrepo* &> /dev/null
+ rm -rf /var/cache/yum/* &> /dev/null
+
+ repoquery --repofrompath=query,$CHROOT_PATH --repoid=query -a --location 2>$ERR_LOG \
+ | cut -c8- > $LATEST_PKGS
+
+ # Remove builds older then $DAYS days and which have newer builds available
+ for SUCCESS in $(find -name success -mtime +$DAYS); do
+ DIR=$(basename $(dirname $SUCCESS))
+ echo "# checking dir: " $DIR
+ for PACKAGE in $(ls $DIR); do
+ grep $PACKAGE $LATEST_PKGS >/dev/null && continue 2
+ done
+ # package was not found in $LATEST_PKGS
+ #rm -rf $DIR
+ #echo -n .
+ echo $DIR
+ done
+
+ rm -f $LATEST_PKGS $ERR_LOG
+ popd >/dev/null
+fi
diff --git a/backend/run/copr_prune_old_builds.sh b/backend/run/copr_prune_old_builds.sh
deleted file mode 100755
index 5b50c4c..0000000
--- a/backend/run/copr_prune_old_builds.sh
+++ /dev/null
@@ -1,52 +0,0 @@
-#!/bin/bash
-
-REPOPATH=$1
-DAYS=$2
-
-
-pushd $REPOPATH >/dev/null
-
-
-for CHROOT in $(ls -d $REPOPATH/*/); do
- pushd $CHROOT >/dev/null
-
- # remove old failed builds
- for FAILED in $(find -name 'fail' -mtime +$DAYS); do
- rm -rf $(dirname $FAILED)
- echo -n .
- done
-
- # query latest (sucessfull) packages
- LATEST_PKGS=$(mktemp)
- ERR_LOG=$(mktemp)
- # "yum clean metadata" does not work on this custom repos
- rm -rf $(find /var/tmp -name query &> /dev/null) &> /dev/null
- rm -rf /var/tmp/createrepo* &> /dev/null
- rm -rf /var/cache/yum/* &> /dev/null
-
- repoquery --repofrompath=query,$REPOPATH/$CHROOT --repoid=query -a --location 2>$ERR_LOG \
- | cut -c8- > $LATEST_PKGS
-
- # remove directory if it does not have repodata at all
- if $(grep --quiet 'Cannot retrieve repository metadata' $ERR_LOG) || $(grep --quiet "Can't download or revert repomd.xml" $ERR_LOG) ; then
- # don't remove when /devel presents
- if [ ! -d $REPOPATH/$CHROOT/devel ]; then
- rm -rf $REPOPATH/$CHROOT/
- fi
- else
- # Remove builds older then $DAYS days and which have newer builds available
- for SUCCESS in $(find -name success -mtime +$DAYS); do
- DIR=$(basename $(dirname $SUCCESS))
- for PACKAGE in $(ls $DIR); do
- grep $PACKAGE $LATEST_PKGS >/dev/null && continue 2
- done
- # package was not found in $LATEST_PKGS
- rm -rf $DIR
- echo -n .
- done
- fi
-
- rm -f $LATEST_PKGS $ERR_LOG
- popd >/dev/null
-done
-popd >/dev/null
diff --git a/backend/run/copr_prune_results.py b/backend/run/copr_prune_results.py
index 70bc20c..9f77f81 100644
--- a/backend/run/copr_prune_results.py
+++ b/backend/run/copr_prune_results.py
@@ -6,9 +6,15 @@ from __future__ import division
from __future__ import absolute_import
import os
+import shutil
import sys
import logging
from subprocess import Popen, PIPE
+import time
+
+
+log = logging.getLogger(__name__)
+
from copr.client.exceptions import CoprException, CoprRequestException
@@ -17,11 +23,9 @@ sys.path.append("/usr/share/copr/")
from backend.helpers import BackendConfigReader, get_auto_createrepo_status
from backend.createrepo import createrepo_unsafe
-log = logging.getLogger(__name__)
-log.addHandler(logging.NullHandler())
DEF_DAYS = 14
-DEF_PRUNE_SCRIPT = "/usr/bin/copr_prune_old_builds.sh"
+DEF_FIND_OBSOLETE_SCRIPT = "/usr/bin/copr_find_obsolete_builds.sh"
def list_subdir(path):
@@ -29,74 +33,131 @@ def list_subdir(path):
return dir_names, map(lambda x: os.path.join(path, x), dir_names)
-def prune_project(opts, path, username, projectname):
- log.debug("Going to prune {}/{}".format(username, projectname))
- # get ACR
- try:
- if not get_auto_createrepo_status(opts.frontend_base_url, username, projectname):
- log.debug("Skipped {}/{} since auto createrepo option is disabled"
- .format(username, projectname))
+class Pruner(object):
+ def __init__(self, opts):
+ self.opts = opts
+ self.days = getattr(self.opts, "prune_days", DEF_DAYS)
+ self.prune_sh_script = getattr(self.opts, "find_obsolete_script", DEF_FIND_OBSOLETE_SCRIPT)
+
+ def prune_failed_builds(self, chroot_path):
+ """
+ Deletes subdirs (project directories) which contains file `fail`
+ with mtime older then self.days
+
+ :param chroot_path: path to the chroot directory
+ """
+ for sub_dir_name in os.listdir(chroot_path):
+ build_path = os.path.join(chroot_path, sub_dir_name)
+ if not os.path.isdir(build_path):
+ # log.debug("Not a project directory, skipping: {}".format(build_path))
+ continue
+
+ fail_file_path = os.path.join(build_path, "fail")
+ if os.path.exists(fail_file_path) and not os.path.exists(os.path.join(build_path, "success")):
+ if time.time() - os.path.getmtime(fail_file_path) > self.days:
+ log.info("Removing failed build: {}".format(build_path))
+ shutil.rmtree(build_path)
+
+ def prune_obsolete_success_builds(self, chroot_path, username, projectname):
+ """
+ Uses bash script which invokes repoquery to find obsolete build_dirs
+
+ :param chroot_path: path to the chroot directory
+ :return:
+ """
+ # import ipdb; ipdb.set_trace()
+ cmd = map(str, [self.prune_sh_script, chroot_path, self.days])
+ handle = Popen(cmd, stdout=PIPE, stderr=PIPE)
+ stdout, stderr = handle.communicate()
+ if handle.returncode != 0:
+ log.error("Failed to prune old builds for copr {}/{}\n STDOUT: \n{}\n STDERR: \n{}\n"
+ .format(username, projectname, stdout.decode(), stderr.decode()))
+ return
+ # log.debug("find obsolete returned:\n {}\n stderr: \n {}".format(stdout, stderr))
+ for line in stdout.split("\n"):
+ if not line.strip() or (len(line) > 0 and line[0] == "#"):
+ continue
+ to_delete = line.strip()
+ log.debug("Obsolete path, check for remove: {}".format(to_delete))
+
+ if to_delete in os.listdir(chroot_path):
+ to_delete_path = os.path.join(chroot_path, to_delete)
+ if os.path.isdir(to_delete_path):
+ log.info("Removing obsolete build: {}".format(to_delete_path))
+ shutil.rmtree(to_delete_path)
+
+ def run(self):
+ results_dir = self.opts.destdir
+ log.info("Pruning results dir: {} ".format(results_dir))
+ user_dir_names, user_dirs = list_subdir(results_dir)
+
+ log.info("Going to process total number: {} of user's directories".format(len(user_dir_names)))
+ log.info("Going to process user's directories: {}".format(user_dir_names))
+
+ counter = 0
+ for username, subpath in zip(user_dir_names, user_dirs):
+ log.debug("For user `{}` exploring path: {}".format(username, subpath))
+ for projectname, project_path in zip(*list_subdir(subpath)):
+ log.debug("Exploring project `{}` with path: {}".format(projectname, project_path))
+ self.prune_project(project_path, username, projectname)
+
+ counter += 1
+ log.info("Pruned {}. projects".format(counter))
+
+ log.info("Pruning finished")
+
+ def prune_project(self, project_path, username, projectname):
+ log.info("Going to prune {}/{}".format(username, projectname))
+ # get ACR
+ try:
+ if not get_auto_createrepo_status(self.opts.frontend_base_url, username, projectname):
+ log.debug("Skipped {}/{} since auto createrepo option is disabled"
+ .format(username, projectname))
+ return
+ except (CoprException, CoprRequestException) as exception:
+ log.debug("Failed to get project details for {}/{} with error: {}".format(
+ username, projectname, exception))
return
- except (CoprException, CoprRequestException) as exception:
- log.debug("Failed to get project details for {}/{} with error: {}".format(
- username, projectname, exception))
- return
-
- # run prune project sh
- days = getattr(opts, "prune_days", DEF_DAYS)
-
- cmd = map(str, [DEF_PRUNE_SCRIPT, path, days])
-
- handle = Popen(cmd, stdout=PIPE, stderr=PIPE)
- stdout, stderr = handle.communicate()
-
- if handle.returncode != 0:
- print("Failed to prune old builds for copr {}/{}".format(username, projectname))
- print("STDOUT: \n{}".format(stdout.decode()))
- print("STDERR: \n{}".format(stderr.decode()))
- return
-
- # run createrepo
- log.debug("Prune done for {}/{}".format(username, projectname))
- try:
- retcode, stdout, stderr = createrepo_unsafe(path)
- if retcode != 0:
- print("Createrepo for {}/{} failed".format(username, projectname))
- print("STDOUT: \n{}".format(stdout.decode()))
- print("STDERR: \n{}".format(stderr.decode()))
- except Exception as exception:
- print("Createrepo for {}/{} failed with error: {}"
- .format(username, projectname, exception))
+
+ # run prune project sh
+ for sub_dir_name in os.listdir(project_path):
+ if sub_dir_name == "repodata":
+ continue
+ chroot_path = os.path.join(project_path, sub_dir_name)
+ if not os.path.isdir(chroot_path):
+ continue
+
+ try:
+ self.prune_failed_builds(chroot_path)
+ self.prune_obsolete_success_builds(chroot_path, username, projectname)
+ except Exception as err:
+ log.exception(err)
+ log.error("Error during prune copr {}/{}".format(username, projectname))
+
+ log.debug("Prune done for {}/{}".format(username, projectname))
+ # run createrepo
+
+ try:
+ retcode, stdout, stderr = createrepo_unsafe(project_path)
+ if retcode != 0:
+ log.error(
+ "Createrepo for copr {}/{}\n STDOUT: \n{}\n STDERR: \n{}\n"
+ .format(username, projectname, stdout.decode(), stderr.decode()))
+ except Exception as exception:
+ log.exception("Createrepo for copr {}/{} failed with error: {}"
+ .format(username, projectname, exception))
+
+ log.info("Prune finished for copr {}/{}".format(username, projectname))
def main():
config_file = os.environ.get("BACKEND_CONFIG", "/etc/copr/copr-be.conf")
- opts = BackendConfigReader(config_file).read()
-
- results_dir = opts.destdir
- log.info("Pruning results dir: {} ".format(results_dir))
- user_dir_names, user_dirs = list_subdir(results_dir)
-
- print("Going to process total number: {} of user's directories".format(len(user_dir_names)))
- log.info("Going to process user's directories: {}".format(user_dir_names))
-
- counter = 0
- for username, subpath in zip(user_dir_names, user_dirs):
- log.debug("For user `{}` exploring path: {}".format(username, subpath))
- for projectname, project_path in zip(*list_subdir(subpath)):
- log.debug("Exploring project `{}` with path: {}".format(projectname, project_path))
- prune_project(opts, project_path, username, projectname)
-
- counter += 1
- print("Pruned {}. projects".format(counter))
-
- print("Pruning finished")
-
+ pruner = Pruner(BackendConfigReader(config_file).read())
+ pruner.run()
if __name__ == "__main__":
- # logging.basicConfig(
- # level=logging.DEBUG,
- # format='[%(asctime)s] {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s',
- # datefmt='%H:%M:%S'
- # )
+ logging.basicConfig(
+ filename="/var/log/copr/prune_old.log",
+ format='[%(asctime)s][%(levelname)6s]: %(message)s',
+ level=logging.INFO)
main()
diff --git a/backend/tests/_resources/foox.tar.gz b/backend/tests/_resources/foox.tar.gz
new file mode 100644
index 0000000..aaebe8f
Binary files /dev/null and b/backend/tests/_resources/foox.tar.gz differ
diff --git a/backend/tests/run/test_copr_prune_results.py b/backend/tests/run/test_copr_prune_results.py
new file mode 100644
index 0000000..074e9dc
--- /dev/null
+++ b/backend/tests/run/test_copr_prune_results.py
@@ -0,0 +1,190 @@
+# coding: utf-8
+import logging
+import os
+import sys
+import shutil
+import tarfile
+import tempfile
+import time
+from bunch import Bunch
+from subprocess import Popen, PIPE
+
+import pytest
+
+import six
+if six.PY3:
+ from unittest import mock
+else:
+ import mock
+ from mock import MagicMock
+
+
+
+sys.path.append("../../run")
+
+
+MODULE_REF = "copr_prune_results"
+
+(a)pytest.yield_fixture
+def mc_popen():
+ with mock.patch("{}.Popen".format(MODULE_REF)) as handle:
+ yield handle
+
+
+from copr_prune_results import Pruner
+
+
+def assert_same_dirs(a, b):
+ proc = Popen(["diff", "-r", a, b], stdout=PIPE, stdin=PIPE)
+ stdout, stderr = proc.communicate()
+
+ if proc.returncode != 0:
+ raise AssertionError("Directories {}, {} aren't equal, diff out: \n {} \n {}"
+ .format(a, b, stdout, stderr))
+
+class TestPruneResults(object):
+
+ def setup_method(self, method):
+ self.tmp_dir_name = None
+ self.test_time = time.time()
+ self.make_temp_dir()
+
+ self.pkg_1 = "copr-backend-1.59-1.git.1.7ed2b8d.fc20"
+ self.pkg_1_obsolete = "copr-backend-1.40-1.fc20"
+ self.pkg_2 = "hello-2.8-1.fc20"
+ self.pkg_2_obsolete = "hello-1.0.fc20"
+ self.prj = "foox"
+ self.chroots = ["fedora-20-i386", "fedora-20-x86_64",
+ "fedora-21-i386", "fedora-21-x86_64"]
+ self.opts = Bunch(
+ days=14,
+
+ find_obsolete_script=os.path.abspath(
+ os.path.join(os.path.dirname(__file__), os.pardir, os.pardir,
+ "run", "copr_find_obsolete_builds.sh"))
+
+
+ )
+
+ self.log_file_path = os.path.join(self.tmp_dir_name, "_unittest.log")
+ logging.basicConfig(
+ # filename=self.log_file_path,
+ stream=sys.stdout,
+ format='[%(asctime)s][%(levelname)6s]: %(message)s',
+ level=logging.DEBUG
+ )
+ self.log = logging.getLogger(__name__)
+ self.log.info("setup method done")
+
+ self.username = "vgologuz"
+ self.coprname = "foox"
+
+ def teardown_method(self, method):
+ self.rm_tmp_dir()
+
+ def rm_tmp_dir(self):
+ if self.tmp_dir_name:
+ shutil.rmtree(self.tmp_dir_name)
+ self.tmp_dir_name = None
+
+ def make_temp_dir(self):
+ root_tmp_dir = tempfile.gettempdir()
+ subdir = "test_action_{}".format(self.test_time)
+ self.tmp_dir_name = os.path.join(root_tmp_dir, subdir)
+
+ os.mkdir(self.tmp_dir_name)
+ self.expect_dir_name = os.path.join(self.tmp_dir_name, "_expect")
+ os.mkdir(self.expect_dir_name)
+
+ self.unpack_resource("foox.tar.gz")
+ self.unpack_resource("foox.tar.gz", self.expect_dir_name)
+
+ return self.tmp_dir_name
+
+ def unpack_resource(self, resource_name, target=None):
+ if self.tmp_dir_name is None:
+ self.make_temp_dir()
+
+ src_path = os.path.join(os.path.dirname(__file__), "..",
+ "_resources", resource_name)
+
+ with tarfile.open(src_path, "r:gz") as tar_file:
+ tar_file.extractall(target or self.tmp_dir_name)
+
+
+ @pytest.fixture
+ def test_pruner(self):
+ self.pruner = Pruner(self.opts)
+
+ return self.pruner
+
+ def test_prune_old_failed(self, test_pruner):
+ """
+ Delete if build with "fail" fail with mtime > 14 days
+
+ """
+ # TODO: what neither fail or success are present?
+
+ shutil.move(
+ os.path.join(self.tmp_dir_name, self.prj, self.chroots[0], self.pkg_1, "success"),
+ os.path.join(self.tmp_dir_name, self.prj, self.chroots[0], self.pkg_1, "fail"),)
+ shutil.move(
+ os.path.join(self.tmp_dir_name, self.prj, self.chroots[0], self.pkg_2, "success"),
+ os.path.join(self.tmp_dir_name, self.prj, self.chroots[0], self.pkg_2, "fail"),)
+ shutil.move(
+ os.path.join(self.expect_dir_name, self.prj, self.chroots[0], self.pkg_2, "success"),
+ os.path.join(self.expect_dir_name, self.prj, self.chroots[0], self.pkg_2, "fail"),)
+
+ os.utime(os.path.join(self.tmp_dir_name, self.prj, self.chroots[0], self.pkg_1, "fail"), (0, 0))
+ os.utime(os.path.join(self.tmp_dir_name, self.prj, self.chroots[0], self.pkg_2, "fail"),
+ (time.time(), time.time()))
+
+ shutil.rmtree(os.path.join(self.expect_dir_name, self.prj, self.chroots[0], self.pkg_1))
+
+ self.pruner.prune_failed_builds(os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]))
+
+ assert_same_dirs(
+ os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]),
+ os.path.join(self.expect_dir_name, self.prj, self.chroots[0]),
+ )
+
+ def test_prune_obsolete_builds(self, test_pruner):
+
+ os.utime(os.path.join(self.tmp_dir_name, self.prj, self.chroots[0], self.pkg_1, "success"), (0, 0))
+ os.utime(os.path.join(self.tmp_dir_name, self.prj, self.chroots[0], self.pkg_2, "success"), (0, 0))
+
+ # old obsolete build folder should be removed
+ os.utime(os.path.join(self.tmp_dir_name, self.prj, self.chroots[0], self.pkg_2_obsolete, "success"), (0, 0))
+ shutil.rmtree(os.path.join(self.expect_dir_name, self.prj, self.chroots[0], self.pkg_2_obsolete))
+
+ # recent obsolete build should remain
+ os.utime(os.path.join(self.tmp_dir_name, self.prj, self.chroots[0], self.pkg_1_obsolete, "success"),
+ (time.time(), time.time()))
+
+ self.pruner.prune_obsolete_success_builds(
+ os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]),
+ self.username, self.coprname
+ )
+
+ assert_same_dirs(
+ os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]),
+ os.path.join(self.expect_dir_name, self.prj, self.chroots[0]),
+ )
+
+ def test_prune_obsolete_builds_handle_script_error(self, test_pruner, mc_popen):
+ mc_handle = MagicMock()
+ mc_popen.return_value = mc_handle
+ mc_handle.returncode = 1
+ mc_handle.communicate.return_value = ("foo", "bar")
+
+ # doesn't touch FS if `find_obsolete_build` produce error return code
+ self.pruner.prune_obsolete_success_builds(
+ os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]),
+ self.username, self.coprname
+ )
+ assert mc_popen.called
+
+ assert_same_dirs(
+ os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]),
+ os.path.join(self.expect_dir_name, self.prj, self.chroots[0]),
+ )
9 years, 1 month
[copr] master: [backend][frontend] Send for delete action only `src_pkg_name` instead of original URL. (7c4eba7)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 7c4eba7867e14cf9628ee319934010a05e284419
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Mon Mar 23 10:49:42 2015 +0100
[backend][frontend] Send for delete action only `src_pkg_name` instead of original URL.
>---------------------------------------------------------------
backend/backend/actions.py | 8 +++---
backend/tests/test_action.py | 8 +++---
.../coprs_frontend/coprs/logic/builds_logic.py | 8 ++++-
.../tests/test_logic/test_builds_logic.py | 26 ++++++++++++++++++++
.../test_views/test_coprs_ns/test_coprs_builds.py | 8 ++++--
5 files changed, 45 insertions(+), 13 deletions(-)
diff --git a/backend/backend/actions.py b/backend/backend/actions.py
index c195100..1c178c8 100644
--- a/backend/backend/actions.py
+++ b/backend/backend/actions.py
@@ -106,12 +106,12 @@ class Action(object):
projectname = ext_data["projectname"]
chroots_requested = set(ext_data["chroots"])
- package_name = os.path.basename(ext_data["pkgs"]).replace(".src.rpm", "")
- # TODO: replace with explicit pkg_name, send cleaned string into the action
- if not package_name:
- self.add_event("Bad package name {}, action data: {} ".format(package_name, ext_data))
+ if not ext_data.get("src_pkg_name"):
+ self.add_event("Delete build action missing `src_pkg_name` field, check frontend version. Raw ext_data: {}"
+ .format(ext_data))
return
+ package_name = ext_data["src_pkg_name"]
path = os.path.join(self.destdir, project)
self.add_event("Deleting package {0}".format(package_name))
diff --git a/backend/tests/test_action.py b/backend/tests/test_action.py
index 2ca6d47..08fb806 100644
--- a/backend/tests/test_action.py
+++ b/backend/tests/test_action.py
@@ -59,7 +59,7 @@ class TestAction(object):
self.pkgs_stripped = ["foo"]
self.ext_data_for_delete_build = json.dumps({
- "pkgs": " ".join(self.pkgs),
+ "src_pkg_name": self.pkgs_stripped[0],
"username": "foo",
"projectname": "bar",
"chroots": ["fedora20", "epel7"]
@@ -514,7 +514,7 @@ class TestAction(object):
"id": 15,
"old_value": "old_dir",
"data": json.dumps({
- "pkgs": "rubygem-log4r-1.1.10-2.fc21.src.rpm",
+ "src_pkg_name": "rubygem-log4r-1.1.10-2.fc21",
"username": "foo",
"projectname": "bar",
"chroots": ["fedora-20-x86_64", "fedora-21-x86_64"]
@@ -583,7 +583,7 @@ class TestAction(object):
"id": 15,
"old_value": "old_dir",
"data": json.dumps({
- "pkgs": "rubygem-log4r-1.1.10-2.fc21.src.rpm",
+ "src_pkg_name": "rubygem-log4r-1.1.10-2.fc21",
"username": "foo",
"projectname": "bar",
"chroots": ["fedora-20-x86_64", "fedora-21-x86_64"]
@@ -636,7 +636,7 @@ class TestAction(object):
"id": 15,
"old_value": "old_dir",
"data": json.dumps({
- "pkgs": "http://example.com/",
+ "src_pkg_name": "",
"username": "foo",
"projectname": "bar",
"chroots": ["fedora-20-x86_64", "fedora-21-x86_64"]
diff --git a/frontend/coprs_frontend/coprs/logic/builds_logic.py b/frontend/coprs_frontend/coprs/logic/builds_logic.py
index 0e839b3..2712e48 100644
--- a/frontend/coprs_frontend/coprs/logic/builds_logic.py
+++ b/frontend/coprs_frontend/coprs/logic/builds_logic.py
@@ -212,6 +212,10 @@ class BuildsLogic(object):
@classmethod
def delete_build(cls, user, build):
+ """
+ :type user: models.User
+ :type build: models.Build
+ """
if not user.can_build_in(build.copr):
raise exceptions.InsufficientRightsException(
"You are not allowed to delete this build.")
@@ -229,8 +233,8 @@ class BuildsLogic(object):
if chroot.state not in ["skipped"]
]
- if chroots_to_delete:
- data_dict = {"pkgs": build.pkgs,
+ if chroots_to_delete and build.src_pkg_name:
+ data_dict = {"src_pkg_name": build.src_pkg_name,
"username": build.copr.owner.name,
"projectname": build.copr.name,
"chroots": chroots_to_delete}
diff --git a/frontend/coprs_frontend/tests/test_logic/test_builds_logic.py b/frontend/coprs_frontend/tests/test_logic/test_builds_logic.py
index e8a375e..9423c36 100644
--- a/frontend/coprs_frontend/tests/test_logic/test_builds_logic.py
+++ b/frontend/coprs_frontend/tests/test_logic/test_builds_logic.py
@@ -120,6 +120,9 @@ class TestBuildsLogic(CoprsTestCase):
def test_delete_build_basic(
self, f_users, f_coprs, f_mock_chroots, f_builds, f_db):
+
+ self.b1.pkgs = "http://example.com/copr-keygen-1.58-1.fc20.src.rpm"
+ self.db.session.add(self.b1)
self.db.session.commit()
expected_chroots_to_delete = set()
@@ -134,11 +137,32 @@ class TestBuildsLogic(CoprsTestCase):
action = ActionsLogic.get_many().one()
delete_data = json.loads(action.data)
assert "chroots" in delete_data
+ assert "copr-keygen-1.58-1.fc20" == delete_data["src_pkg_name"]
assert expected_chroots_to_delete == set(delete_data["chroots"])
with pytest.raises(NoResultFound):
BuildsLogic.get(self.b1.id).one()
+ def test_delete_build_bad_src_pkg(
+ self, f_users, f_coprs, f_mock_chroots, f_builds, f_db):
+
+ self.b1.pkgs = "http://example.com/"
+ self.db.session.add(self.b1)
+ self.db.session.commit()
+
+ expected_chroots_to_delete = set()
+ for bchroot in self.b1_bc:
+ expected_chroots_to_delete.add(bchroot.name)
+
+ assert len(ActionsLogic.get_many().all()) == 0
+ BuildsLogic.delete_build(self.u1, self.b1)
+ self.db.session.commit()
+
+ assert len(ActionsLogic.get_many().all()) == 0
+
+ with pytest.raises(NoResultFound):
+ BuildsLogic.get(self.b1.id).one()
+
def test_delete_build_no_chroots_to_clean(
self, f_users, f_coprs, f_mock_chroots, f_builds, f_db):
@@ -159,6 +183,8 @@ class TestBuildsLogic(CoprsTestCase):
for bchroot in self.b1_bc[1:-1]:
bchroot.status = helpers.StatusEnum("skipped")
+ self.b1.pkgs = "http://example.com/copr-keygen-1.58-1.fc20.src.rpm"
+ self.db.session.add(self.b1)
self.db.session.commit()
assert len(ActionsLogic.get_many().all()) == 0
diff --git a/frontend/coprs_frontend/tests/test_views/test_coprs_ns/test_coprs_builds.py b/frontend/coprs_frontend/tests/test_views/test_coprs_ns/test_coprs_builds.py
index 393b00e..d8cbf08 100644
--- a/frontend/coprs_frontend/tests/test_views/test_coprs_ns/test_coprs_builds.py
+++ b/frontend/coprs_frontend/tests/test_views/test_coprs_ns/test_coprs_builds.py
@@ -104,9 +104,11 @@ class TestCoprDeleteBuild(CoprsTestCase):
f_coprs, f_mock_chroots,
f_builds, f_db):
- self.db.session.add_all([self.u1, self.c1, self.b1])
- pkgs = "one two three"
+ pkgs = "http://example.com/one.src.rpm"
self.b1.pkgs = pkgs
+ self.db.session.add_all([self.u1, self.c1, self.b1])
+ self.db.session.commit()
+
r = self.test_client.post(
"/coprs/{0}/{1}/delete_build/{2}/"
.format(self.u1.name, self.c1.name, self.b1.id),
@@ -120,7 +122,7 @@ class TestCoprDeleteBuild(CoprsTestCase):
act = self.models.Action.query.first()
assert act.object_type == "build"
assert act.old_value == "user1/foocopr"
- assert json.loads(act.data)["pkgs"] == pkgs
+ assert json.loads(act.data)["src_pkg_name"] == self.b1.src_pkg_name
@TransactionDecorator("u2")
def test_copr_build_non_submitter_cannot_delete_build(self, f_users,
9 years, 1 month
[copr] master: Merge branch 'hotfix_1203753' (c9ea561)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit c9ea561667b71563e7f243829d9590c0731933fa
Merge: c8e4d8f 96cfc8a
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Mon Mar 23 09:45:39 2015 +0100
Merge branch 'hotfix_1203753'
>---------------------------------------------------------------
backend/backend/actions.py | 5 ++
backend/copr-backend.spec | 17 +++++-
.../coprs/logic/__init__.py => backend/run/test.sh | 0
backend/tests/test_action.py | 55 ++++++++++++++++++++
rel-eng/packages/copr-backend | 2 +-
5 files changed, 75 insertions(+), 4 deletions(-)
diff --git a/backend/backend/actions.py b/backend/backend/actions.py
index 6b82ebe..c195100 100644
--- a/backend/backend/actions.py
+++ b/backend/backend/actions.py
@@ -107,6 +107,11 @@ class Action(object):
chroots_requested = set(ext_data["chroots"])
package_name = os.path.basename(ext_data["pkgs"]).replace(".src.rpm", "")
+ # TODO: replace with explicit pkg_name, send cleaned string into the action
+ if not package_name:
+ self.add_event("Bad package name {}, action data: {} ".format(package_name, ext_data))
+ return
+
path = os.path.join(self.destdir, project)
self.add_event("Deleting package {0}".format(package_name))
diff --git a/backend/copr-backend.spec b/backend/copr-backend.spec
index 82cb38b..45e5fff 100644
--- a/backend/copr-backend.spec
+++ b/backend/copr-backend.spec
@@ -3,7 +3,7 @@
%endif
Name: copr-backend
-Version: 1.58
+Version: 1.60
Release: 1%{?dist}
Summary: Backend for Copr
@@ -25,6 +25,7 @@ BuildRequires: python-requests
BuildRequires: python2-devel
BuildRequires: python-copr
BuildRequires: systemd
+BuildRequires: redis
%if 0%{?rhel} < 7 && 0%{?rhel} > 0
BuildRequires: python-argparse
%endif
@@ -162,9 +163,12 @@ cp -a conf/playbooks %{buildroot}%{_pkgdocdir}/
%check
-#PYTHONPATH=backend:run:$PYTHONPATH python -B -m pytest \
-# -s --cov-report term-missing --cov ./backend --cov ./run ./tests/
+redis-server --port 7777 &> /dev/null &
+PYTHONPATH=backend:run:$PYTHONPATH python -B -m pytest \
+ -s --cov-report term-missing --cov ./backend --cov ./run ./tests/
+
+kill %1
%pre
getent group copr >/dev/null || groupadd -r copr
@@ -223,6 +227,13 @@ useradd -r -g copr -G lighttpd -s /bin/bash -c "COPR user" copr
%exclude %{_pkgdocdir}/playbooks
%changelog
+* Fri Mar 20 2015 Valentin Gologuzov <vgologuz(a)redhat.com> 1.60-1
+- [backend][spec] start/stop redis server during package build tests
+
+* Fri Mar 20 2015 Valentin Gologuzov <vgologuz(a)redhat.com> 1.59-1
+- [backend][hotfix] 1203753 : don't process delete action if src_pkg is
+ mallformed
+
* Mon Mar 02 2015 Valentin Gologuzov <vgologuz(a)redhat.com> 1.58-1
- [rhbz:#1185959] - RFE: Present statistics about project
popularity. A few more counters for downloads from backend's result
diff --git a/frontend/coprs_frontend/coprs/logic/__init__.py b/backend/run/test.sh
similarity index 100%
copy from frontend/coprs_frontend/coprs/logic/__init__.py
copy to backend/run/test.sh
diff --git a/backend/tests/test_action.py b/backend/tests/test_action.py
index 74a8e2b..2ca6d47 100644
--- a/backend/tests/test_action.py
+++ b/backend/tests/test_action.py
@@ -382,6 +382,8 @@ class TestAction(object):
test_action.run()
assert not os.path.exists(foo_pkg_dir)
assert not os.path.exists(log_path)
+ assert os.path.exists(chroot_1_dir)
+ assert os.path.exists(chroot_2_dir)
assert_what_from_queue(self.test_q, msg_list=[
"Action delete build",
@@ -418,6 +420,10 @@ class TestAction(object):
with pytest.raises(EmptyQueue):
self.test_q.get_nowait()
+ assert os.path.exists(chroot_1_dir)
+ assert os.path.exists(chroot_2_dir)
+
+
create_repo_expected_call = mock.call(
username=u'foo',
projectname=u'bar',
@@ -531,6 +537,10 @@ class TestAction(object):
assert not os.path.isdir(os.path.join(chroot_20_path, "rubygem-log4r-1.1.10-2.fc21"))
assert not os.path.isdir(os.path.join(chroot_21_path, "rubygem-log4r-1.1.10-2.fc21"))
+ assert os.path.exists(chroot_20_path)
+ assert os.path.exists(chroot_21_path)
+
+
@mock.patch("backend.actions.createrepo")
def test_delete_two_chroots_two_remains(self, mc_createrepo_unsafe, mc_time):
"""
@@ -602,6 +612,51 @@ class TestAction(object):
assert os.path.isdir(os.path.join(chroot_20_i386_path, "rubygem-log4r-1.1.10-2.fc21"))
assert os.path.isdir(os.path.join(chroot_21_i386_path, "rubygem-log4r-1.1.10-2.fc21"))
+ @mock.patch("backend.actions.createrepo")
+ def test_delete_build_with_bad_pkg_name(self, mc_createrepo, mc_time):
+ """
+ regression: https://bugzilla.redhat.com/show_bug.cgi?id=1203753
+
+ """
+ mc_time.time.return_value = self.test_time
+
+ resource_name = "1171796_doubled.tar.gz"
+ self.unpack_resource(resource_name)
+
+ chroot_20_path = os.path.join(self.tmp_dir_name, "old_dir", "fedora-20-x86_64")
+ chroot_21_path = os.path.join(self.tmp_dir_name, "old_dir", "fedora-21-x86_64")
+
+ mc_createrepo.return_value = 0, STDOUT, ""
+ mc_front_cb = MagicMock()
+ test_action = Action(
+ action={
+ "action_type": ActionType.DELETE,
+ "object_type": "build",
+ "object_id": 15,
+ "id": 15,
+ "old_value": "old_dir",
+ "data": json.dumps({
+ "pkgs": "http://example.com/",
+ "username": "foo",
+ "projectname": "bar",
+ "chroots": ["fedora-20-x86_64", "fedora-21-x86_64"]
+ }),
+ },
+ events=self.test_q, lock=None,
+ frontend_callback=mc_front_cb,
+ destdir=self.tmp_dir_name,
+ front_url=None,
+ results_root_url=RESULTS_ROOT_URL
+ )
+
+ assert os.path.exists(chroot_20_path)
+ assert os.path.exists(chroot_21_path)
+ test_action.run()
+ # shouldn't touch chroot dirs
+ assert os.path.exists(chroot_20_path)
+ assert os.path.exists(chroot_21_path)
+
+
@mock.patch("backend.actions.createrepo_unsafe")
def test_delete_two_chroots_two_builds_stay_untouched(self, mc_createrepo_unsafe, mc_time):
# TODO: prepare archive
diff --git a/rel-eng/packages/copr-backend b/rel-eng/packages/copr-backend
index d19bf95..0b75fea 100644
--- a/rel-eng/packages/copr-backend
+++ b/rel-eng/packages/copr-backend
@@ -1 +1 @@
-1.58-1 backend/
+1.60-1 backend/
9 years, 1 month