Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
commit 2f547f9cf20f2804336f091dd6f32ebc800a3c33 Author: Valentin Gologuzov vgologuz@redhat.com Date: Thu Nov 6 14:21:23 2014 +0100
[backend] copr-prune-repo should respect auto_createrepo option
backend/backend/createrepo.py | 16 +---- backend/backend/helpers.py | 155 +++++++++++++++++++++++++++++++++- backend/copr-backend.spec | 18 ++-- backend/copr-be.conf.example | 5 + backend/copr-be.local.conf | 4 + backend/copr-prune-repo | 55 ------------ backend/crontab/copr-backend | 2 +- backend/{ => run}/copr-be.py | 102 ++++------------------- backend/run/copr_prune_old_builds.sh | 52 +++++++++++ backend/run/copr_prune_results.py | 103 ++++++++++++++++++++++ 10 files changed, 343 insertions(+), 169 deletions(-)
diff --git a/backend/backend/createrepo.py b/backend/backend/createrepo.py index 3c3998d..0e8e4ac 100644 --- a/backend/backend/createrepo.py +++ b/backend/backend/createrepo.py @@ -1,12 +1,8 @@ -import sys - import os import subprocess from subprocess import Popen
-from copr.client import CoprClient - -from backend.helpers import SortedOptParser +from .helpers import get_auto_createrepo_status
def createrepo_unsafe(path, lock=None, dest_dir=None, base_url=None): @@ -54,16 +50,6 @@ def createrepo_unsafe(path, lock=None, dest_dir=None, base_url=None): return cmd.returncode, out, err
-def get_auto_createrepo_status(front_url, username, projectname): - client = CoprClient(copr_url=front_url) - result = client.get_project_details(projectname, username) - - if "auto_createrepo" in result.data["detail"]: - return bool(result.data["detail"]["auto_createrepo"]) - else: - return True - - def createrepo(path, front_url, username, projectname, base_url=None, lock=None): """ Creates repo depending on the project setting "auto_createrepo". diff --git a/backend/backend/helpers.py b/backend/backend/helpers.py index db30689..0992eb6 100644 --- a/backend/backend/helpers.py +++ b/backend/backend/helpers.py @@ -1,7 +1,17 @@ +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import division +from __future__ import absolute_import + from operator import methodcaller import optparse +import ConfigParser +import os + +from bunch import Bunch +from copr.client import CoprClient
-__author__ = 'vgologuz' +from .exceptions import CoprBackendError
class SortedOptParser(optparse.OptionParser): @@ -10,4 +20,145 @@ class SortedOptParser(optparse.OptionParser):
def format_help(self, formatter=None): self.option_list.sort(key=methodcaller("get_opt_string")) - return optparse.OptionParser.format_help(self, formatter=None) + return optparse.OptionParser.format_help(self) + + +def _get_conf(cp, section, option, default, mode=None): + """ + To make returning items from config parser less irritating + + :param mode: convert obtained value, possible modes: + - None (default): do nothing + - "bool" or "boolean" + - "int" + - "float" + """ + + if cp.has_section(section) and cp.has_option(section, option): + if mode is None: + return cp.get(section, option) + elif mode in ["bool", "boolean"]: + return cp.getboolean(section, option) + elif mode == "int": + return cp.getint(section, option) + elif mode == "float": + return cp.getfloat(section, option) + elif mode == "path": + path = cp.get(section, option) + if path.startswith("~"): + path = os.path.expanduser(path) + path = os.path.abspath(path) + path = os.path.normpath(path) + + return path + return default + + +class BackendConfigReader(object): + def __init__(self, config_file=None, ext_opts=None): + self.config_file = config_file or "/etc/copr/copr-be.conf" + self.ext_opts = ext_opts + + def read(self): + try: + opts = self._read_unsafe() + if self.ext_opts: + for key, value in self.ext_opts.items(): + setattr(opts, key, value) + + if not opts.destdir: + raise CoprBackendError( + "Incomplete Config - must specify" + " destdir in configuration") + + return opts + + except ConfigParser.Error as e: + raise CoprBackendError( + "Error parsing config file: {0}: {1}".format( + self.config_file, e)) + + def _read_unsafe(self): + cp = ConfigParser.ConfigParser() + cp.read(self.config_file) + + opts = Bunch() + opts.results_baseurl = _get_conf( + cp, "backend", "results_baseurl", "http://copr") + + #TODO: this should be built from frontend_base_url + '/backend' + opts.frontend_url = _get_conf( + cp, "backend", "frontend_url", "http://coprs/rest/api") + + # We need this to access public api + opts.frontend_base_url = _get_conf( + cp, "backend", "frontend_base_url", "http://coprs/") + + opts.frontend_auth = _get_conf( + cp, "backend", "frontend_auth", "PASSWORDHERE") + + opts.do_sign = _get_conf( + cp, "backend", "do_sign", False, mode="bool") + + opts.build_groups_count = _get_conf( + cp, "backend", "build_groups", 1, mode="int") + + opts.build_groups = [] + for group_id in range(int(opts.build_groups_count)): + group = { + "id": int(group_id), + "name": _get_conf( + cp, "backend", "group{0}_name".format(group_id), "PC"), + "archs": _get_conf( + cp, "backend", "group{0}_archs".format(group_id), + "i386,x86_64").split(","), + "spawn_playbook": _get_conf( + cp, "backend", "group{0}_spawn_playbook".format(group_id), + "/srv/copr-work/provision/builderpb-PC.yml"), + "terminate_playbook": _get_conf( + cp, "backend", + "group{0}_terminate_playbook".format(group_id), + "/srv/copr-work/provision/terminatepb-PC.yml"), + "max_workers": int(_get_conf( + cp, "backend", "group{0}_max_workers".format(group_id), 8)) + } + opts.build_groups.append(group) + + opts.destdir = _get_conf(cp, "backend", "destdir", None, mode="path") + + + opts.exit_on_worker = _get_conf( + cp, "backend", "exit_on_worker", False, mode="bool") + opts.fedmsg_enabled = _get_conf( + cp, "backend", "fedmsg_enabled", False, mode="bool") + opts.sleeptime = _get_conf( + cp, "backend", "sleeptime", 10, mode="int") + opts.timeout = _get_conf( + cp, "builder", "timeout", 1800, mode="int") + opts.logfile = _get_conf( + cp, "backend", "logfile", "/var/log/copr/backend.log") + opts.verbose = _get_conf( + cp, "backend", "verbose", False, mode="bool") + opts.worker_logdir = _get_conf( + cp, "backend", "worker_logdir", "/var/log/copr/workers/") + opts.spawn_vars = _get_conf(cp, "backend", "spawn_vars", None) + opts.terminate_vars = _get_conf(cp, "backend", "terminate_vars", None) + + opts.prune_days = _get_conf(cp, "backend", "prune_days", None, mode="int") + opts.prune_script = _get_conf(cp, "backend", "prune_script", None, mode="path") + + # thoughts for later + # ssh key for connecting to builders? + # cloud key stuff? + # + return opts + + +def get_auto_createrepo_status(front_url, username, projectname): + client = CoprClient(copr_url=front_url) + result = client.get_project_details(projectname, username) + + if "auto_createrepo" in result.data["detail"]: + return bool(result.data["detail"]["auto_createrepo"]) + else: + return True diff --git a/backend/copr-backend.spec b/backend/copr-backend.spec index 2ac7a41..8b9a8c6 100644 --- a/backend/copr-backend.spec +++ b/backend/copr-backend.spec @@ -104,9 +104,9 @@ install -d %{buildroot}%{_sysconfdir}/cron.daily install -d %{buildroot}%{_sysconfdir}/sudoers.d
cp -a backend/* %{buildroot}%{_datadir}/copr/backend -cp -a copr-be.py %{buildroot}%{_datadir}/copr/ +cp -a run/* %{buildroot}%{_datadir}/copr/ cp -a copr-be.conf.example %{buildroot}%{_sysconfdir}/copr/copr-be.conf -install -p -m 755 copr-prune-repo %{buildroot}%{_sbindir}/copr-prune-repo + install -p -m 755 crontab/copr-backend %{buildroot}%{_sysconfdir}/cron.daily/copr-backend
cp -a dist/lighttpd/* %{buildroot}%{_pkgdocdir}/lighttpd/ @@ -115,6 +115,8 @@ cp -a tmpfiles.d/* %{buildroot}/%{_tmpfilesdir}
# for ghost files touch %{buildroot}%{_var}/log/copr/copr.log +touch %{buildroot}%{_var}/log/copr/prune_old.log + for i in `seq 7`; do touch %{buildroot}%{_var}/log/copr/workers/worker-$i.log done @@ -144,7 +146,8 @@ useradd -r -g copr -G lighttpd -s /bin/bash -c "COPR user" copr
%files %doc LICENSE -%dir %{_datadir}/copr + +%{_datadir}/copr/* %dir %{_sharedstatedir}/copr %dir %attr(0755, copr, copr) %{_sharedstatedir}/copr/jobs/ %dir %attr(0755, copr, copr) %{_sharedstatedir}/copr/public_html/ @@ -153,7 +156,7 @@ useradd -r -g copr -G lighttpd -s /bin/bash -c "COPR user" copr %dir %attr(0755, copr, copr) %{_var}/log/copr/workers %dir %attr(0755, copr, copr) %{_var}/run/copr-backend
-%ghost %{_var}/log/copr/copr.log +%ghost %{_var}/log/copr/*.log %ghost %{_var}/log/copr/workers/worker-*.log %ghost %{_var}/run/copr-backend/copr-be.pid
@@ -165,13 +168,8 @@ useradd -r -g copr -G lighttpd -s /bin/bash -c "COPR user" copr %config(noreplace) %attr(0640, root, copr) %{_sysconfdir}/copr/copr-be.conf %{_unitdir}/copr-backend.service %{_tmpfilesdir}/copr-backend.conf -%{_sbindir}/copr-prune-repo -%config(noreplace) %{_sysconfdir}/cron.daily/copr-backend
- - -%{_datadir}/copr/backend -%{_datadir}/copr/copr-be.py* +%config(noreplace) %{_sysconfdir}/cron.daily/copr-backend
%config(noreplace) %attr(0600, root, root) %{_sysconfdir}/sudoers.d/copr
diff --git a/backend/copr-be.conf.example b/backend/copr-be.conf.example index 0ae59bc..0f1f461 100644 --- a/backend/copr-be.conf.example +++ b/backend/copr-be.conf.example @@ -96,6 +96,11 @@ worker_logdir=/var/log/copr/workers/ # signer host and correct /etc/sign.conf #do_sign=false
+# minimum age for builds to be pruned +prune_days=14 +# path to executable script to clean old build +prune_script=/usr/sbin/copr_prune_old_builds.sh + [builder] # default is 1800 timeout=3600 diff --git a/backend/copr-be.local.conf b/backend/copr-be.local.conf index 3885b04..76df89c 100644 --- a/backend/copr-be.local.conf +++ b/backend/copr-be.local.conf @@ -98,6 +98,10 @@ worker_logdir=/tmp/log/copr/workers/
verbose=true
+# minimum age for builds to be pruned +prune_days=14 +# path to executable script to clean old build +prune_script=./copr_prune_old_builds.sh
[builder] # default is 1800 diff --git a/backend/copr-prune-repo b/backend/copr-prune-repo deleted file mode 100755 index 7b5ac79..0000000 --- a/backend/copr-prune-repo +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/bash - -DAYS=14 -RESULTDIR=$(awk -F "=" '/^destdir/ {print $2}' /etc/copr/copr-be.conf) - -if [ ! -d $RESULTDIR ]; then - echo "ERROR: destdir ($RESULTDIR) in /etc/copr/copr-be.conf does not exist." - exit 1 -fi - -pushd $RESULTDIR >/dev/null || { echo "Could not change to $RESULTDIR"; exit 1; } - -for REPO in $( ls -d */*/* | grep -v '/repodata$' ); do - pushd $REPO >/dev/null || continue - - # Removing failed builds - for FAILED in $(find -name 'fail' -mtime +$DAYS); do - rm -rf $(dirname $FAILED) - echo -n . - done - - # Query latest (sucessfull) packages - LATEST_PKGS=$(mktemp) - ERR_LOG=$(mktemp) - # "yum clean metadata" does not work on this custom repos - rm -rf $(find /var/tmp -name query) - rm -rf /var/tmp/createrepo* - rm -rf /var/cache/yum/* - repoquery --repofrompath=query,$RESULTDIR/$REPO --repoid=query -a --location 2>$ERR_LOG \ - | cut -c8- > $LATEST_PKGS - # remove directory if it does not have repodata at all - grep 'Cannot retrieve repository metadata' $ERR_LOG && rm -rf $RESULTDIR/$REPO - - # Remove builds older then $DAYS days and which have newer builds available - for SUCCESS in $(find -name success -mtime +$DAYS); do - DIR=$(basename $(dirname $SUCCESS)) - for PACKAGE in $(ls $DIR); do - grep $PACKAGE $LATEST_PKGS >/dev/null && continue 2 - done - # package was not found in $LATEST_PKGS - rm -rf $DIR - echo -n . - done - - rm -f $LATEST_PKGS $ERR_LOG - # unless we previosly deleted the repo, regen metadata - if [ -d $(pwd) ]; then - runuser -c "createrepo_c --update -q --ignore-lock $(pwd)" - copr &>/dev/null || runuser -c "createrepo_c --update -q --ignore-lock $(pwd)" - copr - fi - popd >/dev/null -done -popd >/dev/null -rm -rf /var/tmp/createrepo* -echo -echo Done diff --git a/backend/crontab/copr-backend b/backend/crontab/copr-backend index 586d45b..2260f7a 100644 --- a/backend/crontab/copr-backend +++ b/backend/crontab/copr-backend @@ -1,3 +1,3 @@ #!/usr/bin/sh
-/usr/sbin/copr-prune-repo >/var/log/copr.log 2>&1 +python /usr/share/copr/copr_prune_results.py >> /var/log/copr/prune_old.log 2>&1 diff --git a/backend/copr-be.py b/backend/run/copr-be.py similarity index 77% rename from backend/copr-be.py rename to backend/run/copr-be.py index 74d74a1..e1fea3b 100755 --- a/backend/copr-be.py +++ b/backend/run/copr-be.py @@ -1,5 +1,10 @@ #!/usr/bin/python -ttu
+from __future__ import print_function +from __future__ import unicode_literals +from __future__ import division +from __future__ import absolute_import + import ConfigParser import grp import lockfile @@ -25,6 +30,7 @@ from backend.exceptions import CoprBackendError from backend.dispatcher import Worker from backend.actions import Action from backend.callback import FrontendCallback +from backend.helpers import BackendConfigReader
def _get_conf(cp, section, option, default, mode=None): @@ -203,7 +209,12 @@ class CoprBackend(object): self.ext_opts = ext_opts # to stow our cli options for read_conf() self.workers_by_group_id = defaultdict(list) self.max_worker_num_by_group_id = defaultdict(int) - self.opts = self.read_conf() + + #self.opts = self.read_conf() + self.config_reader = BackendConfigReader(self.config_file, self.ext_opts) + self.opts = None + self.update_conf() + self.lock = multiprocessing.Lock()
self.task_queues = [] @@ -240,90 +251,8 @@ class CoprBackend(object): def event(self, what): self.events.put({"when": time.time(), "who": "main", "what": what})
- def read_conf(self): - "read in config file - return Bunch of config data" - opts = Bunch() - cp = ConfigParser.ConfigParser() - try: - cp.read(self.config_file) - opts.results_baseurl = _get_conf( - cp, "backend", "results_baseurl", "http://copr") - - #TODO: this should be built from frontend_base_url + '/backend' - opts.frontend_url = _get_conf( - cp, "backend", "frontend_url", "http://coprs/rest/api") - - # We need this to access public api - opts.frontend_base_url = _get_conf( - cp, "backend", "frontend_base_url", "http://coprs/") - - opts.frontend_auth = _get_conf( - cp, "backend", "frontend_auth", "PASSWORDHERE") - - opts.build_groups_count = _get_conf( - cp, "backend", "build_groups", 1, mode="int") - - opts.do_sign = _get_conf( - cp, "backend", "do_sign", False, mode="bool") - - opts.build_groups = [] - for group_id in range(int(opts.build_groups_count)): - group = { - "id": int(group_id), - "name": _get_conf( - cp, "backend", "group{0}_name".format(group_id), "PC"), - "archs": _get_conf( - cp, "backend", "group{0}_archs".format(group_id), - "i386,x86_64").split(","), - "spawn_playbook": _get_conf( - cp, "backend", "group{0}_spawn_playbook".format(group_id), - "/srv/copr-work/provision/builderpb-PC.yml"), - "terminate_playbook": _get_conf( - cp, "backend", - "group{0}_terminate_playbook".format(group_id), - "/srv/copr-work/provision/terminatepb-PC.yml"), - "max_workers": int(_get_conf( - cp, "backend", "group{0}_max_workers".format(group_id), 8)) - } - opts.build_groups.append(group) - - opts.destdir = _get_conf(cp, "backend", "destdir", None) - opts.exit_on_worker = _get_conf( - cp, "backend", "exit_on_worker", False, mode="bool") - opts.fedmsg_enabled = _get_conf( - cp, "backend", "fedmsg_enabled", False, mode="bool") - opts.sleeptime = _get_conf( - cp, "backend", "sleeptime", 10, mode="int") - opts.timeout = _get_conf( - cp, "builder", "timeout", 1800, mode="int") - opts.logfile = _get_conf( - cp, "backend", "logfile", "/var/log/copr/backend.log") - opts.verbose = _get_conf( - cp, "backend", "verbose", False, mode="bool") - opts.worker_logdir = _get_conf( - cp, "backend", "worker_logdir", "/var/log/copr/workers/") - opts.spawn_vars = _get_conf(cp, "backend", "spawn_vars", None) - opts.terminate_vars = _get_conf(cp, "backend", "terminate_vars", - None) - - # thoughts for later - # ssh key for connecting to builders? - # cloud key stuff? - # - except ConfigParser.Error as e: - raise CoprBackendError( - "Error parsing config file: {0}: {1}".format( - self.config_file, e)) - - if not opts.destdir: - raise CoprBackendError( - "Incomplete Config - must specify" - " destdir in configuration") - - if self.ext_opts: - for v in self.ext_opts: - setattr(opts, v, self.ext_opts.get(v)) - return opts + def update_conf(self): + self.opts = self.config_reader.read()
def clean_task_queues(self): try: @@ -338,7 +267,8 @@ class CoprBackend(object): self.abort = False while not self.abort: # re-read config into opts - self.opts = self.read_conf() + #self.opts = self.read_conf() + self.update_conf()
for group in self.opts.build_groups: group_id = group["id"] diff --git a/backend/run/copr_prune_old_builds.sh b/backend/run/copr_prune_old_builds.sh new file mode 100755 index 0000000..30f280d --- /dev/null +++ b/backend/run/copr_prune_old_builds.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +REPOPATH=$1 +DAYS=$2 + + +pushd $REPOPATH >/dev/null + + +for CHROOT in $(ls $REPOPATH); do + pushd $CHROOT >/dev/null + + # remove old failed builds + for FAILED in $(find -name 'fail' -mtime +$DAYS); do + rm -rf $(dirname $FAILED) + echo -n . + done + + # query latest (sucessfull) packages + LATEST_PKGS=$(mktemp) + ERR_LOG=$(mktemp) + # "yum clean metadata" does not work on this custom repos + rm -rf $(find /var/tmp -name query &> /dev/null) &> /dev/null + rm -rf /var/tmp/createrepo* &> /dev/null + rm -rf /var/cache/yum/* &> /dev/null + + repoquery --repofrompath=query,$REPOPATH/$CHROOT --repoid=query -a --location 2>$ERR_LOG \ + | cut -c8- > $LATEST_PKGS + + # remove directory if it does not have repodata at all + if $(grep --quiet 'Cannot retrieve repository metadata' $ERR_LOG) || $(grep --quiet "Can't download or revert repomd.xml" $ERR_LOG) ; then + # don't remove when /devel presents + if [ ! -d $REPOPATH/$CHROOT/devel ]; then + rm -rf $REPOPATH/$CHROOT/ + fi + else + # Remove builds older then $DAYS days and which have newer builds available + for SUCCESS in $(find -name success -mtime +$DAYS); do + DIR=$(basename $(dirname $SUCCESS)) + for PACKAGE in $(ls $DIR); do + grep $PACKAGE $LATEST_PKGS >/dev/null && continue 2 + done + # package was not found in $LATEST_PKGS + rm -rf $DIR + echo -n . + done + fi + + rm -f $LATEST_PKGS $ERR_LOG + popd >/dev/null +done +popd >/dev/null diff --git a/backend/run/copr_prune_results.py b/backend/run/copr_prune_results.py new file mode 100644 index 0000000..eec6df4 --- /dev/null +++ b/backend/run/copr_prune_results.py @@ -0,0 +1,103 @@ +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import division +from __future__ import absolute_import + +import os +import logging +from subprocess import Popen, PIPE + +from copr.client.exceptions import CoprException, CoprRequestException + +from backend.helpers import BackendConfigReader, get_auto_createrepo_status +from backend.createrepo import createrepo_unsafe + +log = logging.getLogger(__name__) +log.addHandler(logging.NullHandler()) + +DEF_DAYS = 14 +DEF_PRUNE_SCRIPT = "copr_prune_old_builds.sh" +DEF_PRUNE_SCRIPT = "/usr/sbin/copr_prune_old_builds.sh" + + +def list_subdir(path): + dir_names = [d for d in os.listdir(path) if os.path.isdir(os.path.join(path, d))] + return dir_names, map(lambda x: os.path.join(path, x), dir_names) + + +def prune_project(opts, path, username, projectname): + log.debug("Going to prune {}/{}".format(username, projectname)) + # get ACR + try: + if not get_auto_createrepo_status(opts.frontend_base_url, username, projectname): + log.debug("Skipped {}/{} since auto createrepo option is disabled" + .format(username, projectname)) + return + except (CoprException, CoprRequestException) as exception: + log.debug("Failed to get project details for {}/{} with error: {}".format( + username, projectname, exception)) + return + + + # run prune project sh + #cmd = ["/usr/bin/copr_prune.sh", path, DAYS] + days = getattr(opts, "prune_days", DEF_DAYS) + prune_script = getattr(opts, "prune_script", DEF_PRUNE_SCRIPT) + + cmd = map(str, [prune_script, path, days]) + + print("CMD {}".format(cmd)) + handle = Popen(cmd, stdout=PIPE, stderr=PIPE) + stdout, stderr = handle.communicate() + + if handle.returncode != 0: + print("Failed to prune old builds for copr {}/{}".format(username, projectname)) + print("STDOUT: \n{}".format(stdout.decode())) + print("STDERR: \n{}".format(stderr.decode())) + return + + # run createrepo + log.debug("Prune done for {}/{}".format(username, projectname)) + try: + retcode, stdout, stderr = createrepo_unsafe(path) + if retcode != 0: + print("Createrepo for {}/{} failed".format(username, projectname)) + print("STDOUT: \n{}".format(stdout.decode())) + print("STDERR: \n{}".format(stderr.decode())) + except Exception as exception: + print("Createrepo for {}/{} failed with error: {}" + .format(username, projectname, exception)) + + + +def main(): + config_file= os.environ.get("BACKEND_CONFIG", "/etc/copr/copr-be.conf") + opts = BackendConfigReader(config_file).read() + + results_dir = opts.destdir + log.info("Pruning results dir: {} ".format(results_dir)) + user_dir_names, user_dirs = list_subdir(results_dir) + + print("Going to process total number: {} of user's directories".format(len(user_dir_names))) + log.info("Going to process user's directories: {}".format(user_dir_names)) + + counter = 0 + for username, subpath in zip(user_dir_names, user_dirs): + log.debug("For user `{}` exploring path: {}".format(username, subpath)) + for projectname, project_path in zip(*list_subdir(subpath)): + log.debug("Exploring project `{}` with path: {}".format(projectname, project_path)) + prune_project(opts, project_path, username, projectname) + + counter += 1 + print("Pruned {} projects".format(counter)) + + print("Pruning finished") + + +if __name__ == "__main__": + # logging.basicConfig( + # level=logging.DEBUG, + # format='[%(asctime)s] {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s', + # datefmt='%H:%M:%S' + # ) + main()
copr-commits@lists.fedorahosted.org