Repository :
http://git.fedorahosted.org/cgit/copr.git
On branch : master
---------------------------------------------------------------
commit 10909c0dadde9fa30c63e7bc5385df3181974292
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Tue Mar 24 02:00:08 2015 +0100
[backend] safer copr_prune_results script, [WIP]
---------------------------------------------------------------
backend/conf/crontab/copr-backend | 2 +-
backend/run/copr_find_obsolete_builds.sh | 37 +++++
backend/run/copr_prune_old_builds.sh | 52 -------
backend/run/copr_prune_results.py | 193 +++++++++++++++++---------
backend/tests/_resources/foox.tar.gz | Bin 0 -> 17135362 bytes
backend/tests/run/test_copr_prune_results.py | 190 +++++++++++++++++++++++++
6 files changed, 355 insertions(+), 119 deletions(-)
diff --git a/backend/conf/crontab/copr-backend b/backend/conf/crontab/copr-backend
index 68ba1a4..de9f250 100644
--- a/backend/conf/crontab/copr-backend
+++ b/backend/conf/crontab/copr-backend
@@ -1,3 +1,3 @@
#!/usr/bin/sh
-runuser -c "PYTHONUNBUFFERED=1 python /usr/bin/copr_prune_results.py" - copr
>> /var/log/copr/prune_old.log 2>&1
+runuser -c "PYTHONUNBUFFERED=1 python /usr/bin/copr_prune_results.py" - copr
diff --git a/backend/run/copr_find_obsolete_builds.sh
b/backend/run/copr_find_obsolete_builds.sh
new file mode 100755
index 0000000..dcd3e34
--- /dev/null
+++ b/backend/run/copr_find_obsolete_builds.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+CHROOT_PATH=$1
+DAYS=$2
+
+echo "# looking for obsolete files in: " $CHROOT_PATH
+if [ -d $CHROOT_PATH ]
+then
+ pushd $CHROOT_PATH >/dev/null
+
+ # query latest (sucessfull) packages
+ LATEST_PKGS=$(mktemp)
+ ERR_LOG=$(mktemp)
+ # "yum clean metadata" does not work on this custom repos
+ rm -rf $(find /var/tmp -name query &> /dev/null) &> /dev/null
+ rm -rf /var/tmp/createrepo* &> /dev/null
+ rm -rf /var/cache/yum/* &> /dev/null
+
+ repoquery --repofrompath=query,$CHROOT_PATH --repoid=query -a --location
2>$ERR_LOG \
+ | cut -c8- > $LATEST_PKGS
+
+ # Remove builds older then $DAYS days and which have newer builds available
+ for SUCCESS in $(find -name success -mtime +$DAYS); do
+ DIR=$(basename $(dirname $SUCCESS))
+ echo "# checking dir: " $DIR
+ for PACKAGE in $(ls $DIR); do
+ grep $PACKAGE $LATEST_PKGS >/dev/null && continue 2
+ done
+ # package was not found in $LATEST_PKGS
+ #rm -rf $DIR
+ #echo -n .
+ echo $DIR
+ done
+
+ rm -f $LATEST_PKGS $ERR_LOG
+ popd >/dev/null
+fi
diff --git a/backend/run/copr_prune_old_builds.sh b/backend/run/copr_prune_old_builds.sh
deleted file mode 100755
index 5b50c4c..0000000
--- a/backend/run/copr_prune_old_builds.sh
+++ /dev/null
@@ -1,52 +0,0 @@
-#!/bin/bash
-
-REPOPATH=$1
-DAYS=$2
-
-
-pushd $REPOPATH >/dev/null
-
-
-for CHROOT in $(ls -d $REPOPATH/*/); do
- pushd $CHROOT >/dev/null
-
- # remove old failed builds
- for FAILED in $(find -name 'fail' -mtime +$DAYS); do
- rm -rf $(dirname $FAILED)
- echo -n .
- done
-
- # query latest (sucessfull) packages
- LATEST_PKGS=$(mktemp)
- ERR_LOG=$(mktemp)
- # "yum clean metadata" does not work on this custom repos
- rm -rf $(find /var/tmp -name query &> /dev/null) &> /dev/null
- rm -rf /var/tmp/createrepo* &> /dev/null
- rm -rf /var/cache/yum/* &> /dev/null
-
- repoquery --repofrompath=query,$REPOPATH/$CHROOT --repoid=query -a --location
2>$ERR_LOG \
- | cut -c8- > $LATEST_PKGS
-
- # remove directory if it does not have repodata at all
- if $(grep --quiet 'Cannot retrieve repository metadata' $ERR_LOG) || $(grep
--quiet "Can't download or revert repomd.xml" $ERR_LOG) ; then
- # don't remove when /devel presents
- if [ ! -d $REPOPATH/$CHROOT/devel ]; then
- rm -rf $REPOPATH/$CHROOT/
- fi
- else
- # Remove builds older then $DAYS days and which have newer builds available
- for SUCCESS in $(find -name success -mtime +$DAYS); do
- DIR=$(basename $(dirname $SUCCESS))
- for PACKAGE in $(ls $DIR); do
- grep $PACKAGE $LATEST_PKGS >/dev/null && continue 2
- done
- # package was not found in $LATEST_PKGS
- rm -rf $DIR
- echo -n .
- done
- fi
-
- rm -f $LATEST_PKGS $ERR_LOG
- popd >/dev/null
-done
-popd >/dev/null
diff --git a/backend/run/copr_prune_results.py b/backend/run/copr_prune_results.py
index 70bc20c..9f77f81 100644
--- a/backend/run/copr_prune_results.py
+++ b/backend/run/copr_prune_results.py
@@ -6,9 +6,15 @@ from __future__ import division
from __future__ import absolute_import
import os
+import shutil
import sys
import logging
from subprocess import Popen, PIPE
+import time
+
+
+log = logging.getLogger(__name__)
+
from copr.client.exceptions import CoprException, CoprRequestException
@@ -17,11 +23,9 @@ sys.path.append("/usr/share/copr/")
from backend.helpers import BackendConfigReader, get_auto_createrepo_status
from backend.createrepo import createrepo_unsafe
-log = logging.getLogger(__name__)
-log.addHandler(logging.NullHandler())
DEF_DAYS = 14
-DEF_PRUNE_SCRIPT = "/usr/bin/copr_prune_old_builds.sh"
+DEF_FIND_OBSOLETE_SCRIPT = "/usr/bin/copr_find_obsolete_builds.sh"
def list_subdir(path):
@@ -29,74 +33,131 @@ def list_subdir(path):
return dir_names, map(lambda x: os.path.join(path, x), dir_names)
-def prune_project(opts, path, username, projectname):
- log.debug("Going to prune {}/{}".format(username, projectname))
- # get ACR
- try:
- if not get_auto_createrepo_status(opts.frontend_base_url, username,
projectname):
- log.debug("Skipped {}/{} since auto createrepo option is disabled"
- .format(username, projectname))
+class Pruner(object):
+ def __init__(self, opts):
+ self.opts = opts
+ self.days = getattr(self.opts, "prune_days", DEF_DAYS)
+ self.prune_sh_script = getattr(self.opts, "find_obsolete_script",
DEF_FIND_OBSOLETE_SCRIPT)
+
+ def prune_failed_builds(self, chroot_path):
+ """
+ Deletes subdirs (project directories) which contains file `fail`
+ with mtime older then self.days
+
+ :param chroot_path: path to the chroot directory
+ """
+ for sub_dir_name in os.listdir(chroot_path):
+ build_path = os.path.join(chroot_path, sub_dir_name)
+ if not os.path.isdir(build_path):
+ # log.debug("Not a project directory, skipping:
{}".format(build_path))
+ continue
+
+ fail_file_path = os.path.join(build_path, "fail")
+ if os.path.exists(fail_file_path) and not
os.path.exists(os.path.join(build_path, "success")):
+ if time.time() - os.path.getmtime(fail_file_path) > self.days:
+ log.info("Removing failed build: {}".format(build_path))
+ shutil.rmtree(build_path)
+
+ def prune_obsolete_success_builds(self, chroot_path, username, projectname):
+ """
+ Uses bash script which invokes repoquery to find obsolete build_dirs
+
+ :param chroot_path: path to the chroot directory
+ :return:
+ """
+ # import ipdb; ipdb.set_trace()
+ cmd = map(str, [self.prune_sh_script, chroot_path, self.days])
+ handle = Popen(cmd, stdout=PIPE, stderr=PIPE)
+ stdout, stderr = handle.communicate()
+ if handle.returncode != 0:
+ log.error("Failed to prune old builds for copr {}/{}\n STDOUT: \n{}\n
STDERR: \n{}\n"
+ .format(username, projectname, stdout.decode(), stderr.decode()))
+ return
+ # log.debug("find obsolete returned:\n {}\n stderr: \n
{}".format(stdout, stderr))
+ for line in stdout.split("\n"):
+ if not line.strip() or (len(line) > 0 and line[0] == "#"):
+ continue
+ to_delete = line.strip()
+ log.debug("Obsolete path, check for remove: {}".format(to_delete))
+
+ if to_delete in os.listdir(chroot_path):
+ to_delete_path = os.path.join(chroot_path, to_delete)
+ if os.path.isdir(to_delete_path):
+ log.info("Removing obsolete build:
{}".format(to_delete_path))
+ shutil.rmtree(to_delete_path)
+
+ def run(self):
+ results_dir = self.opts.destdir
+ log.info("Pruning results dir: {} ".format(results_dir))
+ user_dir_names, user_dirs = list_subdir(results_dir)
+
+ log.info("Going to process total number: {} of user's
directories".format(len(user_dir_names)))
+ log.info("Going to process user's directories:
{}".format(user_dir_names))
+
+ counter = 0
+ for username, subpath in zip(user_dir_names, user_dirs):
+ log.debug("For user `{}` exploring path: {}".format(username,
subpath))
+ for projectname, project_path in zip(*list_subdir(subpath)):
+ log.debug("Exploring project `{}` with path:
{}".format(projectname, project_path))
+ self.prune_project(project_path, username, projectname)
+
+ counter += 1
+ log.info("Pruned {}. projects".format(counter))
+
+ log.info("Pruning finished")
+
+ def prune_project(self, project_path, username, projectname):
+ log.info("Going to prune {}/{}".format(username, projectname))
+ # get ACR
+ try:
+ if not get_auto_createrepo_status(self.opts.frontend_base_url, username,
projectname):
+ log.debug("Skipped {}/{} since auto createrepo option is
disabled"
+ .format(username, projectname))
+ return
+ except (CoprException, CoprRequestException) as exception:
+ log.debug("Failed to get project details for {}/{} with error:
{}".format(
+ username, projectname, exception))
return
- except (CoprException, CoprRequestException) as exception:
- log.debug("Failed to get project details for {}/{} with error:
{}".format(
- username, projectname, exception))
- return
-
- # run prune project sh
- days = getattr(opts, "prune_days", DEF_DAYS)
-
- cmd = map(str, [DEF_PRUNE_SCRIPT, path, days])
-
- handle = Popen(cmd, stdout=PIPE, stderr=PIPE)
- stdout, stderr = handle.communicate()
-
- if handle.returncode != 0:
- print("Failed to prune old builds for copr {}/{}".format(username,
projectname))
- print("STDOUT: \n{}".format(stdout.decode()))
- print("STDERR: \n{}".format(stderr.decode()))
- return
-
- # run createrepo
- log.debug("Prune done for {}/{}".format(username, projectname))
- try:
- retcode, stdout, stderr = createrepo_unsafe(path)
- if retcode != 0:
- print("Createrepo for {}/{} failed".format(username, projectname))
- print("STDOUT: \n{}".format(stdout.decode()))
- print("STDERR: \n{}".format(stderr.decode()))
- except Exception as exception:
- print("Createrepo for {}/{} failed with error: {}"
- .format(username, projectname, exception))
+
+ # run prune project sh
+ for sub_dir_name in os.listdir(project_path):
+ if sub_dir_name == "repodata":
+ continue
+ chroot_path = os.path.join(project_path, sub_dir_name)
+ if not os.path.isdir(chroot_path):
+ continue
+
+ try:
+ self.prune_failed_builds(chroot_path)
+ self.prune_obsolete_success_builds(chroot_path, username, projectname)
+ except Exception as err:
+ log.exception(err)
+ log.error("Error during prune copr {}/{}".format(username,
projectname))
+
+ log.debug("Prune done for {}/{}".format(username, projectname))
+ # run createrepo
+
+ try:
+ retcode, stdout, stderr = createrepo_unsafe(project_path)
+ if retcode != 0:
+ log.error(
+ "Createrepo for copr {}/{}\n STDOUT: \n{}\n STDERR:
\n{}\n"
+ .format(username, projectname, stdout.decode(),
stderr.decode()))
+ except Exception as exception:
+ log.exception("Createrepo for copr {}/{} failed with error:
{}"
+ .format(username, projectname, exception))
+
+ log.info("Prune finished for copr {}/{}".format(username,
projectname))
def main():
config_file = os.environ.get("BACKEND_CONFIG",
"/etc/copr/copr-be.conf")
- opts = BackendConfigReader(config_file).read()
-
- results_dir = opts.destdir
- log.info("Pruning results dir: {} ".format(results_dir))
- user_dir_names, user_dirs = list_subdir(results_dir)
-
- print("Going to process total number: {} of user's
directories".format(len(user_dir_names)))
- log.info("Going to process user's directories:
{}".format(user_dir_names))
-
- counter = 0
- for username, subpath in zip(user_dir_names, user_dirs):
- log.debug("For user `{}` exploring path: {}".format(username,
subpath))
- for projectname, project_path in zip(*list_subdir(subpath)):
- log.debug("Exploring project `{}` with path:
{}".format(projectname, project_path))
- prune_project(opts, project_path, username, projectname)
-
- counter += 1
- print("Pruned {}. projects".format(counter))
-
- print("Pruning finished")
-
+ pruner = Pruner(BackendConfigReader(config_file).read())
+ pruner.run()
if __name__ == "__main__":
- # logging.basicConfig(
- # level=logging.DEBUG,
- # format='[%(asctime)s] {%(pathname)s:%(lineno)d} %(levelname)s -
%(message)s',
- # datefmt='%H:%M:%S'
- # )
+ logging.basicConfig(
+ filename="/var/log/copr/prune_old.log",
+ format='[%(asctime)s][%(levelname)6s]: %(message)s',
+ level=logging.INFO)
main()
diff --git a/backend/tests/_resources/foox.tar.gz b/backend/tests/_resources/foox.tar.gz
new file mode 100644
index 0000000..aaebe8f
Binary files /dev/null and b/backend/tests/_resources/foox.tar.gz differ
diff --git a/backend/tests/run/test_copr_prune_results.py
b/backend/tests/run/test_copr_prune_results.py
new file mode 100644
index 0000000..074e9dc
--- /dev/null
+++ b/backend/tests/run/test_copr_prune_results.py
@@ -0,0 +1,190 @@
+# coding: utf-8
+import logging
+import os
+import sys
+import shutil
+import tarfile
+import tempfile
+import time
+from bunch import Bunch
+from subprocess import Popen, PIPE
+
+import pytest
+
+import six
+if six.PY3:
+ from unittest import mock
+else:
+ import mock
+ from mock import MagicMock
+
+
+
+sys.path.append("../../run")
+
+
+MODULE_REF = "copr_prune_results"
+
+(a)pytest.yield_fixture
+def mc_popen():
+ with mock.patch("{}.Popen".format(MODULE_REF)) as handle:
+ yield handle
+
+
+from copr_prune_results import Pruner
+
+
+def assert_same_dirs(a, b):
+ proc = Popen(["diff", "-r", a, b], stdout=PIPE, stdin=PIPE)
+ stdout, stderr = proc.communicate()
+
+ if proc.returncode != 0:
+ raise AssertionError("Directories {}, {} aren't equal, diff out: \n {}
\n {}"
+ .format(a, b, stdout, stderr))
+
+class TestPruneResults(object):
+
+ def setup_method(self, method):
+ self.tmp_dir_name = None
+ self.test_time = time.time()
+ self.make_temp_dir()
+
+ self.pkg_1 = "copr-backend-1.59-1.git.1.7ed2b8d.fc20"
+ self.pkg_1_obsolete = "copr-backend-1.40-1.fc20"
+ self.pkg_2 = "hello-2.8-1.fc20"
+ self.pkg_2_obsolete = "hello-1.0.fc20"
+ self.prj = "foox"
+ self.chroots = ["fedora-20-i386", "fedora-20-x86_64",
+ "fedora-21-i386", "fedora-21-x86_64"]
+ self.opts = Bunch(
+ days=14,
+
+ find_obsolete_script=os.path.abspath(
+ os.path.join(os.path.dirname(__file__), os.pardir, os.pardir,
+ "run", "copr_find_obsolete_builds.sh"))
+
+
+ )
+
+ self.log_file_path = os.path.join(self.tmp_dir_name, "_unittest.log")
+ logging.basicConfig(
+ # filename=self.log_file_path,
+ stream=sys.stdout,
+ format='[%(asctime)s][%(levelname)6s]: %(message)s',
+ level=logging.DEBUG
+ )
+ self.log = logging.getLogger(__name__)
+ self.log.info("setup method done")
+
+ self.username = "vgologuz"
+ self.coprname = "foox"
+
+ def teardown_method(self, method):
+ self.rm_tmp_dir()
+
+ def rm_tmp_dir(self):
+ if self.tmp_dir_name:
+ shutil.rmtree(self.tmp_dir_name)
+ self.tmp_dir_name = None
+
+ def make_temp_dir(self):
+ root_tmp_dir = tempfile.gettempdir()
+ subdir = "test_action_{}".format(self.test_time)
+ self.tmp_dir_name = os.path.join(root_tmp_dir, subdir)
+
+ os.mkdir(self.tmp_dir_name)
+ self.expect_dir_name = os.path.join(self.tmp_dir_name, "_expect")
+ os.mkdir(self.expect_dir_name)
+
+ self.unpack_resource("foox.tar.gz")
+ self.unpack_resource("foox.tar.gz", self.expect_dir_name)
+
+ return self.tmp_dir_name
+
+ def unpack_resource(self, resource_name, target=None):
+ if self.tmp_dir_name is None:
+ self.make_temp_dir()
+
+ src_path = os.path.join(os.path.dirname(__file__), "..",
+ "_resources", resource_name)
+
+ with tarfile.open(src_path, "r:gz") as tar_file:
+ tar_file.extractall(target or self.tmp_dir_name)
+
+
+ @pytest.fixture
+ def test_pruner(self):
+ self.pruner = Pruner(self.opts)
+
+ return self.pruner
+
+ def test_prune_old_failed(self, test_pruner):
+ """
+ Delete if build with "fail" fail with mtime > 14 days
+
+ """
+ # TODO: what neither fail or success are present?
+
+ shutil.move(
+ os.path.join(self.tmp_dir_name, self.prj, self.chroots[0], self.pkg_1,
"success"),
+ os.path.join(self.tmp_dir_name, self.prj, self.chroots[0], self.pkg_1,
"fail"),)
+ shutil.move(
+ os.path.join(self.tmp_dir_name, self.prj, self.chroots[0], self.pkg_2,
"success"),
+ os.path.join(self.tmp_dir_name, self.prj, self.chroots[0], self.pkg_2,
"fail"),)
+ shutil.move(
+ os.path.join(self.expect_dir_name, self.prj, self.chroots[0], self.pkg_2,
"success"),
+ os.path.join(self.expect_dir_name, self.prj, self.chroots[0], self.pkg_2,
"fail"),)
+
+ os.utime(os.path.join(self.tmp_dir_name, self.prj, self.chroots[0], self.pkg_1,
"fail"), (0, 0))
+ os.utime(os.path.join(self.tmp_dir_name, self.prj, self.chroots[0], self.pkg_2,
"fail"),
+ (time.time(), time.time()))
+
+ shutil.rmtree(os.path.join(self.expect_dir_name, self.prj, self.chroots[0],
self.pkg_1))
+
+ self.pruner.prune_failed_builds(os.path.join(self.tmp_dir_name, self.prj,
self.chroots[0]))
+
+ assert_same_dirs(
+ os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]),
+ os.path.join(self.expect_dir_name, self.prj, self.chroots[0]),
+ )
+
+ def test_prune_obsolete_builds(self, test_pruner):
+
+ os.utime(os.path.join(self.tmp_dir_name, self.prj, self.chroots[0], self.pkg_1,
"success"), (0, 0))
+ os.utime(os.path.join(self.tmp_dir_name, self.prj, self.chroots[0], self.pkg_2,
"success"), (0, 0))
+
+ # old obsolete build folder should be removed
+ os.utime(os.path.join(self.tmp_dir_name, self.prj, self.chroots[0],
self.pkg_2_obsolete, "success"), (0, 0))
+ shutil.rmtree(os.path.join(self.expect_dir_name, self.prj, self.chroots[0],
self.pkg_2_obsolete))
+
+ # recent obsolete build should remain
+ os.utime(os.path.join(self.tmp_dir_name, self.prj, self.chroots[0],
self.pkg_1_obsolete, "success"),
+ (time.time(), time.time()))
+
+ self.pruner.prune_obsolete_success_builds(
+ os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]),
+ self.username, self.coprname
+ )
+
+ assert_same_dirs(
+ os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]),
+ os.path.join(self.expect_dir_name, self.prj, self.chroots[0]),
+ )
+
+ def test_prune_obsolete_builds_handle_script_error(self, test_pruner, mc_popen):
+ mc_handle = MagicMock()
+ mc_popen.return_value = mc_handle
+ mc_handle.returncode = 1
+ mc_handle.communicate.return_value = ("foo", "bar")
+
+ # doesn't touch FS if `find_obsolete_build` produce error return code
+ self.pruner.prune_obsolete_success_builds(
+ os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]),
+ self.username, self.coprname
+ )
+ assert mc_popen.called
+
+ assert_same_dirs(
+ os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]),
+ os.path.join(self.expect_dir_name, self.prj, self.chroots[0]),
+ )