commit 3cc16616c7eabf920b68f3da800f700c1279fc2b Author: Till Maas opensource@till.name Date: Mon Sep 29 20:47:18 2014 +0200
Add block_retired.py
Helper script to block (and untag/unblock in build tag for EPEL) all retired packages.
Reference: https://fedorahosted.org/rel-eng/ticket/5914
scripts/block_retired.py | 270 ++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 270 insertions(+), 0 deletions(-) --- diff --git a/scripts/block_retired.py b/scripts/block_retired.py new file mode 100755 index 0000000..2556980 --- /dev/null +++ b/scripts/block_retired.py @@ -0,0 +1,270 @@ +#!/usr/bin/python -tt +# vim: fileencoding=utf8 foldmethod=marker +# SPDX-License-Identifier: GPL-2.0+ +# {{{ License header: GPLv2+ +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see https://www.gnu.org/licenses/. +# }}} + +import argparse +import datetime +import getpass +import logging +import subprocess +import time + +import koji +import pkgdb2client + + +log = logging.getLogger(__name__) +RETIRING_BRANCHES = ["el5", "el6", "epel7", "f21", "master"] + + +class ReleaseMapper(object): + BRANCHNAME = 0 + KOJI_TAG = 1 + EPEL_BUILD_TAG = 2 + + def __init__(self): + + # git branchname, koji tag, pkgdb version + self.mapping = ( + ("master", "f22", ""), + ("f21", "f21", ""), + ("f20", "f20", ""), + ("f19", "f19", ""), + ("f18", "f18", ""), + ("epel7", "epel7", "epel7-build"), + ("el6", "dist-6E-epel", "dist-6E-epel-build"), + ("el5", "dist-5E-epel", "dist-5E-epel-build"), + ) + + def branchname(self, key=""): + return self.lookup(key, self.BRANCHNAME) + + def koji_tag(self, key=""): + return self.lookup(key, self.KOJI_TAG) + + def epel_build_tag(self, key=""): + return self.lookup(key, self.EPEL_BUILD_TAG) + + def lookup(self, key, column): + if key: + key = key.lower() + for row in self.mapping: + for c in row: + if c.lower() == key: + return row[column] + else: + return [row[column] for row in self.mapping] + return None + + +def blocked_packages(branch="master"): + mapper = ReleaseMapper() + tag = mapper.koji_tag(branch) + kojisession = koji.ClientSession('https://koji.fedoraproject.org/kojihub') + pkglist = kojisession.listPackages(tagID=tag, inherited=True) + blocked = [p["package_name"] for p in pkglist if p["blocked"]] + return blocked + + +def get_retired_packages(branch="master"): + pkgdb = pkgdb2client.PkgDB() + retiredresponse = pkgdb.get_packages( + "", branches=branch, page="all", status="Retired") + retiredinfo = retiredresponse["packages"] + retiredpkgs = [p["name"] for p in retiredinfo] + return retiredpkgs + + +def pkgdb_retirement_status(package, branch="master"): + """ Returns retirement info for `package` in `branch` + + :returns: dict: retired: True - if retired, False if not, None if + there was an error, status_change: last status change as datetime object + """ + + pkgdb = pkgdb2client.PkgDB() + retired = None + status_change = None + try: + pkgdbresult = pkgdb.get_package(package, branches=branch) + if pkgdbresult["output"] == "ok": + for pkginfo in pkgdbresult["packages"]: + if pkginfo["package"]["name"] == package: + if pkginfo["status"] == "Retired": + retired = True + else: + retired = False + status_change = datetime.datetime.fromtimestamp( + pkginfo["status_change"]) + break + except: + pass + + return dict(retired=retired, status_change=status_change) + + +def get_retirement_info(message): + """ Check whether a message is a retire message. + + :param message: Message to check + :returns: (str, str, bool) or (None, None, None): package name and + branch the package was retired on, bool: True: package was retired, False: + package was unretired + + """ + if message['topic'] == \ + u'org.fedoraproject.prod.pkgdb.package.update.status': + msg = message['msg'] + pkgname = msg['package_listing']['package']['name'] + branch = msg['package_listing']['collection']['branchname'] + res = dict(name=pkgname, branch=branch) + if msg["prev_status"] != "Retired" and msg["status"] == "Retired": + res["retired"] = True + elif msg["prev_status"] == "Retired" and \ + msg["status"] != "Retired": + res["retired"] = False + return res + return None + + +def block_package(packages, branch="master"): + if isinstance(packages, basestring): + packages = [packages] + + if len(packages) == 0: + return None + + mapper = ReleaseMapper() + tag = mapper.koji_tag(branch) + cmd = ["koji", "block-pkg", tag] + packages + log.debug("Running: %s", " ".join(cmd)) + subprocess.check_call(cmd) + + epel_build_tag = mapper.epel_build_tag(branch) + + if epel_build_tag: + cmd = ["koji", "untag-build", "--all", tag] + packages + log.debug("Running: %s", " ".join(cmd)) + subprocess.check_call(cmd) + + cmd = ["koji", "unblock-pkg", epel_build_tag] + packages + log.debug("Running: %s", " ".join(cmd)) + subprocess.check_call(cmd) + + +def handle_message(message, retiring_branches=RETIRING_BRANCHES): + messageinfo = get_retirement_info(message) + msg_id = message["msg_id"] + if messageinfo is None: + return None + + if messageinfo["retired"] is False: + return False + + branch = messageinfo["branch"] + if branch not in retiring_branches: + log.error("Message '%s' for the wrong branch '%s'", msg_id, + branch) + return None + + package = messageinfo["name"] + + pkgdbinfo = pkgdb_retirement_status(package, branch) + + if pkgdbinfo["retired"] is not True: + log.error("Processing '%s', package '%s' not retired", + msg_id, package) + + log.debug("'%s' retired on '%s'", package, pkgdbinfo["status_change"]) + return block_package(package, branch) + + +def block_all_retired(branches=RETIRING_BRANCHES): + for branch in branches: + log.debug("Processing branch %s", branch) + retired = get_retired_packages(branch) + blocked = blocked_packages(branch) + + unblocked = [] + for pkg in retired: + if pkg not in blocked: + unblocked.append(pkg) + + if unblocked: + log.info("Blocked packages %s on %s", unblocked, branch) + block_package(unblocked, branch) + + +class SubjectSMTPHandler(logging.handlers.SMTPHandler): + # Class copied from autosigner.py + subject_prefix = "" + + def getSubject(self, record): + first_line = record.message.split("\n")[0] + fmt = self.subject_prefix + "{0.levelname}: {first_line}" + + return fmt.format(record, first_line=first_line) + + +def setup_logging(debug=False, mail=False): + if debug: + log.setLevel(logging.DEBUG) + else: + log.setLevel(logging.INFO) + + formatter = logging.Formatter( + '%(asctime)s: %(levelname)s: %(message)s', + ) + # Log in UTC + formatter.converter = time.gmtime + + console_logger = logging.StreamHandler() + if debug: + console_logger.setLevel(logging.DEBUG) + else: + console_logger.setLevel(logging.INFO) + console_logger.setFormatter(formatter) + log.addHandler(console_logger) + + if mail: + # FIXME: Make this a config option + fedora_user = getpass.getuser() + mail_logger = SubjectSMTPHandler( + "127.0.0.1", fedora_user, [fedora_user], "block_retired event") + if debug: + mail_logger.setLevel(logging.DEBUG) + mail_logger.setFormatter(formatter) + mail_logger.subject_prefix = "Package Blocker: " + log.addHandler(mail_logger) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser("Block retired packages") + parser.add_argument("--debug", default=False, action="store_true") + parser.add_argument("packages", nargs="*", metavar="package", + help="Packages to block, default all retired packages") + parser.add_argument( + "--branch", default="master", + help="Branch to retire specified packages on, default: %(default)s") + args = parser.parse_args() + + setup_logging(args.debug) + + if not args.packages: + block_all_retired() + else: + block_package(args.packages, args.branch)
rel-eng@lists.fedoraproject.org