From 04681857aeced8100b9e2251bae8a4d3f212aa38 Mon Sep 17 00:00:00 2001 From: Mike Bonnet Date: Wed, 22 Apr 2015 16:06:48 -0400 Subject: [PATCH] mergeScratch(): import rpms from a scratch build into an existing build The mergeScratch() method allows importing rpms built by a scratch build into an existing build, if that build did not produce rpms matching the arch of the scratch build. This is useful for bootstrapping a new arch into existing builds, and avoiding a mass-rebuild to add arch support. --- hub/kojihub.py | 141 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ koji/__init__.py | 3 ++ 2 files changed, 144 insertions(+) diff --git a/hub/kojihub.py b/hub/kojihub.py index 9decf7c..d11ea7d 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4720,6 +4720,117 @@ def _import_wrapper(task_id, build_info, rpm_results): import_build_log(os.path.join(rpm_task_dir, log), build_info, subdir='noarch') +def merge_scratch(task_id): + """Import rpms from a scratch build into an existing build, retaining + buildroot metadata and build logs.""" + task = Task(task_id) + try: + task_info = task.getInfo(request=True) + except koji.GenericError: + raise koji.ImportError, 'invalid task: %s' % task_id + if task_info['state'] != koji.TASK_STATES['CLOSED']: + raise koji.ImportError, 'task %s did not complete successfully' % task_id + if task_info['method'] != 'build': + raise koji.ImportError, 'task %s is not a build task' % task_id + if len(task_info['request']) < 3 or not task_info['request'][2].get('scratch'): + raise koji.ImportError, 'task %s is not a scratch build' % task_id + + # sanity check the task, and extract data required for import + srpm = None + tasks = {} + for child in task.getChildren(): + if child['method'] != 'buildArch': + continue + info = {'rpms': [], + 'logs': []} + for output in list_task_output(child['id']): + if output.endswith('.src.rpm'): + srpm_name = os.path.basename(output) + if not srpm: + srpm = srpm_name + else: + if srpm != srpm_name: + raise koji.ImportError, 'task srpm names do not match: %s, %s' % \ + (srpm, srpm_name) + elif output.endswith('.noarch.rpm'): + continue + elif output.endswith('.rpm'): + rpminfo = koji.parse_NVRA(os.path.basename(output)) + if 'arch' not in info: + info['arch'] = rpminfo['arch'] + elif info['arch'] != rpminfo['arch']: + raise koji.ImportError, 'multiple arches generated by task %s: %s, %s' % \ + (child['id'], info['arch'], rpminfo['arch']) + info['rpms'].append(output) + elif output.endswith('.log'): + info['logs'].append(output) + if not info['rpms']: + raise koji.ImportError, 'no arch-specific rpms produced by task %s' % child['id'] + if not info['logs']: + raise koji.ImportError, 'task %s is missing logs' % child['id'] + buildroots = query_buildroots(taskID=child['id'], + queryOpts={'order': '-id', 'limit': 1}) + if not buildroots: + raise koji.ImportError, 'no buildroot associated with task %s' % child['id'] + info['buildroot_id'] = buildroots[0]['id'] + tasks[child['id']] = info + + # sanity check the build + build_nvr = koji.parse_NVRA(srpm) + build = get_build(build_nvr) + if not build: + raise koji.ImportError, 'no such build: %(name)s-%(version)s-%(release)s' % \ + build_nvr + if build['state'] != koji.BUILD_STATES['COMPLETE']: + raise koji.ImportError, '%s did not complete successfully' % build['nvr'] + if not build['task_id']: + raise koji.ImportError, 'no task for %s' % build['nvr'] + build_task_info = Task(build['task_id']).getInfo(request=True) + # Intentionally skip checking the build task state. + # There are cases where the build can be valid even though the task has failed, + # e.g. tagging failures. + + # compare the task and build and make sure they are compatible with importing + if task_info['request'][0] != build_task_info['request'][0]: + raise koji.ImportError, 'SCM URLs for the task and build do not match: %s, %s' % \ + (task_info['request'][0], build_task_info['request'][0]) + build_arches = set() + for rpm in list_rpms(buildID=build['id']): + if rpm['arch'] == 'src': + build_srpm = '%s.src.rpm' % rpm['nvr'] + if srpm != build_srpm: + raise koji.ImportError, 'task and build srpm names do not match: %s, %s' % \ + (srpm, build_srpm) + elif rpm['arch'] == 'noarch': + continue + else: + build_arches.add(rpm['arch']) + if not build_arches: + raise koji.ImportError, 'no arch-specific rpms found for %s' % build['nvr'] + task_arches = set([t['arch'] for t in tasks.values()]) + overlapping_arches = task_arches.intersection(build_arches) + if overlapping_arches: + raise koji.ImportError, 'task %s and %s produce rpms with the same arches: %s' % \ + (task_info['id'], build['nvr'], ', '.join(overlapping_arches)) + + # everything looks good, do the import + for task_id, info in tasks.items(): + taskpath = koji.pathinfo.task(task_id) + for filename in info['rpms']: + filepath = os.path.realpath(os.path.join(taskpath, filename)) + rpminfo = import_rpm(filepath, build, info['buildroot_id']) + import_rpm_file(filepath, build, rpminfo) + add_rpm_sig(rpminfo['id'], koji.rip_rpm_sighdr(filepath)) + for logname in info['logs']: + logpath = os.path.realpath(os.path.join(taskpath, logname)) + import_build_log(logpath, build, subdir=info['arch']) + + # flag tags whose content has changed, so relevant repos can be regen'ed + for tag in list_tags(build=build['id']): + set_tag_update(tag['id'], 'IMPORT') + + return build['id'] + def get_archive_types(): """Return a list of all supported archivetypes""" select = """SELECT id, name, description, extensions FROM archivetypes @@ -7647,6 +7758,36 @@ class RootExports(object): for tag in list_tags(build=rpminfo['build_id']): set_tag_update(tag['id'], 'IMPORT') + def mergeScratch(self, task_id): + """Import the rpms generated by a scratch build, and associate + them with an existing build. + + To be eligible for import, the build must: + - be successfully completed + - contain at least one arch-specific rpm + + The task must: + - be a 'build' task + - be successfully completed + - use the exact same SCM URL as the build + - contain at least one arch-specific rpm + - have no overlap between the arches of the rpms it contains and + the rpms contained by the build + - contain a .src.rpm whose filename exactly matches the .src.rpm + of the build + + Only arch-specific rpms will be imported. noarch rpms and the src + rpm will be skipped. Build logs and buildroot metadata from the + scratch build will be imported along with the rpms. + + This is useful for bootstrapping a new arch. RPMs can be built + for the new arch using a scratch build and then merged into an + existing build, incrementally expanding arch coverage and avoiding + the need for a mass-rebuild to support the new arch. + """ + context.session.assertPerm('admin') + return merge_scratch(task_id) + def addExternalRPM(self, rpminfo, external_repo, strict=True): """Import an external RPM diff --git a/koji/__init__.py b/koji/__init__.py index d133ed1..c162e7e 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -323,6 +323,9 @@ class ParameterError(GenericError): """Raised when an rpc call receives incorrect arguments""" faultCode = 1019 +class ImportError(GenericError): + """Raised when an import fails""" + faultCode = 1020 class MultiCallInProgress(object): """ -- 1.8.2.1