The attached patchset adds the following functionality to pungi: - building 'full' trees, that include all subpackages for any included package - building 'selfhosting' trees, that solve the package set for, and include, any build dependencies
This is a rebase of some patches sent earlier this year.
bin/pungi.py | 23 ++++++++++ pypungi/__init__.py | 109 ++++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 112 insertions(+), 20 deletions(-)
Bill
Signed-off-by: Bill Nottingham notting@redhat.com --- src/pypungi/__init__.py | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-)
diff --git a/src/pypungi/__init__.py b/src/pypungi/__init__.py index e084054..cc3928f 100644 --- a/src/pypungi/__init__.py +++ b/src/pypungi/__init__.py @@ -201,6 +201,8 @@ class Pungi(pypungi.PungiBase): thisrepo.includepkgs = repo.includepkgs if repo.cost: thisrepo.cost = repo.cost + if repo.ignoregroups: + thisrepo.enablegroups = 0 self.ayum.repos.add(thisrepo) self.ayum.repos.enableRepo(thisrepo.id) self.ayum._getRepos(thisrepo=thisrepo.id, doSetup = True)
Signed-off-by: Bill Nottingham notting@redhat.com --- src/pypungi/__init__.py | 36 +++++++++++++++++++----------------- 1 files changed, 19 insertions(+), 17 deletions(-)
diff --git a/src/pypungi/__init__.py b/src/pypungi/__init__.py index cc3928f..2590775 100644 --- a/src/pypungi/__init__.py +++ b/src/pypungi/__init__.py @@ -133,7 +133,7 @@ class Pungi(pypungi.PungiBase):
self.ksparser = ksparser self.polist = [] - self.srpmlist = [] + self.srpmpolist = [] self.debuginfolist = [] self.resolved_deps = {} # list the deps we've already resolved, short circuit.
@@ -396,15 +396,29 @@ class Pungi(pypungi.PungiBase): self.polist = final_pkgobjs.keys() self.logger.info('Finished gathering package objects.')
+ def getSRPMPo(self, po): + """Given a package object, get a package object for the + corresponding source rpm. Requires yum still configured + and a valid package object.""" + srpm = po.sourcerpm.split('.src.rpm')[0] + (sname, sver, srel) = srpm.rsplit('-', 2) + try: + srpmpo = self.ayum.pkgSack.searchNevra(name=sname, ver=sver, rel=srel, arch='src')[0] + return srpmpo + except IndexError: + print >> sys.stderr, "Error: Cannot find a source rpm for %s" % srpm + sys.exit(1) + def getSRPMList(self): """Cycle through the list of package objects and find the sourcerpm for them. Requires yum still configured and a list of package objects"""
for po in self.polist: - srpm = po.sourcerpm.split('.src.rpm')[0] - if not srpm in self.srpmlist: - self.srpmlist.append(srpm) + srpmpo = self.getSRPMPo(po) + if not srpmpo in self.srpmpolist: + self.logger.info("Adding source package %s.%s" % (srpmpo.name, srpmpo.arch)) + self.srpmpolist.append(srpmpo)
def getDebuginfoList(self): """Cycle through the list of package objects and find @@ -528,20 +542,8 @@ class Pungi(pypungi.PungiBase): """Cycle through the list of srpms and find the package objects for them, Then download them."""
- srpmpolist = [] - - for srpm in self.srpmlist: - (sname, sver, srel) = srpm.rsplit('-', 2) - try: - srpmpo = self.ayum.pkgSack.searchNevra(name=sname, ver=sver, rel=srel, arch='src')[0] - if not srpmpo in srpmpolist: - srpmpolist.append(srpmpo) - except IndexError: - print >> sys.stderr, "Error: Cannot find a source rpm for %s" % srpm - sys.exit(1) - # do the downloads - self._downloadPackageList(srpmpolist, os.path.join('source', 'SRPMS')) + self._downloadPackageList(self.srpmpolist, os.path.join('source', 'SRPMS'))
def downloadDebuginfo(self): """Cycle through the list of debuginfo rpms and
Since each package we add for build dependencies may add a new source rpm to our list, this needs to recurse.
Signed-off-by: Bill Nottingham notting@redhat.com --- src/pypungi/__init__.py | 23 +++++++++++++++++++++++ 1 files changed, 23 insertions(+), 0 deletions(-)
diff --git a/src/pypungi/__init__.py b/src/pypungi/__init__.py index 2590775..3271f26 100644 --- a/src/pypungi/__init__.py +++ b/src/pypungi/__init__.py @@ -420,6 +420,29 @@ class Pungi(pypungi.PungiBase): self.logger.info("Adding source package %s.%s" % (srpmpo.name, srpmpo.arch)) self.srpmpolist.append(srpmpo)
+ def resolvePackageBuildDeps(self): + """Make the package lists self hosting. Requires yum + still configured, a list of package objects, and a + a list of source rpms.""" + for srpm in self.srpmpolist: + self.ayum.tsInfo.addInstall(srpm) + deppass = 1 + checked_srpms = [] + while 1: + self.logger.info("Resolving build dependencies, pass %d" % (deppass)) + prev = list(self.ayum.tsInfo.getMembers()) + for srpm in self.srpmpolist[len(checked_srpms):]: + self.getPackageDeps(srpm) + for txmbr in self.ayum.tsInfo: + if txmbr.po.arch != 'src' and txmbr.po not in self.polist: + self.polist.append(txmbr.po) + # Now that we've resolved deps, refresh the source rpm list + checked_srpms = list(self.srpmpolist) + self.getSRPMList() + deppass = deppass + 1 + if len(prev) == len(self.ayum.tsInfo.getMembers()): + break + def getDebuginfoList(self): """Cycle through the list of package objects and find debuginfo rpms for them. Requires yum still
Signed-off-by: Bill Nottingham notting@redhat.com --- src/bin/pungi.py | 7 ++++++- 1 files changed, 6 insertions(+), 1 deletions(-)
diff --git a/src/bin/pungi.py b/src/bin/pungi.py index 7cc615c..9224713 100755 --- a/src/bin/pungi.py +++ b/src/bin/pungi.py @@ -86,13 +86,16 @@ def main(): mypungi._inityum() # initialize the yum object for things that need it if opts.do_all or opts.do_gather: mypungi.getPackageObjects() + if not opts.nosource or opts.selfhosting: + mypungi.getSRPMList() + if opts.selfhosting: + mypungi.resolvePackageBuildDeps() mypungi.downloadPackages() mypungi.makeCompsFile() if not opts.nodebuginfo: mypungi.getDebuginfoList() mypungi.downloadDebuginfo() if not opts.nosource: - mypungi.getSRPMList() mypungi.downloadSRPMs()
if opts.do_all or opts.do_createrepo: @@ -153,6 +156,8 @@ if __name__ == '__main__': parser.add_option("--bugurl", dest="bugurl", type="string", action="callback", callback=set_config, callback_args=(config, ), help='the url for your bug system (defaults to http://bugzilla.redhat.com)') + parser.add_option("--selfhosting", action="store_true", dest="selfhosting", + help='build a self-hosting tree by following build dependencies (optional)') parser.add_option("--nosource", action="store_true", dest="nosource", help='disable gathering of source packages (optional)') parser.add_option("--nodebuginfo", action="store_true", dest="nodebuginfo",
Signed-off-by: Bill Nottingham notting@redhat.com --- src/pypungi/__init__.py | 2 -- 1 files changed, 0 insertions(+), 2 deletions(-)
diff --git a/src/pypungi/__init__.py b/src/pypungi/__init__.py index 3271f26..bd57bf8 100644 --- a/src/pypungi/__init__.py +++ b/src/pypungi/__init__.py @@ -424,8 +424,6 @@ class Pungi(pypungi.PungiBase): """Make the package lists self hosting. Requires yum still configured, a list of package objects, and a a list of source rpms.""" - for srpm in self.srpmpolist: - self.ayum.tsInfo.addInstall(srpm) deppass = 1 checked_srpms = [] while 1:
This avoids repeating the operation many times later if we do it on demand each time.
Signed-off-by: Bill Nottingham notting@redhat.com --- src/pypungi/__init__.py | 32 ++++++++++++++++++++++++++------ 1 files changed, 26 insertions(+), 6 deletions(-)
diff --git a/src/pypungi/__init__.py b/src/pypungi/__init__.py index bd57bf8..1d2734c 100644 --- a/src/pypungi/__init__.py +++ b/src/pypungi/__init__.py @@ -135,6 +135,9 @@ class Pungi(pypungi.PungiBase): self.polist = [] self.srpmpolist = [] self.debuginfolist = [] + self.srpms_build = [] + self.srpms_fulltree = [] + self.last_po = 0 self.resolved_deps = {} # list the deps we've already resolved, short circuit.
def _inityum(self): @@ -409,33 +412,50 @@ class Pungi(pypungi.PungiBase): print >> sys.stderr, "Error: Cannot find a source rpm for %s" % srpm sys.exit(1)
+ def createSourceHashes(self): + """Create two dicts - one that maps binary POs to source POs, and + one that maps a single source PO to all binary POs it produces. + Requires yum still configured.""" + self.src_by_bin = {} + self.bin_by_src = {} + self.logger.info("Generating source <-> binary package mappings") + (dummy1, everything, dummy2) = yum.packages.parsePackages(self.ayum.pkgSack.returnPackages(), ['*']) + for po in everything: + if po.arch == 'src': + continue + srpmpo = self.getSRPMPo(po) + self.src_by_bin[po] = srpmpo + if self.bin_by_src.has_key(srpmpo): + self.bin_by_src[srpmpo].append(po) + else: + self.bin_by_src[srpmpo] = [po] + def getSRPMList(self): """Cycle through the list of package objects and find the sourcerpm for them. Requires yum still configured and a list of package objects""" - - for po in self.polist: - srpmpo = self.getSRPMPo(po) + for po in self.polist[self.last_po:]: + srpmpo = self.src_by_bin[po] if not srpmpo in self.srpmpolist: self.logger.info("Adding source package %s.%s" % (srpmpo.name, srpmpo.arch)) self.srpmpolist.append(srpmpo) + self.last_po = len(self.polist)
def resolvePackageBuildDeps(self): """Make the package lists self hosting. Requires yum still configured, a list of package objects, and a a list of source rpms.""" deppass = 1 - checked_srpms = [] while 1: self.logger.info("Resolving build dependencies, pass %d" % (deppass)) prev = list(self.ayum.tsInfo.getMembers()) - for srpm in self.srpmpolist[len(checked_srpms):]: + for srpm in self.srpmpolist[len(self.srpms_build):]: self.getPackageDeps(srpm) for txmbr in self.ayum.tsInfo: if txmbr.po.arch != 'src' and txmbr.po not in self.polist: self.polist.append(txmbr.po) + self.srpms_build = list(self.srpmpolist) # Now that we've resolved deps, refresh the source rpm list - checked_srpms = list(self.srpmpolist) self.getSRPMList() deppass = deppass + 1 if len(prev) == len(self.ayum.tsInfo.getMembers()):
In other places, this method could be called No Package Left Behind.
Signed-off-by: Bill Nottingham notting@redhat.com --- src/pypungi/__init__.py | 26 ++++++++++++++++++++++++++ 1 files changed, 26 insertions(+), 0 deletions(-)
diff --git a/src/pypungi/__init__.py b/src/pypungi/__init__.py index 1d2734c..01d7b90 100644 --- a/src/pypungi/__init__.py +++ b/src/pypungi/__init__.py @@ -461,6 +461,32 @@ class Pungi(pypungi.PungiBase): if len(prev) == len(self.ayum.tsInfo.getMembers()): break
+ def completePackageSet(self): + """Cycle through all package objects, and add any + that correspond to a source rpm that we are including. + Requires yum still configured and a list of package + objects.""" + thepass = 1 + while 1: + prevlen = len(self.srpmpolist) + self.logger.info("Completing package set, pass %d" % (thepass,)) + for srpm in self.srpmpolist[len(self.srpms_fulltree):]: + for po in self.bin_by_src[srpm]: + if po not in self.polist: + self.logger.info("Adding %s.%s to complete package set" % (po.name, po.arch)) + self.polist.append(po) + self.getPackageDeps(po) + for txmbr in self.ayum.tsInfo: + if txmbr.po.arch != 'src' and txmbr.po not in self.polist: + self.polist.append(txmbr.po) + self.srpms_fulltree = list(self.srpmpolist) + # Now that we've resolved deps, refresh the source rpm list + self.getSRPMList() + if len(self.srpmpolist) == prevlen: + self.logger.info("Completion finished in %d passes" % (thepass,)) + break + thepass = thepass + 1 + def getDebuginfoList(self): """Cycle through the list of package objects and find debuginfo rpms for them. Requires yum still
Since full trees and build-solved trees can affect each other, if we're doing both we need to loop between them until there are no new packages added.
Signed-off-by: Bill Nottingham notting@redhat.com --- src/bin/pungi.py | 18 +++++++++++++++++- 1 files changed, 17 insertions(+), 1 deletions(-)
diff --git a/src/bin/pungi.py b/src/bin/pungi.py index 9224713..eab54a6 100755 --- a/src/bin/pungi.py +++ b/src/bin/pungi.py @@ -86,10 +86,24 @@ def main(): mypungi._inityum() # initialize the yum object for things that need it if opts.do_all or opts.do_gather: mypungi.getPackageObjects() - if not opts.nosource or opts.selfhosting: + if not opts.nosource or opts.selfhosting or opts.fulltree: + mypungi.createSourceHashes() mypungi.getSRPMList() if opts.selfhosting: mypungi.resolvePackageBuildDeps() + if opts.fulltree: + mypungi.completePackageSet() + if opts.selfhosting and opts.fulltree: + # OUCH. + while 1: + plen = len(mypungi.srpmpolist) + mypungi.resolvePackageBuildDeps() + if plen == len(mypungi.srpmpolist): + break + plen = len(mypungi.srpmpolist) + mypungi.completePackageSet() + if plen == len(mypungi.srpmpolist): + break mypungi.downloadPackages() mypungi.makeCompsFile() if not opts.nodebuginfo: @@ -158,6 +172,8 @@ if __name__ == '__main__': help='the url for your bug system (defaults to http://bugzilla.redhat.com)') parser.add_option("--selfhosting", action="store_true", dest="selfhosting", help='build a self-hosting tree by following build dependencies (optional)') + parser.add_option("--fulltree", action="store_true", dest="fulltree", + help='build a tree that includes all packages built from corresponding source rpms (optional)') parser.add_option("--nosource", action="store_true", dest="nosource", help='disable gathering of source packages (optional)') parser.add_option("--nodebuginfo", action="store_true", dest="nodebuginfo",
On Fri, 2009-04-03 at 15:53 -0400, Bill Nottingham wrote:
Signed-off-by: Bill Nottingham notting@redhat.com
src/pypungi/__init__.py | 2 -- 1 files changed, 0 insertions(+), 2 deletions(-)
diff --git a/src/pypungi/__init__.py b/src/pypungi/__init__.py index 3271f26..bd57bf8 100644 --- a/src/pypungi/__init__.py +++ b/src/pypungi/__init__.py @@ -424,8 +424,6 @@ class Pungi(pypungi.PungiBase): """Make the package lists self hosting. Requires yum still configured, a list of package objects, and a a list of source rpms."""
for srpm in self.srpmpolist:self.ayum.tsInfo.addInstall(srpm) deppass = 1 checked_srpms = [] while 1:
The patch set fails to apply from here out.
Jesse Keating (jkeating@redhat.com) said:
The patch set fails to apply from here out.
This was created from a rebase to head (979c0b5e9470b28d195262d0e2e664460bb9315d) and then sent out with git-format-patch. I've checked out a clean copy with git and applied them all with 'git am'; it works for me.
Bill
Hi,
I'm trying to build my own Koji server and while playing I ran:
/usr/sbin/kojid --fg -v --debug-mock -d --force-lock
--debug-mock uses the mock parameter "--debug" . This parameter does not exist in the mock version I'm running and then raises an error when building The closest parameter I have found is "--trace".
This little patch for /usr/sbin/kojid may fix the problem:
--- kojid.orig 2009-04-13 17:32:44.000000000 -0300 +++ kojid 2009-04-13 17:33:02.000000000 -0300 @@ -375,7 +375,7 @@ mockpath = getattr(options,"mockpath","/usr/bin/mock") cmd = [mockpath, "-r", self.mockcfg] if options.debug_mock: - cmd.append('--debug') + cmd.append('--trace') cmd.extend(args) self.logger.info(' '.join(cmd)) pid = os.fork()
Not sure if this is the right list to send it. My versions on Fedora10: koji-builder-1.3.1-1.fc10.noarch mock-0.9.14-1.fc10.noarch
Regards
Rodrigo Trujillo
buildsys@lists.fedoraproject.org
-
Bill Nottingham -
Jesse Keating -
rotru＠br.ibm.com