utils/cloneByDate.py | 248 ++++++++++++++++++++++++++++++++++++---------------
1 file changed, 177 insertions(+), 71 deletions(-)
New commits:
commit c0caebe95a29451ecccf92f0ca45d849da347e08
Author: Justin Sherrill <jsherril(a)redhat.com>
Date: Fri Jan 27 16:31:14 2012 -0500
moving dep solver to channel tree from individual channel
diff --git a/utils/cloneByDate.py b/utils/cloneByDate.py
index 16bf870..d39438d 100644
--- a/utils/cloneByDate.py
+++ b/utils/cloneByDate.py
@@ -17,7 +17,7 @@
# in this software or its documentation.
#
-import pdb
+
import sys
import time
import copy
@@ -94,9 +94,8 @@ def main(options):
##
# Usage:
-# a = ChannelTreeCloner(channel_hash, xmlrpc, db, to_date)
-# if len(a.needing_channels()) > 0:
-# a.create_channels()
+# a = ChannelTreeCloner(channel_hash, xmlrpc, db, to_date)
+# a.create_channels()
# a.prepare()
# a.clone()
#
@@ -128,25 +127,23 @@ class ChannelTreeCloner:
total += cloner.pending()
return total
- def find_cloner(self, label):
- for cloner in self.cloners:
- if cloner.dest_label() == label:
- return cloner
-
def create_channels(self):
to_create = self.needing_create()
if len(to_create) == 0:
return
dest_parent = self.channel_map[self.src_parent]
+ nvreas = []
if to_create.count(dest_parent) > 0:
self.remote_api.clone_channel(self.src_parent, dest_parent, None)
to_create.remove(dest_parent)
- self.find_cloner(dest_parent).after_create()
- for from_label, to_label in self.channel_map.items():
- if to_create.count(to_label) > 0:
- self.remote_api.clone_channel(from_label, to_label, dest_parent)
- self.find_cloner(to_label).after_create()
-
+ cloner = self.cloners[0]
+ nvreas += [ pkg['nvrea'] for pkg in
cloner.reset_new_pkgs().values()]
+
+ for cloner in self.cloners:
+ if cloner.dest_label() in to_create:
+ self.remote_api.clone_channel(cloner.src_label(), cloner.dest_label(),
dest_parent)
+ nvreas += [ pkg['nvrea'] for pkg in
cloner.reset_new_pkgs().values() ]
+ self.dep_solve(nvreas, labels=to_create)
def validate_source_channels(self):
@@ -194,10 +191,60 @@ class ChannelTreeCloner:
cloner.pre_summary();
def clone(self):
+ added_pkgs = []
for cloner in self.cloners:
cloner.process()
+ added_pkgs += cloner.pkg_diff()
+ self.dep_solve(added_pkgs)
+
+
+ def dep_solve(self, nvrea_list, labels=None):
+ if not labels:
+ labels = self.channel_map.keys()
+ repos = [{"id":label, "relative_path":self.repodata(label)}
for label in labels]
+
+ print "Solving deps"
+
+ solver = DepSolver(repos, nvrea_list)
+ dep_results = solver.processResults(solver.getDependencylist())
+
+ print "Processing"
+ self.process_deps(dep_results)
+
+
+ def process_deps(self, deps):
+ needed_list = dict((label, []) for label in self.channel_map.values())
+ unsolved_deps = []
+ #loop through all the deps and find any that don't exist in the
+ # destination channels
+ for pkg in deps:
+ for dep, solved_list in pkg.items():
+ found = False
+ for cloner in self.cloners:
+ if cloner.src_pkg_exist(solved_list) and not
cloner.dest_pkg_exist(solved_list):
+ needed_list[cloner.dest_label()].append(solved_list[0]) #grab
oldest package
+ found = True
+ if not found:
+ unsolved_deps.append((pkg))
+
+ print "Unsolved deps: %i" % len(unsolved_deps)
+ print "Needed deps: "
+ for label in needed_list.keys():
+ print "%s: %i" % (label, len(needed_list[label]))
+
+ #import pdb; pdb.set_trace()
+ for cloner in self.cloners:
+ needed = needed_list[cloner.dest_label()]
+ if len(needed) > 0:
+ cloner.process_deps(needed)
+
+ def repodata(self, label):
+ repo_dir = "/var/cache/rhn/repodata/%s" % label
+ tmp_dir = tempfile.mkdtemp(suffix="clone-by-date")
+ shutil.copytree(repo_dir, tmp_dir + "/repodata/")
+ return tmp_dir
@@ -209,15 +256,39 @@ class ChannelCloner:
self.from_label = from_label
self.to_label = to_label
self.to_date = to_date
- self.original_packages = None
+
+
+ self.from_pkg_hash = None
+
+ self.new_pkg_hash = {}
+ self.old_pkg_hash = {}
def dest_label(self):
return self.to_label
+
+ def src_label(self):
+ return self.from_label
+
+ def pkg_diff(self):
+ return self.diff_packages(self.old_pkg_hash.values(), self.new_pkg_hash.values())
+
- def prepare(self):
- self.original_packages = self.remote_api.list_packages(self.to_label)
- self.errata_to_clone, self.available_errata = self.get_errata()
+ def reset_original_pkgs(self):
+ self.old_pkg_hash =
self.list_to_hash(self.remote_api.list_packages(self.to_label), 'nvrea')
+ return self.old_pkg_hash
+
+ def reset_new_pkgs(self):
+ self.new_pkg_hash =
self.list_to_hash(self.remote_api.list_packages(self.to_label), 'nvrea')
+ return self.new_pkg_hash
+
+ def reset_from_pkgs(self):
+ self.from_pkg_hash =
self.list_to_hash(self.remote_api.list_packages(self.from_label), 'nvrea')
+
+ def prepare(self):
+ self.reset_original_pkgs()
+ self.errata_to_clone, self.available_errata = self.get_errata()
+
def pending(self):
return len(self.errata_to_clone)
@@ -227,51 +298,14 @@ class ChannelCloner:
def process(self):
self.clone();
- new_packages = self.remote_api.list_packages(self.to_label)
- pkg_diff = self.diff_packages(self.original_packages, new_packages)
- print "New packages added: %i" % len(pkg_diff)
- self.dep_solve(pkg_diff, new_packages)
-
- def after_create(self):
- new_packages = self.remote_api.list_packages(self.to_label)
- self.dep_solve(new_packages, new_packages)
-
- # pkg_list - list of packages to solve deps for
- # trim_list - list of packages that already exist
- def dep_solve(self, pkg_list, trim_list):
- repo_dir = self.repodata(self.from_label)
- print "Solving deps"
- nvreas = [pkg['nvrea'] for pkg in pkg_list]
- solver = DepSolver([{"id":self.from_label,
"relative_path":repo_dir}], nvreas)
- dep_results = solver.processResults(solver.getDependencylist())
-
- print "Processing"
- self.process_deps(trim_list, dep_results)
-
-
- def process_deps(self, new_packages, deps):
- to_pkg_hash = self.list_to_hash(new_packages, 'nvrea')
-
- needed_list = []
-
- for pkg in deps:
- for dep, solved_list in pkg.items():
-# if dep == "qffmpeg-libs = 0.4.9-0.16.20080908.el5_5":
-# import pdb; pdb.set_trace();
- if not self.pkg_exists(solved_list, to_pkg_hash):
- if len(solved_list) > 0:
- needed_list.append(solved_list[0]) #grab oldest package
- else:
- print "No packages %s" % dep
-
- print "Unsolved deps: %i" % len(needed_list)
-
- from_pkg_hash = self.list_to_hash(self.remote_api.list_packages(self.from_label),
'nvrea')
-
+ self.reset_new_pkgs()
+ print "New packages added: %i" % len(len(self.new_pkg_hash) -
len(self.old_pkg_hash))
+
+ def process_deps(self, needed_pkgs):
needed_ids = []
unsolved_deps = []
- for pkg in needed_list:
- found = self.pkg_exists([pkg], from_pkg_hash)
+ for pkg in needed_pkgs:
+ found = self.src_pkg_exist([pkg])
if found:
needed_ids.append(found['id'])
else:
@@ -291,14 +325,21 @@ class ChannelCloner:
pkg_hash[pkg[key]] = pkg
return pkg_hash
+ def src_pkg_exist(self, needed_list):
+ if not self.from_pkg_hash:
+ self.reset_from_pkgs()
+ return self.pkg_exists(needed_list, self.from_pkg_hash)
+
+ def dest_pkg_exist(self, needed_list):
+ return self.pkg_exists(needed_list, self.new_pkg_hash)
- def pkg_exists(self, needed_list, pkg_hash):
+ def pkg_exists(self, needed_list, pkg_list):
"""Given a list of packages in [N, V, E, R, A] format, do any of
them exist
in the pkg_hash with key of N-V-R.A format"""
for i in needed_list:
key = "%s-%s-%s.%s" % (i[0], i[1], i[3], i[4])
- if pkg_hash.has_key(key):
- return pkg_hash[key]
+ if pkg_list.has_key(key):
+ return pkg_list[key]
return False
def diff_packages(self, old, new):
@@ -347,6 +388,7 @@ class ChannelCloner:
""" Returns tuple of all available for cloning, and what falls in
the date range"""
available_errata = self.db_api.applicable_errata(self.from_label, self.to_label)
to_clone = []
+
for err in available_errata:
if err['issue_date'] <= self.to_date:
to_clone.append(err)
@@ -402,11 +444,11 @@ class RemoteApi:
except xmlrpclib.Fault, e:
raise UserError(e.faultString + ": " + label)
- def add_packages(self, label, package_ids):
+ def add_packages(self, label, package_ids):
while(len(package_ids) > 0):
set = package_ids[:20]
del package_ids[:20]
- self.client.channel.software.addPackages(self.auth_token, label,
package_ids)
+ self.client.channel.software.addPackages(self.auth_token, label, set)
def clone_channel(self, original_label, new_label, parent):
details = {'name': new_label, 'label':new_label,
'summary': new_label}
@@ -451,7 +493,8 @@ class DBApi:
where c2.label = :to_label)
""")
h.execute(from_label=from_label, to_label=to_label)
- return h.fetchall_dict()
+ to_ret = h.fetchall_dict() or []
+ return to_ret
class UserError(Exception):
def __init__(self, msg):
commit 7127884dbed67250a8ef0f214169a4617e1367f7
Author: Justin Sherrill <jsherril(a)redhat.com>
Date: Wed Jan 25 09:47:14 2012 -0500
errata date clone - locating and adding deps from dep solver
diff --git a/utils/cloneByDate.py b/utils/cloneByDate.py
index e97fe38..16bf870 100644
--- a/utils/cloneByDate.py
+++ b/utils/cloneByDate.py
@@ -61,8 +61,8 @@ def main(options):
cloners = []
needed_channels = []
- for list in options.channels:
- tree_cloner = ChannelTreeCloner(list, xmlrpc, db, options.to_date)
+ for channel_list in options.channels:
+ tree_cloner = ChannelTreeCloner(channel_list, xmlrpc, db, options.to_date)
cloners.append(tree_cloner)
needed_channels += tree_cloner.needing_create()
@@ -73,23 +73,33 @@ def main(options):
for cloner in cloners:
cloner.create_channels()
-
-
for tree_cloner in cloners:
tree_cloner.prepare();
print "\nBy continuing the following will be cloned:"
+ total = 0
for cloner in cloners:
cloner.pre_summary()
- print "\n"
- confirm("Continue with clone (y/n)?", options)
+ total += cloner.pending()
+ if total == 0:
+ print ("\nNothing to do.")
+ sys.exit(0)
+
+ confirm("\nContinue with clone (y/n)?", options)
for cloner in cloners:
cloner.clone()
-
+##
+# Usage:
+# a = ChannelTreeCloner(channel_hash, xmlrpc, db, to_date)
+# if len(a.needing_channels()) > 0:
+# a.create_channels()
+# a.prepare()
+# a.clone()
+#
class ChannelTreeCloner:
def __init__(self, channels, remote_api, db_api, to_date):
self.remote_api = remote_api
@@ -98,6 +108,11 @@ class ChannelTreeCloner:
self.to_date = to_date
self.cloners = []
self.validate_source_channels()
+
+ for from_label in self.ordered_labels():
+ to_label = self.channel_map[from_label]
+ cloner = ChannelCloner(from_label, to_label, self.to_date, self.remote_api,
self.db_api)
+ self.cloners.append(cloner)
def needing_create(self):
to_ret = []
@@ -107,6 +122,16 @@ class ChannelTreeCloner:
to_ret.append(label)
return to_ret
+ def pending(self):
+ total = 0
+ for cloner in self.cloners:
+ total += cloner.pending()
+ return total
+
+ def find_cloner(self, label):
+ for cloner in self.cloners:
+ if cloner.dest_label() == label:
+ return cloner
def create_channels(self):
to_create = self.needing_create()
@@ -116,10 +141,13 @@ class ChannelTreeCloner:
if to_create.count(dest_parent) > 0:
self.remote_api.clone_channel(self.src_parent, dest_parent, None)
to_create.remove(dest_parent)
+ self.find_cloner(dest_parent).after_create()
for from_label, to_label in self.channel_map.items():
if to_create.count(to_label) > 0:
self.remote_api.clone_channel(from_label, to_label, dest_parent)
-
+ self.find_cloner(to_label).after_create()
+
+
def validate_source_channels(self):
self.channel_details = self.remote_api.channel_details(self.channel_map,
values=False)
@@ -151,17 +179,14 @@ class ChannelTreeCloner:
def ordered_labels(self):
"""Return list of labels with parent first"""
- list = self.channel_map.keys()
- list.remove(self.src_parent)
- list.insert(0,self.src_parent)
- return list
+ labels = self.channel_map.keys()
+ labels.remove(self.src_parent)
+ labels.insert(0,self.src_parent)
+ return labels
- def prepare(self):
- for from_label in self.ordered_labels():
- to_label = self.channel_map[from_label]
- self.validate_dest_channels()
- cloner = ChannelCloner(from_label, to_label, self.to_date, self.remote_api,
self.db_api)
- self.cloners.append(cloner)
+ def prepare(self):
+ self.validate_dest_channels()
+ for cloner in self.cloners:
cloner.prepare()
def pre_summary(self):
@@ -169,7 +194,7 @@ class ChannelTreeCloner:
cloner.pre_summary();
def clone(self):
- for cloner in self.cloners:
+ for cloner in self.cloners:
cloner.process()
@@ -186,58 +211,95 @@ class ChannelCloner:
self.to_date = to_date
self.original_packages = None
+
+ def dest_label(self):
+ return self.to_label
+
def prepare(self):
self.original_packages = self.remote_api.list_packages(self.to_label)
self.errata_to_clone, self.available_errata = self.get_errata()
+ def pending(self):
+ return len(self.errata_to_clone)
+
def pre_summary(self):
print "%s -> %s (%i/%i Errata)" %(self.from_label, self.to_label,
len(self.errata_to_clone), len(self.available_errata))
def process(self):
self.clone();
new_packages = self.remote_api.list_packages(self.to_label)
- pkg_diff = self.diff_packages(self.original_packages, new_packages)
-
+ pkg_diff = self.diff_packages(self.original_packages, new_packages)
print "New packages added: %i" % len(pkg_diff)
+ self.dep_solve(pkg_diff, new_packages)
+ def after_create(self):
+ new_packages = self.remote_api.list_packages(self.to_label)
+ self.dep_solve(new_packages, new_packages)
+
+ # pkg_list - list of packages to solve deps for
+ # trim_list - list of packages that already exist
+ def dep_solve(self, pkg_list, trim_list):
repo_dir = self.repodata(self.from_label)
-
print "Solving deps"
- nvreas = [pkg['nvrea'] for pkg in pkg_diff]
+ nvreas = [pkg['nvrea'] for pkg in pkg_list]
solver = DepSolver([{"id":self.from_label,
"relative_path":repo_dir}], nvreas)
dep_results = solver.processResults(solver.getDependencylist())
- #import pdb; pdb.set_trace()
print "Processing"
- self.process_deps(new_packages, dep_results)
-
- #self.remote_api.add_packages()
+ self.process_deps(trim_list, dep_results)
+
+
+ def process_deps(self, new_packages, deps):
+ to_pkg_hash = self.list_to_hash(new_packages, 'nvrea')
- def process_deps(self, pkg_list, deps):
- pkg_hash = {}
- for pkg in pkg_list:
- pkg_hash[pkg['nvrea']] = pkg
-
needed_list = []
- for pkg in deps:
-
+
+ for pkg in deps:
for dep, solved_list in pkg.items():
- if not self.pkg_exists(solved_list, pkg_hash):
- needed_list.append(solved_list)
- print len(needed_list)
-
+# if dep == "qffmpeg-libs = 0.4.9-0.16.20080908.el5_5":
+# import pdb; pdb.set_trace();
+ if not self.pkg_exists(solved_list, to_pkg_hash):
+ if len(solved_list) > 0:
+ needed_list.append(solved_list[0]) #grab oldest package
+ else:
+ print "No packages %s" % dep
+
+ print "Unsolved deps: %i" % len(needed_list)
+
+ from_pkg_hash = self.list_to_hash(self.remote_api.list_packages(self.from_label),
'nvrea')
+
+ needed_ids = []
+ unsolved_deps = []
+ for pkg in needed_list:
+ found = self.pkg_exists([pkg], from_pkg_hash)
+ if found:
+ needed_ids.append(found['id'])
+ else:
+ unsolved_deps.append(pkg)
+
+ if len(needed_ids) > 0:
+ print "Adding dependencies: %i" % len(needed_ids)
+ self.remote_api.add_packages(self.to_label, needed_ids)
+ if len(unsolved_deps) > 0:
+ print "Unresolved dependencies: %i" % len(unsolved_deps)
+
+
+
+ def list_to_hash(self, pkg_list, key):
+ pkg_hash = {}
+ for pkg in pkg_list:
+ pkg_hash[pkg[key]] = pkg
+ return pkg_hash
+
- def pkg_exists(self, list, pkg_hash):
+ def pkg_exists(self, needed_list, pkg_hash):
"""Given a list of packages in [N, V, E, R, A] format, do any of
them exist
- in the pkg_hash with key of N-V-R.A format"""
- for i in list:
+ in the pkg_hash with key of N-V-R.A format"""
+ for i in needed_list:
key = "%s-%s-%s.%s" % (i[0], i[1], i[3], i[4])
if pkg_hash.has_key(key):
- return True
+ return pkg_hash[key]
return False
-
-
-
def diff_packages(self, old, new):
old_hash = {}
@@ -250,7 +312,7 @@ class ChannelCloner:
new_hash[pkg["id"]] = pkg
id_diff = set(new_hash.keys()) - set(old_hash.keys())
for id in id_diff:
- to_ret.append(new_hash[id])
+ to_ret.append(new_hash[id])
return to_ret
@@ -266,6 +328,7 @@ class ChannelCloner:
del errata_ids[:bunch_size]
self.remote_api.clone_errata(self.to_label, errata_set)
pb.addTo(bunch_size)
+ pb.printIncrement()
pb.printComplete()
def collect(self, items, attribute):
@@ -281,7 +344,7 @@ class ChannelCloner:
return tmp_dir
def get_errata(self):
- """ Returns tuple of all available for cloning, and what falls in
teh date range"""
+ """ Returns tuple of all available for cloning, and what falls in
the date range"""
available_errata = self.db_api.applicable_errata(self.from_label, self.to_label)
to_clone = []
for err in available_errata:
@@ -301,16 +364,15 @@ class RemoteApi:
self.auth_token = self.client.auth.login(username, password)
except xmlrpclib.Fault, e:
raise UserError(e.faultString)
-
-
+
def list_channel_labels(self):
key = "chan_labels"
if self.cache.has_key(key):
return self.cache[key]
- list = self.client.channel.listAllChannels(self.auth_token)
+ chan_list = self.client.channel.listAllChannels(self.auth_token)
to_ret = []
- for item in list:
+ for item in chan_list:
to_ret.append(item["label"])
self.cache[key] = to_ret
return to_ret
@@ -367,7 +429,8 @@ class DBApi:
def applicable_errata(self, from_label, to_label):
- """list of errata that is applicable to be cloned, used db because
we need to exclude cloned errata too"""
+ """list of errata that is applicable to be cloned, used db because
we
+ need to exclude cloned errata too"""
h = rhnSQL.prepare("""
select e.id, e.advisory_name, e.advisory_type, e.issue_date
from rhnErrata e inner join