backend/satellite_tools/satsync.py | 79 +++++++++++++++++++++++-------- backend/satellite_tools/syncCache.py | 16 ++---- backend/satellite_tools/sync_handlers.py | 3 + backend/satellite_tools/xmlSource.py | 12 ++-- backend/server/importlib/backendLib.py | 2 5 files changed, 75 insertions(+), 37 deletions(-)
New commits: commit dd70feb83dec51eb7b96b68cf34a17c738f6ec06 Author: Michael Mraka michael.mraka@redhat.com Date: Tue May 18 14:07:01 2010 +0200
exceptions are very expensive; if instance(..) is 60% faster
diff --git a/backend/satellite_tools/xmlSource.py b/backend/satellite_tools/xmlSource.py index 7901f13..4b77fbd 100644 --- a/backend/satellite_tools/xmlSource.py +++ b/backend/satellite_tools/xmlSource.py @@ -342,18 +342,18 @@ def _stringify(data): # Accelerate the most common cases if isinstance(data, types.StringType): return data - try: return data.encode('UTF8') - except AttributeError: pass + elif isinstance(data, types.UnicodeType): + return data.encode('UTF8') return str(data)
def _dict_to_utf8(d): # Convert the dictionary to have non-unocide key-value pairs ret = {} for k, v in d.items(): - try: k = k.encode('UTF8') - except AttributeError: pass - try: v = v.encode('UTF8') - except AttributeError: pass + if isinstance(k,types.UnicodeType): + k = k.encode('UTF8') + if isinstance(v, types.UnicodeType): + v = v.encode('UTF8') ret[k] = v return ret
commit c1af4b827cbbd2d7347541aa34d58b1d8f6cac73 Author: Michael Mraka michael.mraka@redhat.com Date: Tue May 18 10:39:37 2010 +0200
time.strftime is 25% faster
save couple of minutes here
diff --git a/backend/server/importlib/backendLib.py b/backend/server/importlib/backendLib.py index f3f890d..260b2e5 100644 --- a/backend/server/importlib/backendLib.py +++ b/backend/server/importlib/backendLib.py @@ -31,7 +31,7 @@ def localtime(timestamp): return _format_time(time.localtime(float(timestamp)))
def _format_time(time_tuple): - return "%d-%02d-%02d %02d:%02d:%02d" % time_tuple[:6] + return time.strftime("%Y-%m-%d %H:%M:%S", time_tuple)
# Database datatypes class DBtype:
commit 8dfcb8bdcc0bc86295532832a8ffc9a135e553b4 Author: Michael Mraka michael.mraka@redhat.com Date: Fri May 14 22:21:28 2010 +0200
made _hash_object_id 3 times faster
diff --git a/backend/satellite_tools/syncCache.py b/backend/satellite_tools/syncCache.py index 2d7c0b5..c8662fb 100644 --- a/backend/satellite_tools/syncCache.py +++ b/backend/satellite_tools/syncCache.py @@ -56,17 +56,13 @@ class BaseCache:
def _hash_object_id(self, object_id, factor): """Given an object id (assumed to be <label>-<number>), returns the - last few digits for the number. For instance, (812345, 1000) should + last few digits for the number. For instance, (812345, 3) should return 345""" - object_id = str(object_id) - # Compute the number of digits we need from the hash - format = int(math.ceil(math.log10(factor))) - # Format the template - format = '%0' + str(format) + 'd' # Grab the digits after - num_id = string.split(object_id, '-')[-1] - num_id = int(num_id) % factor - return format % num_id + # get last 'factor' numbers + num_id = num_id[-factor:] + return num_id.rjust(factor,'0')
class ChannelCache(BaseCache): def _get_key(self, object_id): @@ -75,7 +71,7 @@ class ChannelCache(BaseCache): class BasePackageCache(BaseCache): _subdir = "__unknown__" def _get_key(self, object_id): - hash_val = self._hash_object_id(object_id, 100) + hash_val = self._hash_object_id(object_id, 2) return os.path.join("satsync", self._subdir, hash_val, str(object_id))
class ShortPackageCache(BasePackageCache): @@ -91,7 +87,7 @@ class SourcePackageCache(BasePackageCache): class ErratumCache(BaseCache): _subdir = "errata" def _get_key(self, object_id): - hash_val = self._hash_object_id(object_id, 10) + hash_val = self._hash_object_id(object_id, 1) return os.path.join("satsync", self._subdir, hash_val, str(object_id))
class KickstartableTreesCache(BaseCache):
commit dddbef429d26067d482df5ade2909c276cacee56 Author: Michael Mraka michael.mraka@redhat.com Date: Tue May 18 16:03:50 2010 +0200
partially reverted commit 9ad7fbe8fd05accec78d8de5e65fb192e1ba4f6f and ressurected short_packages
diff --git a/backend/satellite_tools/satsync.py b/backend/satellite_tools/satsync.py index 8c8adef..5f9fb35 100644 --- a/backend/satellite_tools/satsync.py +++ b/backend/satellite_tools/satsync.py @@ -88,6 +88,7 @@ class Runner: 'channels' : ['channel-families'], 'channel-families' : ['blacklists'], 'blacklists' : ['arches'], + 'short' : [''], 'download-errata' : ['errata'], 'download-packages' : [''], 'download-source-packages' : [''], @@ -102,6 +103,7 @@ class Runner: 'arches', 'channels', 'blacklists', + 'short', 'download-packages', 'rpms', 'packages', @@ -298,6 +300,16 @@ class Runner: def _step_blacklists(self): return self.syncer.processBlacklists()
+ def _step_short(self): + try: + return self.syncer.processShortPackages() + except xmlDiskSource.MissingXmlDiskSourceFileError, e: + msg= "ERROR: The dump is missing package data, use --no-rpms to skip this step or fix the content to include package data." + log2disk(-1, msg) + log2stderr(-1, msg, cleanYN=1) + sys.exit(25) + + def _step_download_packages(self): return self.syncer.download_package_metadata()
@@ -840,6 +852,37 @@ Please contact your RHN representative""" % (generation, sat_cert.generation)) self._channel_packages[chn] = package_ids self._avail_channel_packages[chn] = avail_package_ids
+ def processShortPackages(self): + log(1, ["", "Retrieving short package metadata (used for indexing)"]) + + # Compute the unique packages and populate self._channel_packages + self._compute_unique_packages() + + h = sync_handlers.get_short_package_handler() + stream_loader = StreamProducer(h) + if self.mountpoint: + s = xmlDiskSource.ShortPackageDiskSource(self.mountpoint) + stream_loader.set_disk_loader(s) + else: + s = self.xmlWireServer.getChannelShortPackagesXmlStream + stream_loader.set_wire_loader(s) + + # OK, now uq_channel_packages only has the unique packages + for channel_label, package_ids in self._channel_packages.items(): + # Pretend we fetch all packages + log(1, " Retrieving / parsing short package metadata: %s (%s)" % + (channel_label, len(package_ids))) + + lm = self._channel_collection.get_channel_timestamp(channel_label) + channel_last_modified = int(rhnLib.timestamp(lm)) + + stream_loader.set_args(channel_label, channel_last_modified) + stream_loader.process(package_ids) + + h.close() + + self._diff_packages() + _query_compare_packages = """ select p.id, c.checksum_type, c.checksum, p.path, p.package_size, TO_CHAR(p.last_modified, 'YYYYMMDDHH24MISS') last_modified @@ -853,7 +896,7 @@ Please contact your RHN representative""" % (generation, sat_cert.generation)) """ # XXX the "is null" condition will have to change in multiorg satellites def _diff_packages(self): - package_collection = sync_handlers.PackageCollection() + package_collection = sync_handlers.ShortPackageCollection() h = rhnSQL.prepare(self._query_compare_packages)
missing_channel_packages = {} @@ -987,6 +1030,7 @@ Please contact your RHN representative""" % (generation, sat_cert.generation)) if errcode: # file doesn't match fs_package = package_id + channel_package = package_id path = db_path else: # upload package and reimport metadata @@ -1024,6 +1068,7 @@ Please contact your RHN representative""" % (generation, sat_cert.generation)) missing_packages = {}
# First, determine what has to be downloaded + short_package_collection = sync_handlers.ShortPackageCollection() package_collection = sync_handlers.PackageCollection() for channel, pids in self._missing_channel_packages.items(): missing_packages[channel] = mp = [] @@ -1034,7 +1079,7 @@ Please contact your RHN representative""" % (generation, sat_cert.generation))
for pid in pids: # XXX Catch errors - timestamp = package_collection.get_package_timestamp(pid) + timestamp = short_package_collection.get_package_timestamp(pid) if not package_collection.has_package(pid, timestamp): # not in the cache mp.append(pid) @@ -1042,9 +1087,9 @@ Please contact your RHN representative""" % (generation, sat_cert.generation)) return missing_packages
def download_package_metadata(self): - - # Compute the unique packages and populate self._channel_packages - self._compute_unique_packages() + log(1, ["", "Downloading package metadata"]) + # Get the missing but uncached packages + missing_packages = self._missing_not_cached_packages()
h = sync_handlers.get_package_handler() stream_loader = StreamProducer(h) @@ -1055,12 +1100,7 @@ Please contact your RHN representative""" % (generation, sat_cert.generation)) s = self.xmlWireServer.getPackageXmlStream stream_loader.set_wire_loader(s)
- log(1, ["", "Downloading package metadata"]) - # Get the missing but uncached packages -# missing_packages = self._missing_not_cached_packages() - - - for channel, pids in self._channel_packages.items(): + for channel, pids in missing_packages.items(): package_count = len(pids)
log(1, messages.package_parsing % (channel, @@ -1075,9 +1115,6 @@ Please contact your RHN representative""" % (generation, sat_cert.generation)) pb.redrawYN = 0 pb.printAll(1)
- lm = self._channel_collection.get_channel_timestamp(channel) - channel_last_modified = int(rhnLib.timestamp(lm)) - ss = SequenceServer(pids[:], nevermorethan=self._batch_size) while not ss.doneYN(): chunk = ss.getChunk() @@ -1090,8 +1127,6 @@ Please contact your RHN representative""" % (generation, sat_cert.generation))
h.close()
- self._diff_packages() - # Double-check that we got all the packages missing_packages = self._missing_not_cached_packages() for channel, pids in missing_packages.items(): @@ -1726,11 +1761,12 @@ Please contact your RHN representative""" % (generation, sat_cert.generation))
def _link_channel_packages(self): log(1, ["", messages.link_channel_packages]) + short_package_collection = sync_handlers.ShortPackageCollection() package_collection = sync_handlers.PackageCollection() uq_packages = {} for chn, package_ids in self._missing_channel_packages.items(): for pid in package_ids: - timestamp = package_collection.get_package_timestamp(pid) + timestamp = short_package_collection.get_package_timestamp(pid) package = package_collection.get_package(pid, timestamp) assert package is not None channel_obj = {'label' : chn} @@ -1764,13 +1800,14 @@ Please contact your RHN representative""" % (generation, sat_cert.generation)) """ short-circuit the most common case""" if not chunk: return [] + short_package_collection = sync_handlers.ShortPackageCollection() if sources: package_collection = sync_handlers.SourcePackageCollection() else: package_collection = sync_handlers.PackageCollection() batch = [] for pid in chunk: - timestamp = package_collection.get_package_timestamp(pid) + timestamp = short_package_collection.get_package_timestamp(pid) package = package_collection.get_package(pid, timestamp) if package is None: # not in the cache @@ -1817,7 +1854,7 @@ Please contact your RHN representative""" % (generation, sat_cert.generation))
def _fix_erratum(self, erratum): """ Replace the list of packages with references to short packages""" - sp_coll = sync_handlers.PackageCollection() + sp_coll = sync_handlers.ShortPackageCollection() pids = unique(erratum['packages']) # map all the pkgs objects to the erratum packages = [] @@ -1873,6 +1910,7 @@ Please contact your RHN representative""" % (generation, sat_cert.generation))
def _fetch_packages(self, channel, missing_fs_packages, sources=0): + short_package_collection = sync_handlers.ShortPackageCollection() if sources: # acronym = "SRPM" package_collection = sync_handlers.SourcePackageCollection() @@ -1887,7 +1925,7 @@ Please contact your RHN representative""" % (generation, sat_cert.generation)) cfg = config.initUp2dateConfig() for package_id, path in missing_fs_packages: pkg_current = pkg_current + 1 - timestamp = package_collection.get_package_timestamp(package_id) + timestamp = short_package_collection.get_package_timestamp(package_id) package = package_collection.get_package(package_id, timestamp)
checksum_type = package['checksum_type'] @@ -2351,6 +2389,7 @@ def processCommandline():
if actionDict['no-packages']: actionDict['packages'] = 0 + actionDict['short'] = 0 actionDict['download-packages'] = 0 actionDict['rpms'] = 0
diff --git a/backend/satellite_tools/sync_handlers.py b/backend/satellite_tools/sync_handlers.py index 59c74f8..7e79c3c 100644 --- a/backend/satellite_tools/sync_handlers.py +++ b/backend/satellite_tools/sync_handlers.py @@ -288,6 +288,9 @@ class PackageCollection(ShortPackageCollection): def _init_cache(self): self._cache = syncCache.PackageCache()
+ def get_package_timestamp(self, package_id): + raise NotImplementedError + class PackageContainer(xmlSource.PackageContainer):
def endItemCallback(self):