backend/satellite_tools/satsync.py | 79 +++++++++++++++++++++++--------
backend/satellite_tools/syncCache.py | 16 ++----
backend/satellite_tools/sync_handlers.py | 3 +
backend/satellite_tools/xmlSource.py | 12 ++--
backend/server/importlib/backendLib.py | 2
5 files changed, 75 insertions(+), 37 deletions(-)
New commits:
commit dd70feb83dec51eb7b96b68cf34a17c738f6ec06
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Tue May 18 14:07:01 2010 +0200
exceptions are very expensive; if instance(..) is 60% faster
diff --git a/backend/satellite_tools/xmlSource.py b/backend/satellite_tools/xmlSource.py
index 7901f13..4b77fbd 100644
--- a/backend/satellite_tools/xmlSource.py
+++ b/backend/satellite_tools/xmlSource.py
@@ -342,18 +342,18 @@ def _stringify(data):
# Accelerate the most common cases
if isinstance(data, types.StringType):
return data
- try: return data.encode('UTF8')
- except AttributeError: pass
+ elif isinstance(data, types.UnicodeType):
+ return data.encode('UTF8')
return str(data)
def _dict_to_utf8(d):
# Convert the dictionary to have non-unocide key-value pairs
ret = {}
for k, v in d.items():
- try: k = k.encode('UTF8')
- except AttributeError: pass
- try: v = v.encode('UTF8')
- except AttributeError: pass
+ if isinstance(k,types.UnicodeType):
+ k = k.encode('UTF8')
+ if isinstance(v, types.UnicodeType):
+ v = v.encode('UTF8')
ret[k] = v
return ret
commit c1af4b827cbbd2d7347541aa34d58b1d8f6cac73
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Tue May 18 10:39:37 2010 +0200
time.strftime is 25% faster
save couple of minutes here
diff --git a/backend/server/importlib/backendLib.py
b/backend/server/importlib/backendLib.py
index f3f890d..260b2e5 100644
--- a/backend/server/importlib/backendLib.py
+++ b/backend/server/importlib/backendLib.py
@@ -31,7 +31,7 @@ def localtime(timestamp):
return _format_time(time.localtime(float(timestamp)))
def _format_time(time_tuple):
- return "%d-%02d-%02d %02d:%02d:%02d" % time_tuple[:6]
+ return time.strftime("%Y-%m-%d %H:%M:%S", time_tuple)
# Database datatypes
class DBtype:
commit 8dfcb8bdcc0bc86295532832a8ffc9a135e553b4
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Fri May 14 22:21:28 2010 +0200
made _hash_object_id 3 times faster
diff --git a/backend/satellite_tools/syncCache.py b/backend/satellite_tools/syncCache.py
index 2d7c0b5..c8662fb 100644
--- a/backend/satellite_tools/syncCache.py
+++ b/backend/satellite_tools/syncCache.py
@@ -56,17 +56,13 @@ class BaseCache:
def _hash_object_id(self, object_id, factor):
"""Given an object id (assumed to be
<label>-<number>), returns the
- last few digits for the number. For instance, (812345, 1000) should
+ last few digits for the number. For instance, (812345, 3) should
return 345"""
- object_id = str(object_id)
- # Compute the number of digits we need from the hash
- format = int(math.ceil(math.log10(factor)))
- # Format the template
- format = '%0' + str(format) + 'd'
# Grab the digits after -
num_id = string.split(object_id, '-')[-1]
- num_id = int(num_id) % factor
- return format % num_id
+ # get last 'factor' numbers
+ num_id = num_id[-factor:]
+ return num_id.rjust(factor,'0')
class ChannelCache(BaseCache):
def _get_key(self, object_id):
@@ -75,7 +71,7 @@ class ChannelCache(BaseCache):
class BasePackageCache(BaseCache):
_subdir = "__unknown__"
def _get_key(self, object_id):
- hash_val = self._hash_object_id(object_id, 100)
+ hash_val = self._hash_object_id(object_id, 2)
return os.path.join("satsync", self._subdir, hash_val, str(object_id))
class ShortPackageCache(BasePackageCache):
@@ -91,7 +87,7 @@ class SourcePackageCache(BasePackageCache):
class ErratumCache(BaseCache):
_subdir = "errata"
def _get_key(self, object_id):
- hash_val = self._hash_object_id(object_id, 10)
+ hash_val = self._hash_object_id(object_id, 1)
return os.path.join("satsync", self._subdir, hash_val, str(object_id))
class KickstartableTreesCache(BaseCache):
commit dddbef429d26067d482df5ade2909c276cacee56
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Tue May 18 16:03:50 2010 +0200
partially reverted commit 9ad7fbe8fd05accec78d8de5e65fb192e1ba4f6f
and ressurected short_packages
diff --git a/backend/satellite_tools/satsync.py b/backend/satellite_tools/satsync.py
index 8c8adef..5f9fb35 100644
--- a/backend/satellite_tools/satsync.py
+++ b/backend/satellite_tools/satsync.py
@@ -88,6 +88,7 @@ class Runner:
'channels' : ['channel-families'],
'channel-families' : ['blacklists'],
'blacklists' : ['arches'],
+ 'short' : [''],
'download-errata' : ['errata'],
'download-packages' : [''],
'download-source-packages' : [''],
@@ -102,6 +103,7 @@ class Runner:
'arches',
'channels',
'blacklists',
+ 'short',
'download-packages',
'rpms',
'packages',
@@ -298,6 +300,16 @@ class Runner:
def _step_blacklists(self):
return self.syncer.processBlacklists()
+ def _step_short(self):
+ try:
+ return self.syncer.processShortPackages()
+ except xmlDiskSource.MissingXmlDiskSourceFileError, e:
+ msg= "ERROR: The dump is missing package data, use --no-rpms to skip
this step or fix the content to include package data."
+ log2disk(-1, msg)
+ log2stderr(-1, msg, cleanYN=1)
+ sys.exit(25)
+
+
def _step_download_packages(self):
return self.syncer.download_package_metadata()
@@ -840,6 +852,37 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
self._channel_packages[chn] = package_ids
self._avail_channel_packages[chn] = avail_package_ids
+ def processShortPackages(self):
+ log(1, ["", "Retrieving short package metadata (used for
indexing)"])
+
+ # Compute the unique packages and populate self._channel_packages
+ self._compute_unique_packages()
+
+ h = sync_handlers.get_short_package_handler()
+ stream_loader = StreamProducer(h)
+ if self.mountpoint:
+ s = xmlDiskSource.ShortPackageDiskSource(self.mountpoint)
+ stream_loader.set_disk_loader(s)
+ else:
+ s = self.xmlWireServer.getChannelShortPackagesXmlStream
+ stream_loader.set_wire_loader(s)
+
+ # OK, now uq_channel_packages only has the unique packages
+ for channel_label, package_ids in self._channel_packages.items():
+ # Pretend we fetch all packages
+ log(1, " Retrieving / parsing short package metadata: %s (%s)" %
+ (channel_label, len(package_ids)))
+
+ lm = self._channel_collection.get_channel_timestamp(channel_label)
+ channel_last_modified = int(rhnLib.timestamp(lm))
+
+ stream_loader.set_args(channel_label, channel_last_modified)
+ stream_loader.process(package_ids)
+
+ h.close()
+
+ self._diff_packages()
+
_query_compare_packages = """
select p.id, c.checksum_type, c.checksum, p.path, p.package_size,
TO_CHAR(p.last_modified, 'YYYYMMDDHH24MISS') last_modified
@@ -853,7 +896,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
"""
# XXX the "is null" condition will have to change in multiorg satellites
def _diff_packages(self):
- package_collection = sync_handlers.PackageCollection()
+ package_collection = sync_handlers.ShortPackageCollection()
h = rhnSQL.prepare(self._query_compare_packages)
missing_channel_packages = {}
@@ -987,6 +1030,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
if errcode:
# file doesn't match
fs_package = package_id
+ channel_package = package_id
path = db_path
else:
# upload package and reimport metadata
@@ -1024,6 +1068,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
missing_packages = {}
# First, determine what has to be downloaded
+ short_package_collection = sync_handlers.ShortPackageCollection()
package_collection = sync_handlers.PackageCollection()
for channel, pids in self._missing_channel_packages.items():
missing_packages[channel] = mp = []
@@ -1034,7 +1079,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
for pid in pids:
# XXX Catch errors
- timestamp = package_collection.get_package_timestamp(pid)
+ timestamp = short_package_collection.get_package_timestamp(pid)
if not package_collection.has_package(pid, timestamp):
# not in the cache
mp.append(pid)
@@ -1042,9 +1087,9 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
return missing_packages
def download_package_metadata(self):
-
- # Compute the unique packages and populate self._channel_packages
- self._compute_unique_packages()
+ log(1, ["", "Downloading package metadata"])
+ # Get the missing but uncached packages
+ missing_packages = self._missing_not_cached_packages()
h = sync_handlers.get_package_handler()
stream_loader = StreamProducer(h)
@@ -1055,12 +1100,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
s = self.xmlWireServer.getPackageXmlStream
stream_loader.set_wire_loader(s)
- log(1, ["", "Downloading package metadata"])
- # Get the missing but uncached packages
-# missing_packages = self._missing_not_cached_packages()
-
-
- for channel, pids in self._channel_packages.items():
+ for channel, pids in missing_packages.items():
package_count = len(pids)
log(1, messages.package_parsing % (channel,
@@ -1075,9 +1115,6 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
pb.redrawYN = 0
pb.printAll(1)
- lm = self._channel_collection.get_channel_timestamp(channel)
- channel_last_modified = int(rhnLib.timestamp(lm))
-
ss = SequenceServer(pids[:], nevermorethan=self._batch_size)
while not ss.doneYN():
chunk = ss.getChunk()
@@ -1090,8 +1127,6 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
h.close()
- self._diff_packages()
-
# Double-check that we got all the packages
missing_packages = self._missing_not_cached_packages()
for channel, pids in missing_packages.items():
@@ -1726,11 +1761,12 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
def _link_channel_packages(self):
log(1, ["", messages.link_channel_packages])
+ short_package_collection = sync_handlers.ShortPackageCollection()
package_collection = sync_handlers.PackageCollection()
uq_packages = {}
for chn, package_ids in self._missing_channel_packages.items():
for pid in package_ids:
- timestamp = package_collection.get_package_timestamp(pid)
+ timestamp = short_package_collection.get_package_timestamp(pid)
package = package_collection.get_package(pid, timestamp)
assert package is not None
channel_obj = {'label' : chn}
@@ -1764,13 +1800,14 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
""" short-circuit the most common case"""
if not chunk:
return []
+ short_package_collection = sync_handlers.ShortPackageCollection()
if sources:
package_collection = sync_handlers.SourcePackageCollection()
else:
package_collection = sync_handlers.PackageCollection()
batch = []
for pid in chunk:
- timestamp = package_collection.get_package_timestamp(pid)
+ timestamp = short_package_collection.get_package_timestamp(pid)
package = package_collection.get_package(pid, timestamp)
if package is None:
# not in the cache
@@ -1817,7 +1854,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
def _fix_erratum(self, erratum):
""" Replace the list of packages with references to short
packages"""
- sp_coll = sync_handlers.PackageCollection()
+ sp_coll = sync_handlers.ShortPackageCollection()
pids = unique(erratum['packages'])
# map all the pkgs objects to the erratum
packages = []
@@ -1873,6 +1910,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
def _fetch_packages(self, channel, missing_fs_packages, sources=0):
+ short_package_collection = sync_handlers.ShortPackageCollection()
if sources:
# acronym = "SRPM"
package_collection = sync_handlers.SourcePackageCollection()
@@ -1887,7 +1925,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
cfg = config.initUp2dateConfig()
for package_id, path in missing_fs_packages:
pkg_current = pkg_current + 1
- timestamp = package_collection.get_package_timestamp(package_id)
+ timestamp = short_package_collection.get_package_timestamp(package_id)
package = package_collection.get_package(package_id, timestamp)
checksum_type = package['checksum_type']
@@ -2351,6 +2389,7 @@ def processCommandline():
if actionDict['no-packages']:
actionDict['packages'] = 0
+ actionDict['short'] = 0
actionDict['download-packages'] = 0
actionDict['rpms'] = 0
diff --git a/backend/satellite_tools/sync_handlers.py
b/backend/satellite_tools/sync_handlers.py
index 59c74f8..7e79c3c 100644
--- a/backend/satellite_tools/sync_handlers.py
+++ b/backend/satellite_tools/sync_handlers.py
@@ -288,6 +288,9 @@ class PackageCollection(ShortPackageCollection):
def _init_cache(self):
self._cache = syncCache.PackageCache()
+ def get_package_timestamp(self, package_id):
+ raise NotImplementedError
+
class PackageContainer(xmlSource.PackageContainer):
def endItemCallback(self):