backend/rhn-conf/rhn_server.conf | 2
backend/satellite_tools/exporter/exportLib.py | 15
backend/satellite_tools/satsync.py | 477 +++++++++-----------------
backend/satellite_tools/sync_handlers.py | 3
backend/satellite_tools/xmlSource.py | 48 ++
backend/server/importlib/importLib.py | 56 +--
6 files changed, 253 insertions(+), 348 deletions(-)
New commits:
commit 9ad7fbe8fd05accec78d8de5e65fb192e1ba4f6f
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Tue May 4 13:25:11 2010 +0200
short packages doesn't have enough information about checksums, let's use
packages instead
diff --git a/backend/satellite_tools/satsync.py b/backend/satellite_tools/satsync.py
index 7107e66..f046fb7 100644
--- a/backend/satellite_tools/satsync.py
+++ b/backend/satellite_tools/satsync.py
@@ -88,7 +88,6 @@ class Runner:
'channels' : ['channel-families'],
'channel-families' : ['blacklists'],
'blacklists' : ['arches'],
- 'short' : [''],
'download-errata' : ['errata'],
'download-packages' : [''],
'download-source-packages' : [''],
@@ -103,7 +102,6 @@ class Runner:
'arches',
'channels',
'blacklists',
- 'short',
'download-packages',
'rpms',
'packages',
@@ -300,16 +298,6 @@ class Runner:
def _step_blacklists(self):
return self.syncer.processBlacklists()
- def _step_short(self):
- try:
- return self.syncer.processShortPackages()
- except xmlDiskSource.MissingXmlDiskSourceFileError, e:
- msg= "ERROR: The dump is missing package data, use --no-rpms to skip
this step or fix the content to include package data."
- log2disk(-1, msg)
- log2stderr(-1, msg, cleanYN=1)
- sys.exit(25)
-
-
def _step_download_packages(self):
return self.syncer.download_package_metadata()
@@ -852,37 +840,6 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
self._channel_packages[chn] = package_ids
self._avail_channel_packages[chn] = avail_package_ids
- def processShortPackages(self):
- log(1, ["", "Retrieving short package metadata (used for
indexing)"])
-
- # Compute the unique packages and populate self._channel_packages
- self._compute_unique_packages()
-
- h = sync_handlers.get_short_package_handler()
- stream_loader = StreamProducer(h)
- if self.mountpoint:
- s = xmlDiskSource.ShortPackageDiskSource(self.mountpoint)
- stream_loader.set_disk_loader(s)
- else:
- s = self.xmlWireServer.getChannelShortPackagesXmlStream
- stream_loader.set_wire_loader(s)
-
- # OK, now uq_channel_packages only has the unique packages
- for channel_label, package_ids in self._channel_packages.items():
- # Pretend we fetch all packages
- log(1, " Retrieving / parsing short package metadata: %s (%s)" %
- (channel_label, len(package_ids)))
-
- lm = self._channel_collection.get_channel_timestamp(channel_label)
- channel_last_modified = int(rhnLib.timestamp(lm))
-
- stream_loader.set_args(channel_label, channel_last_modified)
- stream_loader.process(package_ids)
-
- h.close()
-
- self._diff_packages()
-
_query_compare_packages = """
select p.id, c.checksum_type, c.checksum, p.path, p.package_size,
TO_CHAR(p.last_modified, 'YYYYMMDDHH24MISS') last_modified
@@ -896,7 +853,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
"""
# XXX the "is null" condition will have to change in multiorg satellites
def _diff_packages(self):
- package_collection = sync_handlers.ShortPackageCollection()
+ package_collection = sync_handlers.PackageCollection()
h = rhnSQL.prepare(self._query_compare_packages)
missing_channel_packages = {}
@@ -1067,7 +1024,6 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
missing_packages = {}
# First, determine what has to be downloaded
- short_package_collection = sync_handlers.ShortPackageCollection()
package_collection = sync_handlers.PackageCollection()
for channel, pids in self._missing_channel_packages.items():
missing_packages[channel] = mp = []
@@ -1078,7 +1034,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
for pid in pids:
# XXX Catch errors
- timestamp = short_package_collection.get_package_timestamp(pid)
+ timestamp = package_collection.get_package_timestamp(pid)
if not package_collection.has_package(pid, timestamp):
# not in the cache
mp.append(pid)
@@ -1086,9 +1042,9 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
return missing_packages
def download_package_metadata(self):
- log(1, ["", "Downloading package metadata"])
- # Get the missing but uncached packages
- missing_packages = self._missing_not_cached_packages()
+
+ # Compute the unique packages and populate self._channel_packages
+ self._compute_unique_packages()
h = sync_handlers.get_package_handler()
stream_loader = StreamProducer(h)
@@ -1099,7 +1055,12 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
s = self.xmlWireServer.getPackageXmlStream
stream_loader.set_wire_loader(s)
- for channel, pids in missing_packages.items():
+ log(1, ["", "Downloading package metadata"])
+ # Get the missing but uncached packages
+# missing_packages = self._missing_not_cached_packages()
+
+
+ for channel, pids in self._channel_packages.items():
package_count = len(pids)
log(1, messages.package_parsing % (channel,
@@ -1114,6 +1075,9 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
pb.redrawYN = 0
pb.printAll(1)
+ lm = self._channel_collection.get_channel_timestamp(channel)
+ channel_last_modified = int(rhnLib.timestamp(lm))
+
ss = SequenceServer(pids[:], nevermorethan=self._batch_size)
while not ss.doneYN():
chunk = ss.getChunk()
@@ -1126,6 +1090,8 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
h.close()
+ self._diff_packages()
+
# Double-check that we got all the packages
missing_packages = self._missing_not_cached_packages()
for channel, pids in missing_packages.items():
@@ -1760,12 +1726,11 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
def _link_channel_packages(self):
log(1, ["", messages.link_channel_packages])
- short_package_collection = sync_handlers.ShortPackageCollection()
package_collection = sync_handlers.PackageCollection()
uq_packages = {}
for chn, package_ids in self._channel_packages.items():
for pid in package_ids:
- timestamp = short_package_collection.get_package_timestamp(pid)
+ timestamp = package_collection.get_package_timestamp(pid)
package = package_collection.get_package(pid, timestamp)
assert package is not None
channel_obj = {'label' : chn}
@@ -1799,14 +1764,13 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
""" short-circuit the most common case"""
if not chunk:
return []
- short_package_collection = sync_handlers.ShortPackageCollection()
if sources:
package_collection = sync_handlers.SourcePackageCollection()
else:
package_collection = sync_handlers.PackageCollection()
batch = []
for pid in chunk:
- timestamp = short_package_collection.get_package_timestamp(pid)
+ timestamp = package_collection.get_package_timestamp(pid)
package = package_collection.get_package(pid, timestamp)
if package is None:
# not in the cache
@@ -1853,7 +1817,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
def _fix_erratum(self, erratum):
""" Replace the list of packages with references to short
packages"""
- sp_coll = sync_handlers.ShortPackageCollection()
+ sp_coll = sync_handlers.PackageCollection()
pids = unique(erratum['packages'])
# map all the pkgs objects to the erratum
packages = []
@@ -1909,7 +1873,6 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
def _fetch_packages(self, channel, missing_fs_packages, sources=0):
- short_package_collection = sync_handlers.ShortPackageCollection()
if sources:
# acronym = "SRPM"
package_collection = sync_handlers.SourcePackageCollection()
@@ -1924,7 +1887,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
cfg = config.initUp2dateConfig()
for package_id, path in missing_fs_packages:
pkg_current = pkg_current + 1
- timestamp = short_package_collection.get_package_timestamp(package_id)
+ timestamp = package_collection.get_package_timestamp(package_id)
package = package_collection.get_package(package_id, timestamp)
checksum_type = package['checksum_type']
@@ -2419,7 +2382,6 @@ def processCommandline():
if actionDict['no-packages']:
actionDict['packages'] = 0
- actionDict['short'] = 0
actionDict['download-packages'] = 0
actionDict['rpms'] = 0
diff --git a/backend/satellite_tools/sync_handlers.py
b/backend/satellite_tools/sync_handlers.py
index 7e79c3c..59c74f8 100644
--- a/backend/satellite_tools/sync_handlers.py
+++ b/backend/satellite_tools/sync_handlers.py
@@ -288,9 +288,6 @@ class PackageCollection(ShortPackageCollection):
def _init_cache(self):
self._cache = syncCache.PackageCache()
- def get_package_timestamp(self, package_id):
- raise NotImplementedError
-
class PackageContainer(xmlSource.PackageContainer):
def endItemCallback(self):
commit 7d79306c59cea1f2040ceb3da6f71bddac34f838
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Mon May 3 23:35:04 2010 +0200
fixed global name 'l_timestamp' is not defined
diff --git a/backend/satellite_tools/satsync.py b/backend/satellite_tools/satsync.py
index 00165d7..7107e66 100644
--- a/backend/satellite_tools/satsync.py
+++ b/backend/satellite_tools/satsync.py
@@ -1942,7 +1942,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
package['path'] = path
package_collection.add_package(package)
- errcode = self._verify_file(path, l_timestamp, package_size,
+ errcode = self._verify_file(path, rhnLib.timestamp(timestamp), package_size,
checksum_type, checksum)
if errcode == 0:
# file is already there
commit 014ac47b005e0101077ffddbe554468888b252ee
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Mon May 3 23:33:42 2010 +0200
created get_nevra_dict() so it can be reused
diff --git a/backend/satellite_tools/satsync.py b/backend/satellite_tools/satsync.py
index e3d6451..00165d7 100644
--- a/backend/satellite_tools/satsync.py
+++ b/backend/satellite_tools/satsync.py
@@ -61,7 +61,7 @@ from server.importlib.errataCache import schedule_errata_cache_update
from server.importlib.importLib import InvalidChannelFamilyError
from server.importlib.importLib import MissingParentChannelError
-from server.importlib.importLib import get_nevra
+from server.importlib.importLib import get_nevra, get_nevra_dict
import satCerts
import req_channels
@@ -897,7 +897,6 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
# XXX the "is null" condition will have to change in multiorg satellites
def _diff_packages(self):
package_collection = sync_handlers.ShortPackageCollection()
- nvrea_keys = ['name', 'epoch', 'version',
'release', 'arch']
h = rhnSQL.prepare(self._query_compare_packages)
missing_channel_packages = {}
@@ -923,17 +922,13 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
l_timestamp = rhnLib.timestamp(p_timestamp)
package = package_collection.get_package(pid, p_timestamp)
assert package is not None
- nevra = {}
- for t in nvrea_keys:
- nevra[t] = package[t] or ""
if package['org_id'] is not None:
- nevra['org_id'] = OPTIONS.orgid or DEFAULT_ORG
package['org_id'] = OPTIONS.orgid or DEFAULT_ORG
- else:
- nevra['org_id'] = package['org_id']
+ nevra = get_nevra_dict(package)
+ nevra['org_id'] = package['org_id']
- apply(h.execute, (), nevra)
+ h.execute(**nevra)
row = None
for r in (h.fetchall_dict() or []):
# let's check which checksum we have in database
diff --git a/backend/server/importlib/importLib.py
b/backend/server/importlib/importLib.py
index 76fe3be..4d2601e 100644
--- a/backend/server/importlib/importLib.py
+++ b/backend/server/importlib/importLib.py
@@ -844,9 +844,17 @@ def copy_package(fd, basedir, relpath, checksum_type, checksum,
force=None):
# Returns a list of containing nevra for the given RPM header
+NEVRA_TAGS = ['name', 'epoch', 'version', 'release',
'arch']
def get_nevra(header):
# Get nevra
nevra = []
- for tag in ['name', 'epoch', 'version', 'release',
'arch']:
+ for tag in NEVRA_TAGS:
nevra.append(header[tag])
return nevra
+
+def get_nevra_dict(header):
+ # Get nevra
+ nevra = {}
+ for tag in NEVRA_TAGS:
+ nevra[tag] = header[tag]
+ return nevra
commit 4664d0dc8240f412c93fdbd6f9564c90df6ace5a
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Mon May 3 22:48:15 2010 +0200
reuse get_nevra()
diff --git a/backend/satellite_tools/satsync.py b/backend/satellite_tools/satsync.py
index 5785c53..e3d6451 100644
--- a/backend/satellite_tools/satsync.py
+++ b/backend/satellite_tools/satsync.py
@@ -61,6 +61,7 @@ from server.importlib.errataCache import schedule_errata_cache_update
from server.importlib.importLib import InvalidChannelFamilyError
from server.importlib.importLib import MissingParentChannelError
+from server.importlib.importLib import get_nevra
import satCerts
import req_channels
@@ -1935,9 +1936,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
checksum = package['checksum']
package_size = package['package_size']
if not path:
- nevra = []
- for t in ['name', 'epoch', 'version',
'release', 'arch']:
- nevra.append(package[t])
+ nevra = get_nevra(package)
orgid = None
if package['org_id']:
orgid = OPTIONS.orgid or DEFAULT_ORG
commit e2738df2ca0b6e489fc7d8eacc5ec6468ff7a5ea
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Mon May 3 19:54:07 2010 +0200
removed unused method
moreover it looks completely wrong as it returns name twice
diff --git a/backend/server/importlib/importLib.py
b/backend/server/importlib/importLib.py
index 4fc23a2..76fe3be 100644
--- a/backend/server/importlib/importLib.py
+++ b/backend/server/importlib/importLib.py
@@ -318,10 +318,6 @@ class IncompletePackage(BaseInformation):
return "%s-%s-%s.%s.rpm" % (self.name, self.evr[1], self.evr[2],
self.arch)
- def get_nevrao(self):
- return (self.name, self['name'], self['epoch'],
self['version'],
- self['release'], self['arch'], self['org_id'])
-
class Package(IncompletePackage):
"""
A package is a hash of attributes
commit a00c35c49f93eaefeb84ead9114cf54578fa0c95
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Mon May 3 16:47:58 2010 +0200
simplified errata uniquify
diff --git a/backend/satellite_tools/satsync.py b/backend/satellite_tools/satsync.py
index 0470f97..5785c53 100644
--- a/backend/satellite_tools/satsync.py
+++ b/backend/satellite_tools/satsync.py
@@ -401,7 +401,7 @@ class Syncer:
self._failed_fs_packages = {}
self._extinct_packages = {}
- self._uq_channel_errata = {}
+ self._channel_errata = {}
self._missing_channel_errata = {}
self._channel_source_packages = {}
@@ -1537,7 +1537,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
# First, determine what has to be downloaded
errata_collection = sync_handlers.ErrataCollection()
- for channel, errata in self._uq_channel_errata.items():
+ for channel, errata in self._channel_errata.items():
missing_errata[channel] = mp = []
if not errata:
@@ -1620,19 +1620,8 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
channel_errata[chn] = errata
# Uniquify the errata
- self._uq_channel_errata = uq_channel_errata = {}
- uq_errata = {}
for channel, errata in channel_errata.items():
- ch_erratum_ids = uq_channel_errata[channel] = []
- for eid, timestamp, advisory_name in errata:
- if uq_errata.has_key(eid):
- # Saw this erratum already
- continue
- ch_erratum_ids.append((eid, timestamp, advisory_name))
- uq_errata[eid] = None
- # Be nice enough to sort the list
- ch_erratum_ids.sort()
- del uq_errata
+ self._channel_errata[channel] = sorted(list(set(errata)))
def _diff_db_errata(self):
""" Compute errata that are missing from the satellite
@@ -1642,7 +1631,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
errata_collection = sync_handlers.ErrataCollection()
self._missing_channel_errata = missing_channel_errata = {}
db_channel_errata = self._get_db_channel_errata()
- for channel, errata in self._uq_channel_errata.items():
+ for channel, errata in self._channel_errata.items():
ch_erratum_ids = missing_channel_errata[channel] = []
for eid, timestamp, advisory_name in errata:
if timestamp is not None:
commit 163a0cd2b59940eef3566fe2c6301c85675a9783
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Mon May 3 16:33:54 2010 +0200
_verify_file() returns only errcode now
diff --git a/backend/satellite_tools/satsync.py b/backend/satellite_tools/satsync.py
index f3a808f..0470f97 100644
--- a/backend/satellite_tools/satsync.py
+++ b/backend/satellite_tools/satsync.py
@@ -1498,7 +1498,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
dest_path = os.path.join(base_path, relative_path)
timestamp = rhnLib.timestamp(f['last_modified'])
file_size = f['file_size']
- (errcode, ret_path) = self._verify_file(dest_path,
+ errcode = self._verify_file(dest_path,
timestamp, file_size, f['checksum_type'],
f['checksum'])
if errcode != 0:
# Have to download it
commit f4c3fbb166e7d05a2ec8221f7179050af3088844
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Mon May 3 16:23:12 2010 +0200
simplified _compute_unique_kickstarts()
diff --git a/backend/satellite_tools/satsync.py b/backend/satellite_tools/satsync.py
index 4d80a4a..f3a808f 100644
--- a/backend/satellite_tools/satsync.py
+++ b/backend/satellite_tools/satsync.py
@@ -407,7 +407,6 @@ class Syncer:
self._channel_source_packages = {}
self._channel_kickstarts = {}
- self._uq_channel_kickstarts = {}
def initialize(self):
"""Initialization that requires IO, etc."""
@@ -1318,22 +1317,13 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
def _compute_unique_kickstarts(self):
""" process package metadata for one channel at a
time"""
relevant = self._channel_req.get_requested_channels()
- self._channel_kickstarts = channel_kickstarts = {}
- self._uq_channel_kickstarts = uq_channel_kickstarts = {}
- uq_kickstarts = {}
+ self._channel_kickstarts = {}
for chn in relevant:
timestamp = self._get_channel_timestamp(chn)
channel_obj = self._channel_collection.get_channel(chn, timestamp)
- kickstart_trees = channel_obj['kickstartable_trees']
- cks = channel_kickstarts[chn] = kickstart_trees
- uq_cks = uq_channel_kickstarts[chn] = []
- for kt in kickstart_trees:
- if uq_kickstarts.has_key(kt):
- # Found this one already
- continue
- uq_kickstarts[kt] = None
- uq_cks.append(kt)
+ self._channel_kickstarts[chn] = \
+ sorted(set(channel_obj['kickstartable_trees'] or []))
_query_get_kickstarts = rhnSQL.Statement("""
select TO_CHAR(last_modified, 'YYYYMMDDHH24MISS') last_modified
@@ -1400,7 +1390,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
s = self.xmlWireServer.getKickstartsXmlStream
stream_loader.set_wire_loader(s)
- for channel, ktids in self._uq_channel_kickstarts.items():
+ for channel, ktids in self._channel_kickstarts.items():
kt_count = len(ktids)
log(1, messages.kickstart_parsing % (channel,
@@ -1494,7 +1484,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
missing_ks_files = {}
# download files for the ks trees
- for channel, ktids in self._uq_channel_kickstarts.items():
+ for channel, ktids in self._channel_kickstarts.items():
missing_ks_files[channel] = missing = []
for ktid in ktids:
# No timestamp for kickstartable trees
commit 966552317ffbc11232134d71fcc7c671b8db496d
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Mon May 3 15:52:14 2010 +0200
rpms have to go before packages
we need to know rpm path in package import step
diff --git a/backend/satellite_tools/satsync.py b/backend/satellite_tools/satsync.py
index eb571cd..4d80a4a 100644
--- a/backend/satellite_tools/satsync.py
+++ b/backend/satellite_tools/satsync.py
@@ -104,8 +104,8 @@ class Runner:
'blacklists',
'short',
'download-packages',
- 'packages',
'rpms',
+ 'packages',
'srpms',
'download-source-packages',
'download-errata',
commit db7fbc3cee201b56671c303e32294644a2bf943c
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Mon May 3 15:57:07 2010 +0200
moved file verification to rpm download step
because checksum might be unknown in package diff step
diff --git a/backend/satellite_tools/satsync.py b/backend/satellite_tools/satsync.py
index 1a2c228..eb571cd 100644
--- a/backend/satellite_tools/satsync.py
+++ b/backend/satellite_tools/satsync.py
@@ -1034,33 +1034,21 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
package_size, checksum_type, checksum)
if errcode:
# file doesn't match
- fs_package = (package_id, db_path)
-
- # package is missing from the DB
- channel_package = package_id
-
- if not fs_package:
- nevra = []
- for t in ['name', 'epoch', 'version',
'release', 'arch']:
- nevra.append(package[t])
-
- if package['org_id'] is not None:
- orgid = OPTIONS.orgid or DEFAULT_ORG
- else:
- orgid = package['org_id']
-
- path = self._get_rel_package_path(nevra, orgid, source,
- checksum_type, checksum)
- errcode = self._verify_file(path,
- l_timestamp, package_size, checksum_type, checksum)
- if errcode:
- # file doesn't match
- fs_package = (package_id, path)
+ fs_package = package_id
+ path = db_path
+ else:
+ # upload package and reimport metadata
+ channel_package = package_id
+ fs_package = package_id
+ else:
+ # package is missing from the DB
+ channel_package = package_id
+ fs_package = package_id
if channel_package:
m_channel_packages.append(channel_package)
if fs_package:
- m_fs_packages.append(fs_package)
+ m_fs_packages.append((fs_package, path))
return
def download_rpms(self):
@@ -1849,8 +1837,6 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
# not in the cache
raise Exception("Package Not Found in Cache, Clear the Cache to \
Regenerate it.")
- # Now set the path
-# FIXME package['path'] = file_path
batch.append(package)
return batch
@@ -1948,12 +1934,13 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
def _fetch_packages(self, channel, missing_fs_packages, sources=0):
+ short_package_collection = sync_handlers.ShortPackageCollection()
if sources:
# acronym = "SRPM"
package_collection = sync_handlers.SourcePackageCollection()
else:
# acronym = "RPM"
- package_collection = sync_handlers.ShortPackageCollection()
+ package_collection = sync_handlers.PackageCollection()
self._failed_fs_packages.clear()
self._extinct_packages.clear()
@@ -1962,9 +1949,32 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
cfg = config.initUp2dateConfig()
for package_id, path in missing_fs_packages:
pkg_current = pkg_current + 1
- timestamp = package_collection.get_package_timestamp(package_id)
+ timestamp = short_package_collection.get_package_timestamp(package_id)
package = package_collection.get_package(package_id, timestamp)
+ checksum_type = package['checksum_type']
+ checksum = package['checksum']
+ package_size = package['package_size']
+ if not path:
+ nevra = []
+ for t in ['name', 'epoch', 'version',
'release', 'arch']:
+ nevra.append(package[t])
+ orgid = None
+ if package['org_id']:
+ orgid = OPTIONS.orgid or DEFAULT_ORG
+ path = self._get_rel_package_path(nevra, orgid, sources,
+ checksum_type, checksum)
+
+ # update package path
+ package['path'] = path
+ package_collection.add_package(package)
+
+ errcode = self._verify_file(path, l_timestamp, package_size,
+ checksum_type, checksum)
+ if errcode == 0:
+ # file is already there
+ continue
+
rpmManip = RpmManip(package, path)
nvrea = rpmManip.nvrea()
commit 0d427b6e63adca1d86af75af2522c57c5f85daa4
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Fri Apr 30 15:44:45 2010 +0200
removed unused variable
diff --git a/backend/satellite_tools/satsync.py b/backend/satellite_tools/satsync.py
index 13d26b8..1a2c228 100644
--- a/backend/satellite_tools/satsync.py
+++ b/backend/satellite_tools/satsync.py
@@ -2149,7 +2149,6 @@ def _validate_package_org(batch):
If custom and package org is not valid default to org 1
"""
orgid = OPTIONS.orgid or None
- orgs = map(lambda a: a['id'], satCerts.get_all_orgs())
for pkg in batch:
if not pkg['org_id'] or pkg['org_id'] == 'None':
# default to Null so do nothing
commit 048ecb8d454a2e1f947d01b012b7f7bbad1a5795
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Fri Apr 30 14:18:54 2010 +0200
straightened logic in _process_package() and _verify_file()
diff --git a/backend/satellite_tools/satsync.py b/backend/satellite_tools/satsync.py
index e1d5bbb..13d26b8 100644
--- a/backend/satellite_tools/satsync.py
+++ b/backend/satellite_tools/satsync.py
@@ -972,105 +972,96 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
source=source, checksum_type=checksum_type, checksum=checksum)
def _verify_file(self, path, mtime, size, checksum_type, checksum):
- """Verifies if the file is on the filesystem and matches the mtime
and
- checksum
- Computing the checksum is costly, that's why we rely on mtime
- comparisons.
- Returns a tuple (error_code, ret_path) where:
- if the file has the specified mtime and checksum, error_code is 0
- and ret_path is None
- if the file has the checksum, the function sets mtime, error_code is
- 0 and ret_path is path
- if the file exists but has a different checksum, error_code is the
- file's current checksum and ret_path is path
- if the file does not exist at all, error_code is 1 and ret_path is
- null
- The idea is that error_code is 0 if the file exists or something else
- otherwise
+ """
+ Verifies if the file is on the filesystem and matches the mtime and checksum.
+ Computing the checksum is costly, that's why we rely on mtime comparisons.
+ Returns errcode:
+ 0 - file is ok, it has either the specified mtime and size
+ or checksum matches (then function sets mtime)
+ 1 - file does not exist at all
+ 2 - file has a different checksum
"""
if not path:
- return (1, None)
+ return 1
abs_path = os.path.join(CFG.MOUNT_POINT, path)
try:
stat_info = os.stat(abs_path)
except OSError:
# File is missing completely
- return (1, None)
+ return 1
l_mtime = stat_info[stat.ST_MTIME]
l_size = stat_info[stat.ST_SIZE]
if l_mtime == mtime and l_size == size:
# Same mtime, and size, assume identity
- return (0, None)
+ return 0
# Have to check checksum
l_checksum = getFileChecksum(checksum_type, filename=abs_path)
if l_checksum != checksum:
- # Different checksums
- return (l_checksum, path)
+ return 2
# Set the mtime
os.utime(abs_path, (mtime, mtime))
- return (0, path)
+ return 0
def _process_package(self, package_id, package, l_timestamp, row,
m_channel_packages, m_fs_packages, source=0):
- nevra = []
- for t in ['name', 'epoch', 'version', 'release',
'arch']:
- nevra.append(package[t])
- package_size = package['package_size']
-
- if package['org_id'] is not None:
- orgid = OPTIONS.orgid or DEFAULT_ORG
- else:
- orgid = package['org_id']
-
- if not row:
- # Package is missing completely from the DB
- m_channel_packages.append(package_id)
- #m_fs_packages.append((package_id, path))
- return
+ path = None
+ channel_package = None
+ fs_package = None
+ if row:
+ # package found in the DB
+ checksum_type = row['checksum_type']
+ if checksum_type in package['checksums']:
+ checksum = package['checksums'][row['checksum_type']]
+ package_size = package['package_size']
- # Package found in the DB
- checksum_type = row['checksum_type']
- if checksum_type in package['checksums']:
- checksum = package['checksums'][row['checksum_type']]
-
- db_timestamp = int(rhnLib.timestamp(row['last_modified']))
- db_checksum = row['checksum']
- db_package_size = row['package_size']
- db_path = row['path']
- final_path = db_path
-
- path = self._get_rel_package_path(nevra, orgid, source, checksum_type,
checksum)
- # Check the filesystem
- # This is one ugly piece of code
- (errcode, ret_path) = self._verify_file(db_path, l_timestamp,
- package_size, checksum_type, checksum)
- if errcode != 0:
- if errcode != 1 or path == db_path:
- # Package is modified; fix it
- m_fs_packages.append((package_id, path))
- else:
- # Package is missing, and the DB path is, for some
- # reason, not the same as the computed path.
- (errcode, ret_path) = self._verify_file(path,
- l_timestamp, package_size, checksum_type, checksum)
- if errcode != 1:
- # Use the computed path
- final_path = path
- if errcode != 0:
- # file is modified too; re-download
- m_fs_packages.append((package_id, final_path))
-
- if (l_timestamp <= db_timestamp and
- checksum == db_checksum and
- package_size == db_package_size and final_path == db_path):
- # Same package
- return
- # Have to re-import the package - this may be just because the
- # path has changed
- m_channel_packages.append(package_id)
+ db_timestamp = int(rhnLib.timestamp(row['last_modified']))
+ db_checksum = row['checksum']
+ db_package_size = row['package_size']
+ db_path = row['path']
+
+ if not (l_timestamp <= db_timestamp and
+ checksum == db_checksum and
+ package_size == db_package_size):
+ # package doesn't match
+ channel_package = package_id
+
+ if db_path:
+ # check the filesystem
+ errcode = self._verify_file(db_path, l_timestamp,
+ package_size, checksum_type, checksum)
+ if errcode:
+ # file doesn't match
+ fs_package = (package_id, db_path)
+
+ # package is missing from the DB
+ channel_package = package_id
+
+ if not fs_package:
+ nevra = []
+ for t in ['name', 'epoch', 'version',
'release', 'arch']:
+ nevra.append(package[t])
+
+ if package['org_id'] is not None:
+ orgid = OPTIONS.orgid or DEFAULT_ORG
+ else:
+ orgid = package['org_id']
+
+ path = self._get_rel_package_path(nevra, orgid, source,
+ checksum_type, checksum)
+ errcode = self._verify_file(path,
+ l_timestamp, package_size, checksum_type, checksum)
+ if errcode:
+ # file doesn't match
+ fs_package = (package_id, path)
+
+ if channel_package:
+ m_channel_packages.append(channel_package)
+ if fs_package:
+ m_fs_packages.append(fs_package)
+ return
def download_rpms(self):
log(1, ["", "Downloading rpm packages"])
commit 8bc0b88a8528d15ddebdadec0a273a39605390f1
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Thu Apr 29 23:20:19 2010 +0200
made _compute_unique_packages() even more simple and straightforward
diff --git a/backend/satellite_tools/satsync.py b/backend/satellite_tools/satsync.py
index b0bf855..e1d5bbb 100644
--- a/backend/satellite_tools/satsync.py
+++ b/backend/satellite_tools/satsync.py
@@ -847,13 +847,10 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
timestamp = self._get_channel_timestamp(chn)
channel_obj = self._channel_collection.get_channel(chn, timestamp)
- package_ids = channel_obj['all-packages']
- avail_package_ids = channel_obj['packages']
- if package_ids is None:
- # Not an incremental
- package_ids = avail_package_ids
- self._channel_packages[chn] = set(package_ids or [])
- self._avail_channel_packages[chn] = set(avail_package_ids or [])
+ avail_package_ids = sorted(set(channel_obj['packages'] or []))
+ package_ids = sorted(set(channel_obj['all-packages'] or [])) or
avail_package_ids
+ self._channel_packages[chn] = package_ids
+ self._avail_channel_packages[chn] = avail_package_ids
def processShortPackages(self):
log(1, ["", "Retrieving short package metadata (used for
indexing)"])
commit 2b8ba63777b4b00e295259139fc8fa71acc66fd2
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Thu Apr 29 23:14:30 2010 +0200
moved repeated code into function
diff --git a/backend/satellite_tools/satsync.py b/backend/satellite_tools/satsync.py
index 09ba88d..b0bf855 100644
--- a/backend/satellite_tools/satsync.py
+++ b/backend/satellite_tools/satsync.py
@@ -830,17 +830,21 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
except Exception:
None
+ def _get_channel_timestamp(self, channel):
+ try:
+ timestamp = self._channel_collection.get_channel_timestamp(channel)
+ except KeyError:
+ # XXX Do something with this exception
+ raise
+ return timestamp
+
def _compute_unique_packages(self):
""" process package metadata for one channel at a time
"""
relevant = self._channel_req.get_requested_channels()
self._channel_packages = {}
self._avail_channel_packages = {}
for chn in relevant:
- try:
- timestamp = self._channel_collection.get_channel_timestamp(chn)
- except KeyError:
- # XXX Do something with this exception
- raise
+ timestamp = self._get_channel_timestamp(chn)
channel_obj = self._channel_collection.get_channel(chn, timestamp)
package_ids = channel_obj['all-packages']
@@ -1184,11 +1188,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
self._channel_source_packages = {}
self._avail_channel_source_packages = {}
for chn in relevant:
- try:
- timestamp = self._channel_collection.get_channel_timestamp(chn)
- except KeyError:
- # XXX Do something with this exception
- raise
+ timestamp = self._get_channel_timestamp(chn)
channel_obj = self._channel_collection.get_channel(chn, timestamp)
sps = set(channel_obj['source_packages'])
@@ -1346,11 +1346,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
self._uq_channel_kickstarts = uq_channel_kickstarts = {}
uq_kickstarts = {}
for chn in relevant:
- try:
- timestamp = self._channel_collection.get_channel_timestamp(chn)
- except KeyError:
- # XXX Do something with this exception
- raise
+ timestamp = self._get_channel_timestamp(chn)
channel_obj = self._channel_collection.get_channel(chn, timestamp)
kickstart_trees = channel_obj['kickstartable_trees']
@@ -1376,11 +1372,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
missing_kickstarts = {}
h = rhnSQL.Statement(self._query_get_kickstarts)
for chn in relevant:
- try:
- timestamp = self._channel_collection.get_channel_timestamp(chn)
- except KeyError:
- # XXX Do something with this exception
- raise
+ timestamp = self._get_channel_timestamp(chn)
channel_obj = self._channel_collection.get_channel(chn, timestamp)
kickstart_trees = channel_obj['kickstartable_trees']
@@ -1635,11 +1627,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
channel_errata = {}
for chn in relevant:
db_ce = db_channel_errata[chn]
- try:
- timestamp = self._channel_collection.get_channel_timestamp(chn)
- except KeyError:
- # XXX Do something with this exception
- raise
+ timestamp = self._get_channel_timestamp(chn)
channel_obj = self._channel_collection.get_channel(chn, timestamp)
errata_timestamps = channel_obj['errata_timestamps']
commit bd61764a494504c62609a35665d0860f11456140
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Thu Apr 29 16:03:45 2010 +0200
simplified _compute_unique_source_packages()
diff --git a/backend/satellite_tools/satsync.py b/backend/satellite_tools/satsync.py
index 218fff1..09ba88d 100644
--- a/backend/satellite_tools/satsync.py
+++ b/backend/satellite_tools/satsync.py
@@ -405,7 +405,6 @@ class Syncer:
self._missing_channel_errata = {}
self._channel_source_packages = {}
- self._uq_channel_source_packages = {}
self._channel_kickstarts = {}
self._uq_channel_kickstarts = {}
@@ -1182,10 +1181,8 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
def _compute_unique_source_packages(self):
""" process package metadata for one channel at a
time"""
relevant = self._channel_req.get_requested_channels()
- self._channel_source_packages = channel_sp = {}
- self._avail_channel_source_packages = avail_channel_source_packages = {}
- self._uq_channel_source_packages = uq_sp = {}
- uq = {}
+ self._channel_source_packages = {}
+ self._avail_channel_source_packages = {}
for chn in relevant:
try:
timestamp = self._channel_collection.get_channel_timestamp(chn)
@@ -1194,7 +1191,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
raise
channel_obj = self._channel_collection.get_channel(chn, timestamp)
- sps = channel_obj['source_packages']
+ sps = set(channel_obj['source_packages'])
if not sps:
# No source package info
continue
@@ -1207,29 +1204,15 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
ret_sps.append((sp['id'], sp['last_modified']))
del sps
ret_sps.sort()
- channel_sp[chn] = ret_sps
-
- avail_channel_source_packages[chn] = ret_sps
-
- # Uniquify source packages
- for channel_label, sps in channel_sp.items():
- ch_sp_ids = uq_sp[channel_label] = []
- for sp_id, timestamp in sps:
- if uq.has_key(sp_id):
- # Saw this source package already
- continue
- ch_sp_ids.append((sp_id, timestamp))
- uq[sp_id] = None
- # Be nice enough to sort the list
- ch_sp_ids.sort()
- del uq
+ self._channel_source_packages[chn] = ret_sps
+ self._avail_channel_source_packages[chn] = ret_sps
def _compute_not_cached_source_packages(self):
missing_sps = {}
# First, determine what has to be downloaded
sp_collection = sync_handlers.SourcePackageCollection()
- for channel, sps in self._uq_channel_source_packages.items():
+ for channel, sps in self._channel_source_packages.items():
missing_sps[channel] = mp = []
if not sps:
@@ -1263,7 +1246,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
missing_channel_source_packages = {}
missing_fs_source_packages = {}
- for channel_label, upids in self._uq_channel_source_packages.items():
+ for channel_label, upids in self._channel_source_packages.items():
log(1, "Diffing source package metadata (what's missing locally?):
%s" % channel_label)
m_channel_source_packages = missing_channel_source_packages[channel_label] =
[]
m_fs_source_packages = missing_fs_source_packages[channel_label] = []
commit 7e775347bf70ee73ba5e69d21e772dc90c4c8f36
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Thu Apr 29 15:31:30 2010 +0200
modified rhn-package-file to import new checksum correctly
diff --git a/backend/satellite_tools/xmlSource.py b/backend/satellite_tools/xmlSource.py
index d6d6546..a82d9cb 100644
--- a/backend/satellite_tools/xmlSource.py
+++ b/backend/satellite_tools/xmlSource.py
@@ -483,6 +483,11 @@ class BaseChecksummedItem(BaseItem):
for csum in item['checksum_list']:
item['checksums'][csum['type']] = csum['value']
del(item['checksum_list'])
+ for ctype in CFG.CHECKSUM_PRIORITY_LIST:
+ if ctype in item['checksums']:
+ item['checksum_type'] = ctype
+ item['checksum'] = item['checksums'][ctype]
+ break
return item
addItem(BaseChecksummedItem)
@@ -551,14 +556,9 @@ class PackageItem(IncompletePackageItem):
item = IncompletePackageItem.populate(self, attributes, elements)
# find out "primary" checksum
have_filedigests = len([1 for i in item['requires'] if i['name']
== 'rpmlib(FileDigests)'])
- if have_filedigests:
- for ctype in CFG.CHECKSUM_PRIORITY_LIST:
- if ctype in item['checksums']:
- item['checksum_type'] = ctype
- break
- else:
+ if not have_filedigests:
item['checksum_type'] = 'md5'
- item['checksum'] =
item['checksums'][item['checksum_type']]
+ item['checksum'] = item['checksums']['md5']
return item
addItem(PackageItem)
@@ -632,16 +632,11 @@ addItem(ObsoletesItem)
class FileItem(BaseChecksummedItem):
item_name = 'rhn-package-file'
item_class = importLib.File
- tagMap = {
- 'md5' : 'md5sum',
- 'checksum-type': 'checksum_type',
- 'checksum' : 'checksum',
- }
def populate(self, attributes, elements):
+ if 'md5' in attributes:
+ attributes['checksum_type'] = 'md5'
+ attributes['checksum'] = attributes['md5']
item = BaseChecksummedItem.populate(self, attributes, elements)
- if not item['checksum_type']:
- item['checksum_type'] = item['checksums'].keys()[0]
- item['checksum'] =
item['checksums'][item['checksum_type']]
return item
addItem(FileItem)
@@ -769,7 +764,6 @@ class KickstartFileItem(BaseChecksummedItem):
'relative-path' : 'relative_path',
'file-size' : 'file_size',
'last-modified' : 'last_modified',
- 'checksum-type' : 'checksum_type',
}
addItem(KickstartFileItem)
diff --git a/backend/server/importlib/importLib.py
b/backend/server/importlib/importLib.py
index 29dfabc..4fc23a2 100644
--- a/backend/server/importlib/importLib.py
+++ b/backend/server/importlib/importLib.py
@@ -239,10 +239,8 @@ class File(Item):
'flags' : IntType,
'verifyflags' : IntType,
'lang' : StringType,
- # those attributes are mutualy exclusive
- 'md5sum' : StringType, # xml dumps < 3.5
- 'checksum' : StringType, # xml dumps >= 3.5
- 'checksum_type' : StringType, # xml dumps >= 3.5
+ 'checksum' : StringType,
+ 'checksum_type' : StringType,
}
def __init__(self):
Item.__init__(self, self.attributeTypes)
commit 6dac065ac479e58d450de49938008ff94db77422
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Thu Apr 29 11:10:26 2010 +0200
made checksum priority list configurable
diff --git a/backend/rhn-conf/rhn_server.conf b/backend/rhn-conf/rhn_server.conf
index 8a5c3fa..c6808fa 100644
--- a/backend/rhn-conf/rhn_server.conf
+++ b/backend/rhn-conf/rhn_server.conf
@@ -131,3 +131,5 @@ repomd_path_prefix = rhn/repodata
# Default to not using taskomatic for repomd
use_taskomatic_repomd = 1
+# list of checksum types, most prefered first
+checksum_priority_list = sha512, sha384, sha256, sha1, md5
diff --git a/backend/satellite_tools/xmlSource.py b/backend/satellite_tools/xmlSource.py
index 020224e..d6d6546 100644
--- a/backend/satellite_tools/xmlSource.py
+++ b/backend/satellite_tools/xmlSource.py
@@ -22,7 +22,7 @@ import string
from xml.sax import make_parser, SAXParseException, ContentHandler, \
ErrorHandler
-from common import log_debug, Traceback, rhnFlags
+from common import log_debug, Traceback, rhnFlags, CFG
from server.importlib import importLib, backendLib
@@ -552,7 +552,7 @@ class PackageItem(IncompletePackageItem):
# find out "primary" checksum
have_filedigests = len([1 for i in item['requires'] if i['name']
== 'rpmlib(FileDigests)'])
if have_filedigests:
- for ctype in
['sha512','sha384','sha256','sha1','md5']:
+ for ctype in CFG.CHECKSUM_PRIORITY_LIST:
if ctype in item['checksums']:
item['checksum_type'] = ctype
break
commit 96d243d2e98381c5faab066c48f1540cf936c3a7
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Wed Apr 28 17:13:43 2010 +0200
ShortPackageCollection doesn't have enough information about checksum
diff --git a/backend/satellite_tools/satsync.py b/backend/satellite_tools/satsync.py
index 722b142..218fff1 100644
--- a/backend/satellite_tools/satsync.py
+++ b/backend/satellite_tools/satsync.py
@@ -1839,11 +1839,12 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
def _link_channel_packages(self):
log(1, ["", messages.link_channel_packages])
short_package_collection = sync_handlers.ShortPackageCollection()
+ package_collection = sync_handlers.PackageCollection()
uq_packages = {}
for chn, package_ids in self._channel_packages.items():
for pid in package_ids:
timestamp = short_package_collection.get_package_timestamp(pid)
- package = short_package_collection.get_package(pid, timestamp)
+ package = package_collection.get_package(pid, timestamp)
assert package is not None
channel_obj = {'label' : chn}
if uq_packages.has_key(pid):
commit a47dbb4789ac898cba72e47401b6c0abf9867dc6
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Wed Apr 28 16:33:03 2010 +0200
removed unused code and its comment
diff --git a/backend/satellite_tools/satsync.py b/backend/satellite_tools/satsync.py
index b3a8c64..722b142 100644
--- a/backend/satellite_tools/satsync.py
+++ b/backend/satellite_tools/satsync.py
@@ -1839,8 +1839,6 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
def _link_channel_packages(self):
log(1, ["", messages.link_channel_packages])
short_package_collection = sync_handlers.ShortPackageCollection()
- # XXX: don't use this next line for anything. Why is it here?
- #relevant = self._channel_req.get_requested_channels()
uq_packages = {}
for chn, package_ids in self._channel_packages.items():
for pid in package_ids:
commit 587c345ae82b1fc0cd5cba946132577355d7db19
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Tue Apr 27 16:45:58 2010 +0200
removed _ChecksumEntryDumper(): can be simply rewriten via EmptyDumper()
diff --git a/backend/satellite_tools/exporter/exportLib.py
b/backend/satellite_tools/exporter/exportLib.py
index 6c68926..4b290c6 100644
--- a/backend/satellite_tools/exporter/exportLib.py
+++ b/backend/satellite_tools/exporter/exportLib.py
@@ -838,19 +838,12 @@ class _ChecksumDumper(BaseDumper):
tag_name = 'checksums'
def dump_subelement(self, data):
- c = _ChecksumEntryDumper(self._writer, data)
+ c = EmptyDumper(self._writer, 'checksum', attributes={
+ 'type' : data['type'],
+ 'value': data['value'],
+ })
c.dump()
-class _ChecksumEntryDumper(BaseRowDumper):
- tag_name = 'checksum'
-
- def set_attributes(self):
- attrs = ['type', 'value']
- attrdict = {}
- for attr in attrs:
- attrdict[attr] = self._row[attr]
- return attrdict
-
##
class _ChangelogDumper(BaseDumper):
tag_name = 'rhn-package-changelog'
commit ae9705bfdb895160fb6d9ff0cb236b9f1a8b1b54
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Mon Apr 26 15:12:34 2010 +0200
let the comment make sense
diff --git a/backend/satellite_tools/satsync.py b/backend/satellite_tools/satsync.py
index 953b222..b3a8c64 100644
--- a/backend/satellite_tools/satsync.py
+++ b/backend/satellite_tools/satsync.py
@@ -2323,7 +2323,7 @@ def processCommandline():
global OPTIONS
OPTIONS, args = optionParser.parse_args()
- # we take no extra commandline arguments that are not linked to an option
+ # we take extra commandline arguments that are not linked to an option
if args:
msg = "ERROR: these arguments make no sense in this context (try --help):
%s" % repr(args)
log2stderr(-1, msg, 1,1)
commit cd53bea124d3749f8e6a2c8e31f57caf839b000d
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Mon Apr 26 15:00:21 2010 +0200
removed an extra condition
diff --git a/backend/satellite_tools/satsync.py b/backend/satellite_tools/satsync.py
index 8004e4a..953b222 100644
--- a/backend/satellite_tools/satsync.py
+++ b/backend/satellite_tools/satsync.py
@@ -660,10 +660,12 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
dataServer.setChannel(c)
stream = dataServer.load()
h.process(stream)
+ doEOSYN=0
else:
dataServer = self.xmlWireServer.getChannelXmlStream
stream = dataServer([])
h.process(stream)
+ doEOSYN=1
h.close()
@@ -676,10 +678,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
#satellite-sync is running with the --mount-point (-m) option. If it
#did, it would incorrectly list channels as end-of-service if they had been
#synced already but aren't in the channel dump.
- if self.mountpoint:
- self._printChannelTree(doEOSYN=0)
- else:
- self._printChannelTree(doEOSYN=1)
+ self._printChannelTree(doEOSYN=doEOSYN)
if self.listChannelsYN:
# We're done here
commit f436d8bfe8d25aa13796a097de349de9ce88afd5
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Mon Apr 26 14:53:08 2010 +0200
removed an extra condition and made code slightly readable
diff --git a/backend/satellite_tools/satsync.py b/backend/satellite_tools/satsync.py
index 7785a51..8004e4a 100644
--- a/backend/satellite_tools/satsync.py
+++ b/backend/satellite_tools/satsync.py
@@ -422,16 +422,17 @@ class Syncer:
self.xmlWireServer = xmlWireSource.MetadataWireSource(self.systemid,
self.sslYN, self.xml_dump_version)
if CFG.ISS_PARENT:
- url = self.xmlWireServer.schemeAndUrl(CFG.ISS_PARENT)
+ sync_parent = CFG.ISS_PARENT
+ is_iss = 1
else:
- url = self.xmlWireServer.schemeAndUrl(CFG.RHN_PARENT)
+ sync_parent = CFG.RHN_PARENT
+ is_iss = 0
+
+ url = self.xmlWireServer.schemeAndUrl(sync_parent)
log(1, ['Red Hat Network Satellite - live synchronization',
' url: %s' % url,
' debug/output level: %s' % CFG.DEBUG])
- if CFG.ISS_PARENT:
- self.xmlWireServer.setServerHandler(isIss=1)
- else:
- self.xmlWireServer.setServerHandler()
+ self.xmlWireServer.setServerHandler(isIss=is_iss)
if not self.mountpoint:
# check and fetch systemid (NOTE: systemid kept in memory... may or may not
commit f0ae657e1cac57136d7fdaa12bcc459ae6eb2c1b
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Mon Apr 26 14:31:57 2010 +0200
package metadata have to go before rpms now
(we need whether package is md5 or sha256)
diff --git a/backend/satellite_tools/satsync.py b/backend/satellite_tools/satsync.py
index 4ff9548..7785a51 100644
--- a/backend/satellite_tools/satsync.py
+++ b/backend/satellite_tools/satsync.py
@@ -103,13 +103,13 @@ class Runner:
'channels',
'blacklists',
'short',
+ 'download-packages',
+ 'packages',
'rpms',
'srpms',
- 'download-packages',
'download-source-packages',
'download-errata',
'download-kickstarts',
- 'packages',
'source-packages',
'errata',
'kickstarts',
commit e94ebbd78cf08b7dadadce51c3cc866ac4aa7a4c
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Mon Apr 26 14:29:19 2010 +0200
check whether database contains one of the checksums we got from dump
diff --git a/backend/satellite_tools/satsync.py b/backend/satellite_tools/satsync.py
index fefbabe..4ff9548 100644
--- a/backend/satellite_tools/satsync.py
+++ b/backend/satellite_tools/satsync.py
@@ -1033,39 +1033,41 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
# Package found in the DB
checksum_type = row['checksum_type']
- checksum = package['checksum']
- db_timestamp = int(rhnLib.timestamp(row['last_modified']))
- db_checksum = row['checksum']
- db_package_size = row['package_size']
- db_path = row['path']
- final_path = db_path
-
- path = self._get_rel_package_path(nevra, orgid, source, checksum_type, checksum)
- # Check the filesystem
- # This is one ugly piece of code
- (errcode, ret_path) = self._verify_file(db_path, l_timestamp,
- package_size, checksum_type, checksum)
- if errcode != 0:
- if errcode != 1 or path == db_path:
- # Package is modified; fix it
- m_fs_packages.append((package_id, path))
- else:
- # Package is missing, and the DB path is, for some
- # reason, not the same as the computed path.
- (errcode, ret_path) = self._verify_file(path,
- l_timestamp, package_size, checksum_type, checksum)
- if errcode != 1:
- # Use the computed path
- final_path = path
- if errcode != 0:
- # file is modified too; re-download
- m_fs_packages.append((package_id, final_path))
-
- if (l_timestamp <= db_timestamp and
- checksum == db_checksum and
- package_size == db_package_size and final_path == db_path):
- # Same package
- return
+ if checksum_type in package['checksums']:
+ checksum = package['checksums'][row['checksum_type']]
+
+ db_timestamp = int(rhnLib.timestamp(row['last_modified']))
+ db_checksum = row['checksum']
+ db_package_size = row['package_size']
+ db_path = row['path']
+ final_path = db_path
+
+ path = self._get_rel_package_path(nevra, orgid, source, checksum_type,
checksum)
+ # Check the filesystem
+ # This is one ugly piece of code
+ (errcode, ret_path) = self._verify_file(db_path, l_timestamp,
+ package_size, checksum_type, checksum)
+ if errcode != 0:
+ if errcode != 1 or path == db_path:
+ # Package is modified; fix it
+ m_fs_packages.append((package_id, path))
+ else:
+ # Package is missing, and the DB path is, for some
+ # reason, not the same as the computed path.
+ (errcode, ret_path) = self._verify_file(path,
+ l_timestamp, package_size, checksum_type, checksum)
+ if errcode != 1:
+ # Use the computed path
+ final_path = path
+ if errcode != 0:
+ # file is modified too; re-download
+ m_fs_packages.append((package_id, final_path))
+
+ if (l_timestamp <= db_timestamp and
+ checksum == db_checksum and
+ package_size == db_package_size and final_path == db_path):
+ # Same package
+ return
# Have to re-import the package - this may be just because the
# path has changed
m_channel_packages.append(package_id)
commit e59e8f13eb3d2d3d5a8baa3f4790a94fcaa8ebfd
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Mon Apr 26 14:17:07 2010 +0200
path may be None if package has been synced with --no-rpms
diff --git a/backend/satellite_tools/satsync.py b/backend/satellite_tools/satsync.py
index c437cc5..fefbabe 100644
--- a/backend/satellite_tools/satsync.py
+++ b/backend/satellite_tools/satsync.py
@@ -988,6 +988,8 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
The idea is that error_code is 0 if the file exists or something else
otherwise
"""
+ if not path:
+ return (1, None)
abs_path = os.path.join(CFG.MOUNT_POINT, path)
try:
stat_info = os.stat(abs_path)
commit 4bb4793ca32abac24afa658181c9fd01dea0e9cc
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Mon Apr 26 12:16:49 2010 +0200
choose primary checksum according to rpm capability
fix rpm-package-file's checksum
diff --git a/backend/satellite_tools/xmlSource.py b/backend/satellite_tools/xmlSource.py
index a96dab4..020224e 100644
--- a/backend/satellite_tools/xmlSource.py
+++ b/backend/satellite_tools/xmlSource.py
@@ -547,6 +547,19 @@ class PackageItem(IncompletePackageItem):
'rhn-package-changelog' : 'changelog',
}
tagMap.update(IncompletePackageItem.tagMap)
+ def populate(self, attributes, elements):
+ item = IncompletePackageItem.populate(self, attributes, elements)
+ # find out "primary" checksum
+ have_filedigests = len([1 for i in item['requires'] if i['name']
== 'rpmlib(FileDigests)'])
+ if have_filedigests:
+ for ctype in
['sha512','sha384','sha256','sha1','md5']:
+ if ctype in item['checksums']:
+ item['checksum_type'] = ctype
+ break
+ else:
+ item['checksum_type'] = 'md5'
+ item['checksum'] =
item['checksums'][item['checksum_type']]
+ return item
addItem(PackageItem)
class IncompleteSourcePackageItem(BaseItem):
@@ -621,7 +634,15 @@ class FileItem(BaseChecksummedItem):
item_class = importLib.File
tagMap = {
'md5' : 'md5sum',
+ 'checksum-type': 'checksum_type',
+ 'checksum' : 'checksum',
}
+ def populate(self, attributes, elements):
+ item = BaseChecksummedItem.populate(self, attributes, elements)
+ if not item['checksum_type']:
+ item['checksum_type'] = item['checksums'].keys()[0]
+ item['checksum'] =
item['checksums'][item['checksum_type']]
+ return item
addItem(FileItem)
class DistItem(BaseItem):
commit 39635cb117feda28e062fe09ccfe563fe5c9e555
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Fri Apr 23 11:21:51 2010 +0200
path has been removed from the collection
diff --git a/backend/satellite_tools/satsync.py b/backend/satellite_tools/satsync.py
index 67c4a3c..c437cc5 100644
--- a/backend/satellite_tools/satsync.py
+++ b/backend/satellite_tools/satsync.py
@@ -1880,7 +1880,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
else:
package_collection = sync_handlers.PackageCollection()
batch = []
- for pid, file_path in chunk:
+ for pid in chunk:
timestamp = short_package_collection.get_package_timestamp(pid)
package = package_collection.get_package(pid, timestamp)
if package is None:
@@ -1888,7 +1888,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
raise Exception("Package Not Found in Cache, Clear the Cache to \
Regenerate it.")
# Now set the path
- package['path'] = file_path
+# FIXME package['path'] = file_path
batch.append(package)
return batch
commit ac116e50ced003ff24e05ec51632ce84af08a6ab
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Fri Apr 23 11:18:53 2010 +0200
fixed variables changed in b2b2398f7b78938841a2e9c67164dd3780f4516d
diff --git a/backend/satellite_tools/satsync.py b/backend/satellite_tools/satsync.py
index 3f474ae..67c4a3c 100644
--- a/backend/satellite_tools/satsync.py
+++ b/backend/satellite_tools/satsync.py
@@ -849,8 +849,8 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
if package_ids is None:
# Not an incremental
package_ids = avail_package_ids
- channel_packages[chn] = set(package_ids or [])
- avail_channel_packages[chn] = set(avail_package_ids or [])
+ self._channel_packages[chn] = set(package_ids or [])
+ self._avail_channel_packages[chn] = set(avail_package_ids or [])
def processShortPackages(self):
log(1, ["", "Retrieving short package metadata (used for
indexing)"])
commit bea4a180f0220a75e065dc2141de5fa214e6906d
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Fri Apr 23 11:10:59 2010 +0200
fixed typo
diff --git a/backend/satellite_tools/satsync.py b/backend/satellite_tools/satsync.py
index 83faa81..3f474ae 100644
--- a/backend/satellite_tools/satsync.py
+++ b/backend/satellite_tools/satsync.py
@@ -935,7 +935,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
apply(h.execute, (), nevra)
row = None
- for r in (h.fetchall_dict() or []):
+ for r in (h.fetchall_dict() or []):
# let's check which checksum we have in database
if package['checksums'][r['checksum_type']] ==
r['checksum']:
row = r
commit 503e6a858b5d6ada430a654307b04796e971f095
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Fri Apr 23 10:58:51 2010 +0200
modified rhn-package, rhn-erratum-file and rhn-kickstart-file to import checksum
element
diff --git a/backend/server/importlib/importLib.py
b/backend/server/importlib/importLib.py
index fc5c597..29dfabc 100644
--- a/backend/server/importlib/importLib.py
+++ b/backend/server/importlib/importLib.py
@@ -349,13 +349,8 @@ class Package(IncompletePackage):
'header_start' : IntType,
'header_end' : IntType,
'path' : StringType,
- # these attributes are mutualy exclusive
'md5sum' : StringType, # xml dumps < 3.5
- 'checksum' : StringType, # xml dumps >= 3.5
- 'checksum_type' : StringType, # xml dumps >= 3.5
- 'sigmd5' : StringType, # xml dumps < 3.5 and rpms
- 'sigchecksum_type' : StringType, # xml dumps >= 3.5
- 'sigchecksum' : StringType, # xml dumps >= 3.5
+ 'sigmd5' : StringType,
# These attributes are lists of objects
'files' : [File],
'requires' : [Dependency],
@@ -364,6 +359,7 @@ class Package(IncompletePackage):
'obsoletes' : [Dependency],
'changelog' : [ChangeLog],
'channels' : [StringType],
+ 'checksum_list' : [Checksum],
}
def __init__(self):
# Inherit from IncompletePackage
@@ -389,11 +385,8 @@ class SourcePackage(IncompletePackage):
'last_modified' : DateType,
# these attributes are mutualy exclusive
'md5sum' : StringType, # xml dumps < 3.5
- 'checksum' : StringType, # xml dumps >= 3.5
- 'checksum_type' : StringType, # xml dumps >= 3.5
'sigmd5' : StringType, # xml dumps < 3.5 and rpms
- 'sigchecksum_type' : StringType, # xml dumps >= 3.5
- 'sigchecksum' : StringType, # xml dumps >= 3.5
+ 'checksum_list' : [Checksum],
}
def __init__(self):
# Inherit from IncompletePackage
@@ -414,8 +407,7 @@ class SourcePackageFile(Information):
'org_id' : IntType,
# these attributes are mutualy exclusive
'md5sum' : StringType, # xml dumps < 3.5
- 'checksum' : StringType, # xml dumps >= 3.5
- 'checksum_type' : StringType, # xml dumps >= 3.5
+ 'checksum_list' : [Checksum],
}
@@ -434,8 +426,7 @@ class ErrataFile(Information):
'package_id' : IntType,
# these attributes are mutualy exclusive
'md5sum' : StringType, # xml dumps < 3.5
- 'checksum' : StringType, # xml dumps >= 3.5
- 'checksum_type' : StringType, # xml dumps >= 3.5
+ 'checksum_list' : [Checksum],
}
@@ -544,10 +535,8 @@ class KickstartFile(Information):
'relative_path' : StringType,
'last_modified' : DateType,
'file_size' : IntType,
- # these attributes are mutualy exclusive
'md5sum' : StringType, # xml dumps < 3.5
- 'checksum' : StringType, # xml dumps >= 3.5
- 'checksum_type' : StringType, # xml dumps >= 3.5
+ 'checksum_list' : [Checksum],
}
class KickstartableTree(Information):
commit 0a3d13f3b3771c6593a3256f2d620e3584c0867a
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Fri Apr 23 10:21:41 2010 +0200
check if any of checksums we got matches to checksum of the package we have
diff --git a/backend/satellite_tools/satsync.py b/backend/satellite_tools/satsync.py
index a4c616e..83faa81 100644
--- a/backend/satellite_tools/satsync.py
+++ b/backend/satellite_tools/satsync.py
@@ -893,14 +893,11 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
and (p.org_id = :org_id or
(p.org_id is null and :org_id is null))
and p.checksum_id = c.id
- and c.checksum = :checksum
- and c.checksum_type = :checksum_type
"""
# XXX the "is null" condition will have to change in multiorg satellites
def _diff_packages(self):
package_collection = sync_handlers.ShortPackageCollection()
- nvrea_keys = ['name', 'epoch', 'version', \
- 'release', 'arch', 'checksum',
'checksum_type']
+ nvrea_keys = ['name', 'epoch', 'version',
'release', 'arch']
h = rhnSQL.prepare(self._query_compare_packages)
missing_channel_packages = {}
@@ -937,7 +934,13 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
nevra['org_id'] = package['org_id']
apply(h.execute, (), nevra)
- row = h.fetchone_dict()
+ row = None
+ for r in (h.fetchall_dict() or []):
+ # let's check which checksum we have in database
+ if package['checksums'][r['checksum_type']] ==
r['checksum']:
+ row = r
+ break
+
# Update the progress bar
pb.addTo(1)
pb.printIncrement()
@@ -955,8 +958,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
In an incremental approach, one may request packages that are actually
not available in the current dump, probably because of applying an
incremental to the wrong base"""
- for channel_label, objs in missing_channel_packages.items():
- pids = map(lambda x: x[0], objs)
+ for channel_label, pids in missing_channel_packages.items():
if sources:
avail_pids = map(lambda x: x[0], self._avail_channel_source_packages[channel_label])
else:
@@ -1014,8 +1016,6 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
nevra = []
for t in ['name', 'epoch', 'version', 'release',
'arch']:
nevra.append(package[t])
- checksum_type = package['checksum_type']
- checksum = package['checksum']
package_size = package['package_size']
if package['org_id'] is not None:
@@ -1023,27 +1023,22 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
else:
orgid = package['org_id']
- path = self._get_rel_package_path(nevra, orgid, source, checksum_type, checksum)
if not row:
# Package is missing completely from the DB
- m_channel_packages.append((package_id, path))
- (errcode, ret_path) = self._verify_file(path,
- l_timestamp, package_size, checksum_type, checksum)
- if errcode == 0:
- # Package on the filesystem, and matches
- return
- # we have to update this rpm
- m_fs_packages.append((package_id, path))
+ m_channel_packages.append(package_id)
+ #m_fs_packages.append((package_id, path))
return
# Package found in the DB
+ checksum_type = row['checksum_type']
+ checksum = package['checksum']
db_timestamp = int(rhnLib.timestamp(row['last_modified']))
- db_checksum_type = row['checksum_type']
db_checksum = row['checksum']
db_package_size = row['package_size']
db_path = row['path']
final_path = db_path
+ path = self._get_rel_package_path(nevra, orgid, source, checksum_type, checksum)
# Check the filesystem
# This is one ugly piece of code
(errcode, ret_path) = self._verify_file(db_path, l_timestamp,
@@ -1065,13 +1060,13 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
m_fs_packages.append((package_id, final_path))
if (l_timestamp <= db_timestamp and
- checksum_type == db_checksum_type and checksum == db_checksum and
+ checksum == db_checksum and
package_size == db_package_size and final_path == db_path):
# Same package
return
# Have to re-import the package - this may be just because the
# path has changed
- m_channel_packages.append((package_id, final_path))
+ m_channel_packages.append(package_id)
def download_rpms(self):
log(1, ["", "Downloading rpm packages"])
@@ -1103,7 +1098,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
# Nothing to see here
continue
- for (pid, path) in pids:
+ for pid in pids:
# XXX Catch errors
timestamp = short_package_collection.get_package_timestamp(pid)
if not package_collection.has_package(pid, timestamp):
commit ab6f2343ee881d88e15c2b26a6be7b7e8dc962bb
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Wed Apr 21 15:49:24 2010 +0200
modified importer to read new checksum element from dump
<rhn-package-short md5sum="ba311903b5549a2ed382ea463f6" ... >
<checksums>
<checksum type="md5" value="ba311903b5549a2ed382ea463f6"/>
<checksum type="sha256"
value="1908d9bfd416739c34cae407b03b9ee046084a9dfd858103b0bf79dd8dcb9129"/>
</checksums>
</rhn-package-short>
diff --git a/backend/satellite_tools/xmlSource.py b/backend/satellite_tools/xmlSource.py
index 13ae0ad..a96dab4 100644
--- a/backend/satellite_tools/xmlSource.py
+++ b/backend/satellite_tools/xmlSource.py
@@ -474,12 +474,15 @@ addItem(ChannelItem)
class BaseChecksummedItem(BaseItem):
def populate(self, attributes, elements):
item = BaseItem.populate(self, attributes, elements)
+ item['checksums'] = {}
if 'md5sum' in item:
- if type(item['checksum']) != types.StringType:
- # xml dumps < 3.5 (aka pre-sha256)
- item['checksum_type'] = 'md5'
- item['checksum'] = item['md5sum']
+ # xml dumps < 3.6 (aka pre-sha256)
+ item['checksums']['md5'] = item['md5sum']
del(item['md5sum'])
+ if 'checksum_list' in item and item['checksum_list']:
+ for csum in item['checksum_list']:
+ item['checksums'][csum['type']] = csum['value']
+ del(item['checksum_list'])
return item
addItem(BaseChecksummedItem)
@@ -492,10 +495,20 @@ class IncompletePackageItem(BaseChecksummedItem):
'last-modified' : 'last_modified',
'package-arch' : 'arch',
'org-id' : 'org_id',
- 'checksum-type' : 'checksum_type', # xml dump 3.5
(sha256)
+ 'checksums' : 'checksum_list',
}
addItem(IncompletePackageItem)
+class ChecksumItem(BaseItem):
+ item_name = 'checksum'
+ item_class = importLib.Checksum
+ tagMap = {
+ 'checksum-type' : 'type',
+ 'checksum-value' : 'value',
+ }
+addItem(ChecksumItem)
+
+
class PackageItem(IncompletePackageItem):
item_name = 'rhn-package'
item_class = importLib.Package
diff --git a/backend/server/importlib/importLib.py
b/backend/server/importlib/importLib.py
index c0b2b64..fc5c597 100644
--- a/backend/server/importlib/importLib.py
+++ b/backend/server/importlib/importLib.py
@@ -268,6 +268,15 @@ class ChangeLog(Item):
Item.__init__(self, self.attributeTypes)
+class Checksum(Item):
+ attributeTypes = {
+ 'type' : StringType,
+ 'value' : StringType,
+ }
+ def __init__(self):
+ Item.__init__(self, self.attributeTypes)
+
+
class IncompletePackage(BaseInformation):
attributeTypes = {
'package_id' : StringType, # RH db id
@@ -279,12 +288,10 @@ class IncompletePackage(BaseInformation):
'org_id' : IntType,
'package_size' : IntType,
'last_modified' : DateType,
- # those attributes are mutualy exclusive
'md5sum' : StringType, # xml dumps < 3.5
- 'checksum' : StringType, # xml dumps >= 3.5
- 'checksum_type' : StringType, # xml dumps >= 3.5
# These attributes are lists of objects
'channels' : [StringType],
+ 'checksum_list' : [Checksum],
}
def __init__(self):
BaseInformation.__init__(self, IncompletePackage.attributeTypes)
commit bae794c9d7f815b1db0569a381778379fd1a4253
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Wed Apr 21 10:46:28 2010 +0200
changed class and procedure comments to docstrings
diff --git a/backend/satellite_tools/satsync.py b/backend/satellite_tools/satsync.py
index 02522a5..a4c616e 100644
--- a/backend/satellite_tools/satsync.py
+++ b/backend/satellite_tools/satsync.py
@@ -259,7 +259,7 @@ class Runner:
def _run_syncer_step(self, function, step_name):
- # Runs a function, and catches the most common error cases
+ """ Runs a function, and catches the most common error cases
"""
try:
ret = function()
except (xmlDiskSource.MissingXmlDiskSourceDirError,
@@ -344,7 +344,7 @@ class Runner:
self.syncer.import_kickstarts()
def sendMail(forceEmail=0):
- ### Send email summary
+ """ Send email summary """
if forceEmail or (OPTIONS is not None and OPTIONS.email):
body = dumpEMAIL_LOG()
if body:
@@ -411,7 +411,7 @@ class Syncer:
self._uq_channel_kickstarts = {}
def initialize(self):
- "Initialization that requires IO, etc."
+ """Initialization that requires IO, etc."""
# Sync from filesystem:
if self.mountpoint:
@@ -542,8 +542,9 @@ class Syncer:
return self._process_cert(cert)
def _process_cert(self, cert, store_cert=1):
- # Give the cert a check - if it's broken xml we'd better find it out
- # now
+ """Give the cert a check - if it's broken xml we'd better
find it out
+ now
+ """
log2(1, 4, " - parsing for sanity")
sat_cert = satellite_cert.SatelliteCert()
try:
@@ -642,8 +643,9 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
self._set_comps_for_channel(backend, data[label]['id'], comps_path,
timestamp)
def process_channels(self):
- # push channels, channel-family and dist. map information
- # as well upon parsing.
+ """ push channels, channel-family and dist. map information
+ as well upon parsing.
+ """
log(1, ["", "Retrieving / parsing channel data"])
h = sync_handlers.get_channel_handler()
@@ -830,7 +832,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
None
def _compute_unique_packages(self):
- ### process package metadata for one channel at a time
+ """ process package metadata for one channel at a time
"""
relevant = self._channel_req.get_requested_channels()
self._channel_packages = {}
self._avail_channel_packages = {}
@@ -1179,7 +1181,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
log(1, "Processing srpm packages complete")
def _compute_unique_source_packages(self):
- ### process package metadata for one channel at a time
+ """ process package metadata for one channel at a
time"""
relevant = self._channel_req.get_requested_channels()
self._channel_source_packages = channel_sp = {}
self._avail_channel_source_packages = avail_channel_source_packages = {}
@@ -1356,7 +1358,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
raise ReprocessingNeeded
def _compute_unique_kickstarts(self):
- ### process package metadata for one channel at a time
+ """ process package metadata for one channel at a
time"""
relevant = self._channel_req.get_requested_channels()
self._channel_kickstarts = channel_kickstarts = {}
self._uq_channel_kickstarts = uq_channel_kickstarts = {}
@@ -1386,7 +1388,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
""")
def _compute_missing_kickstarts(self):
- ### process package metadata for one channel at a time
+ """ process package metadata for one channel at a
time"""
relevant = self._channel_req.get_requested_channels()
coll = sync_handlers.KickstartableTreesCollection()
missing_kickstarts = {}
@@ -1642,7 +1644,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
return db_channel_errata
def _diff_errata(self):
- # Fetch the errata for this channel
+ """ Fetch the errata for this channel"""
db_channel_errata = self._get_db_channel_errata()
relevant = self._channel_req.get_requested_channels()
@@ -1697,9 +1699,10 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
del uq_errata
def _diff_db_errata(self):
- # Compute errata that are missing from the satellite
- # Kind of similar to diff_errata, if we had the timestamp and advisory
- # information available
+ """ Compute errata that are missing from the satellite
+ Kind of similar to diff_errata, if we had the timestamp and advisory
+ information available
+ """
errata_collection = sync_handlers.ErrataCollection()
self._missing_channel_errata = missing_channel_errata = {}
db_channel_errata = self._get_db_channel_errata()
@@ -1873,7 +1876,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
return importer.affected_channels
def _get_cached_package_batch(self, chunk, sources=0):
- # short-circuit the most common case
+ """ short-circuit the most common case"""
if not chunk:
return []
short_package_collection = sync_handlers.ShortPackageCollection()
@@ -1931,7 +1934,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
pb.printComplete()
def _fix_erratum(self, erratum):
- # Replace the list of packages with references to short packages
+ """ Replace the list of packages with references to short
packages"""
sp_coll = sync_handlers.ShortPackageCollection()
pids = unique(erratum['packages'])
# map all the pkgs objects to the erratum
commit 164fad7cad3098ddf3413afe3111fd1b1c0532d7
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Wed Apr 21 10:38:20 2010 +0200
made _compute_unique_packages() simple and straightforward
diff --git a/backend/satellite_tools/satsync.py b/backend/satellite_tools/satsync.py
index 9a5a65d..02522a5 100644
--- a/backend/satellite_tools/satsync.py
+++ b/backend/satellite_tools/satsync.py
@@ -393,7 +393,6 @@ class Syncer:
self.systemid = None
self._channel_packages = {}
- self._uq_channel_packages = {}
self._avail_channel_packages = {}
self._missing_channel_packages = None
@@ -833,10 +832,8 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
def _compute_unique_packages(self):
### process package metadata for one channel at a time
relevant = self._channel_req.get_requested_channels()
- self._channel_packages = channel_packages = {}
- self._avail_channel_packages = avail_channel_packages = {}
- uq_package_ids = {}
- self._uq_channel_packages = uq_channel_packages = {}
+ self._channel_packages = {}
+ self._avail_channel_packages = {}
for chn in relevant:
try:
timestamp = self._channel_collection.get_channel_timestamp(chn)
@@ -850,27 +847,13 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
if package_ids is None:
# Not an incremental
package_ids = avail_package_ids
- channel_packages[chn] = package_ids
- avail_channel_packages[chn] = avail_package_ids
-
- # Uniquify the packages
- for channel_label, package_ids in channel_packages.items():
- ch_package_ids = uq_channel_packages[channel_label] = []
- for pid in package_ids:
- if uq_package_ids.has_key(pid):
- # Saw this package already
- continue
- ch_package_ids.append(pid)
- uq_package_ids[pid] = None
- # Be nice enough to sort the list
- ch_package_ids.sort()
- del uq_package_ids
+ channel_packages[chn] = set(package_ids or [])
+ avail_channel_packages[chn] = set(avail_package_ids or [])
def processShortPackages(self):
log(1, ["", "Retrieving short package metadata (used for
indexing)"])
- # Compute the unique packages and populate self._channel_packages and
- # self._uq_channel_packages
+ # Compute the unique packages and populate self._channel_packages
self._compute_unique_packages()
h = sync_handlers.get_short_package_handler()
@@ -921,7 +904,7 @@ Please contact your RHN representative""" %
(generation, sat_cert.generation))
missing_channel_packages = {}
missing_fs_packages = {}
- for channel_label, upids in self._uq_channel_packages.items():
+ for channel_label, upids in self._channel_packages.items():
log(1, "Diffing package metadata (what's missing locally?): %s"
%
channel_label)
m_channel_packages = missing_channel_packages[channel_label] = []