backend/satellite_tools/repo_plugins/yum_src.py | 6 backend/satellite_tools/reposync.py | 156 +++++++++++------------- backend/server/rhnPackage.py | 83 +++++------- 3 files changed, 111 insertions(+), 134 deletions(-)
New commits: commit ab07eddd4caf3de6a04c7e35ae9217eda101bffa Author: Michael Mraka michael.mraka@redhat.com Date: Wed Feb 2 22:30:35 2011 +0100
shortened and narrowed package sync logic
process packages in one loop
diff --git a/backend/satellite_tools/reposync.py b/backend/satellite_tools/reposync.py index f2efd0f..f1ba44d 100644 --- a/backend/satellite_tools/reposync.py +++ b/backend/satellite_tools/reposync.py @@ -133,69 +133,62 @@ class RepoSync:
def import_packages(self, plug, url): packages = plug.list_packages() - to_link = [] - to_download = [] + to_process = [] self.print_msg("Repo " + url + " has " + str(len(packages)) + " packages.") for pack in packages: - # we have a few scenarios here: - # 1. package is not on the server (link and download) - # 2. package is in the server, but not in the channel (just link if we can) - # 3. package is in the server and channel, but not on the file system (just download) path, package_channel = rhnPackage.get_path_for_package( [pack.name, pack.version, pack.release, pack.epoch, pack.arch], self.channel_label)
- if path: + to_download = False + to_link = False + if not path: + # package is not on disk + to_download = True + else: + # a different package is on disk pack.path = os.path.join(CFG.MOUNT_POINT, path) if not self.match_package_checksum(pack.path, - pack.checksum_type, pack.checksum)): - to_download.append(pack) - else: - to_download.append(pack) + pack.checksum_type, pack.checksum): + to_download = True
if package_channel != self.channel_label: - to_link.append(pack) + # package is not in the channel + to_link = True
- if len(to_download) == 0: - self.print_msg("No new packages to download.") - else: - self.regen=True + if to_download or to_link: + to_process.append((pack, to_download, to_link)) + + num_to_process = len(to_process) + if num_to_process == 0: + self.print_msg("No new packages to sync.") + return + + self.regen=True is_non_local_repo = (url.find("file://") < 0) - for (index, pack) in enumerate(to_download): - """download each package""" - # try/except/finally doesn't work in python 2.4 (RHEL5), so here's a hack + # try/except/finally doesn't work in python 2.4 (RHEL5), so here's a hack + def finally_remove(path): + if is_non_local_repo: + os.remove(path) + + for (index, what) in enumerate(to_process): + pack, to_download, to_link = what try: - try: - self.print_msg(str(index+1) + "/" + str(len(to_download)) + " : "+ \ - pack.getNVREA()) + self.print_msg("%d/%d : %s" % (index+1, num_to_process, pack.getNVREA())) + if to_download: pack.path = localpath = plug.get_package(pack) - pack.load_checksum_from_header() + pack.load_checksum_from_header() + if to_download: self.upload_package(pack) - if pack in to_link: - self.associate_package(pack) - except KeyboardInterrupt: - raise - except Exception, e: - self.error_msg(e) - if self.fail: - raise - continue - finally: - if is_non_local_repo: - os.remove(localpath) - - if len(to_link) != 0: - self.regen=True - for (index, pack) in enumerate(to_link): - """Link each package that wasn't already linked in the previous step""" - try: - if pack not in to_download: - pack.load_checksum_from_header() + finally_remove(localpath) + if to_link: self.associate_package(pack) except KeyboardInterrupt: + finally_remove(localpath) raise except Exception, e: self.error_msg(e) + finally_remove(localpath) if self.fail: raise continue @@ -305,7 +298,7 @@ class ContentPackage: return self.name + '-' + self.version + '-' + self.release + '.' + self.arch
def load_checksum_from_header(self): - if self.path is None: + if self.path is None: raise rhnFault(50, "Unable to load package", explain=0) self.file = open(self.path, 'rb') self.header, self.payload_stream, self.header_start, self.header_end = \
commit d09d119267834048004b2aecc98ec1adba91edf4 Author: Michael Mraka michael.mraka@redhat.com Date: Wed Feb 2 21:59:58 2011 +0100
moved checksum logic into ContentPackage
yum repo metadata have precomputed checksum we can use to compare the package on disk; unfortunately it's a different from the one we want to store so we have to compute the second one on our own
diff --git a/backend/satellite_tools/repo_plugins/yum_src.py b/backend/satellite_tools/repo_plugins/yum_src.py index 982ff48..649388e 100644 --- a/backend/satellite_tools/repo_plugins/yum_src.py +++ b/backend/satellite_tools/repo_plugins/yum_src.py @@ -79,8 +79,10 @@ class ContentSource: new_pack.setNVREA(pack.name, pack.version, pack.release, pack.epoch, pack.arch) new_pack.unique_id = pack - for cs in pack.checksums: - new_pack.checksums[cs[0]] = cs[1] + new_pack.checksum_type = pack.checksums[0][0] + if new_pack.checksum_type == 'sha': + new_pack.checksum_type = 'sha1' + new_pack.checksum = pack.checksums[0][1] to_return.append(new_pack) return to_return
diff --git a/backend/satellite_tools/reposync.py b/backend/satellite_tools/reposync.py index 81ccc56..f2efd0f 100644 --- a/backend/satellite_tools/reposync.py +++ b/backend/satellite_tools/reposync.py @@ -16,7 +16,7 @@ import sys, os, time import hashlib from optparse import OptionParser from spacewalk.server import rhnPackage, rhnSQL, rhnChannel, rhnPackageUpload -from spacewalk.common import CFG, initCFG, rhnLog +from spacewalk.common import CFG, initCFG, rhnLog, rhnFault from spacewalk.common.checksum import getFileChecksum from spacewalk.server.importlib.importLib import IncompletePackage from spacewalk.server.importlib.packageImport import ChannelPackageSubscription @@ -145,9 +145,14 @@ class RepoSync: [pack.name, pack.version, pack.release, pack.epoch, pack.arch], self.channel_label)
- if not (path and - self.match_package_checksum(path, pack.checksum_type, pack.pkgId)): + if path: + pack.path = os.path.join(CFG.MOUNT_POINT, path) + if not self.match_package_checksum(pack.path, + pack.checksum_type, pack.checksum)): + to_download.append(pack) + else: to_download.append(pack) + if package_channel != self.channel_label: to_link.append(pack)
@@ -163,8 +168,9 @@ class RepoSync: try: self.print_msg(str(index+1) + "/" + str(len(to_download)) + " : "+ \ pack.getNVREA()) - path = plug.get_package(pack) - self.upload_package(pack, path) + pack.path = localpath = plug.get_package(pack) + pack.load_checksum_from_header() + self.upload_package(pack) if pack in to_link: self.associate_package(pack) except KeyboardInterrupt: @@ -176,7 +182,7 @@ class RepoSync: continue finally: if is_non_local_repo: - os.remove(path) + os.remove(localpath)
if len(to_link) != 0: self.regen=True @@ -184,6 +190,7 @@ class RepoSync: """Link each package that wasn't already linked in the previous step""" try: if pack not in to_download: + pack.load_checksum_from_header() self.associate_package(pack) except KeyboardInterrupt: raise @@ -193,30 +200,23 @@ class RepoSync: raise continue
- def match_package_checksum(self, path, checksum_type, checksum): - abspath = os.path.join(CFG.MOUNT_POINT, path) + def match_package_checksum(self, abspath, checksum_type, checksum): if (os.path.exists(abspath) and getFileChecksum(checksum_type, filename=abspath) == checksum): return 1 return 0
- def upload_package(self, package, path): - temp_file = open(path, 'rb') - header, payload_stream, header_start, header_end = \ - rhnPackageUpload.load_package(temp_file) - package.checksum_type = header.checksum_type() - package.checksum = getFileChecksum(package.checksum_type, file=temp_file) - + def upload_package(self, package): rel_package_path = rhnPackageUpload.relative_path_from_header( - header, self.channel['org_id'], + package.header, self.channel['org_id'], package.checksum_type, package.checksum) - package_dict, diff_level = rhnPackageUpload.push_package(header, - payload_stream, package.checksum_type, package.checksum, + package_dict, diff_level = rhnPackageUpload.push_package(package.header, + package.payload_stream, package.checksum_type, package.checksum, force=False, - header_start=header_start, header_end=header_end, + header_start=package.header_start, header_end=package.header_end, relative_path=rel_package_path, org_id=self.channel['org_id']) - temp_file.close() + package.file.close()
def associate_package(self, pack): caller = "server.app.yumreposync" @@ -272,7 +272,6 @@ class ContentPackage:
def __init__(self): # map of checksums - self.checksums = {} self.checksum_type = None self.checksum = None
@@ -285,6 +284,13 @@ class ContentPackage: self.epoch = None self.arch = None
+ self.path = None + self.file = None + self.header = None + self.payload_stream = None + self.header_start = None + self.header_end = None + def setNVREA(self, name, version, release, epoch, arch): self.name = name self.version = version @@ -298,3 +304,11 @@ class ContentPackage: else: return self.name + '-' + self.version + '-' + self.release + '.' + self.arch
+ def load_checksum_from_header(self): + if self.path is None: + raise rhnFault(50, "Unable to load package", explain=0) + self.file = open(self.path, 'rb') + self.header, self.payload_stream, self.header_start, self.header_end = \ + rhnPackageUpload.load_package(self.file) + self.checksum_type = self.header.checksum_type() + self.checksum = getFileChecksum(self.checksum_type, file=self.file)
commit f404a3f4958754d473a5114faa34c1397e11515f Author: Michael Mraka michael.mraka@redhat.com Date: Wed Feb 2 19:22:12 2011 +0100
yum repo metadata says epoch="0" even if it's NULL
diff --git a/backend/server/rhnPackage.py b/backend/server/rhnPackage.py index 8444b4f..3a2b1ed 100644 --- a/backend/server/rhnPackage.py +++ b/backend/server/rhnPackage.py @@ -283,13 +283,14 @@ def get_path_for_package(pkg, channel_label): params = {'name': pkg[0], 'ver': pkg[1], 'rel': pkg[2], + 'epoch': pkg[3], 'arch': pkg[4], 'label': channel_label} - if pkg[3] == "": - epochStatement = "is null" + # yum repo has epoch="0" not only when epoch is "0" but also if it's NULL + if pkg[3] == '0' or pkg[3] == '': + epochStatement = "(epoch is null or epoch = :epoch)" else: - epochStatement = "= :epoch" - params.update({'epoch': pkg[3]}) + epochStatement = "epoch = :epoch" statement = """ select p.path, c.label as channel_label from rhnPackage p @@ -308,7 +309,7 @@ def get_path_for_package(pkg, channel_label): where pn.name = :name and pe.version = :ver and pe.release = :rel - and pe.epoch %s + and %s and pa.label = :arch order by c.label nulls last """ % epochStatement
commit 2eb0373b3a9f4fab31922c1966a2719b046344b4 Author: Michael Mraka michael.mraka@redhat.com Date: Wed Feb 2 16:40:07 2011 +0100
reformated sql query
diff --git a/backend/server/rhnPackage.py b/backend/server/rhnPackage.py index 09d38c8..8444b4f 100644 --- a/backend/server/rhnPackage.py +++ b/backend/server/rhnPackage.py @@ -291,29 +291,26 @@ def get_path_for_package(pkg, channel_label): epochStatement = "= :epoch" params.update({'epoch': pkg[3]}) statement = """ - select - p.path, c.label as channel_label - from - rhnPackage p - left join rhnChannelPackage cp - on p.id = cp.package_id - left join rhnChannel c - on cp.channel_id = c.id - and p.org_id = c.org_id - and c.label = :label, - rhnPackageName pn, - rhnPackageEVR pe, - rhnPackageArch pa - where - p.name_id = pn.id - and pn.name = :name - and p.evr_id = pe.id - and pe.version = :ver - and pe.release = :rel - and pe.epoch %s - and p.package_arch_id = pa.id - and pa.label = :arch - order by c.label nulls last + select p.path, c.label as channel_label + from rhnPackage p + join rhnPackageName pn + on p.name_id = pn.id + join rhnPackageEVR pe + on p.evr_id = pe.id + join rhnPackageArch pa + on p.package_arch_id = pa.id + left join rhnChannelPackage cp + on p.id = cp.package_id + left join rhnChannel c + on cp.channel_id = c.id + and p.org_id = c.org_id + and c.label = :label + where pn.name = :name + and pe.version = :ver + and pe.release = :rel + and pe.epoch %s + and pa.label = :arch + order by c.label nulls last """ % epochStatement h = rhnSQL.prepare(statement) h.execute(**params)
commit ed04aba148f36dd1dfed83e9ade72dfa22e5b216 Author: Michael Mraka michael.mraka@redhat.com Date: Wed Feb 2 16:33:52 2011 +0100
fixed indentation
diff --git a/backend/satellite_tools/reposync.py b/backend/satellite_tools/reposync.py index 4333cdc..81ccc56 100644 --- a/backend/satellite_tools/reposync.py +++ b/backend/satellite_tools/reposync.py @@ -137,19 +137,19 @@ class RepoSync: to_download = [] self.print_msg("Repo " + url + " has " + str(len(packages)) + " packages.") for pack in packages: - # we have a few scenarios here: - # 1. package is not on the server (link and download) - # 2. package is in the server, but not in the channel (just link if we can) - # 3. package is in the server and channel, but not on the file system (just download) - path, package_channel = rhnPackage.get_path_for_package( - [pack.name, pack.version, pack.release, pack.epoch, pack.arch], - self.channel_label) - - if not (path and - self.match_package_checksum(path, pack.checksum_type, pack.pkgId)): - to_download.append(pack) - if package_channel != self.channel_label: - to_link.append(pack) + # we have a few scenarios here: + # 1. package is not on the server (link and download) + # 2. package is in the server, but not in the channel (just link if we can) + # 3. package is in the server and channel, but not on the file system (just download) + path, package_channel = rhnPackage.get_path_for_package( + [pack.name, pack.version, pack.release, pack.epoch, pack.arch], + self.channel_label) + + if not (path and + self.match_package_checksum(path, pack.checksum_type, pack.pkgId)): + to_download.append(pack) + if package_channel != self.channel_label: + to_link.append(pack)
if len(to_download) == 0: self.print_msg("No new packages to download.")
commit 16dbafff04c457cef2e0327cb8ee0adbfe816df8 Author: Michael Mraka michael.mraka@redhat.com Date: Wed Feb 2 16:30:58 2011 +0100
spacewalk-repo-sync should not download package which is already on disk
diff --git a/backend/satellite_tools/reposync.py b/backend/satellite_tools/reposync.py index f8182b3..4333cdc 100644 --- a/backend/satellite_tools/reposync.py +++ b/backend/satellite_tools/reposync.py @@ -141,38 +141,15 @@ class RepoSync: # 1. package is not on the server (link and download) # 2. package is in the server, but not in the channel (just link if we can) # 3. package is in the server and channel, but not on the file system (just download) - path = rhnPackage.get_path_for_package([pack.name, pack.version, pack.release,\ - pack.epoch, pack.arch], self.channel_label) - if not path: - path = rhnPackage.get_path_for_package([pack.name, pack.version, pack.release,\ - '', pack.arch], self.channel_label) - - if path: - if os.path.exists(os.path.join(CFG.MOUNT_POINT, path)): - continue - else: - to_download.append(pack) - continue - - # we know that it's not in the channel, lets try to check the server by checksum! - #for some repos (sha256), we can check to see if we have them by - # checksum and not bother downloading. For older repos, we only - # have sha1, which satellite doesn't track - # regardless we have to link the package - to_link.append(pack) - - found = False - for type,sum in pack.checksums.items(): - if type == 'sha': #we use sha1 (instead of sha) - type = 'sha1' - path = rhnPackage.get_path_for_checksum(self.channel['org_id'],\ - type, sum) - if path and os.path.exists(os.path.join(CFG.MOUNT_POINT, path)): - found = True - break - if not found: - to_download.append(pack) + path, package_channel = rhnPackage.get_path_for_package( + [pack.name, pack.version, pack.release, pack.epoch, pack.arch], + self.channel_label)
+ if not (path and + self.match_package_checksum(path, pack.checksum_type, pack.pkgId)): + to_download.append(pack) + if package_channel != self.channel_label: + to_link.append(pack)
if len(to_download) == 0: self.print_msg("No new packages to download.") @@ -201,6 +178,8 @@ class RepoSync: if is_non_local_repo: os.remove(path)
+ if len(to_link) != 0: + self.regen=True for (index, pack) in enumerate(to_link): """Link each package that wasn't already linked in the previous step""" try: @@ -214,6 +193,12 @@ class RepoSync: raise continue
+ def match_package_checksum(self, path, checksum_type, checksum): + abspath = os.path.join(CFG.MOUNT_POINT, path) + if (os.path.exists(abspath) and + getFileChecksum(checksum_type, filename=abspath) == checksum): + return 1 + return 0
def upload_package(self, package, path): temp_file = open(path, 'rb') diff --git a/backend/server/rhnPackage.py b/backend/server/rhnPackage.py index f94f9e1..09d38c8 100644 --- a/backend/server/rhnPackage.py +++ b/backend/server/rhnPackage.py @@ -277,33 +277,13 @@ def get_source_package_path_by_name(server_id, packageName): rhnFlags.set("Download-Accelerator-Path", rs['path']) return filePath
-def get_path_for_checksum(org_id, checksum_type, checksum): - statement = """ - select - p.path - from - rhnPackage p, - rhnChecksumView c - where p.org_id = :org_id - and p.checksum_id = c.id - and c.checksum = :checksum - and c.checksum_type = :checksumtype - """ - h = rhnSQL.prepare(statement) - h.execute(org_id=org_id, checksum=checksum, checksumtype=checksum_type) - ret = h.fetchone_dict() - if not ret: - return None - return ret['path'] - - def get_path_for_package(pkg, channel_label): log_debug(3, pkg) pkg = map(str, pkg) params = {'name': pkg[0], 'ver': pkg[1], 'rel': pkg[2], - 'arch': pkg[4] + 'arch': pkg[4], 'label': channel_label} if pkg[3] == "": epochStatement = "is null" @@ -312,14 +292,18 @@ def get_path_for_package(pkg, channel_label): params.update({'epoch': pkg[3]}) statement = """ select - P.path + p.path, c.label as channel_label from - rhnChannel c, - rhnPackage p, + rhnPackage p + left join rhnChannelPackage cp + on p.id = cp.package_id + left join rhnChannel c + on cp.channel_id = c.id + and p.org_id = c.org_id + and c.label = :label, rhnPackageName pn, rhnPackageEVR pe, - rhnPackageArch pa, - rhnChannelPackage cp + rhnPackageArch pa where p.name_id = pn.id and pn.name = :name @@ -329,17 +313,15 @@ def get_path_for_package(pkg, channel_label): and pe.epoch %s and p.package_arch_id = pa.id and pa.label = :arch - and p.id = cp.package_id - and cp.channel_id = c.id - and C.label = :label + order by c.label nulls last """ % epochStatement h = rhnSQL.prepare(statement) h.execute(**params)
ret = h.fetchone_dict() if not ret: - return None - return ret['path'] + return None, None + return ret['path'], ret['channel_label']
def _none2emptyString(foo):
commit b177778a548ef9ad2b9090b994fa49b3a1967149 Author: Michael Mraka michael.mraka@redhat.com Date: Wed Feb 2 14:28:19 2011 +0100
fixed duplicated code
diff --git a/backend/server/rhnPackage.py b/backend/server/rhnPackage.py index 563510f..f94f9e1 100644 --- a/backend/server/rhnPackage.py +++ b/backend/server/rhnPackage.py @@ -300,10 +300,16 @@ def get_path_for_checksum(org_id, checksum_type, checksum): def get_path_for_package(pkg, channel_label): log_debug(3, pkg) pkg = map(str, pkg) + params = {'name': pkg[0], + 'ver': pkg[1], + 'rel': pkg[2], + 'arch': pkg[4] + 'label': channel_label} if pkg[3] == "": epochStatement = "is null" else: epochStatement = "= :epoch" + params.update({'epoch': pkg[3]}) statement = """ select P.path @@ -328,10 +334,7 @@ def get_path_for_package(pkg, channel_label): and C.label = :label """ % epochStatement h = rhnSQL.prepare(statement) - if pkg[3] == '': - h.execute(name = pkg[0], ver = pkg[1], rel = pkg[2], arch = pkg[4], label = channel_label) - else: - h.execute(name = pkg[0], ver = pkg[1], rel = pkg[2], arch = pkg[4], epoch=pkg[3], label = channel_label) + h.execute(**params)
ret = h.fetchone_dict() if not ret:
spacewalk-commits@lists.fedorahosted.org