backend/satellite_tools/repo_plugins/yum_src.py | 6
backend/satellite_tools/reposync.py | 156 +++++++++++-------------
backend/server/rhnPackage.py | 83 +++++-------
3 files changed, 111 insertions(+), 134 deletions(-)
New commits:
commit ab07eddd4caf3de6a04c7e35ae9217eda101bffa
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Wed Feb 2 22:30:35 2011 +0100
shortened and narrowed package sync logic
process packages in one loop
diff --git a/backend/satellite_tools/reposync.py b/backend/satellite_tools/reposync.py
index f2efd0f..f1ba44d 100644
--- a/backend/satellite_tools/reposync.py
+++ b/backend/satellite_tools/reposync.py
@@ -133,69 +133,62 @@ class RepoSync:
def import_packages(self, plug, url):
packages = plug.list_packages()
- to_link = []
- to_download = []
+ to_process = []
self.print_msg("Repo " + url + " has " + str(len(packages)) +
" packages.")
for pack in packages:
- # we have a few scenarios here:
- # 1. package is not on the server (link and download)
- # 2. package is in the server, but not in the channel (just link if we can)
- # 3. package is in the server and channel, but not on the file system (just
download)
path, package_channel = rhnPackage.get_path_for_package(
[pack.name, pack.version, pack.release, pack.epoch, pack.arch],
self.channel_label)
- if path:
+ to_download = False
+ to_link = False
+ if not path:
+ # package is not on disk
+ to_download = True
+ else:
+ # a different package is on disk
pack.path = os.path.join(CFG.MOUNT_POINT, path)
if not self.match_package_checksum(pack.path,
- pack.checksum_type, pack.checksum)):
- to_download.append(pack)
- else:
- to_download.append(pack)
+ pack.checksum_type, pack.checksum):
+ to_download = True
if package_channel != self.channel_label:
- to_link.append(pack)
+ # package is not in the channel
+ to_link = True
- if len(to_download) == 0:
- self.print_msg("No new packages to download.")
- else:
- self.regen=True
+ if to_download or to_link:
+ to_process.append((pack, to_download, to_link))
+
+ num_to_process = len(to_process)
+ if num_to_process == 0:
+ self.print_msg("No new packages to sync.")
+ return
+
+ self.regen=True
is_non_local_repo = (url.find("file://") < 0)
- for (index, pack) in enumerate(to_download):
- """download each package"""
- # try/except/finally doesn't work in python 2.4 (RHEL5), so here's a
hack
+ # try/except/finally doesn't work in python 2.4 (RHEL5), so here's a
hack
+ def finally_remove(path):
+ if is_non_local_repo:
+ os.remove(path)
+
+ for (index, what) in enumerate(to_process):
+ pack, to_download, to_link = what
try:
- try:
- self.print_msg(str(index+1) + "/" + str(len(to_download)) +
" : "+ \
- pack.getNVREA())
+ self.print_msg("%d/%d : %s" % (index+1, num_to_process,
pack.getNVREA()))
+ if to_download:
pack.path = localpath = plug.get_package(pack)
- pack.load_checksum_from_header()
+ pack.load_checksum_from_header()
+ if to_download:
self.upload_package(pack)
- if pack in to_link:
- self.associate_package(pack)
- except KeyboardInterrupt:
- raise
- except Exception, e:
- self.error_msg(e)
- if self.fail:
- raise
- continue
- finally:
- if is_non_local_repo:
- os.remove(localpath)
-
- if len(to_link) != 0:
- self.regen=True
- for (index, pack) in enumerate(to_link):
- """Link each package that wasn't already linked in the
previous step"""
- try:
- if pack not in to_download:
- pack.load_checksum_from_header()
+ finally_remove(localpath)
+ if to_link:
self.associate_package(pack)
except KeyboardInterrupt:
+ finally_remove(localpath)
raise
except Exception, e:
self.error_msg(e)
+ finally_remove(localpath)
if self.fail:
raise
continue
@@ -305,7 +298,7 @@ class ContentPackage:
return self.name + '-' + self.version + '-' + self.release +
'.' + self.arch
def load_checksum_from_header(self):
- if self.path is None:
+ if self.path is None:
raise rhnFault(50, "Unable to load package", explain=0)
self.file = open(self.path, 'rb')
self.header, self.payload_stream, self.header_start, self.header_end = \
commit d09d119267834048004b2aecc98ec1adba91edf4
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Wed Feb 2 21:59:58 2011 +0100
moved checksum logic into ContentPackage
yum repo metadata have precomputed checksum we can use to compare the
package on disk; unfortunately it's a different from the one we want to
store so we have to compute the second one on our own
diff --git a/backend/satellite_tools/repo_plugins/yum_src.py
b/backend/satellite_tools/repo_plugins/yum_src.py
index 982ff48..649388e 100644
--- a/backend/satellite_tools/repo_plugins/yum_src.py
+++ b/backend/satellite_tools/repo_plugins/yum_src.py
@@ -79,8 +79,10 @@ class ContentSource:
new_pack.setNVREA(pack.name, pack.version, pack.release,
pack.epoch, pack.arch)
new_pack.unique_id = pack
- for cs in pack.checksums:
- new_pack.checksums[cs[0]] = cs[1]
+ new_pack.checksum_type = pack.checksums[0][0]
+ if new_pack.checksum_type == 'sha':
+ new_pack.checksum_type = 'sha1'
+ new_pack.checksum = pack.checksums[0][1]
to_return.append(new_pack)
return to_return
diff --git a/backend/satellite_tools/reposync.py b/backend/satellite_tools/reposync.py
index 81ccc56..f2efd0f 100644
--- a/backend/satellite_tools/reposync.py
+++ b/backend/satellite_tools/reposync.py
@@ -16,7 +16,7 @@ import sys, os, time
import hashlib
from optparse import OptionParser
from spacewalk.server import rhnPackage, rhnSQL, rhnChannel, rhnPackageUpload
-from spacewalk.common import CFG, initCFG, rhnLog
+from spacewalk.common import CFG, initCFG, rhnLog, rhnFault
from spacewalk.common.checksum import getFileChecksum
from spacewalk.server.importlib.importLib import IncompletePackage
from spacewalk.server.importlib.packageImport import ChannelPackageSubscription
@@ -145,9 +145,14 @@ class RepoSync:
[pack.name, pack.version, pack.release, pack.epoch, pack.arch],
self.channel_label)
- if not (path and
- self.match_package_checksum(path, pack.checksum_type, pack.pkgId)):
+ if path:
+ pack.path = os.path.join(CFG.MOUNT_POINT, path)
+ if not self.match_package_checksum(pack.path,
+ pack.checksum_type, pack.checksum)):
+ to_download.append(pack)
+ else:
to_download.append(pack)
+
if package_channel != self.channel_label:
to_link.append(pack)
@@ -163,8 +168,9 @@ class RepoSync:
try:
self.print_msg(str(index+1) + "/" + str(len(to_download)) +
" : "+ \
pack.getNVREA())
- path = plug.get_package(pack)
- self.upload_package(pack, path)
+ pack.path = localpath = plug.get_package(pack)
+ pack.load_checksum_from_header()
+ self.upload_package(pack)
if pack in to_link:
self.associate_package(pack)
except KeyboardInterrupt:
@@ -176,7 +182,7 @@ class RepoSync:
continue
finally:
if is_non_local_repo:
- os.remove(path)
+ os.remove(localpath)
if len(to_link) != 0:
self.regen=True
@@ -184,6 +190,7 @@ class RepoSync:
"""Link each package that wasn't already linked in the
previous step"""
try:
if pack not in to_download:
+ pack.load_checksum_from_header()
self.associate_package(pack)
except KeyboardInterrupt:
raise
@@ -193,30 +200,23 @@ class RepoSync:
raise
continue
- def match_package_checksum(self, path, checksum_type, checksum):
- abspath = os.path.join(CFG.MOUNT_POINT, path)
+ def match_package_checksum(self, abspath, checksum_type, checksum):
if (os.path.exists(abspath) and
getFileChecksum(checksum_type, filename=abspath) == checksum):
return 1
return 0
- def upload_package(self, package, path):
- temp_file = open(path, 'rb')
- header, payload_stream, header_start, header_end = \
- rhnPackageUpload.load_package(temp_file)
- package.checksum_type = header.checksum_type()
- package.checksum = getFileChecksum(package.checksum_type, file=temp_file)
-
+ def upload_package(self, package):
rel_package_path = rhnPackageUpload.relative_path_from_header(
- header, self.channel['org_id'],
+ package.header, self.channel['org_id'],
package.checksum_type, package.checksum)
- package_dict, diff_level = rhnPackageUpload.push_package(header,
- payload_stream, package.checksum_type, package.checksum,
+ package_dict, diff_level = rhnPackageUpload.push_package(package.header,
+ package.payload_stream, package.checksum_type, package.checksum,
force=False,
- header_start=header_start, header_end=header_end,
+ header_start=package.header_start, header_end=package.header_end,
relative_path=rel_package_path,
org_id=self.channel['org_id'])
- temp_file.close()
+ package.file.close()
def associate_package(self, pack):
caller = "server.app.yumreposync"
@@ -272,7 +272,6 @@ class ContentPackage:
def __init__(self):
# map of checksums
- self.checksums = {}
self.checksum_type = None
self.checksum = None
@@ -285,6 +284,13 @@ class ContentPackage:
self.epoch = None
self.arch = None
+ self.path = None
+ self.file = None
+ self.header = None
+ self.payload_stream = None
+ self.header_start = None
+ self.header_end = None
+
def setNVREA(self, name, version, release, epoch, arch):
self.name = name
self.version = version
@@ -298,3 +304,11 @@ class ContentPackage:
else:
return self.name + '-' + self.version + '-' + self.release +
'.' + self.arch
+ def load_checksum_from_header(self):
+ if self.path is None:
+ raise rhnFault(50, "Unable to load package", explain=0)
+ self.file = open(self.path, 'rb')
+ self.header, self.payload_stream, self.header_start, self.header_end = \
+ rhnPackageUpload.load_package(self.file)
+ self.checksum_type = self.header.checksum_type()
+ self.checksum = getFileChecksum(self.checksum_type, file=self.file)
commit f404a3f4958754d473a5114faa34c1397e11515f
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Wed Feb 2 19:22:12 2011 +0100
yum repo metadata says epoch="0" even if it's NULL
diff --git a/backend/server/rhnPackage.py b/backend/server/rhnPackage.py
index 8444b4f..3a2b1ed 100644
--- a/backend/server/rhnPackage.py
+++ b/backend/server/rhnPackage.py
@@ -283,13 +283,14 @@ def get_path_for_package(pkg, channel_label):
params = {'name': pkg[0],
'ver': pkg[1],
'rel': pkg[2],
+ 'epoch': pkg[3],
'arch': pkg[4],
'label': channel_label}
- if pkg[3] == "":
- epochStatement = "is null"
+ # yum repo has epoch="0" not only when epoch is "0" but also if
it's NULL
+ if pkg[3] == '0' or pkg[3] == '':
+ epochStatement = "(epoch is null or epoch = :epoch)"
else:
- epochStatement = "= :epoch"
- params.update({'epoch': pkg[3]})
+ epochStatement = "epoch = :epoch"
statement = """
select p.path, c.label as channel_label
from rhnPackage p
@@ -308,7 +309,7 @@ def get_path_for_package(pkg, channel_label):
where pn.name = :name
and pe.version = :ver
and pe.release = :rel
- and pe.epoch %s
+ and %s
and pa.label = :arch
order by c.label nulls last
""" % epochStatement
commit 2eb0373b3a9f4fab31922c1966a2719b046344b4
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Wed Feb 2 16:40:07 2011 +0100
reformated sql query
diff --git a/backend/server/rhnPackage.py b/backend/server/rhnPackage.py
index 09d38c8..8444b4f 100644
--- a/backend/server/rhnPackage.py
+++ b/backend/server/rhnPackage.py
@@ -291,29 +291,26 @@ def get_path_for_package(pkg, channel_label):
epochStatement = "= :epoch"
params.update({'epoch': pkg[3]})
statement = """
- select
- p.path, c.label as channel_label
- from
- rhnPackage p
- left join rhnChannelPackage cp
- on p.id = cp.package_id
- left join rhnChannel c
- on cp.channel_id = c.id
- and p.org_id = c.org_id
- and c.label = :label,
- rhnPackageName pn,
- rhnPackageEVR pe,
- rhnPackageArch pa
- where
- p.name_id = pn.id
- and pn.name = :name
- and p.evr_id = pe.id
- and pe.version = :ver
- and pe.release = :rel
- and pe.epoch %s
- and p.package_arch_id = pa.id
- and pa.label = :arch
- order by c.label nulls last
+ select p.path, c.label as channel_label
+ from rhnPackage p
+ join rhnPackageName pn
+ on p.name_id = pn.id
+ join rhnPackageEVR pe
+ on p.evr_id = pe.id
+ join rhnPackageArch pa
+ on p.package_arch_id = pa.id
+ left join rhnChannelPackage cp
+ on p.id = cp.package_id
+ left join rhnChannel c
+ on cp.channel_id = c.id
+ and p.org_id = c.org_id
+ and c.label = :label
+ where pn.name = :name
+ and pe.version = :ver
+ and pe.release = :rel
+ and pe.epoch %s
+ and pa.label = :arch
+ order by c.label nulls last
""" % epochStatement
h = rhnSQL.prepare(statement)
h.execute(**params)
commit ed04aba148f36dd1dfed83e9ade72dfa22e5b216
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Wed Feb 2 16:33:52 2011 +0100
fixed indentation
diff --git a/backend/satellite_tools/reposync.py b/backend/satellite_tools/reposync.py
index 4333cdc..81ccc56 100644
--- a/backend/satellite_tools/reposync.py
+++ b/backend/satellite_tools/reposync.py
@@ -137,19 +137,19 @@ class RepoSync:
to_download = []
self.print_msg("Repo " + url + " has " + str(len(packages)) +
" packages.")
for pack in packages:
- # we have a few scenarios here:
- # 1. package is not on the server (link and download)
- # 2. package is in the server, but not in the channel (just link if we
can)
- # 3. package is in the server and channel, but not on the file system
(just download)
- path, package_channel = rhnPackage.get_path_for_package(
- [pack.name, pack.version, pack.release, pack.epoch, pack.arch],
- self.channel_label)
-
- if not (path and
- self.match_package_checksum(path, pack.checksum_type, pack.pkgId)):
- to_download.append(pack)
- if package_channel != self.channel_label:
- to_link.append(pack)
+ # we have a few scenarios here:
+ # 1. package is not on the server (link and download)
+ # 2. package is in the server, but not in the channel (just link if we can)
+ # 3. package is in the server and channel, but not on the file system (just
download)
+ path, package_channel = rhnPackage.get_path_for_package(
+ [pack.name, pack.version, pack.release, pack.epoch, pack.arch],
+ self.channel_label)
+
+ if not (path and
+ self.match_package_checksum(path, pack.checksum_type, pack.pkgId)):
+ to_download.append(pack)
+ if package_channel != self.channel_label:
+ to_link.append(pack)
if len(to_download) == 0:
self.print_msg("No new packages to download.")
commit 16dbafff04c457cef2e0327cb8ee0adbfe816df8
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Wed Feb 2 16:30:58 2011 +0100
spacewalk-repo-sync should not download package which is already on disk
diff --git a/backend/satellite_tools/reposync.py b/backend/satellite_tools/reposync.py
index f8182b3..4333cdc 100644
--- a/backend/satellite_tools/reposync.py
+++ b/backend/satellite_tools/reposync.py
@@ -141,38 +141,15 @@ class RepoSync:
# 1. package is not on the server (link and download)
# 2. package is in the server, but not in the channel (just link if we
can)
# 3. package is in the server and channel, but not on the file system
(just download)
- path = rhnPackage.get_path_for_package([pack.name, pack.version,
pack.release,\
- pack.epoch, pack.arch], self.channel_label)
- if not path:
- path = rhnPackage.get_path_for_package([pack.name, pack.version,
pack.release,\
- '', pack.arch], self.channel_label)
-
- if path:
- if os.path.exists(os.path.join(CFG.MOUNT_POINT, path)):
- continue
- else:
- to_download.append(pack)
- continue
-
- # we know that it's not in the channel, lets try to check the server
by checksum!
- #for some repos (sha256), we can check to see if we have them by
- # checksum and not bother downloading. For older repos, we only
- # have sha1, which satellite doesn't track
- # regardless we have to link the package
- to_link.append(pack)
-
- found = False
- for type,sum in pack.checksums.items():
- if type == 'sha': #we use sha1 (instead of sha)
- type = 'sha1'
- path =
rhnPackage.get_path_for_checksum(self.channel['org_id'],\
- type, sum)
- if path and os.path.exists(os.path.join(CFG.MOUNT_POINT, path)):
- found = True
- break
- if not found:
- to_download.append(pack)
+ path, package_channel = rhnPackage.get_path_for_package(
+ [pack.name, pack.version, pack.release, pack.epoch, pack.arch],
+ self.channel_label)
+ if not (path and
+ self.match_package_checksum(path, pack.checksum_type, pack.pkgId)):
+ to_download.append(pack)
+ if package_channel != self.channel_label:
+ to_link.append(pack)
if len(to_download) == 0:
self.print_msg("No new packages to download.")
@@ -201,6 +178,8 @@ class RepoSync:
if is_non_local_repo:
os.remove(path)
+ if len(to_link) != 0:
+ self.regen=True
for (index, pack) in enumerate(to_link):
"""Link each package that wasn't already linked in the
previous step"""
try:
@@ -214,6 +193,12 @@ class RepoSync:
raise
continue
+ def match_package_checksum(self, path, checksum_type, checksum):
+ abspath = os.path.join(CFG.MOUNT_POINT, path)
+ if (os.path.exists(abspath) and
+ getFileChecksum(checksum_type, filename=abspath) == checksum):
+ return 1
+ return 0
def upload_package(self, package, path):
temp_file = open(path, 'rb')
diff --git a/backend/server/rhnPackage.py b/backend/server/rhnPackage.py
index f94f9e1..09d38c8 100644
--- a/backend/server/rhnPackage.py
+++ b/backend/server/rhnPackage.py
@@ -277,33 +277,13 @@ def get_source_package_path_by_name(server_id, packageName):
rhnFlags.set("Download-Accelerator-Path", rs['path'])
return filePath
-def get_path_for_checksum(org_id, checksum_type, checksum):
- statement = """
- select
- p.path
- from
- rhnPackage p,
- rhnChecksumView c
- where p.org_id = :org_id
- and p.checksum_id = c.id
- and c.checksum = :checksum
- and c.checksum_type = :checksumtype
- """
- h = rhnSQL.prepare(statement)
- h.execute(org_id=org_id, checksum=checksum, checksumtype=checksum_type)
- ret = h.fetchone_dict()
- if not ret:
- return None
- return ret['path']
-
-
def get_path_for_package(pkg, channel_label):
log_debug(3, pkg)
pkg = map(str, pkg)
params = {'name': pkg[0],
'ver': pkg[1],
'rel': pkg[2],
- 'arch': pkg[4]
+ 'arch': pkg[4],
'label': channel_label}
if pkg[3] == "":
epochStatement = "is null"
@@ -312,14 +292,18 @@ def get_path_for_package(pkg, channel_label):
params.update({'epoch': pkg[3]})
statement = """
select
- P.path
+ p.path, c.label as channel_label
from
- rhnChannel c,
- rhnPackage p,
+ rhnPackage p
+ left join rhnChannelPackage cp
+ on p.id = cp.package_id
+ left join rhnChannel c
+ on cp.channel_id = c.id
+ and p.org_id = c.org_id
+ and c.label = :label,
rhnPackageName pn,
rhnPackageEVR pe,
- rhnPackageArch pa,
- rhnChannelPackage cp
+ rhnPackageArch pa
where
p.name_id = pn.id
and pn.name = :name
@@ -329,17 +313,15 @@ def get_path_for_package(pkg, channel_label):
and pe.epoch %s
and p.package_arch_id = pa.id
and pa.label = :arch
- and p.id = cp.package_id
- and cp.channel_id = c.id
- and C.label = :label
+ order by c.label nulls last
""" % epochStatement
h = rhnSQL.prepare(statement)
h.execute(**params)
ret = h.fetchone_dict()
if not ret:
- return None
- return ret['path']
+ return None, None
+ return ret['path'], ret['channel_label']
def _none2emptyString(foo):
commit b177778a548ef9ad2b9090b994fa49b3a1967149
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Wed Feb 2 14:28:19 2011 +0100
fixed duplicated code
diff --git a/backend/server/rhnPackage.py b/backend/server/rhnPackage.py
index 563510f..f94f9e1 100644
--- a/backend/server/rhnPackage.py
+++ b/backend/server/rhnPackage.py
@@ -300,10 +300,16 @@ def get_path_for_checksum(org_id, checksum_type, checksum):
def get_path_for_package(pkg, channel_label):
log_debug(3, pkg)
pkg = map(str, pkg)
+ params = {'name': pkg[0],
+ 'ver': pkg[1],
+ 'rel': pkg[2],
+ 'arch': pkg[4]
+ 'label': channel_label}
if pkg[3] == "":
epochStatement = "is null"
else:
epochStatement = "= :epoch"
+ params.update({'epoch': pkg[3]})
statement = """
select
P.path
@@ -328,10 +334,7 @@ def get_path_for_package(pkg, channel_label):
and C.label = :label
""" % epochStatement
h = rhnSQL.prepare(statement)
- if pkg[3] == '':
- h.execute(name = pkg[0], ver = pkg[1], rel = pkg[2], arch = pkg[4], label =
channel_label)
- else:
- h.execute(name = pkg[0], ver = pkg[1], rel = pkg[2], arch = pkg[4], epoch=pkg[3],
label = channel_label)
+ h.execute(**params)
ret = h.fetchone_dict()
if not ret: