backend/common/fileutils.py | 31 +++++++ backend/common/rhn_mpm.py | 4 backend/satellite_tools/repo_plugins/yum_src.py | 2 backend/satellite_tools/reposync.py | 2 backend/server/handlers/app/packages.py | 51 ------------ backend/server/importlib/importLib.py | 34 +++----- backend/server/rhnPackageUpload.py | 10 +- backend/upload_server/handlers/package_push/package_push.py | 9 +- 8 files changed, 62 insertions(+), 81 deletions(-)
New commits: commit 580b4e6015ddf7a3402ea6e8b7f97197d12123e5 Author: Michael Mraka michael.mraka@redhat.com Date: Fri Dec 2 19:46:14 2011 +0100
676369 - unlink old file when forced
diff --git a/backend/server/importlib/importLib.py b/backend/server/importlib/importLib.py index 9bfc38f..3572ed0 100644 --- a/backend/server/importlib/importLib.py +++ b/backend/server/importlib/importLib.py @@ -793,13 +793,16 @@ def move_package(filename, basedir, relpath, checksum_type, checksum, force=None """ packagePath = basedir + "/" + relpath # Is the file there already? - if os.path.isfile(packagePath) and not force: - # Get its checksum - localsum = getFileChecksum(checksum_type, packagePath) - if checksum == localsum: - # Same file, so get outa here - return - raise FileConflictError(os.path.basename(packagePath)) + if os.path.isfile(packagePath): + if force: + os.unlink(packagePath) + else: + # Get its checksum + localsum = getFileChecksum(checksum_type, packagePath) + if checksum == localsum: + # Same file, so get outa here + return + raise FileConflictError(os.path.basename(packagePath))
dir = os.path.dirname(packagePath) # Create the directory where the file will reside
commit c9d894c679bea0e556774d0708cfa593eaebea28 Author: Michael Mraka michael.mraka@redhat.com Date: Fri Dec 2 18:36:53 2011 +0100
676369 - sync packages directly to /var/satellite
diff --git a/backend/satellite_tools/repo_plugins/yum_src.py b/backend/satellite_tools/repo_plugins/yum_src.py index f0949da..67b9189 100644 --- a/backend/satellite_tools/repo_plugins/yum_src.py +++ b/backend/satellite_tools/repo_plugins/yum_src.py @@ -17,6 +17,7 @@ import yum import shutil import sys import gzip +import os.path from yum.update_md import UpdateMetadata, UpdateNoticeException, UpdateNotice from yum.yumRepo import YumRepository try: @@ -99,6 +100,7 @@ class ContentSource: repo.mirrorlist = url repo.baseurl = [url] repo.basecachedir = self.cache_dir + repo._dirSetAttr('pkgdir', os.path.join(CFG.MOUNT_POINT, CFG.PREPENDED_DIR, '1')) if hasattr(repo, 'base_persistdir'): repo.base_persistdir = self.cache_dir if self.proxy_url is not None:
commit 2dbe4deade884d410f93bfcd30947bb77dd18e88 Author: Michael Mraka michael.mraka@redhat.com Date: Fri Dec 2 18:01:00 2011 +0100
676369 - move package to the final location
diff --git a/backend/satellite_tools/reposync.py b/backend/satellite_tools/reposync.py index 2ba1d7d..832876c 100644 --- a/backend/satellite_tools/reposync.py +++ b/backend/satellite_tools/reposync.py @@ -416,7 +416,7 @@ class RepoSync: package.header, self.channel['org_id'], package.checksum_type, package.checksum) package_dict, diff_level = rhnPackageUpload.push_package(package.header, - package.payload_stream, package.checksum_type, package.checksum, + package.path, package.payload_stream, package.checksum_type, package.checksum, force=False, header_start=package.header_start, header_end=package.header_end, relative_path=rel_package_path, diff --git a/backend/server/importlib/importLib.py b/backend/server/importlib/importLib.py index f44f657..9bfc38f 100644 --- a/backend/server/importlib/importLib.py +++ b/backend/server/importlib/importLib.py @@ -784,7 +784,7 @@ def removeNone(list):
# Assorted functions for various things
-def copy_package(fd, basedir, relpath, checksum_type, checksum, force=None): +def move_package(filename, basedir, relpath, checksum_type, checksum, force=None): """ Copies the information from the file descriptor to a file Checks the file's checksum, raising FileConflictErrror if it's different @@ -805,20 +805,9 @@ def copy_package(fd, basedir, relpath, checksum_type, checksum, force=None): # Create the directory where the file will reside if not os.path.exists(dir): createPath(dir) - pkgfd = os.open(packagePath, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) - os.lseek(fd, 0, 0) - while 1: - buffer = os.read(fd, 65536) - if not buffer: - break - n = os.write(pkgfd, buffer) - if n != len(buffer): - # Error writing to the file - raise IOError, "Wrote %s out of %s bytes in file %s" % ( - n, len(buffer), packagePath) - os.close(pkgfd) + os.rename(filename, packagePath) # set the path perms readable by all users - setPermsPath(packagePath, chmod=0644) + os.chmod(packagePath, 0644)
# Returns a list of containing nevra for the given RPM header diff --git a/backend/server/rhnPackageUpload.py b/backend/server/rhnPackageUpload.py index cd7db76..8f3dd43 100644 --- a/backend/server/rhnPackageUpload.py +++ b/backend/server/rhnPackageUpload.py @@ -127,7 +127,7 @@ def relative_path_from_nevra_without_package_name(nevra, org_id, checksum_type, return get_package_path_without_package_name(nevra, org_id, CFG.PREPENDED_DIR, checksum_type, checksum)
-def push_package(header, payload_stream, checksum_type, checksum, org_id=None, force=None, +def push_package(header, temp_path, payload_stream, checksum_type, checksum, org_id=None, force=None, header_start=None, header_end=None, channels=[], relative_path=None): """Uploads an RPM package """ @@ -140,7 +140,7 @@ def push_package(header, payload_stream, checksum_type, checksum, org_id=None, f
# First write the package to the filesystem to final location try: - importLib.copy_package(payload_stream.fileno(), basedir=CFG.MOUNT_POINT, + importLib.move_package(temp_path, basedir=CFG.MOUNT_POINT, relpath=relative_path, checksum_type=checksum_type, checksum=checksum, force=1) except OSError, e: raise rhnFault(50, "Package upload failed: %s" % e), None, sys.exc_info()[2] diff --git a/backend/upload_server/handlers/package_push/package_push.py b/backend/upload_server/handlers/package_push/package_push.py index 640b99e..44d09f8 100644 --- a/backend/upload_server/handlers/package_push/package_push.py +++ b/backend/upload_server/handlers/package_push/package_push.py @@ -135,7 +135,7 @@ class PackagePush(basePackageUpload.BasePackageUpload): raise rhnFault(104, "Mismatching information")
package_dict, diff_level = rhnPackageUpload.push_package(header, - temp_stream, checksum_type, checksum, force=self.force, + temp_stream.name, payload_stream, checksum_type, checksum, force=self.force, header_start=header_start, header_end=header_end, relative_path=self.rel_package_path, org_id=self.org_id) temp_stream.delete=False
commit dc547088d8b9b85be4d0ec57bee54fc81e10cdfd Author: Michael Mraka michael.mraka@redhat.com Date: Fri Dec 2 17:43:07 2011 +0100
676369 - create tempfile under /var/satellite
so we can move it (not copy) to the final location
diff --git a/backend/server/rhnPackageUpload.py b/backend/server/rhnPackageUpload.py index ecd03ac..cd7db76 100644 --- a/backend/server/rhnPackageUpload.py +++ b/backend/server/rhnPackageUpload.py @@ -33,14 +33,16 @@ from spacewalk.server.rhnLib import get_package_path, \ from spacewalk.server.rhnServer import server_packages
-def write_temp_file(req, buffer_size, packaging=None): +def write_temp_file(req, buffer_size, dir, prefix, packaging=None): """ Write request to temporary file (write max. buffer_size at once). Returns the file object. """ suffix = '' if packaging == 'deb': suffix = '.deb' - t = tempfile.NamedTemporaryFile(suffix=suffix) + t = tempfile.NamedTemporaryFile( + dir=dir, prefix=prefix + '.', + suffix=suffix) while 1: buf = req.read(buffer_size) if not buf: diff --git a/backend/upload_server/handlers/package_push/package_push.py b/backend/upload_server/handlers/package_push/package_push.py index aa84a4d..640b99e 100644 --- a/backend/upload_server/handlers/package_push/package_push.py +++ b/backend/upload_server/handlers/package_push/package_push.py @@ -108,7 +108,11 @@ class PackagePush(basePackageUpload.BasePackageUpload): if ret != apache.OK: return ret
- temp_stream = rhnPackageUpload.write_temp_file(req, 16384, self.packaging) + temp_stream = rhnPackageUpload.write_temp_file(req, 16384, + os.path.join(CFG.MOUNT_POINT, CFG.PREPENDED_DIR, str(self.org_id)), + '-'.join((self.package_name, self.package_version, + self.package_release, self.package_arch)), + self.packaging)
header, payload_stream, header_start, header_end = \ rhnPackageUpload.load_package(temp_stream) @@ -131,9 +135,10 @@ class PackagePush(basePackageUpload.BasePackageUpload): raise rhnFault(104, "Mismatching information")
package_dict, diff_level = rhnPackageUpload.push_package(header, - payload_stream, checksum_type, checksum, force=self.force, + temp_stream, checksum_type, checksum, force=self.force, header_start=header_start, header_end=header_end, relative_path=self.rel_package_path, org_id=self.org_id) + temp_stream.delete=False
if diff_level: return self._send_package_diff(req, diff_level, package_dict)
commit 1548a90b8082f2c9f7314f03bd7daf08de845d2f Author: Michael Mraka michael.mraka@redhat.com Date: Fri Dec 2 14:47:13 2011 +0100
uploadPackage* function have been deprecated long time before Satellite 4.0
diff --git a/backend/server/handlers/app/packages.py b/backend/server/handlers/app/packages.py index b897c57..1d39487 100644 --- a/backend/server/handlers/app/packages.py +++ b/backend/server/handlers/app/packages.py @@ -65,8 +65,6 @@ class Packages(RPC_Base): self.functions.append('listChannelSourceBySession') self.functions.append('listMissingSourcePackages') self.functions.append('listMissingSourcePackagesBySession') - self.functions.append('uploadPackage') - self.functions.append('uploadPackageBySession') self.functions.append('channelPackageSubscription') self.functions.append('channelPackageSubscriptionBySession') self.functions.append('no_op') @@ -251,55 +249,6 @@ class Packages(RPC_Base):
return missing_packages
- - def uploadPackage(self, username, password, info): - """ Uploads an RPM package. """ - log_debug(3) - - channels = info.get('channels', []) - force = info.get('force', 0) - - org_id, force = rhnPackageUpload.authenticate(username, password, - channels=channels, force=force) - return self._uploadPackage(channels, org_id, force, info) - - def uploadPackageBySession(self, session_string, info): - log_debug(3) - channels = info.get('channels', []) - force = info.get('force', 0) - org_id, force = rhnPackageUpload.authenticate_session(session_string, - channels=channels, force=force) - return self._uploadPackage(channels, org_id, force, info) - - def _uploadPackage(self, channels, org_id, force, info): - """ Write the bits to a temporary file """ - packageBits = info['package'] - - package_stream = tempfile.TemporaryFile() - package_stream.write(packageBits) - package_stream.seek(0, 0) - del packageBits - - header, payload_stream, header_start, header_end = \ - rhnPackageUpload.load_package(package_stream) - - checksum_type = header.checksum_type() - checksum = getFileChecksum(header.checksum_type(), file=payload_stream) - - relative_path = rhnPackageUpload.relative_path_from_header( - header, org_id=org_id,checksum=checksum, checksum_type=checksum_type) - - package_dict, diff_level = rhnPackageUpload.push_package( - header, payload_stream, checksum_type, checksum, org_id=org_id, force=force, - header_start=header_start, header_end=header_end, - relative_path=relative_path) - - if diff_level: - return package_dict, diff_level - - return 0 - - def channelPackageSubscription(self, username, password, info): """ Uploads an RPM package. """ log_debug(3)
commit 9bced2c78c6a5cb5a2dd4b6293459199da219877 Author: Michael Mraka michael.mraka@redhat.com Date: Fri Dec 2 13:41:49 2011 +0100
read payload directly from inputstream
there's really no point in copying GiB of payload to tempfile
diff --git a/backend/common/rhn_mpm.py b/backend/common/rhn_mpm.py index cb06f5f..1d7941e 100644 --- a/backend/common/rhn_mpm.py +++ b/backend/common/rhn_mpm.py @@ -20,6 +20,7 @@ import tempfile import xmlrpclib import struct import sys +import fileutils
from types import ListType, TupleType
@@ -170,8 +171,7 @@ class MPM_Package: # Read the header input_stream.seek(128, 0) header_data = self._read_bytes(input_stream, header_len) - payload_stream = tempfile.TemporaryFile() - self.stream_copy(input_stream, payload_stream) + payload_stream = fileutils.payload(input_stream.name, input_stream.tell())
self._read_header(header_data, header_flags) self._read_payload(payload_stream, payload_flags)
commit 32486e4145c44fff73a07f97e17c060a717b428f Author: Michael Mraka michael.mraka@redhat.com Date: Fri Dec 2 13:36:13 2011 +0100
implemented file payload object
diff --git a/backend/common/fileutils.py b/backend/common/fileutils.py index 5b8008f..43d3f45 100644 --- a/backend/common/fileutils.py +++ b/backend/common/fileutils.py @@ -417,3 +417,34 @@ def ostr_to_sym(octstr, ftype): def f_date(dbiDate): return "%04d-%02d-%02d %02d:%02d:%02d" % (dbiDate.year, dbiDate.month, dbiDate.day, dbiDate.hour, dbiDate.minute, dbiDate.second) + + +class payload: + """ this class implements simple file like object usable for reading payload + from rpm, mpm, etc. + it skips first 'skip' bytes of header + """ + def __init__(self, filename, skip=0): + self.fileobj = open(filename, 'r') + self.skip = skip + self.seek(0) + + def seek(self, offset, whence=0): + if whence == 0: + offset += self.skip + return self.fileobj.seek(offset, whence) + + def tell(self): + return self.fileobj.tell() - self.skip + + def truncate(self, size=-1): + raise AttributeError("'Payload' object do not implement this method") + + def write(self, str): + raise AttributeError("'Payload' object do not implement this method") + + def writelines(self, seq): + raise AttributeError("'Payload' object do not implement this method") + + def __getattr__(self, x): + return getattr(self.fileobj, x)
spacewalk-commits@lists.fedorahosted.org
-
Michael Mraka