backend/common/fileutils.py | 31 +++++++
backend/common/rhn_mpm.py | 4
backend/satellite_tools/repo_plugins/yum_src.py | 2
backend/satellite_tools/reposync.py | 2
backend/server/handlers/app/packages.py | 51 ------------
backend/server/importlib/importLib.py | 34 +++-----
backend/server/rhnPackageUpload.py | 10 +-
backend/upload_server/handlers/package_push/package_push.py | 9 +-
8 files changed, 62 insertions(+), 81 deletions(-)
New commits:
commit 580b4e6015ddf7a3402ea6e8b7f97197d12123e5
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Fri Dec 2 19:46:14 2011 +0100
676369 - unlink old file when forced
diff --git a/backend/server/importlib/importLib.py b/backend/server/importlib/importLib.py
index 9bfc38f..3572ed0 100644
--- a/backend/server/importlib/importLib.py
+++ b/backend/server/importlib/importLib.py
@@ -793,13 +793,16 @@ def move_package(filename, basedir, relpath, checksum_type, checksum, force=None
"""
packagePath = basedir + "/" + relpath
# Is the file there already?
- if os.path.isfile(packagePath) and not force:
- # Get its checksum
- localsum = getFileChecksum(checksum_type, packagePath)
- if checksum == localsum:
- # Same file, so get outa here
- return
- raise FileConflictError(os.path.basename(packagePath))
+ if os.path.isfile(packagePath):
+ if force:
+ os.unlink(packagePath)
+ else:
+ # Get its checksum
+ localsum = getFileChecksum(checksum_type, packagePath)
+ if checksum == localsum:
+ # Same file, so get outa here
+ return
+ raise FileConflictError(os.path.basename(packagePath))
dir = os.path.dirname(packagePath)
# Create the directory where the file will reside
commit c9d894c679bea0e556774d0708cfa593eaebea28
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Fri Dec 2 18:36:53 2011 +0100
676369 - sync packages directly to /var/satellite
diff --git a/backend/satellite_tools/repo_plugins/yum_src.py b/backend/satellite_tools/repo_plugins/yum_src.py
index f0949da..67b9189 100644
--- a/backend/satellite_tools/repo_plugins/yum_src.py
+++ b/backend/satellite_tools/repo_plugins/yum_src.py
@@ -17,6 +17,7 @@ import yum
import shutil
import sys
import gzip
+import os.path
from yum.update_md import UpdateMetadata, UpdateNoticeException, UpdateNotice
from yum.yumRepo import YumRepository
try:
@@ -99,6 +100,7 @@ class ContentSource:
repo.mirrorlist = url
repo.baseurl = [url]
repo.basecachedir = self.cache_dir
+ repo._dirSetAttr('pkgdir', os.path.join(CFG.MOUNT_POINT, CFG.PREPENDED_DIR, '1'))
if hasattr(repo, 'base_persistdir'):
repo.base_persistdir = self.cache_dir
if self.proxy_url is not None:
commit 2dbe4deade884d410f93bfcd30947bb77dd18e88
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Fri Dec 2 18:01:00 2011 +0100
676369 - move package to the final location
diff --git a/backend/satellite_tools/reposync.py b/backend/satellite_tools/reposync.py
index 2ba1d7d..832876c 100644
--- a/backend/satellite_tools/reposync.py
+++ b/backend/satellite_tools/reposync.py
@@ -416,7 +416,7 @@ class RepoSync:
package.header, self.channel['org_id'],
package.checksum_type, package.checksum)
package_dict, diff_level = rhnPackageUpload.push_package(package.header,
- package.payload_stream, package.checksum_type, package.checksum,
+ package.path, package.payload_stream, package.checksum_type, package.checksum,
force=False,
header_start=package.header_start, header_end=package.header_end,
relative_path=rel_package_path,
diff --git a/backend/server/importlib/importLib.py b/backend/server/importlib/importLib.py
index f44f657..9bfc38f 100644
--- a/backend/server/importlib/importLib.py
+++ b/backend/server/importlib/importLib.py
@@ -784,7 +784,7 @@ def removeNone(list):
# Assorted functions for various things
-def copy_package(fd, basedir, relpath, checksum_type, checksum, force=None):
+def move_package(filename, basedir, relpath, checksum_type, checksum, force=None):
"""
Copies the information from the file descriptor to a file
Checks the file's checksum, raising FileConflictErrror if it's different
@@ -805,20 +805,9 @@ def copy_package(fd, basedir, relpath, checksum_type, checksum, force=None):
# Create the directory where the file will reside
if not os.path.exists(dir):
createPath(dir)
- pkgfd = os.open(packagePath, os.O_WRONLY | os.O_CREAT | os.O_TRUNC)
- os.lseek(fd, 0, 0)
- while 1:
- buffer = os.read(fd, 65536)
- if not buffer:
- break
- n = os.write(pkgfd, buffer)
- if n != len(buffer):
- # Error writing to the file
- raise IOError, "Wrote %s out of %s bytes in file %s" % (
- n, len(buffer), packagePath)
- os.close(pkgfd)
+ os.rename(filename, packagePath)
# set the path perms readable by all users
- setPermsPath(packagePath, chmod=0644)
+ os.chmod(packagePath, 0644)
# Returns a list of containing nevra for the given RPM header
diff --git a/backend/server/rhnPackageUpload.py b/backend/server/rhnPackageUpload.py
index cd7db76..8f3dd43 100644
--- a/backend/server/rhnPackageUpload.py
+++ b/backend/server/rhnPackageUpload.py
@@ -127,7 +127,7 @@ def relative_path_from_nevra_without_package_name(nevra, org_id, checksum_type,
return get_package_path_without_package_name(nevra, org_id,
CFG.PREPENDED_DIR, checksum_type, checksum)
-def push_package(header, payload_stream, checksum_type, checksum, org_id=None, force=None,
+def push_package(header, temp_path, payload_stream, checksum_type, checksum, org_id=None, force=None,
header_start=None, header_end=None, channels=[], relative_path=None):
"""Uploads an RPM package
"""
@@ -140,7 +140,7 @@ def push_package(header, payload_stream, checksum_type, checksum, org_id=None, f
# First write the package to the filesystem to final location
try:
- importLib.copy_package(payload_stream.fileno(), basedir=CFG.MOUNT_POINT,
+ importLib.move_package(temp_path, basedir=CFG.MOUNT_POINT,
relpath=relative_path, checksum_type=checksum_type, checksum=checksum, force=1)
except OSError, e:
raise rhnFault(50, "Package upload failed: %s" % e), None, sys.exc_info()[2]
diff --git a/backend/upload_server/handlers/package_push/package_push.py b/backend/upload_server/handlers/package_push/package_push.py
index 640b99e..44d09f8 100644
--- a/backend/upload_server/handlers/package_push/package_push.py
+++ b/backend/upload_server/handlers/package_push/package_push.py
@@ -135,7 +135,7 @@ class PackagePush(basePackageUpload.BasePackageUpload):
raise rhnFault(104, "Mismatching information")
package_dict, diff_level = rhnPackageUpload.push_package(header,
- temp_stream, checksum_type, checksum, force=self.force,
+ temp_stream.name, payload_stream, checksum_type, checksum, force=self.force,
header_start=header_start, header_end=header_end,
relative_path=self.rel_package_path, org_id=self.org_id)
temp_stream.delete=False
commit dc547088d8b9b85be4d0ec57bee54fc81e10cdfd
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Fri Dec 2 17:43:07 2011 +0100
676369 - create tempfile under /var/satellite
so we can move it (not copy) to the final location
diff --git a/backend/server/rhnPackageUpload.py b/backend/server/rhnPackageUpload.py
index ecd03ac..cd7db76 100644
--- a/backend/server/rhnPackageUpload.py
+++ b/backend/server/rhnPackageUpload.py
@@ -33,14 +33,16 @@ from spacewalk.server.rhnLib import get_package_path, \
from spacewalk.server.rhnServer import server_packages
-def write_temp_file(req, buffer_size, packaging=None):
+def write_temp_file(req, buffer_size, dir, prefix, packaging=None):
""" Write request to temporary file (write max. buffer_size at once).
Returns the file object.
"""
suffix = ''
if packaging == 'deb':
suffix = '.deb'
- t = tempfile.NamedTemporaryFile(suffix=suffix)
+ t = tempfile.NamedTemporaryFile(
+ dir=dir, prefix=prefix + '.',
+ suffix=suffix)
while 1:
buf = req.read(buffer_size)
if not buf:
diff --git a/backend/upload_server/handlers/package_push/package_push.py b/backend/upload_server/handlers/package_push/package_push.py
index aa84a4d..640b99e 100644
--- a/backend/upload_server/handlers/package_push/package_push.py
+++ b/backend/upload_server/handlers/package_push/package_push.py
@@ -108,7 +108,11 @@ class PackagePush(basePackageUpload.BasePackageUpload):
if ret != apache.OK:
return ret
- temp_stream = rhnPackageUpload.write_temp_file(req, 16384, self.packaging)
+ temp_stream = rhnPackageUpload.write_temp_file(req, 16384,
+ os.path.join(CFG.MOUNT_POINT, CFG.PREPENDED_DIR, str(self.org_id)),
+ '-'.join((self.package_name, self.package_version,
+ self.package_release, self.package_arch)),
+ self.packaging)
header, payload_stream, header_start, header_end = \
rhnPackageUpload.load_package(temp_stream)
@@ -131,9 +135,10 @@ class PackagePush(basePackageUpload.BasePackageUpload):
raise rhnFault(104, "Mismatching information")
package_dict, diff_level = rhnPackageUpload.push_package(header,
- payload_stream, checksum_type, checksum, force=self.force,
+ temp_stream, checksum_type, checksum, force=self.force,
header_start=header_start, header_end=header_end,
relative_path=self.rel_package_path, org_id=self.org_id)
+ temp_stream.delete=False
if diff_level:
return self._send_package_diff(req, diff_level, package_dict)
commit 1548a90b8082f2c9f7314f03bd7daf08de845d2f
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Fri Dec 2 14:47:13 2011 +0100
uploadPackage* function have been deprecated long time before Satellite 4.0
diff --git a/backend/server/handlers/app/packages.py b/backend/server/handlers/app/packages.py
index b897c57..1d39487 100644
--- a/backend/server/handlers/app/packages.py
+++ b/backend/server/handlers/app/packages.py
@@ -65,8 +65,6 @@ class Packages(RPC_Base):
self.functions.append('listChannelSourceBySession')
self.functions.append('listMissingSourcePackages')
self.functions.append('listMissingSourcePackagesBySession')
- self.functions.append('uploadPackage')
- self.functions.append('uploadPackageBySession')
self.functions.append('channelPackageSubscription')
self.functions.append('channelPackageSubscriptionBySession')
self.functions.append('no_op')
@@ -251,55 +249,6 @@ class Packages(RPC_Base):
return missing_packages
-
- def uploadPackage(self, username, password, info):
- """ Uploads an RPM package. """
- log_debug(3)
-
- channels = info.get('channels', [])
- force = info.get('force', 0)
-
- org_id, force = rhnPackageUpload.authenticate(username, password,
- channels=channels, force=force)
- return self._uploadPackage(channels, org_id, force, info)
-
- def uploadPackageBySession(self, session_string, info):
- log_debug(3)
- channels = info.get('channels', [])
- force = info.get('force', 0)
- org_id, force = rhnPackageUpload.authenticate_session(session_string,
- channels=channels, force=force)
- return self._uploadPackage(channels, org_id, force, info)
-
- def _uploadPackage(self, channels, org_id, force, info):
- """ Write the bits to a temporary file """
- packageBits = info['package']
-
- package_stream = tempfile.TemporaryFile()
- package_stream.write(packageBits)
- package_stream.seek(0, 0)
- del packageBits
-
- header, payload_stream, header_start, header_end = \
- rhnPackageUpload.load_package(package_stream)
-
- checksum_type = header.checksum_type()
- checksum = getFileChecksum(header.checksum_type(), file=payload_stream)
-
- relative_path = rhnPackageUpload.relative_path_from_header(
- header, org_id=org_id,checksum=checksum, checksum_type=checksum_type)
-
- package_dict, diff_level = rhnPackageUpload.push_package(
- header, payload_stream, checksum_type, checksum, org_id=org_id, force=force,
- header_start=header_start, header_end=header_end,
- relative_path=relative_path)
-
- if diff_level:
- return package_dict, diff_level
-
- return 0
-
-
def channelPackageSubscription(self, username, password, info):
""" Uploads an RPM package. """
log_debug(3)
commit 9bced2c78c6a5cb5a2dd4b6293459199da219877
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Fri Dec 2 13:41:49 2011 +0100
read payload directly from inputstream
there's really no point in copying GiB of payload to tempfile
diff --git a/backend/common/rhn_mpm.py b/backend/common/rhn_mpm.py
index cb06f5f..1d7941e 100644
--- a/backend/common/rhn_mpm.py
+++ b/backend/common/rhn_mpm.py
@@ -20,6 +20,7 @@ import tempfile
import xmlrpclib
import struct
import sys
+import fileutils
from types import ListType, TupleType
@@ -170,8 +171,7 @@ class MPM_Package:
# Read the header
input_stream.seek(128, 0)
header_data = self._read_bytes(input_stream, header_len)
- payload_stream = tempfile.TemporaryFile()
- self.stream_copy(input_stream, payload_stream)
+ payload_stream = fileutils.payload(input_stream.name, input_stream.tell())
self._read_header(header_data, header_flags)
self._read_payload(payload_stream, payload_flags)
commit 32486e4145c44fff73a07f97e17c060a717b428f
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Fri Dec 2 13:36:13 2011 +0100
implemented file payload object
diff --git a/backend/common/fileutils.py b/backend/common/fileutils.py
index 5b8008f..43d3f45 100644
--- a/backend/common/fileutils.py
+++ b/backend/common/fileutils.py
@@ -417,3 +417,34 @@ def ostr_to_sym(octstr, ftype):
def f_date(dbiDate):
return "%04d-%02d-%02d %02d:%02d:%02d" % (dbiDate.year, dbiDate.month,
dbiDate.day, dbiDate.hour, dbiDate.minute, dbiDate.second)
+
+
+class payload:
+ """ this class implements simple file like object usable for reading payload
+ from rpm, mpm, etc.
+ it skips first 'skip' bytes of header
+ """
+ def __init__(self, filename, skip=0):
+ self.fileobj = open(filename, 'r')
+ self.skip = skip
+ self.seek(0)
+
+ def seek(self, offset, whence=0):
+ if whence == 0:
+ offset += self.skip
+ return self.fileobj.seek(offset, whence)
+
+ def tell(self):
+ return self.fileobj.tell() - self.skip
+
+ def truncate(self, size=-1):
+ raise AttributeError("'Payload' object do not implement this method")
+
+ def write(self, str):
+ raise AttributeError("'Payload' object do not implement this method")
+
+ def writelines(self, seq):
+ raise AttributeError("'Payload' object do not implement this method")
+
+ def __getattr__(self, x):
+ return getattr(self.fileobj, x)
