From: Mathieu Bridon <bochecha(a)daitauha.fr>
This is one more chunk of code which gets moved from pyrpkg.Commands to
pyrpkg.lookaside.CGILookasideCache, where it gets entirely tested.
It also gains some better error handling, as well as the capacity to use
a client-side certificate and/or a custom CA certificate: none are used
by default in pyrpkg, but a client like fedpkg automatically benefits
from this, without needing to do anything on its own.
The Commands._create_curl method is now completely unused, and is marked
as deprecated. However its implementation code is maintained, in case
downstreams were actually using it.
---
src/pyrpkg/__init__.py | 58 ++++---------------
src/pyrpkg/errors.py | 5 ++
src/pyrpkg/lookaside.py | 62 +++++++++++++++++++-
test/test_lookaside.py | 147 +++++++++++++++++++++++++++++++++++++++++++++++-
4 files changed, 222 insertions(+), 50 deletions(-)
diff --git a/src/pyrpkg/__init__.py b/src/pyrpkg/__init__.py
index 6d818dd..14eb808 100644
--- a/src/pyrpkg/__init__.py
+++ b/src/pyrpkg/__init__.py
@@ -26,7 +26,6 @@ import logging
import git
import ConfigParser
import stat
-import StringIO
import tempfile
import fnmatch
import urlparse
@@ -774,11 +773,10 @@ class Commands(object):
# Define some helper functions, they start with _
def _create_curl(self):
- """
- Common curl setup options used for all requests to lookaside.
- """
- curl = pycurl.Curl()
+ warn_deprecated(self, '_create_curl',
+ 'lookasidecache.remote_file_exists')
+ curl = pycurl.Curl()
curl.setopt(pycurl.URL, self.lookaside_cgi)
return curl
@@ -1657,48 +1655,10 @@ class Commands(object):
return
def file_exists(self, pkg_name, filename, checksum):
- """
- Return True if the given file exists in the lookaside cache, False
- if not.
-
- A rpkgError will be thrown if the request looks bad or something
- goes wrong. (i.e. the lookaside URL cannot be reached, or the package
- named does not exist)
- """
-
- # String buffer, used to receive output from the curl request:
- buf = StringIO.StringIO()
-
- # Setup the POST data for lookaside CGI request. The use of
- # 'filename' here appears to be what differentiates this
- # request from an actual file upload.
- post_data = [('name', pkg_name),
- ('%ssum' % self.lookasidehash, checksum),
- ('filename', filename)]
-
- curl = self._create_curl()
- curl.setopt(pycurl.WRITEFUNCTION, buf.write)
- curl.setopt(pycurl.HTTPPOST, post_data)
-
- try:
- curl.perform()
- except Exception, e:
- raise rpkgError('Lookaside failure: %s' % e)
- curl.close()
- output = buf.getvalue().strip()
-
- # Lookaside CGI script returns these strings depending on whether
- # or not the file exists:
- if output == "Available":
- return True
- if output == "Missing":
- return False
-
- # Something unexpected happened, will trigger if the lookaside URL
- # cannot be reached, the package named does not exist, and probably
- # some other scenarios as well.
- raise rpkgError("Error checking for %s at: %s"
- % (filename, self.lookaside_cgi))
+ warn_deprecated(self, '_verify_file',
+ 'lookasidecache.remote_file_exists')
+ return self.lookasidecache.remote_file_exists(pkg_name, filename,
+ checksum)
def build(self, skip_tag=False, scratch=False, background=False,
url=None, chain=None, arches=None, sets=False, nvr_check=True):
@@ -2268,9 +2228,11 @@ class Commands(object):
if not gitignore.match(file_basename):
gitignore.add('/%s' % file_basename)
- if self.file_exists(self.module_name, file_basename, file_hash):
+ if self.lookasidecache.remote_file_exists(
+ self.module_name, file_basename, file_hash):
# Already uploaded, skip it:
self.log.info("File already uploaded: %s" % file_basename)
+
else:
# Ensure the new file is readable:
os.chmod(f, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
diff --git a/src/pyrpkg/errors.py b/src/pyrpkg/errors.py
index 9547702..5232f63 100644
--- a/src/pyrpkg/errors.py
+++ b/src/pyrpkg/errors.py
@@ -42,3 +42,8 @@ class InvalidHashType(rpkgError):
class DownloadError(rpkgError):
"""Raised when something went wrong during a
download"""
pass
+
+
+class UploadError(rpkgError):
+ """Raised when something went wrong during an
upload"""
+ pass
diff --git a/src/pyrpkg/lookaside.py b/src/pyrpkg/lookaside.py
index e455c40..240bd6a 100644
--- a/src/pyrpkg/lookaside.py
+++ b/src/pyrpkg/lookaside.py
@@ -15,13 +15,14 @@ way it is done by Fedora, RHEL, and other distributions maintainers.
import hashlib
+import io
import logging
import os
import sys
import pycurl
-from .errors import DownloadError, InvalidHashType
+from .errors import DownloadError, InvalidHashType, UploadError
class CGILookasideCache(object):
@@ -176,3 +177,62 @@ class CGILookasideCache(object):
if not self.file_is_valid(outfile, hash, hashtype=hashtype):
raise DownloadError('%s failed checksum' % filename)
+
+ def remote_file_exists(self, name, filename, hash):
+ """Verify whether a file exists on the lookaside cache
+
+ Args:
+ name: The name of the module. (usually the name of the SRPM)
+ filename: The name of the file to check for.
+ hash: The known good hash of the file.
+ """
+ post_data = [('name', name),
+ ('%ssum' % self.hashtype, hash),
+ ('filename', filename)]
+
+ with io.BytesIO() as buf:
+ c = pycurl.Curl()
+ c.setopt(pycurl.URL, self.upload_url)
+ c.setopt(pycurl.WRITEFUNCTION, buf.write)
+ c.setopt(pycurl.HTTPPOST, post_data)
+
+ if self.client_cert is not None:
+ if os.path.exists(self.client_cert):
+ c.setopt(pycurl.SSLCERT, self.client_cert)
+ else:
+ self.log.warn("Missing certificate: %s"
+ % self.client_cert)
+
+ if self.ca_cert is not None:
+ if os.path.exists(self.ca_cert):
+ c.setopt(pycurl.CAINFO, self.ca_cert)
+ else:
+ self.log.warn("Missing certificate: %s" % self.ca_cert)
+
+ try:
+ c.perform()
+ status = c.getinfo(pycurl.RESPONSE_CODE)
+
+ except Exception as e:
+ raise UploadError(e)
+
+ finally:
+ c.close()
+
+ output = buf.getvalue().strip()
+
+ if status != 200:
+ raise UploadError(output)
+
+ # Lookaside CGI script returns these strings depending on whether
+ # or not the file exists:
+ if output == b'Available':
+ return True
+
+ if output == b'Missing':
+ return False
+
+ # Something unexpected happened
+ self.log.debug(output)
+ raise UploadError('Error checking for %s at %s'
+ % (filename, self.upload_url))
diff --git a/test/test_lookaside.py b/test/test_lookaside.py
index 4d987c1..b6cc71d 100644
--- a/test/test_lookaside.py
+++ b/test/test_lookaside.py
@@ -21,7 +21,7 @@ old_path = list(sys.path)
src_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../src')
sys.path.insert(0, src_path)
from pyrpkg.lookaside import CGILookasideCache
-from pyrpkg.errors import DownloadError, InvalidHashType
+from pyrpkg.errors import DownloadError, InvalidHashType, UploadError
sys.path = old_path
@@ -309,3 +309,148 @@ class CGILookasideCacheTestCase(unittest.TestCase):
lc = CGILookasideCache('_', '_', '_')
lc.print_progress(0.0, 0.0, 0.0, 0.0)
self.assertEqual(len(written_lines), 0)
+
+ @mock.patch('pyrpkg.lookaside.pycurl.Curl')
+ def test_remote_file_exists(self, mock_curl):
+ def mock_perform():
+ curlopts[pycurl.WRITEFUNCTION](b'Available')
+
+ def mock_setopt(opt, value):
+ curlopts[opt] = value
+
+ curlopts = {}
+ curl = mock_curl.return_value
+ curl.getinfo.return_value = 200
+ curl.perform.side_effect = mock_perform
+ curl.setopt.side_effect = mock_setopt
+
+ lc = CGILookasideCache('_', '_', '_')
+ exists = lc.remote_file_exists('pyrpkg', 'pyrpkg-0.tar.xz',
'thehash')
+ self.assertTrue(exists)
+
+ @mock.patch('pyrpkg.lookaside.pycurl.Curl')
+ def test_remote_file_does_not_exist(self, mock_curl):
+ def mock_perform():
+ curlopts[pycurl.WRITEFUNCTION](b'Missing')
+
+ def mock_setopt(opt, value):
+ curlopts[opt] = value
+
+ curlopts = {}
+ curl = mock_curl.return_value
+ curl.getinfo.return_value = 200
+ curl.perform.side_effect = mock_perform
+ curl.setopt.side_effect = mock_setopt
+
+ lc = CGILookasideCache('_', '_', '_')
+ exists = lc.remote_file_exists('pyrpkg', 'pyrpkg-0.tar.xz',
'thehash')
+ self.assertFalse(exists)
+
+ @mock.patch('pyrpkg.lookaside.pycurl.Curl')
+ def test_remote_file_exists_with_custom_certs(self, mock_curl):
+ def mock_perform():
+ curlopts[pycurl.WRITEFUNCTION](b'Available')
+
+ def mock_setopt(opt, value):
+ curlopts[opt] = value
+
+ curlopts = {}
+ curl = mock_curl.return_value
+ curl.getinfo.return_value = 200
+ curl.perform.side_effect = mock_perform
+ curl.setopt.side_effect = mock_setopt
+
+ client_cert = os.path.join(self.workdir, 'my-client-cert.cert')
+ with open(client_cert, 'w'):
+ pass
+
+ ca_cert = os.path.join(self.workdir, 'my-custom-cacert.cert')
+ with open(ca_cert, 'w'):
+ pass
+
+ lc = CGILookasideCache('_', '_', '_',
client_cert=client_cert,
+ ca_cert=ca_cert)
+ lc.remote_file_exists('pyrpkg', 'pyrpkg-0.tar.xz',
'thehash')
+ self.assertEqual(curlopts[pycurl.SSLCERT], client_cert)
+ self.assertEqual(curlopts[pycurl.CAINFO], ca_cert)
+
+ @mock.patch('pyrpkg.lookaside.logging.getLogger')
+ @mock.patch('pyrpkg.lookaside.pycurl.Curl')
+ def test_remote_file_exists_missing_custom_certs(self, mock_curl,
+ mock_logger):
+ def mock_perform():
+ curlopts[pycurl.WRITEFUNCTION](b'Available')
+
+ def mock_setopt(opt, value):
+ curlopts[opt] = value
+
+ def mock_warn(msg):
+ warn_messages.append(msg)
+
+ curlopts = {}
+ curl = mock_curl.return_value
+ curl.getinfo.return_value = 200
+ curl.perform.side_effect = mock_perform
+ curl.setopt.side_effect = mock_setopt
+
+ warn_messages = []
+ log = mock_logger.return_value
+ log.warn.side_effect = mock_warn
+
+ client_cert = os.path.join(self.workdir, 'my-client-cert.cert')
+ ca_cert = os.path.join(self.workdir, 'my-custom-cacert.cert')
+
+ lc = CGILookasideCache('_', '_', '_',
client_cert=client_cert,
+ ca_cert=ca_cert)
+ lc.remote_file_exists('pyrpkg', 'pyrpkg-0.tar.xz',
'thehash')
+ self.assertTrue(pycurl.SSLCERT not in curlopts)
+ self.assertTrue(pycurl.CAINFO not in curlopts)
+ self.assertEqual(len(warn_messages), 2)
+ self.assertTrue('Missing certificate: ' in warn_messages[0])
+ self.assertTrue('Missing certificate: ' in warn_messages[1])
+
+ @mock.patch('pyrpkg.lookaside.pycurl.Curl')
+ def test_remote_file_exists_check_failed(self, mock_curl):
+ curl = mock_curl.return_value
+ curl.perform.side_effect = Exception(
+ 'Could not resolve host: example.com')
+
+ lc = CGILookasideCache('_', '_', '_')
+ self.assertRaises(UploadError, lc.remote_file_exists, 'pyrpkg',
+ 'pyrpkg-0.tar.xz', 'thehash')
+
+ @mock.patch('pyrpkg.lookaside.pycurl.Curl')
+ def test_remote_file_exists_check_failed_status_code(self, mock_curl):
+ def mock_perform():
+ curlopts[pycurl.WRITEFUNCTION](b'Available')
+
+ def mock_setopt(opt, value):
+ curlopts[opt] = value
+
+ curlopts = {}
+ curl = mock_curl.return_value
+ curl.getinfo.return_value = 500
+ curl.perform.side_effect = mock_perform
+ curl.setopt.side_effect = mock_setopt
+
+ lc = CGILookasideCache('_', '_', '_')
+ self.assertRaises(UploadError, lc.remote_file_exists, 'pyrpkg',
+ 'pyrpkg-0.0.tar.xz', 'thehash')
+
+ @mock.patch('pyrpkg.lookaside.pycurl.Curl')
+ def test_remote_file_exists_check_unexpected_error(self, mock_curl):
+ def mock_perform():
+ curlopts[pycurl.WRITEFUNCTION]('Something unexpected')
+
+ def mock_setopt(opt, value):
+ curlopts[opt] = value
+
+ curlopts = {}
+ curl = mock_curl.return_value
+ curl.getinfo.return_value = 200
+ curl.perform.side_effect = mock_perform
+ curl.setopt.side_effect = mock_setopt
+
+ lc = CGILookasideCache('_', '_', '_')
+ self.assertRaises(UploadError, lc.remote_file_exists, 'pyrpkg',
+ 'pyrpkg-0.tar.xz', 'thehash')
--
2.1.0