Repository :
http://git.fedorahosted.org/cgit/copr.git
On branch : master
---------------------------------------------------------------
commit 7c1aa0c74c7bf5c40cf5f83f2dfb9c63d2228d21
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Thu Aug 20 12:12:00 2015 +0200
[dist-git] replace urllib.urlretrieve with requests.get to catch non-200 HTTP status
codes
---------------------------------------------------------------
dist-git/dist_git/dist_git_importer.py | 36 ++++++++++++++++++++++-------
dist-git/tests/test_dist_git_importer.py | 26 ++++++++++-----------
2 files changed, 39 insertions(+), 23 deletions(-)
diff --git a/dist-git/dist_git/dist_git_importer.py
b/dist-git/dist_git/dist_git_importer.py
index 52ab501..701e030 100755
--- a/dist-git/dist_git/dist_git_importer.py
+++ b/dist-git/dist_git/dist_git_importer.py
@@ -94,6 +94,32 @@ class ImportTask(object):
}
+def do_srpm_fetch(task, target_path):
+ """
+ :param ImportTask task:
+ :param str target_path:
+ :raises PackageDownloadException:
+ """
+ log.debug("download the package")
+ try:
+ r = get(task.package_url, stream=True)
+ except Exception:
+ raise PackageDownloadException("Unexpected error during URL fetch: {}"
+ .format(task.package_url))
+
+ if 200 <= r.status_code < 400:
+ try:
+ with open(target_path, 'wb') as f:
+ for chunk in r.iter_content(1024):
+ f.write(chunk)
+ except Exception:
+ raise PackageDownloadException("Unexpected error during URL retrieval:
{}"
+ .format(task.package_url))
+ else:
+ raise PackageDownloadException("Failed to fetch: {} with HTTP status:
{}"
+ .format(task.package_url, r.status_code))
+
+
class DistGitImporter(object):
def __init__(self, opts):
self.is_running = False
@@ -121,14 +147,6 @@ class DistGitImporter(object):
log.exception("Failed acquire new packages for import")
return
- @staticmethod
- def fetch_srpm(task, fetched_srpm_path):
- log.debug("download the package")
- try:
- urlretrieve(task.package_url, fetched_srpm_path)
- except IOError:
- raise PackageDownloadException()
-
def git_import_srpm(self, task, filepath):
"""
Imports a source rpm file into local dist git.
@@ -208,7 +226,7 @@ class DistGitImporter(object):
fetched_srpm_path = os.path.join(tmp_root, "package.src.rpm")
try:
- self.fetch_srpm(task, fetched_srpm_path)
+ do_srpm_fetch(task, fetched_srpm_path)
task.package_name, task.package_version =
self.pkg_name_evr(fetched_srpm_path)
self.before_git_import(task)
diff --git a/dist-git/tests/test_dist_git_importer.py
b/dist-git/tests/test_dist_git_importer.py
index 7ca9260..9f35726 100644
--- a/dist-git/tests/test_dist_git_importer.py
+++ b/dist-git/tests/test_dist_git_importer.py
@@ -32,21 +32,25 @@ def mc_dgcr():
with mock.patch("{}.DistGitConfigReader".format(MODULE_REF)) as handle:
yield handle
-# @pytest.yield_fixture
-# def mc_dgi():
-# with mock.patch("{}.DistGitImporter".format(MODULE_REF)) as handle:
-# yield handle
@pytest.yield_fixture
def mc_time():
with mock.patch("{}.time".format(MODULE_REF)) as handle:
yield handle
+
+(a)pytest.yield_fixture
+def mc_do_srpm_fetch():
+ with mock.patch("{}.do_srpm_fetch".format(MODULE_REF)) as handle:
+ yield handle
+
+
@pytest.yield_fixture
def mc_popen():
with mock.patch("{}.Popen".format(MODULE_REF)) as handle:
yield handle
+
@pytest.yield_fixture
def mc_call():
with mock.patch("{}.call".format(MODULE_REF)) as handle:
@@ -64,11 +68,13 @@ def mc_post():
with mock.patch("{}.post".format(MODULE_REF)) as handle:
yield handle
+
@pytest.yield_fixture
def mc_urlretrieve():
with mock.patch("{}.urlretrieve".format(MODULE_REF)) as handle:
yield handle
+
@pytest.yield_fixture
def mc_pyrpkg_commands():
with mock.patch("{}.Commands".format(MODULE_REF)) as handle:
@@ -177,14 +183,6 @@ class TestDistGitImporter(object):
mc_get.return_value.json.return_value = {"builds": [task_data]}
assert self.dgi.try_to_obtain_new_task() is None
- def test_fetch_srpm_on(self, mc_urlretrieve):
- self.dgi.fetch_srpm(self.task_1, self.fetched_srpm_path)
- assert mc_urlretrieve.called
-
- mc_urlretrieve.side_effect = IOError
- with pytest.raises(PackageDownloadException):
- self.dgi.fetch_srpm(self.task_1, self.fetched_srpm_path)
-
# def test_my_upload(self):
# filename = "source"
# source_path = os.path.join(self.tmp_dir_name, filename)
@@ -288,9 +286,9 @@ class TestDistGitImporter(object):
self.dgi.post_back_safe(dd)
assert mc_post.called
- def test_do_import(self):
+ def test_do_import(self, mc_do_srpm_fetch):
internal_methods = [
- "fetch_srpm",
+ # "fetch_srpm",
"pkg_name_evr",
"before_git_import",
"git_import_srpm",