[f22-branch 1/2] Revert "Replace python-urlgrabber with python-requests (#1141242)"

dashea installerbot-noreply at redhat.com
Tue Mar 17 20:28:32 UTC 2015


From: David Shea <dshea at redhat.com>

This reverts commit 28df390ed0108aa05f22fb1ee56455d2dde23082.

python-requests supports only http, and the solutions for using
python-requests with ftp:// and file:// URLs are incomplete at best.
Revert back to using urlgrabber for F22.
---
 anaconda.spec.in                    |  3 ++
 pyanaconda/kickstart.py             | 13 +++-----
 pyanaconda/packaging/__init__.py    | 36 ++++++++++----------
 pyanaconda/packaging/livepayload.py | 66 +++++++++++++++++--------------------
 4 files changed, 56 insertions(+), 62 deletions(-)

diff --git a/anaconda.spec.in b/anaconda.spec.in
index 2fa8d34..cfdc1af 100644
--- a/anaconda.spec.in
+++ b/anaconda.spec.in
@@ -31,6 +31,7 @@ Source0: %{name}-%{version}.tar.bz2
 %define yumutilsver 1.1.11-3
 %define mehver 0.23-1
 %define firewalldver 0.3.5-1
+%define pythonurlgrabberver 3.9.1-5
 %define utillinuxver 2.15.1
 %define dracutver 034-7
 %define isomd5sum 1.0.10
@@ -60,6 +61,7 @@ BuildRequires: pykickstart >= %{pykickstartver}
 BuildRequires: python-bugzilla
 %endif
 BuildRequires: python-devel
+BuildRequires: python-urlgrabber >= %{pythonurlgrabberver}
 BuildRequires: python-nose
 BuildRequires: systemd
 BuildRequires: rpm-devel >= %{rpmver}
@@ -90,6 +92,7 @@ Requires: rpm-python >= %{rpmver}
 Requires: parted >= %{partedver}
 Requires: pyparted >= %{pypartedver}
 Requires: yum >= %{yumver}
+Requires: python-urlgrabber >= %{pythonurlgrabberver}
 Requires: python-requests
 Requires: pykickstart >= %{pykickstartver}
 Requires: langtable-data >= %{langtablever}
diff --git a/pyanaconda/kickstart.py b/pyanaconda/kickstart.py
index b64bd09..49dba5c 100644
--- a/pyanaconda/kickstart.py
+++ b/pyanaconda/kickstart.py
@@ -45,8 +45,8 @@
 from pyanaconda.flags import flags, can_touch_runtime_system
 from pyanaconda.constants import ADDON_PATHS, IPMI_ABORTED
 import shlex
-import requests
 import sys
+import urlgrabber
 import pykickstart.commands as commands
 from pyanaconda import keyboard
 from pyanaconda import ntp
@@ -164,18 +164,15 @@ def getEscrowCertificate(escrowCerts, url):
     log.info("escrow: downloading %s", url)
 
     try:
-        request = requests.get(url, verify=True)
-    except requests.exceptions.SSLError as e:
-        msg = _("SSL error while downloading the escrow certificate:\n\n%s") % e
-        raise KickstartError(msg)
-    except requests.exceptions.RequestException as e:
+        f = urlgrabber.urlopen(url)
+    except urlgrabber.grabber.URLGrabError as e:
         msg = _("The following error was encountered while downloading the escrow certificate:\n\n%s") % e
         raise KickstartError(msg)
 
     try:
-        escrowCerts[url] = request.content
+        escrowCerts[url] = f.read()
     finally:
-        request.close()
+        f.close()
 
     return escrowCerts[url]
 
diff --git a/pyanaconda/packaging/__init__.py b/pyanaconda/packaging/__init__.py
index 7f93b5e..8cbeb13 100644
--- a/pyanaconda/packaging/__init__.py
+++ b/pyanaconda/packaging/__init__.py
@@ -28,7 +28,8 @@
 
 """
 import os
-import requests
+from urlgrabber.grabber import URLGrabber
+from urlgrabber.grabber import URLGrabError
 import ConfigParser
 import shutil
 from glob import glob
@@ -67,7 +68,8 @@
 from blivet.platform import platform
 
 from pyanaconda.product import productName, productVersion
-USER_AGENT = "%s (anaconda)/%s" %(productName, productVersion)
+import urlgrabber
+urlgrabber.grabber.default_grabber.opts.user_agent = "%s (anaconda)/%s" %(productName, productVersion)
 
 from distutils.version import LooseVersion
 
@@ -337,6 +339,9 @@ def _getTreeInfo(self, url, proxy_url, sslverify):
         log.debug("retrieving treeinfo from %s (proxy: %s ; sslverify: %s)",
                   url, proxy_url, sslverify)
 
+        ugopts = {"ssl_verify_peer": sslverify,
+                  "ssl_verify_host": sslverify}
+
         proxies = {}
         if proxy_url:
             try:
@@ -347,26 +352,21 @@ def _getTreeInfo(self, url, proxy_url, sslverify):
                 log.info("Failed to parse proxy for _getTreeInfo %s: %s",
                          proxy_url, e)
 
-        response = None
-        headers = {"user-agent": USER_AGENT}
+        ug = URLGrabber()
         try:
-            response = requests.get("%s/.treeinfo" % url, headers=headers, proxies=proxies, verify=sslverify)
-        except requests.exceptions.RequestException as e:
+            treeinfo = ug.urlgrab("%s/.treeinfo" % url,
+                                  "/tmp/.treeinfo", copy_local=True,
+                                  proxies=proxies, **ugopts)
+        except URLGrabError as e:
             try:
-                response = requests.get("%s/treeinfo" % url, headers=headers, proxies=proxies, verify=sslverify)
-            except requests.exceptions.RequestException as e:
+                treeinfo = ug.urlgrab("%s/treeinfo" % url,
+                                      "/tmp/.treeinfo", copy_local=True,
+                                      proxies=proxies, **ugopts)
+            except URLGrabError as e:
                 log.info("Error downloading treeinfo: %s", e)
-                response = None
-
-        if response:
-            # write the local treeinfo file
-            with open("/tmp/.treeinfo", "w") as f:
-                f.write(response.text)
+                treeinfo = None
 
-            # and also return the treeinfo contents as a string
-            return response.text
-        else:
-            return None
+        return treeinfo
 
     def _getReleaseVersion(self, url):
         """ Return the release version of the tree at the specified URL. """
diff --git a/pyanaconda/packaging/livepayload.py b/pyanaconda/packaging/livepayload.py
index f582219..ab7714b 100644
--- a/pyanaconda/packaging/livepayload.py
+++ b/pyanaconda/packaging/livepayload.py
@@ -34,6 +34,8 @@
 from time import sleep
 from threading import Lock
 import requests
+from urlgrabber.grabber import URLGrabber
+from urlgrabber.grabber import URLGrabError
 from pyanaconda.iutil import ProxyString, ProxyStringError, lowerASCII
 import hashlib
 import glob
@@ -195,19 +197,27 @@ def _updateKernelVersionList(self):
     def kernelVersionList(self):
         return self._kernelVersionList
 
-class DownloadProgress(object):
-    """ Provide methods for download progress reporting."""
-
-    def start(self, url, size):
-        """ Start of download
-
-            :param url:      url of the download
-            :type url:       str
+class URLGrabberProgress(object):
+    """ Provide methods for urlgrabber progress."""
+    def start(self, filename, url, basename, size, text):
+        """ Start of urlgrabber download
+
+            :param filename: path and file that download will be saved to
+            :type filename:  string
+            :param url:      url to download from
+            :type url:       string
+            :param basename: file that it will be saved to
+            :type basename:  string
             :param size:     length of the file
             :type size:      int
+            :param text:     unknown
+            :type text:      unknown
         """
+        self.filename = filename
         self.url = url
+        self.basename = basename
         self.size = size
+        self.text = text
         self._pct = -1
 
     def update(self, bytes_read):
@@ -223,6 +233,7 @@ def update(self, bytes_read):
         if pct == self._pct:
             return
         self._pct = pct
+
         progressQ.send_message(_("Downloading %(url)s (%(pct)d%%)") % \
                 {"url" : self.url, "pct" : pct})
 
@@ -315,37 +326,20 @@ def unsetup(self):
         ImagePayload.unsetup(self)
 
     def _preInstall_url_image(self):
-        """ Download the image using Requests with progress reporting"""
+        """ Download the image using urlgrabber """
+        # Setup urlgrabber and call back to download image to sysroot
+        progress = URLGrabberProgress()
+        ugopts = {"ssl_verify_peer": not self.data.method.noverifyssl,
+                  "ssl_verify_host": not self.data.method.noverifyssl,
+                  "proxies" : self._proxies,
+                  "progress_obj" : progress,
+                  "copy_local" : True}
 
         error = None
-        progress = DownloadProgress()
         try:
-            log.info("Starting image download")
-            with open(self.image_path, "wb") as f:
-                ssl_verify = not self.data.method.noverifyssl
-                response = requests.get(self.data.method.url, proxies=self._proxies, verify=ssl_verify, stream=True)
-                total_length = response.headers.get('content-length')
-                if total_length is None:  # no content length header
-                    # just download the file in one go and fake the progress reporting once done
-                    log.warning("content-length header is missing for the installation image, "
-                                "download progress reporting will not be available")
-                    f.write(response.content)
-                    size = f.tell()
-                    progress.start(self.data.method.url, size)
-                    progress.end(size)
-                else:
-                    # requests return headers as strings, so convert total_length to int
-                    progress.start(self.data.method.url, int(total_length))
-                    bytes_read = 0
-                    for buf in response.iter_content(1024 * 1024):  # 1 MB chunks
-                        if buf:
-                            f.write(buf)
-                            f.flush()
-                            bytes_read += len(buf)
-                            progress.update(bytes_read)
-                    progress.end(bytes_read)
-                log.info("Image download finished")
-        except requests.exceptions.RequestException as e:
+            ug = URLGrabber()
+            ug.urlgrab(self.data.method.url, self.image_path, **ugopts)
+        except URLGrabError as e:
             log.error("Error downloading liveimg: %s", e)
             error = e
         else:


-- 
To view this commit on github, visit https://github.com/rhinstaller/anaconda/commit/37c4bd68c715a259d5a4acacd06251c352a83eea


More information about the anaconda-patches mailing list