[PATCH] Replace python-urlgrabber with python-requests (#1141242)

Martin Kolman mkolman at redhat.com
Thu Jan 22 18:38:55 UTC 2015


Python urlgrabber is Python 2 only and kinda crufty.
Replace it with the Python 3 compatible python-requests module,
which provides a much nicer interface, is in active development
and is very widely used.

Signed-off-by: Martin Kolman <mkolman at redhat.com>
---
 anaconda.spec.in                    |  5 ++-
 pyanaconda/kickstart.py             | 13 +++++---
 pyanaconda/packaging/__init__.py    | 36 ++++++++++-----------
 pyanaconda/packaging/livepayload.py | 64 ++++++++++++++++++++-----------------
 4 files changed, 63 insertions(+), 55 deletions(-)

diff --git a/anaconda.spec.in b/anaconda.spec.in
index 6a81c97..964bee3 100644
--- a/anaconda.spec.in
+++ b/anaconda.spec.in
@@ -33,7 +33,6 @@ Source0: %{name}-%{version}.tar.bz2
 %define mehver 0.23-1
 %define sckeyboardver 1.3.1
 %define firewalldver 0.3.5-1
-%define pythonurlgrabberver 3.9.1-5
 %define utillinuxver 2.15.1
 %define dracutver 034-7
 %define isomd5sum 1.0.10
@@ -64,7 +63,7 @@ BuildRequires: pykickstart >= %{pykickstartver}
 BuildRequires: python-bugzilla
 %endif
 BuildRequires: python-devel
-BuildRequires: python-urlgrabber >= %{pythonurlgrabberver}
+BuildRequires: python-requests
 BuildRequires: python-nose
 BuildRequires: systemd
 BuildRequires: yum >= %{yumver}
@@ -100,7 +99,7 @@ Requires: rpm-python >= %{rpmver}
 Requires: parted >= %{partedver}
 Requires: pyparted >= %{pypartedver}
 Requires: yum >= %{yumver}
-Requires: python-urlgrabber >= %{pythonurlgrabberver}
+Requires: python-requests
 Requires: pykickstart >= %{pykickstartver}
 Requires: langtable-data >= %{langtablever}
 Requires: langtable-python >= %{langtablever}
diff --git a/pyanaconda/kickstart.py b/pyanaconda/kickstart.py
index 885bcd6..49c8613 100644
--- a/pyanaconda/kickstart.py
+++ b/pyanaconda/kickstart.py
@@ -44,8 +44,8 @@ import tempfile
 from pyanaconda.flags import flags, can_touch_runtime_system
 from pyanaconda.constants import ADDON_PATHS, IPMI_ABORTED
 import shlex
+import requests
 import sys
-import urlgrabber
 import pykickstart.commands as commands
 from pyanaconda import keyboard
 from pyanaconda import ntp
@@ -159,15 +159,18 @@ def getEscrowCertificate(escrowCerts, url):
     log.info("escrow: downloading %s", url)
 
     try:
-        f = urlgrabber.urlopen(url)
-    except urlgrabber.grabber.URLGrabError as e:
+        request = requests.get(url, verify=True)
+    except requests.exceptions.SSLError as e:
+        msg = _("SSL error while downloading the escrow certificate:\n\n%s") % e
+        raise KickstartError(msg)
+    except requests.exceptions.RequestException as e:
         msg = _("The following error was encountered while downloading the escrow certificate:\n\n%s") % e
         raise KickstartError(msg)
 
     try:
-        escrowCerts[url] = f.read()
+        escrowCerts[url] = request.content
     finally:
-        f.close()
+        request.close()
 
     return escrowCerts[url]
 
diff --git a/pyanaconda/packaging/__init__.py b/pyanaconda/packaging/__init__.py
index 2438b6f..4a0c28e 100644
--- a/pyanaconda/packaging/__init__.py
+++ b/pyanaconda/packaging/__init__.py
@@ -29,8 +29,7 @@
 """
 from __future__ import print_function
 import os, sys
-from urlgrabber.grabber import URLGrabber
-from urlgrabber.grabber import URLGrabError
+import requests
 import ConfigParser
 import shutil
 import time
@@ -69,8 +68,7 @@ import blivet.arch
 from blivet.platform import platform
 
 from pyanaconda.product import productName, productVersion
-import urlgrabber
-urlgrabber.grabber.default_grabber.opts.user_agent = "%s (anaconda)/%s" %(productName, productVersion)
+USER_AGENT = "%s (anaconda)/%s" %(productName, productVersion)
 
 from distutils.version import LooseVersion
 
@@ -340,9 +338,6 @@ class Payload(object):
         log.debug("retrieving treeinfo from %s (proxy: %s ; sslverify: %s)",
                   url, proxy_url, sslverify)
 
-        ugopts = {"ssl_verify_peer": sslverify,
-                  "ssl_verify_host": sslverify}
-
         proxies = {}
         if proxy_url:
             try:
@@ -353,21 +348,26 @@ class Payload(object):
                 log.info("Failed to parse proxy for _getTreeInfo %s: %s",
                          proxy_url, e)
 
-        ug = URLGrabber()
+        response = None
+        headers = {"user-agent": USER_AGENT}
         try:
-            treeinfo = ug.urlgrab("%s/.treeinfo" % url,
-                                  "/tmp/.treeinfo", copy_local=True,
-                                  proxies=proxies, **ugopts)
-        except URLGrabError as e:
+            response = requests.get("%s/.treeinfo" % url, headers=headers, proxies=proxies, verify=sslverify)
+        except requests.exceptions.RequestException as e:
             try:
-                treeinfo = ug.urlgrab("%s/treeinfo" % url,
-                                      "/tmp/.treeinfo", copy_local=True,
-                                      proxies=proxies, **ugopts)
-            except URLGrabError as e:
+                response = requests.get("%s/treeinfo" % url, headers=headers, proxies=proxies, verify=sslverify)
+            except requests.exceptions.RequestException as e:
                 log.info("Error downloading treeinfo: %s", e)
-                treeinfo = None
+                response = None
+
+        if response:
+            # write the the local treeinfo file
+            with open("/tmp/.treeinfo", "w") as f:
+                f.write(response.text())
 
-        return treeinfo
+            # and also return the treeinfo contents as a string
+            return response.text
+        else:
+            return None
 
     def _getReleaseVersion(self, url):
         """ Return the release version of the tree at the specified URL. """
diff --git a/pyanaconda/packaging/livepayload.py b/pyanaconda/packaging/livepayload.py
index 615bb61..d363ded 100644
--- a/pyanaconda/packaging/livepayload.py
+++ b/pyanaconda/packaging/livepayload.py
@@ -33,8 +33,7 @@ import os
 import stat
 from time import sleep
 from threading import Lock
-from urlgrabber.grabber import URLGrabber
-from urlgrabber.grabber import URLGrabError
+import requests
 from pyanaconda.iutil import ProxyString, ProxyStringError, lowerASCII
 import urllib
 import hashlib
@@ -197,27 +196,19 @@ class LiveImagePayload(ImagePayload):
     def kernelVersionList(self):
         return self._kernelVersionList
 
-class URLGrabberProgress(object):
-    """ Provide methods for urlgrabber progress."""
-    def start(self, filename, url, basename, size, text):
-        """ Start of urlgrabber download
-
-            :param filename: path and file that download will be saved to
-            :type filename:  string
-            :param url:      url to download from
-            :type url:       string
-            :param basename: file that it will be saved to
-            :type basename:  string
+class DownloadProgress(object):
+    """ Provide methods for download progress reporting."""
+
+    def start(self, url, size):
+        """ Start of download
+
+            :param url:      url of the download
+            :type url:       str
             :param size:     length of the file
             :type size:      int
-            :param text:     unknown
-            :type text:      unknown
         """
-        self.filename = filename
         self.url = url
-        self.basename = basename
         self.size = size
-        self.text = text
         self._pct = -1
 
     def update(self, bytes_read):
@@ -326,20 +317,35 @@ class LiveImageKSPayload(LiveImagePayload):
         ImagePayload.unsetup(self)
 
     def _preInstall_url_image(self):
-        """ Download the image using urlgrabber """
-        # Setup urlgrabber and call back to download image to sysroot
-        progress = URLGrabberProgress()
-        ugopts = {"ssl_verify_peer": not self.data.method.noverifyssl,
-                  "ssl_verify_host": not self.data.method.noverifyssl,
-                  "proxies" : self._proxies,
-                  "progress_obj" : progress,
-                  "copy_local" : True}
+        """ Download the image using Requests with progress reporting"""
 
         error = None
+        progress = DownloadProgress()
         try:
-            ug = URLGrabber()
-            ug.urlgrab(self.data.method.url, self.image_path, **ugopts)
-        except URLGrabError as e:
+            with open(self.image_path, "wb") as f:
+                ssl_verify = not self.data.method.noverifyssl
+                response = requests.get(self.data.method.url, proxies=self._proxies, verify=ssl_verify)
+                total_length = response.headers.get('content-length')
+                if total_length is None:  # no content length header
+                    # just download the file in one go and fake the progress reporting once done
+                    log.warning("content-length header is missing for the installation image, "
+                                "download progress reporting will not be available")
+                    f.write(response.content)
+                    size = f.tell()
+                    progress.start(self.data.method.url, size)
+                    progress.end(size)
+                else:
+                    # requests return headers as strings, so convert total_length to int
+                    progress.start(self.data.method.url, int(total_length))
+                    bytes_read = 0
+                    for buf in response.iter_content(1024 * 1024):  # 1 MB chunks
+                        if buf:
+                            f.write(buf)
+                            bytes_read += len(buf)
+                            progress.update(bytes_read)
+                    progress.end(bytes_read)
+
+        except requests.exceptions.RequestException as e:
             log.error("Error downloading liveimg: %s", e)
             error = e
         else:
-- 
2.1.0



More information about the anaconda-patches mailing list