[PATCH] Properly retry package downloads (#924860)

Martin Kolman mkolman at redhat.com
Tue Feb 11 20:33:49 UTC 2014


There are basically two cases where package download can fail:
- when YUM is populating the transaction
  (this actually just checks the file exists)
- when the transaction is started and packages are downloaded and
  installed

Previously Anaconda did not attempt to retry the first one at all
and made an unlimited number of attempts in the second one.

This has been changed and Anaconda now does 10 retries separated
by a pause that progressively grows from 0.5 to 256 seconds for
the final wait before giving up. With this mechanism it should
be possible to successfully install even with a very shaky connection.

Signed-off-by: Martin Kolman <mkolman at redhat.com>
---
 scripts/anaconda-yum | 105 +++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 86 insertions(+), 19 deletions(-)

diff --git a/scripts/anaconda-yum b/scripts/anaconda-yum
index 12a562f..259b0b7 100755
--- a/scripts/anaconda-yum
+++ b/scripts/anaconda-yum
@@ -22,6 +22,7 @@ import logging
 import os
 import sys
 import argparse
+import time
 import rpm
 import rpmUtils
 import yum
@@ -29,6 +30,19 @@ from urlgrabber.grabber import URLGrabError
 
 YUM_PLUGINS = ["fastestmirror", "langpacks"]
 
+MAX_DOWNLOAD_RETRIES = 10
+
+
+def get_retry_delay(retry_number):
+    """ The retry delay start short and gets longer as the
+        number of retries increases, for 10 retries, the delay increases
+        from 0.5 to 256 seconds for the final delay before giving up
+
+        :param int retry_number: retry counter
+        :returns float: time to wait in seconds
+    """
+    return 0.25*(2**retry_number)
+
 def setup_parser():
     """ Setup argparse with supported arguments
 
@@ -109,12 +123,26 @@ def run_yum_transaction(release, arch, yum_conf, install_root, ts_file, script_l
             yb.ts.ts.setColor(3)
 
         print("DEBUG: populate transaction set")
-        try:
-            # uses dsCallback.transactionPopulation
-            yb.populateTs(keepold=0)
-        except RepoError as e:
-            print("ERROR: error populating transaction: %s" % e)
-            print("QUIT:")
+        for retry_count in xrange(1, MAX_DOWNLOAD_RETRIES+1):
+            # retry count == 0 -> first attempt
+            # retry count > 0  -> retry
+            if retry_count:
+                # retry after waiting a bit
+                time.sleep(get_retry_delay(retry_count))
+                print("DEBUG: error populating transaction, retrying (%d/%d)"
+                      % (retry_count, MAX_DOWNLOAD_RETRIES))
+            try:
+                # uses dsCallback.transactionPopulation
+                yb.populateTs(keepold=0)
+                break
+            except RepoError as e:
+                continue
+        else:
+            # else = no break called = no successful attempt
+            print("ERROR: error populating transaction after %d retries: %s"
+                  % (retry_count, e))
+            # we don't need to print "QUIT:" there, the finally clause
+            # of the toplevel try-block will do that for us
             return
 
         print("DEBUG: check transaction set")
@@ -255,7 +283,17 @@ class RPMCallback(object):
             raise Exception("rpmcallback getRepo failed")
 
         self.package_file = None
-        while self.package_file is None:
+        retry_message = ""
+        error_message = ""
+        exception_message = ""
+
+        for retry_count in xrange(0, MAX_DOWNLOAD_RETRIES+1):
+            # retry count == 0 -> first attempt
+            # retry count > 0  -> retry
+            if retry_count and retry_message:
+                time.sleep(get_retry_delay(retry_count))  # wait a bit before retry
+                print("DEBUG: %s (%d/%d)" % (retry_message, retry_count, MAX_DOWNLOAD_RETRIES))
+
             try:
                 # checkfunc gets passed to yum's use of URLGrabber which
                 # then calls it with the file being fetched. verifyPkg
@@ -271,21 +309,50 @@ class RPMCallback(object):
                 if self.debug:
                     print("DEBUG: getPackage %s" % txmbr.name)
                 package_path = repo.getPackage(txmbr.po, checkfunc=checkfunc)
+                break
             except URLGrabError as e:
-                print("ERROR: URLGrabError: %s" % e)
-                raise Exception("rpmcallback failed")
+                # url grabber should have an internal retry mechanism,
+                # so if we get this exception, it probably already exhausted
+                # all retry attempts and we don't need to retry ourselves
+                if retry_count < MAX_DOWNLOAD_RETRIES:
+                    retry_message = "rpmcallback failed (URLGrabError), retrying"
+                else:
+                    # run out of retries
+                    error_message = "rpmcallback failed (URLGrabError) after %d retries: %s" % \
+                                    (retry_count, e)
+                    exception_message = "rpmcallback failed"
+
             except (yum.Errors.NoMoreMirrorsRepoError, IOError) as e:
-                if os.path.exists(txmbr.po.localPkg()):
-                    os.unlink(txmbr.po.localPkg())
-                    print("DEBUG: retrying download of %s" % txmbr.po)
-                    continue
-                print("ERROR: getPackage error: %s" % e)
-                raise Exception("getPackage failed")
-            except yum.Errors.RepoError as e:
-                print("DEBUG: RepoError: %s" % e)
-                continue
+                # for some reason, this is the exception you will get if
+                # the package file you want to download vanishes, not URLGrabError
+
+                if retry_count < MAX_DOWNLOAD_RETRIES:
+                    retry_message = "retrying download of %s" % txmbr.po
+                    # remove any unfinished downloads of this package
+                    if os.path.exists(txmbr.po.localPkg()):
+                        os.unlink(txmbr.po.localPkg())
+                else:
+                    # run out of retries
+                    error_message = "getPackage error after %d retries: %s" % \
+                                    (retry_count, e)
+                    exception_message = "getPackage failed"
 
-            self.package_file = open(package_path)
+            except yum.Errors.RepoError as e:
+                if retry_count < MAX_DOWNLOAD_RETRIES:
+                    retry_message = "RepoError, retrying: %s" % e
+                else:
+                    # run out of retries
+                    error_message = "RepoError after %d retries: %s" % \
+                                    (retry_count, e)
+                    exception_message = "too many (%d) consecutive repo errors" % \
+                                        retry_count
+
+        else:  # report what went wrong & abort installation
+            print("ERROR: %s" % error_message)
+            raise Exception(exception_message)
+
+        # if we got this far, there should be a package available
+        self.package_file = open(package_path)
 
         if self.debug:
             print("DEBUG: opening package %s" % self.package_file.name)
-- 
1.8.5.3



More information about the anaconda-patches mailing list