[yum-commits] urlgrabber/grabber.py urlgrabber/mirror.py

[zpavlas at osuosl.org]

 urlgrabber/grabber.py |   10 +++++++---
 urlgrabber/mirror.py  |   24 +++++++++++++++++-------
 2 files changed, 24 insertions(+), 10 deletions(-)

New commits:
commit 501e80136b4741c64441dc0c3c00f59600592241
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date:   Tue Aug 21 17:25:10 2012 +0200

    Set .errors attribute on exceptions from MirrorGroup failures.

diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index 87d8b59..ffd5a10 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -2198,13 +2198,14 @@ def parallel_wait(meter=None):
                 continue
 
             if opts.mirror_group:
-                mg, failed, removed = opts.mirror_group
+                mg, errors, failed, removed = opts.mirror_group
+                errors.append((opts.url, str(ug_err)))
                 failed[key] = failed.get(key, 0) + 1
                 opts.mirror = key
                 opts.exception = ug_err
                 action = mg.default_action or {}
                 if mg.failure_callback:
-                    opts.tries = sum(failed.values())
+                    opts.tries = len(errors)
                     action = dict(action) # update only the copy
                     action.update(_run_callback(mg.failure_callback, opts))
                 if not action.get('fail', 0):
@@ -2213,6 +2214,8 @@ def parallel_wait(meter=None):
                         removed.add(key)
                     _async_queue.append(opts)
                     continue
+                # fail=1 from callback
+                ug_err.errors = errors
 
             # urlgrab failed
             opts.exception = ug_err
@@ -2237,7 +2240,7 @@ def parallel_wait(meter=None):
                 perform()
 
             if opts.mirror_group:
-                mg, failed, removed = opts.mirror_group
+                mg, errors, failed, removed = opts.mirror_group
 
                 # find the best mirror
                 best = None
@@ -2258,6 +2261,7 @@ def parallel_wait(meter=None):
 
                 if best is None:
                     opts.exception = URLGrabError(256, _('No more mirrors to try.'))
+                    opts.exception.errors = errors
                     _run_callback(opts.failfunc, opts)
                     continue
 
diff --git a/urlgrabber/mirror.py b/urlgrabber/mirror.py
index edbbbe8..b17be17 100644
--- a/urlgrabber/mirror.py
+++ b/urlgrabber/mirror.py
@@ -76,8 +76,8 @@ CUSTOMIZATION
        'grabber' is omitted, the default grabber will be used.  If
        kwargs are omitted, then (duh) they will not be used.
 
-       kwarg 'max_connections' is used to store the max connection
-       limit of this mirror.
+       kwarg 'max_connections' limits the number of concurrent
+       connections to this mirror.
 
     3) Pass keyword arguments when instantiating the mirror group.
        See, for example, the failure_callback argument.
@@ -90,6 +90,7 @@ CUSTOMIZATION
 """
 
 
+import sys
 import random
 import thread  # needed for locking to make this threadsafe
 
@@ -130,7 +131,9 @@ class MirrorGroup:
         files)
 
       * if the local list is ever exhausted, a URLGrabError will be
-        raised (errno=256, no more mirrors)
+        raised (errno=256, No more mirrors).  The 'errors' attribute
+        holds a list of (full_url, errmsg) tuples.  This contains
+        all URLs tried and the corresponding error messages.
 
     OPTIONS
 
@@ -157,7 +160,8 @@ class MirrorGroup:
 
         The 'fail' option will cause immediate failure by re-raising
         the exception and no further attempts to get the current
-        download.
+        download.  As in the "No more mirrors" case, the 'errors'
+        attribute is set in the exception object.
 
         This dict can be set at instantiation time,
           mg = MirrorGroup(grabber, mirrors, default_action={'fail':1})
@@ -286,7 +290,9 @@ class MirrorGroup:
         #   return a random mirror so that multiple mirrors get used
         #   even without failures.
         if not gr.mirrors:
-            raise URLGrabError(256, _('No more mirrors to try.'))
+            e = URLGrabError(256, _('No more mirrors to try.'))
+            e.errors = gr.errors
+            raise e
         return gr.mirrors[gr._next]
 
     def _failure(self, gr, cb_obj):
@@ -313,7 +319,9 @@ class MirrorGroup:
         a.update(action)
         action = a
         self.increment_mirror(gr, action)
-        if action and action.get('fail', 0): raise
+        if action and action.get('fail', 0):
+            sys.exc_info()[1].errors = gr.errors
+            raise
 
     def increment_mirror(self, gr, action={}):
         """Tell the mirror object increment the mirror index
@@ -383,6 +391,7 @@ class MirrorGroup:
         gr.url  = url
         gr.kw   = dict(kw)
         self._load_gr(gr)
+        gr.errors = []
 
         for k in self.options:
             try: del kw[k]
@@ -402,6 +411,7 @@ class MirrorGroup:
                 return func_ref( *(fullurl,), opts=opts, **kw )
             except URLGrabError, e:
                 if DEBUG: DEBUG.info('MIRROR: failed')
+                gr.errors.append((fullurl, str(e)))
                 obj = CallbackObject()
                 obj.exception = e
                 obj.mirror = mirrorchoice['mirror']
@@ -415,7 +425,7 @@ class MirrorGroup:
         kw['filename'] = filename
         if kw.get('async'):
             # enable mirror failovers in async path
-            kw['mirror_group'] = self, {}, set()
+            kw['mirror_group'] = self, [], {}, set()
             kw['relative_url'] = url
         else:
             kw.pop('failfunc', None)