[yum-commits] urlgrabber/grabber.py urlgrabber/mirror.py

[zpavlas at osuosl.org]

 urlgrabber/grabber.py |   21 +++++++++++++++------
 urlgrabber/mirror.py  |    6 +++++-
 2 files changed, 20 insertions(+), 7 deletions(-)

New commits:
commit adb37b9f9d1390c6f1d634ae6899d499bbe84138
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date:   Wed Jul 18 17:12:16 2012 +0200

    send 'tries' counter to mirror failure callback
    
    Make the removal of a failed mirror in async path conditional,
    so it's in line with synchronous urlgrab.  Mirror failures are
    counted and the mirror selection always uses mirror with least
    failures.  (overrides speed estimation)

diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index 057cbbe..83823ea 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -2176,13 +2176,18 @@ def parallel_wait(meter = 'text'):
                 continue
 
             if opts.mirror_group:
-                mg, failed = opts.mirror_group
+                mg, failed, removed = opts.mirror_group
+                failed[key] = failed.get(key, 0) + 1
                 opts.mirror = key
                 opts.exception = ug_err
-                action = _run_callback(mg.failure_callback, opts)
-                if not (action and action.get('fail')):
+                action = mg.default_action or {}
+                if mg.failure_callback:
+                    opts.tries = sum(failed.values())
+                    action.update(_run_callback(mg.failure_callback, opts))
+                if not action.get('fail', 0):
                     # mask this mirror and retry
-                    failed.add(key)
+                    if action.get('remove', 1):
+                        removed.add(key)
                     _async_queue.append(opts)
                     continue
 
@@ -2209,17 +2214,21 @@ def parallel_wait(meter = 'text'):
                 perform()
 
             if opts.mirror_group:
-                mg, failed = opts.mirror_group
+                mg, failed, removed = opts.mirror_group
 
                 # find the best mirror
                 best = None
+                best_speed = None
                 for mirror in mg.mirrors:
                     key = mirror['mirror']
-                    if key in failed: continue
+                    if key in removed: continue
 
                     # estimate mirror speed
                     speed = _TH.estimate(key)
                     speed /= 1 + host_con.get(key, 0)
+
+                    # 2-tuple to select mirror with least failures
+                    speed = -failed.get(key, 0), speed
                     if best is None or speed > best_speed:
                         best = mirror
                         best_speed = speed
diff --git a/urlgrabber/mirror.py b/urlgrabber/mirror.py
index 675c2bf..ac78b34 100644
--- a/urlgrabber/mirror.py
+++ b/urlgrabber/mirror.py
@@ -188,6 +188,7 @@ class MirrorGroup:
 
            obj.exception    = < exception that was raised >
            obj.mirror       = < the mirror that was tried >
+           obj.tries        = < the number of mirror tries so far >
            obj.relative_url = < url relative to the mirror >
            obj.url          = < full url that failed >
                               # .url is just the combination of .mirror
@@ -387,7 +388,9 @@ class MirrorGroup:
             try: del kw[k]
             except KeyError: pass
 
+        tries = 0
         while 1:
+            tries += 1
             mirrorchoice = self._get_mirror(gr)
             fullurl = self._join_url(mirrorchoice['mirror'], gr.url)
             kwargs = dict(mirrorchoice.get('kwargs', {}))
@@ -404,6 +407,7 @@ class MirrorGroup:
                 obj.mirror = mirrorchoice['mirror']
                 obj.relative_url = gr.url
                 obj.url = fullurl
+                obj.tries = tries
                 self._failure(gr, obj)
 
     def urlgrab(self, url, filename=None, **kwargs):
@@ -411,7 +415,7 @@ class MirrorGroup:
         kw['filename'] = filename
         if kw.get('async'):
             # enable mirror failovers in async path
-            kw['mirror_group'] = self, set()
+            kw['mirror_group'] = self, {}, set()
             kw['relative_url'] = url
         else:
             kw.pop('failfunc', None)