[Yum-devel] [PATCH 9/9] Implement 'failfunc' callback.

[Zdeněk Pavlas]

This callback is called when urlgrab request fails.
If grab is wrapped in a mirror group, only the mirror
group issues the callback.
---
 urlgrabber/grabber.py |   20 +++++++++++++++++++-
 urlgrabber/mirror.py  |   11 ++++++++++-
 2 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index f0f4c6b..dc6a342 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -339,6 +339,15 @@ RETRY RELATED ARGUMENTS
     but it cannot (without severe trickiness) prevent the exception
     from being raised.
 
+  failfunc = None
+
+    The callback that gets called when urlgrab request fails.
+    If defined, urlgrab() calls it instead of raising URLGrabError.
+    Callback syntax is identical to failure_callback.
+
+    Contrary to failure_callback, it's called only once.  It's primary
+    purpose is to use urlgrab() without a try/except block.
+
   interrupt_callback = None
 
     This callback is called if KeyboardInterrupt is received at any
@@ -1031,7 +1040,16 @@ class URLGrabber(object):
                 fo.close()
             return filename
         
-        return self._retry(opts, retryfunc, url, filename)
+        try: return self._retry(opts, retryfunc, url, filename)
+        except URLGrabError, e:
+            if hasattr(opts, 'mirror_group'):
+                raise e # MG tries other mirror
+
+            # urlgrab failed
+            if hasattr(opts, 'failfunc'):
+                opts.exception = e
+                return _callback(opts.failfunc, opts)
+            raise e
     
     def urlread(self, url, limit=None, **kwargs):
         """read the url into a string, up to 'limit' bytes
diff --git a/urlgrabber/mirror.py b/urlgrabber/mirror.py
index 8731aed..222a519 100644
--- a/urlgrabber/mirror.py
+++ b/urlgrabber/mirror.py
@@ -91,6 +91,7 @@ import random
 import thread  # needed for locking to make this threadsafe
 
 from grabber import URLGrabError, CallbackObject, DEBUG, _to_utf8
+from grabber import _callback
 
 def _(st): 
     return st
@@ -391,6 +392,7 @@ class MirrorGroup:
             grabber = mirrorchoice.get('grabber') or self.grabber
             func_ref = getattr(grabber, func)
             if DEBUG: DEBUG.info('MIRROR: trying %s -> %s', url, fullurl)
+            kwargs['mirror_group'] = self, gr, mirrorchoice
             try:
                 return func_ref( *(fullurl,), **kwargs )
             except URLGrabError, e:
@@ -406,7 +408,14 @@ class MirrorGroup:
         kw = dict(kwargs)
         kw['filename'] = filename
         func = 'urlgrab'
-        return self._mirror_try(func, url, kw)
+        try: return self._mirror_try(func, url, kw)
+        except URLGrabError, e:
+            opts = CallbackObject(url=url, exception=e, **kw)
+
+            # urlgrab failed
+            if hasattr(opts, 'failfunc'):
+                return _callback(opts.failfunc, opts)
+            raise e
     
     def urlopen(self, url, **kwargs):
         kw = dict(kwargs)
-- 
1.7.4.4