[Yum-devel] [PATCH 02/10] Implement 'failfunc' callback.

Zdeněk Pavlas zpavlas at redhat.com
Mon Nov 7 10:55:09 UTC 2011


This callback is called when urlgrab request fails.
If grab is wrapped in a mirror group, only the mirror
group issues the callback.
---
 urlgrabber/grabber.py |   25 ++++++++++++++++++++++++-
 urlgrabber/mirror.py  |    9 +++++++--
 2 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index 5614bcd..e4a7eba 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -339,6 +339,15 @@ RETRY RELATED ARGUMENTS
     but it cannot (without severe trickiness) prevent the exception
     from being raised.
 
+  failfunc = None
+
+    The callback that gets called when urlgrab request fails.
+    If defined, urlgrab() calls it instead of raising URLGrabError.
+    Callback syntax is identical to failure_callback.
+
+    Contrary to failure_callback, it's called only once.  It's primary
+    purpose is to use urlgrab() without a try/except block.
+
   interrupt_callback = None
 
     This callback is called if KeyboardInterrupt is received at any
@@ -838,6 +847,7 @@ class URLGrabberOptions:
         self.retry = None
         self.retrycodes = [-1,2,4,5,6,7]
         self.checkfunc = None
+        self.failfunc = _do_raise
         self.copy_local = 0
         self.close_connection = 0
         self.range = None
@@ -891,6 +901,15 @@ class URLGrabberOptions:
         s = s + indent + '}'
         return s
 
+def _do_raise(obj):
+    raise obj.exception
+
+def _run_callback(cb, obj):
+    if callable(cb):
+        return cb(obj)
+    cb, arg, karg = cb
+    return cb(obj, *arg, **karg)
+
 class URLGrabber(object):
     """Provides easy opening of URLs with a variety of options.
     
@@ -1023,7 +1042,11 @@ class URLGrabber(object):
                 fo.close()
             return filename
         
-        return self._retry(opts, retryfunc, url, filename)
+        try:
+            return self._retry(opts, retryfunc, url, filename)
+        except URLGrabError, e:
+            opts.exception = e
+            return _run_callback(opts.failfunc, opts)
     
     def urlread(self, url, limit=None, **kwargs):
         """read the url into a string, up to 'limit' bytes
diff --git a/urlgrabber/mirror.py b/urlgrabber/mirror.py
index 8731aed..575ef7b 100644
--- a/urlgrabber/mirror.py
+++ b/urlgrabber/mirror.py
@@ -91,6 +91,7 @@ import random
 import thread  # needed for locking to make this threadsafe
 
 from grabber import URLGrabError, CallbackObject, DEBUG, _to_utf8
+from grabber import _run_callback, _do_raise
 
 def _(st): 
     return st
@@ -254,7 +255,7 @@ class MirrorGroup:
     # if these values are found in **kwargs passed to one of the urlXXX
     # methods, they will be stripped before getting passed on to the
     # grabber
-    options = ['default_action', 'failure_callback']
+    options = ['default_action', 'failure_callback', 'failfunc']
     
     def _process_kwargs(self, kwargs):
         self.failure_callback = kwargs.get('failure_callback')
@@ -406,7 +407,11 @@ class MirrorGroup:
         kw = dict(kwargs)
         kw['filename'] = filename
         func = 'urlgrab'
-        return self._mirror_try(func, url, kw)
+        try:
+            return self._mirror_try(func, url, kw)
+        except URLGrabError, e:
+            obj = CallbackObject(url=url, filename=filename, exception=e, **kwargs)
+            return _run_callback(kwargs.get('failfunc', _do_raise), obj)
     
     def urlopen(self, url, **kwargs):
         kw = dict(kwargs)
-- 
1.7.4.4



More information about the Yum-devel mailing list