[yum-commits] urlgrabber/grabber.py urlgrabber/mirror.py

[zpavlas at osuosl.org]

 urlgrabber/grabber.py |   18 +++++++++++-------
 urlgrabber/mirror.py  |    6 +++---
 2 files changed, 14 insertions(+), 10 deletions(-)

New commits:
commit f580027821486b42c76bf2c1c16c828b76ae496e
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date:   Mon Aug 20 14:03:39 2012 +0200

    Use mirror kwargs and grabber options properly
    
    Don't modify urlgrab kwargs, since we can't undo that later.
    Instead, push mirror kwargs on top of grabber.opts.
    
    In multi downloader, update the grabber options when moving
    to the next mirror.

diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index bdcdfe3..b3071ce 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -1046,7 +1046,7 @@ class URLGrabber(object):
                                      retrycode, opts.retrycodes)
                 raise
     
-    def urlopen(self, url, **kwargs):
+    def urlopen(self, url, opts=None, **kwargs):
         """open the url and return a file object
         If a progress object or throttle value specified when this 
         object was created, then  a special file object will be 
@@ -1054,7 +1054,7 @@ class URLGrabber(object):
         like any other file object.
         """
         url = _to_utf8(url)
-        opts = self.opts.derive(**kwargs)
+        opts = (opts or self.opts).derive(**kwargs)
         if DEBUG: DEBUG.debug('combined options: %s' % repr(opts))
         (url,parts) = opts.urlparser.parse(url, opts) 
         opts.find_proxy(url, parts[0])
@@ -1062,14 +1062,14 @@ class URLGrabber(object):
             return PyCurlFileObject(url, filename=None, opts=opts)
         return self._retry(opts, retryfunc, url)
     
-    def urlgrab(self, url, filename=None, **kwargs):
+    def urlgrab(self, url, filename=None, opts=None, **kwargs):
         """grab the file at <url> and make a local copy at <filename>
         If filename is none, the basename of the url is used.
         urlgrab returns the filename of the local file, which may be 
         different from the passed-in filename if copy_local == 0.
         """
         url = _to_utf8(url)
-        opts = self.opts.derive(**kwargs)
+        opts = (opts or self.opts).derive(**kwargs)
         if DEBUG: DEBUG.debug('combined options: %s' % repr(opts))
         (url,parts) = opts.urlparser.parse(url, opts) 
         (scheme, host, path, parm, query, frag) = parts
@@ -1129,7 +1129,7 @@ class URLGrabber(object):
             opts.exception = e
             return _run_callback(opts.failfunc, opts)
     
-    def urlread(self, url, limit=None, **kwargs):
+    def urlread(self, url, limit=None, opts=None, **kwargs):
         """read the url into a string, up to 'limit' bytes
         If the limit is exceeded, an exception will be thrown.  Note
         that urlread is NOT intended to be used as a way of saying 
@@ -1137,7 +1137,7 @@ class URLGrabber(object):
         into memory, but don't use too much'
         """
         url = _to_utf8(url)
-        opts = self.opts.derive(**kwargs)
+        opts = (opts or self.opts).derive(**kwargs)
         if DEBUG: DEBUG.debug('combined options: %s' % repr(opts))
         (url,parts) = opts.urlparser.parse(url, opts) 
         opts.find_proxy(url, parts[0])
@@ -2213,7 +2213,7 @@ def parallel_wait(meter = 'text'):
             idx += 1
 
             # check global limit
-            while len(dl.running) >= opts.max_connections:
+            while len(dl.running) >= default_grabber.opts.max_connections:
                 perform()
 
             if opts.mirror_group:
@@ -2241,6 +2241,10 @@ def parallel_wait(meter = 'text'):
                     _run_callback(opts.failfunc, opts)
                     continue
 
+                # update the grabber object, apply mirror kwargs
+                grabber = best.get('grabber') or mg.grabber
+                opts.delegate = grabber.opts.derive(**best.get('kwargs', {}))
+
                 # update the current mirror and limit
                 key = best['mirror']
                 limit = best.get('kwargs', {}).get('max_connections', 2)
diff --git a/urlgrabber/mirror.py b/urlgrabber/mirror.py
index ac78b34..edbbbe8 100644
--- a/urlgrabber/mirror.py
+++ b/urlgrabber/mirror.py
@@ -393,13 +393,13 @@ class MirrorGroup:
             tries += 1
             mirrorchoice = self._get_mirror(gr)
             fullurl = self._join_url(mirrorchoice['mirror'], gr.url)
-            kwargs = dict(mirrorchoice.get('kwargs', {}))
-            kwargs.update(kw)
             grabber = mirrorchoice.get('grabber') or self.grabber
+            # apply mirrorchoice kwargs on top of grabber.opts
+            opts = grabber.opts.derive(**mirrorchoice.get('kwargs', {}))
             func_ref = getattr(grabber, func)
             if DEBUG: DEBUG.info('MIRROR: trying %s -> %s', url, fullurl)
             try:
-                return func_ref( *(fullurl,), **kwargs )
+                return func_ref( *(fullurl,), opts=opts, **kw )
             except URLGrabError, e:
                 if DEBUG: DEBUG.info('MIRROR: failed')
                 obj = CallbackObject()