[yum-commits] Branch 'multi-downloader' - 4 commits - urlgrabber/grabber.py

[zpavlas at osuosl.org]

 urlgrabber/grabber.py |   12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

New commits:
commit bef05297d3b9533ce19e76cb64eaf99a6c1e7b1b
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date:   Fri May 18 10:37:04 2012 +0200

    URL sanity checking.  BZ 822632
    
    Ignore bogus hostnames, and make parsing of timedhosts file
    a bit more robust.

diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index 094be77..3b0e238 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -2268,7 +2268,7 @@ class _TH:
         if filename and _TH.dirty is None:
             try:
                 for line in open(filename):
-                    host, speed, fail, ts = line.split()
+                    host, speed, fail, ts = line.split(' ', 3)
                     _TH.hosts[host] = int(speed), int(fail), int(ts)
             except IOError: pass
             _TH.dirty = False
@@ -2291,6 +2291,9 @@ class _TH:
     def update(url, dl_size, dl_time, ug_err):
         _TH.load()
         host = urlparse.urlsplit(url).netloc
+        if not host or ' ' in host:
+            if DEBUG: DEBUG.warn('malformed url: %s', repr(url))
+            return
         speed, fail, ts = _TH.hosts.get(host) or (0, 0, 0)
         now = time.time()
 
commit 71a6132f22f8ec9972689ac46b42ca4daa0988cb
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date:   Thu Apr 19 17:20:03 2012 +0200

    use proxy='' to disable default proxies
    
    Instead of ignoring proxy settings '_none_' or '',
    use it to disable default proxies in libcurl.

diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index f89b5d7..094be77 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -866,8 +866,9 @@ class URLGrabberOptions:
                     proxy = self.proxies.get('https')
                 elif scheme == 'https':
                     proxy = self.proxies.get('http')
-            if proxy != '_none_':
-                self.proxy = proxy
+            if proxy == '_none_':
+                proxy = ''
+            self.proxy = proxy
             return
 
         if self.libproxy:
@@ -1375,7 +1376,7 @@ class PyCurlFileObject(object):
             self.curl_obj.setopt(pycurl.MAX_RECV_SPEED_LARGE, int(opts.raw_throttle()))
             
         # proxy
-        if opts.proxy:
+        if opts.proxy is not None:
             self.curl_obj.setopt(pycurl.PROXY, opts.proxy)
             self.curl_obj.setopt(pycurl.PROXYAUTH, pycurl.HTTPAUTH_ANY)
 
commit b2548b16ea1345d47a22a0d40d303aeb08592966
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date:   Thu Apr 19 17:14:27 2012 +0200

    Revert "'proxy=_none_' should disable default proxies"
    
    This reverts commit d4aab164239b916b6a7c1da67b5e72fb088fc579.

diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index aef9269..f89b5d7 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -866,7 +866,8 @@ class URLGrabberOptions:
                     proxy = self.proxies.get('https')
                 elif scheme == 'https':
                     proxy = self.proxies.get('http')
-            self.proxy = proxy
+            if proxy != '_none_':
+                self.proxy = proxy
             return
 
         if self.libproxy:
@@ -1374,9 +1375,7 @@ class PyCurlFileObject(object):
             self.curl_obj.setopt(pycurl.MAX_RECV_SPEED_LARGE, int(opts.raw_throttle()))
             
         # proxy
-        if opts.proxy == '_none_':
-            self.curl_obj.setopt(pycurl.PROXY, '')
-        elif opts.proxy:
+        if opts.proxy:
             self.curl_obj.setopt(pycurl.PROXY, opts.proxy)
             self.curl_obj.setopt(pycurl.PROXYAUTH, pycurl.HTTPAUTH_ANY)
 
commit 24c594111314b267e00aeac1850a3fa7ef028293
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date:   Thu Apr 19 15:07:49 2012 +0200

    'proxy=_none_' should disable default proxies
    
    libcurl uses default proxies.  to disable that, we have to set
    it explicitly to ''. '_none_' was once used for this purpose
    so make it work again.

diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index f89b5d7..aef9269 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -866,8 +866,7 @@ class URLGrabberOptions:
                     proxy = self.proxies.get('https')
                 elif scheme == 'https':
                     proxy = self.proxies.get('http')
-            if proxy != '_none_':
-                self.proxy = proxy
+            self.proxy = proxy
             return
 
         if self.libproxy:
@@ -1375,7 +1374,9 @@ class PyCurlFileObject(object):
             self.curl_obj.setopt(pycurl.MAX_RECV_SPEED_LARGE, int(opts.raw_throttle()))
             
         # proxy
-        if opts.proxy:
+        if opts.proxy == '_none_':
+            self.curl_obj.setopt(pycurl.PROXY, '')
+        elif opts.proxy:
             self.curl_obj.setopt(pycurl.PROXY, opts.proxy)
             self.curl_obj.setopt(pycurl.PROXYAUTH, pycurl.HTTPAUTH_ANY)