[yum-commits] urlgrabber/grabber.py

[zpavlas at osuosl.org]

 urlgrabber/grabber.py |    7 +++++++
 1 file changed, 7 insertions(+)

New commits:
commit e08effe45ca403f935dd06c4dbbf79731fa87c6f
Author: Zdenek Pavlas <zpavlas at redhat.com>
Date:   Mon Jun 17 16:50:27 2013 +0200

    Switch to max_connections=1 after timing out.  BZ 853432

diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index 74178d7..6b409e3 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -2151,6 +2151,7 @@ def parallel_wait(meter=None):
 
     dl = _ExternalDownloaderPool()
     host_con = {} # current host connection counts
+    single = set() # hosts in single connection mode
 
     def start(opts, tries):
         opts.tries = tries
@@ -2197,6 +2198,10 @@ def parallel_wait(meter=None):
 
             if ug_err is None:
                 continue
+            if ug_err.errno == pycurl.E_OPERATION_TIMEOUTED:
+                # One possible cause is connection-limited server.
+                # Turn on the max_connections=1 override. BZ 853432
+                single.add(key)
 
             retry = opts.retry or 0
             if opts.failure_callback:
@@ -2297,6 +2302,8 @@ def parallel_wait(meter=None):
 
             # check host limit, then start
             key, limit = opts.async
+            if key in single:
+                limit = 1
             while host_con.get(key, 0) >= limit:
                 perform()
             if DEBUG: