[Yum-devel] [PATCH] Make _curl_cache thread-local.

Zdeněk Pavlas zpavlas at redhat.com
Mon Jun 27 15:29:43 UTC 2011


While libcurl is (supposed to be) thread-safe, pycurl.Curl() object cannot
be shared between multiple threads.
---
 urlgrabber/grabber.py |   14 ++++++++------
 1 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index f6f57bd..26c0d0b 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -1458,7 +1458,11 @@ class PyCurlFileObject(object):
                 raise err
 
     def _do_open(self):
-        self.curl_obj = _curl_cache
+        # Curl() object is thread-local
+        i = thread.get_ident()
+        if not i in _curl_cache:
+            _curl_cache[i] = pycurl.Curl()
+        self.curl_obj = _curl_cache[i]
         self.curl_obj.reset() # reset all old settings away, just in case
         # setup any ranges
         self._set_opts()
@@ -1769,14 +1773,12 @@ class PyCurlFileObject(object):
             urllib.addinfourl, via. urllib.URLopener.* """
         return self.url
         
-_curl_cache = pycurl.Curl() # make one and reuse it over and over and over
+_curl_cache = {}
 
 def reset_curl_obj():
     """To make sure curl has reread the network/dns info we force a reload"""
-    global _curl_cache
-    _curl_cache.close()
-    _curl_cache = pycurl.Curl()
-
+    for c in _curl_cache.values(): c.close()
+    _curl_cache.clear()
 
     
 
-- 
1.7.4.4



More information about the Yum-devel mailing list