[Yum-devel] Fastest mirror selection plugin

[Luke Macken]

On Sat, Aug 13, 2005 at 12:27:40PM -0400, Luke Macken wrote:
| Updated patch attached.
| 
| Fixed:
|  o No longer need the 'thread' class; use 'threading' Lock objects
|    instead.
|  o Made add_result and poll_mirrors private functions.
| 
| Things to think about:
| 
|  o MGSortable is very ambiguous.  Maybe something to the effect of
|    MGSpeedSortable and MGSpeedSorted ?
| 
|  o Maybe embed the PollThread class inside MGSortable.
| 
|  o I don't know how persistent these MirrorGroup objects are throughout
|    the yum install/update process, but a way to avoid the polling the
|    mirrors multiple times would be to make a singleton MGSortable object
|    (or mirrorlist) if not persistent in memory, then pickled out to disk
|    somewhere.  There are most likely easier ways, but I don't know
|    enough about yum/urlgrabber to say so :)

Fixed typo.
-------------- next part --------------
--- mirror.py.orig	2005-08-13 12:13:10.000000000 -0400
+++ mirror.py	2005-08-13 12:18:40.000000000 -0400
@@ -89,7 +89,10 @@
 # $Id: mirror.py,v 1.12 2004/09/07 21:19:54 mstenner Exp $
 
 import random
-import thread  # needed for locking to make this threadsafe
+import threading
+import socket
+import urlparse
+import time
 
 from grabber import URLGrabError, CallbackObject
 
@@ -253,7 +256,7 @@
         self.grabber = grabber
         self.mirrors = self._parse_mirrors(mirrors)
         self._next = 0
-        self._lock = thread.allocate_lock()
+        self._lock = threading.Lock()
         self.default_action = None
         self._process_kwargs(kwargs)
 
@@ -457,5 +460,67 @@
         MirrorGroup.__init__(self, grabber, mirrors, **kwargs)
         random.shuffle(self.mirrors)
 
+class MGSortable(MirrorGroup):
+
+    def __init__(self, grabber, mirrors, **kwargs):
+        MirrorGroup.__init__(self, grabber, mirrors, **kwargs)
+        self._mirrortimes = {}
+        self._threads = []
+        # XXX make this configurable
+        socket.setdefaulttimeout(3)
+
+    def sort(self):
+        self._poll_mirrors()
+        mirrors = [(v, k) for k, v in self._mirrortimes.items()]
+        mirrors.sort()
+        self.mirrors = []
+        for mtime, murl in mirrors:
+            md = {'mirror': murl}
+            self.mirrors.append(md)
+
+    def _poll_mirrors(self):
+        for mirror in self.mirrors:
+            pollThread = PollThread(self, mirror)
+            pollThread.start()
+            self._threads.append(pollThread)
+        while len(self._threads) > 0:
+            if self._threads[0].isAlive():
+                self._threads[0].join()
+            del (self._threads[0])
+
+    def _add_result(self, mirror, host, time):
+        self._lock.acquire()
+        if DEBUG:
+            DBPRINT(" * %s : %f secs" % (host, time))
+        self._mirrortimes[mirror['mirror']] = time
+        self._lock.release()
+
+class MGFastest(MGSortable):
+
+    def __init__(self, grabber, mirrors, **kwargs):
+        MGSortable.__init__(self, grabber, mirrors, **kwargs)
+        self.sort()
+
+class PollThread(threading.Thread):
+
+    def __init__(self, parent, mirror):
+        threading.Thread.__init__(self)
+        self.parent = parent
+        self.mirror = mirror
+        self.host = urlparse.urlparse(mirror['mirror'])[1]
+
+    def run(self):
+        try:
+            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            time_before = time.time()
+            sock.connect((self.host, 80))
+            result = time.time() - time_before
+            sock.close()
+            self.parent._add_result(self.mirror, self.host, result)
+        except:
+            if DEBUG:
+                DBPRINT(" * %s : dead" % self.host)
+
+
 if __name__ == '__main__':
     pass