[Yum-devel] Selecting mirrors on the "freshness" of their repomd, not just their speed
Willem Riede
wrrhdev at riede.org
Mon Dec 5 01:57:48 UTC 2005
In
https://www.redhat.com/archives/fedora-test-list/2005-November/msg00610.html
Seth told me that submitting code was the only way to help make the subject
happen, so I created a variant of the fastestmirror plugin that also checks
the age of repomd.xml on the site.
A non-up-to-date mirror, however fast, is not going to get the most recent
updates to me, so it is of less use than a slightly slower, but up-to-date
mirror.
A full version is attached, a diff (probably mangled by my mail client)
follows in-line. It works for me - YMMV.
Comments? Willem Riede.
--- fastestmirror.py 2005-11-27 19:43:04.000000000 -0500
+++ bestmirror.py 2005-12-04 20:33:32.000000000 -0500
@@ -1,6 +1,6 @@
#!/usr/bin/env python
#
-# Version: 0.2.2
+# Version: 0.2.2b
#
# A plugin for the Yellowdog Updater Modified which sorts each repo's
# mirrorlist by connection speed prior to metadata download.
@@ -9,7 +9,7 @@
# make sure you have 'plugins=1' in your /etc/yum.conf.
#
# Configuration Options
-# /etc/yum/pluginconf.d/fastestmirror.conf:
+# /etc/yum/pluginconf.d/bestmirror.conf:
# [main]
# enabled=1
# verbose=1
@@ -27,6 +27,8 @@
# GNU General Public License for more details.
#
# Changes
+# * Dec 04 2005 Willem Riede <wrrhdev at riede.org>
+# - clone to BestMirror and check repo "freshness"
# * Nov 26 2005 Luke Macken <lmacken at redhat.com> - 0.2.2
# - Merge Panu's persistent changes to cache timings
# - Add 'hostfilepath' as configuration string
@@ -45,10 +47,15 @@
import sys
import time
import socket
+import os, stat
import urlparse
+import rfc822
import threading
import string
+from urlgrabber.grabber import URLGrabber
+from urlgrabber.grabber import URLGrabError
+
from yum.plugins import TYPE_INTERFACE, TYPE_CORE
from yum.plugins import PluginYumExit
@@ -70,11 +77,11 @@
def postreposetup_hook(conduit):
read_timedhosts()
repomirrors = {}
- conduit.info(2, "Determining fastest mirrors")
+ conduit.info(2, "Determining best mirrors")
repos = conduit.getRepos()
for repo in repos.listEnabled():
if not repomirrors.has_key(str(repo)):
- repomirrors[str(repo)] = FastestMirror(repo.urls).get_mirrorlist()
+ repomirrors[str(repo)] = BestMirror(repo.urls).get_mirrorlist()
repo.set('urls', repomirrors[str(repo)])
repo.set('failovermethod', 'priority')
repo.check()
@@ -86,8 +93,8 @@
try:
hostfile = file(hostfilepath)
for line in hostfile.readlines():
- host, time = line.split()
- timedhosts[host] = float(time)
+ host, time, age = line.split()
+ timedhosts[host] = [ float(time), float(age) ]
hostfile.close()
except IOError:
pass
@@ -96,10 +103,10 @@
global timedhosts
hostfile = file(hostfilepath, 'w')
for host in timedhosts.keys():
- hostfile.write('%s %s\n' % (host, timedhosts[host]))
+ hostfile.write('%s %s %s\n' % ( host, timedhosts[host][0] ,
timedhosts[host][1] ))
hostfile.close()
-class FastestMirror:
+class BestMirror:
def __init__(self, mirrorlist):
self.mirrorlist = mirrorlist
@@ -124,12 +131,12 @@
self.threads[0].join()
del(self.threads[0])
- def _add_result(self, mirror, host, time):
+ def _add_result(self, mirror, host, time, age):
global timedhosts
self.lock.acquire()
- if verbose: print " * %s : %f secs" % (host, time)
- self.results[mirror] = time
- timedhosts[host] = time
+ if verbose: print " * %s : latency %f secs, age %f hour" % (host,
time, age)
+ self.results[mirror] = time + age
+ timedhosts[host] = [ time , age ]
self.lock.release()
class PollThread(threading.Thread):
@@ -148,18 +155,25 @@
self.port = 21
elif uService == "file":
self.host = "127.0.0.1"
+ self.filename = mirror[7:]+'repodata/repomd.xml'
else:
self.port = -2
def run(self):
try:
if timedhosts.has_key(self.host):
- result = timedhosts[self.host]
+ latency, age = timedhosts[self.host]
if verbose:
- print "%s already timed: %s" % (self.host, result)
+ print "%s already timed: %s,%s" % (self.host, latency,
age)
else:
if self.host == "127.0.0.1" :
- result = 0
+ latency = 0
+ try:
+ s = os.stat(self.filename)
+ except OSError:
+ age = 99999000000
+ else:
+ age = ( time.time() - float(s[8]) ) / 3600
else:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
uPort = string.find(self.host,":")
@@ -168,13 +182,28 @@
self.host = self.host[:uPort]
time_before = time.time()
sock.connect((self.host, self.port))
- result = time.time() - time_before
+ latency = time.time() - time_before
sock.close()
- self.parent._add_result(self.mirror, self.host, result)
+ try:
+ g = URLGrabber(prefix=self.mirror,
+ timeout=5,
+ reget='simple')
+ f = g.urlopen('repodata/repomd.xml')
+ except URLGrabError, e:
+ print e #### print '[Errno %i] %s' % (e.errno,
e.strerror)
+ age = 9999900000
+ else:
+ hdr = f.info()
+ mt = hdr.getdate_tz('last-modified')
+ ms = rfc822.mktime_tz(mt)
+ f.close()
+ age = (time.time() - ms) / 3600
+
+ self.parent._add_result(self.mirror, self.host, latency, age)
except:
if verbose:
print " * %s : dead" % self.host
- self.parent._add_result(self.mirror, self.host, 99999999999)
+ self.parent._add_result(self.mirror, self.host, 999999,
99999000000)
def main():
global verbose
@@ -190,7 +219,7 @@
for arg in sys.argv:
mirrorlist.append(arg)
- print "Result: " + str(FastestMirror(mirrorlist).get_mirrorlist())
+ print "Result: " + str(BestMirror(mirrorlist).get_mirrorlist())
if __name__ == '__main__':
main()
-------------- next part --------------
A non-text attachment was scrubbed...
Name: bestmirror.py
Type: text/x-python
Size: 7572 bytes
Desc: not available
Url : http://lists.baseurl.org/pipermail/yum-devel/attachments/20051205/0ce4a991/attachment.py
More information about the Yum-devel
mailing list