[yum-commits] 12 commits - docs/yum.conf.5 yum/config.py yum/__init__.py yum/metalink.py yum/repos.py yum/yumRepo.py
zpavlas at osuosl.org
zpavlas at osuosl.org
Thu May 3 11:34:54 UTC 2012
docs/yum.conf.5 | 11 ++++++
yum/__init__.py | 57 +++++++++++++++++++-------------
yum/config.py | 2 +
yum/metalink.py | 7 +--
yum/repos.py | 42 +++++++++++++++++++++++
yum/yumRepo.py | 99 ++++++++++++++++++++++++++++++++++----------------------
6 files changed, 154 insertions(+), 64 deletions(-)
New commits:
commit 02528bc021950e0dfea3fc32681dc44b32ac6a62
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date: Tue Apr 17 09:45:15 2012 +0200
smart evaluation of repo.async option
False => don't use parallel downloader.
True => use it "if possible".
This is meant to automatically disable parallel
downloading for repositories created by plugins,
which would likely break.
YumRepository instances must set 'self._async'
explicitly to support parallel downloading.
diff --git a/docs/yum.conf.5 b/docs/yum.conf.5
index 14668f7..b456074 100644
--- a/docs/yum.conf.5
+++ b/docs/yum.conf.5
@@ -939,8 +939,8 @@ for any given command. Defaults to False.
.IP
\fBasync \fR
-If set (the default) and urlgrabber supports it, yum will use parallel downloader
-for packages from this repo.
+If set to True Yum will download packages and metadata from this repo in
+parallel, if possible. Defaults to True.
.SH "URL INCLUDE SYNTAX"
.LP
diff --git a/yum/__init__.py b/yum/__init__.py
index b189d02..7e67d20 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -581,7 +581,8 @@ class YumBase(depsolve.Depsolve):
repo.basecachedir = self.conf.cachedir
repo.yumvar.update(self.conf.yumvar)
repo.cfg = parser
-
+ # Enable parallel downloading
+ repo._async = repo.async
return repo
def disablePlugins(self):
@@ -2260,7 +2261,7 @@ class YumBase(depsolve.Depsolve):
text = os.path.basename(po.relativepath)
kwargs = {}
- if async and po.repo.async:
+ if async and po.repo._async:
kwargs['failfunc'] = lambda obj, po=po: adderror(po, exception2msg(obj.exception))
kwargs['async'] = True
elif not (i == 1 and not local_size[0] and remote_size == po.size):
diff --git a/yum/repos.py b/yum/repos.py
index 9ebf895..bd8f1a4 100644
--- a/yum/repos.py
+++ b/yum/repos.py
@@ -78,7 +78,7 @@ class RepoStorage:
repos = []
for repo in self.listEnabled():
- if repo.async and repo._commonLoadRepoXML(repo):
+ if repo._async and repo._commonLoadRepoXML(repo):
mdtypes = repo._mdpolicy2mdtypes()
downloading = repo._commonRetrieveDataMD_list(mdtypes)
repos.append((repo, downloading, [False]))
@@ -325,7 +325,7 @@ class RepoStorage:
if hasattr(urlgrabber.grabber, 'parallel_wait'):
# download all metadata in parallel
for repo in myrepos:
- if repo.async:
+ if repo._async:
sack = repo.getPackageSack()
sack._retrieve_async(repo, data)
urlgrabber.grabber.parallel_wait()
diff --git a/yum/yumRepo.py b/yum/yumRepo.py
index 124ad59..76e05d2 100644
--- a/yum/yumRepo.py
+++ b/yum/yumRepo.py
@@ -307,6 +307,7 @@ class YumRepository(Repository, config.RepoConf):
self._grabfunc = None
self._grab = None
+ self._async = False
def __cmp__(self, other):
""" Sort yum repos. by cost, and then by alphanumeric on their id. """
commit 2ce91e9b5e79bb9d1a4d643a73a5abedbafc8f85
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date: Wed Mar 28 16:44:38 2012 +0200
populateSack(): parallel metadata downloading
diff --git a/yum/repos.py b/yum/repos.py
index d7f7f09..9ebf895 100644
--- a/yum/repos.py
+++ b/yum/repos.py
@@ -322,6 +322,14 @@ class RepoStorage:
else:
data = [ mdtype ]
+ if hasattr(urlgrabber.grabber, 'parallel_wait'):
+ # download all metadata in parallel
+ for repo in myrepos:
+ if repo.async:
+ sack = repo.getPackageSack()
+ sack._retrieve_async(repo, data)
+ urlgrabber.grabber.parallel_wait()
+
for repo in myrepos:
sack = repo.getPackageSack()
try:
diff --git a/yum/yumRepo.py b/yum/yumRepo.py
index d156c74..124ad59 100644
--- a/yum/yumRepo.py
+++ b/yum/yumRepo.py
@@ -124,6 +124,25 @@ class YumPackageSack(packageSack.PackageSack):
# umm, wtf?
pass
+ def _retrieve_async(self, repo, data):
+ """ Just schedule the metadata downloads """
+
+ for item in data:
+ if item in self.added.get(repo, []):
+ continue
+ if item == 'metadata':
+ mydbtype = 'primary_db'
+ elif item == 'filelists':
+ mydbtype = 'filelists_db'
+ elif item == 'otherdata':
+ mydbtype = 'other_db'
+ else:
+ continue
+
+ if self._check_db_version(repo, mydbtype):
+ if not self._check_uncompressed_db(repo, mydbtype):
+ repo._retrieveMD(mydbtype, async=True, failfunc=None)
+
def populate(self, repo, mdtype='metadata', callback=None, cacheonly=0):
if mdtype == 'all':
data = ['metadata', 'filelists', 'otherdata']
commit a522869c21768d53c3861be9c0a2394a3930ad66
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date: Wed Mar 28 15:59:26 2012 +0200
retrieveAllMD(): parallel metadata downloading
Metadata are downloaded in parallel just before postreposetup.
This implements the reverting behavior.
diff --git a/yum/repos.py b/yum/repos.py
index 3793bad..d7f7f09 100644
--- a/yum/repos.py
+++ b/yum/repos.py
@@ -22,6 +22,7 @@ import misc
import Errors
from packageSack import MetaSack
+import urlgrabber.grabber
from weakref import proxy as weakref
@@ -67,6 +68,38 @@ class RepoStorage:
self._cache_enabled_repos = []
self.quick_enable_disable = {}
+ def retrieveAllMD(self):
+ """ Download metadata for all enabled repositories,
+ based on mdpolicy.
+ """
+
+ if not hasattr(urlgrabber.grabber, 'parallel_wait'):
+ return
+
+ repos = []
+ for repo in self.listEnabled():
+ if repo.async and repo._commonLoadRepoXML(repo):
+ mdtypes = repo._mdpolicy2mdtypes()
+ downloading = repo._commonRetrieveDataMD_list(mdtypes)
+ repos.append((repo, downloading, [False]))
+
+ # with sizes first, then without sizes..
+ for no_size in (False, True):
+ for repo, downloading, error in repos:
+ def failfunc(obj, error=error):
+ error[0] = True
+ for (ndata, nmdtype) in downloading:
+ if (ndata.size is None) == no_size:
+ repo._retrieveMD(nmdtype, async=True, failfunc=failfunc)
+ urlgrabber.grabber.parallel_wait()
+
+ # done or revert
+ for repo, downloading, error in repos:
+ if error[0]: # some MD failed?
+ repo._revertOldRepoXML()
+ else:
+ repo._commonRetrieveDataMD_done(downloading)
+
def doSetup(self, thisrepo = None):
self.ayum.plugins.run('prereposetup')
@@ -89,6 +122,7 @@ class RepoStorage:
self.disableRepo(repo.id)
self._setup = True
+ self.retrieveAllMD()
self.ayum.plugins.run('postreposetup')
def __str__(self):
commit ae5055823318efb3b4bcefeb22bbe0a34f0b9a43
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date: Wed Mar 28 12:56:36 2012 +0200
_commonRetrieveDataMD(): split the code
Add _commonRetrieveDataMD_list() that returns the list of metadata
objects to be downloaded.
Add _commonRetrieveDataMD_done() to unpack and commit new files
after they have been succesfully downloaded.
diff --git a/yum/yumRepo.py b/yum/yumRepo.py
index 2592df1..d156c74 100644
--- a/yum/yumRepo.py
+++ b/yum/yumRepo.py
@@ -1326,6 +1326,17 @@ Insufficient space in download directory %s
into the delete list, this means metadata can change filename
without us leaking it. """
+ downloading = self._commonRetrieveDataMD_list(mdtypes)
+ for (ndata, nmdtype) in downloading:
+ if not self._retrieveMD(nmdtype, retrieve_can_fail=True):
+ self._revertOldRepoXML()
+ return False
+ self._commonRetrieveDataMD_done(downloading)
+ return True
+
+ def _commonRetrieveDataMD_list(self, mdtypes):
+ """ Return a list of metadata to be retrieved """
+
def _mdtype_eq(omdtype, odata, nmdtype, ndata):
""" Check if two returns from _get_mdtype_data() are equal. """
if ndata is None:
@@ -1357,8 +1368,7 @@ Insufficient space in download directory %s
# Inited twice atm. ... sue me
self._oldRepoMDData['new_MD_files'] = []
- downloading_with_size = []
- downloading_no_size = []
+ downloading = []
for mdtype in all_mdtypes:
(nmdtype, ndata) = self._get_mdtype_data(mdtype)
@@ -1395,43 +1405,20 @@ Insufficient space in download directory %s
# No old repomd data, but we might still have uncompressed MD
if self._groupCheckDataMDValid(ndata, nmdtype, mdtype):
continue
+ downloading.append((ndata, nmdtype))
+ return downloading
- if ndata.size is None:
- downloading_no_size.append((ndata, nmdtype))
- else:
- downloading_with_size.append((ndata, nmdtype))
-
- if len(downloading_with_size) == 1:
- downloading_no_size.extend(downloading_with_size)
- downloading_with_size = []
-
- remote_size = 0
- local_size = 0
- for (ndata, nmdtype) in downloading_with_size: # Get total size...
- remote_size += int(ndata.size)
+ def _commonRetrieveDataMD_done(self, downloading):
+ """ Uncompress the downloaded metadata """
- for (ndata, nmdtype) in downloading_with_size:
- urlgrabber.progress.text_meter_total_size(remote_size, local_size)
- if not self._retrieveMD(nmdtype, retrieve_can_fail=True):
- self._revertOldRepoXML()
- return False
- local_size += int(ndata.size)
- urlgrabber.progress.text_meter_total_size(0)
- for (ndata, nmdtype) in downloading_no_size:
- if not self._retrieveMD(nmdtype, retrieve_can_fail=True):
- self._revertOldRepoXML()
- return False
-
- for (ndata, nmdtype) in downloading_with_size + downloading_no_size:
+ for (ndata, nmdtype) in downloading:
local = self._get_mdtype_fname(ndata, False)
if nmdtype.endswith("_db"): # Uncompress any compressed files
dl_local = local
local = misc.decompress(dl_local)
misc.unlink_f(dl_local)
self._oldRepoMDData['new_MD_files'].append(local)
-
self._doneOldRepoXML()
- return True
def _groupLoadRepoXML(self, text=None, mdtypes=None):
""" Retrieve the new repomd.xml from the repository, then check it
commit 9da352220ac6b9532ce9a43d654f74388d5c66bd
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date: Mon Mar 19 17:19:10 2012 +0100
_retrieveMD(): make it useable for async downloading.
diff --git a/yum/yumRepo.py b/yum/yumRepo.py
index 26b05fb..2592df1 100644
--- a/yum/yumRepo.py
+++ b/yum/yumRepo.py
@@ -1579,7 +1579,7 @@ Insufficient space in download directory %s
mdtype can be 'primary', 'filelists', 'other' or 'group'."""
return self._retrieveMD(mdtype)
- def _retrieveMD(self, mdtype, retrieve_can_fail=False):
+ def _retrieveMD(self, mdtype, retrieve_can_fail=False, **kwargs):
""" Internal function, use .retrieveMD() from outside yum. """
# Note that this can raise Errors.RepoMDError if mdtype doesn't exist
# for this repo.
@@ -1617,7 +1617,9 @@ Insufficient space in download directory %s
return local # it's the same return the local one
try:
- checkfunc = (self.checkMD, (mdtype,), {})
+ def checkfunc(obj):
+ self.checkMD(obj, mdtype)
+ self.retrieved[mdtype] = 1
text = "%s/%s" % (self.id, mdtype)
if thisdata.size is None:
reget = None
@@ -1633,7 +1635,8 @@ Insufficient space in download directory %s
checkfunc=checkfunc,
text=text,
cache=self.http_caching == 'all',
- size=thisdata.size)
+ size=thisdata.size,
+ **kwargs)
except Errors.RepoError:
if retrieve_can_fail:
return None
@@ -1644,7 +1647,6 @@ Insufficient space in download directory %s
raise Errors.RepoError, \
"Could not retrieve %s matching remote checksum from %s" % (local, self)
else:
- self.retrieved[mdtype] = 1
return local
commit 0282b38f1d5be2c07771b4570b91dc220b256d6c
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date: Thu Feb 2 12:50:51 2012 +0100
max_connections: man page update
diff --git a/docs/yum.conf.5 b/docs/yum.conf.5
index 90aece3..14668f7 100644
--- a/docs/yum.conf.5
+++ b/docs/yum.conf.5
@@ -348,8 +348,8 @@ Determines how yum resolves host names.
\fBmax_connections \fR
The maximum number of simultaneous connections. This overrides the urlgrabber
-default of 5 connections. Note that there are also per-mirror limits, and the
-downloader honors these too.
+default of 5 connections. Note that there are also implicit per-mirror limits
+and the downloader honors these too.
.IP
\fBsslcacert \fR
commit 9fdc18d828565ebddf5ccd993b93fffd4be519a7
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date: Thu Jan 26 16:25:27 2012 +0100
enable timedhosts
Q: Make it configurable?
diff --git a/yum/__init__.py b/yum/__init__.py
index b3cc4f5..b189d02 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -377,6 +377,7 @@ class YumBase(depsolve.Depsolve):
mc = self._conf.max_connections
if mc > 0:
default_grabber.opts.max_connections = mc
+ default_grabber.opts.timedhosts = self._conf.cachedir + '/timedhosts'
# We don't want people accessing/altering preconf after it becomes
# worthless. So we delete it, and thus. it'll raise AttributeError
commit 7e8c76173133fa3bb581fb0765c93c0497b1344a
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date: Thu Jan 26 11:52:30 2012 +0100
add prefix '(%s/%s): ' to blocking urlgrabs only
diff --git a/yum/__init__.py b/yum/__init__.py
index 84537ef..b3cc4f5 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -2257,16 +2257,14 @@ class YumBase(depsolve.Depsolve):
if po in errors:
del errors[po]
+ text = os.path.basename(po.relativepath)
kwargs = {}
if async and po.repo.async:
kwargs['failfunc'] = lambda obj, po=po: adderror(po, exception2msg(obj.exception))
kwargs['async'] = True
+ elif not (i == 1 and not local_size[0] and remote_size == po.size):
+ text = '(%s/%s): %s' % (i, len(remote_pkgs), text)
try:
- if i == 1 and not local_size[0] and remote_size == po.size:
- text = os.path.basename(po.relativepath)
- else:
- text = '(%s/%s): %s' % (i, len(remote_pkgs),
- os.path.basename(po.relativepath))
po.repo.getPackage(po,
checkfunc=checkfunc,
text=text,
commit 4857fd3a0a487912a84b65a11b560d83e6e5ec77
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date: Thu Jan 26 11:32:50 2012 +0100
downloadPkgs: use parallel downloader
diff --git a/yum/__init__.py b/yum/__init__.py
index e8f28f6..84537ef 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -2223,6 +2223,7 @@ class YumBase(depsolve.Depsolve):
i = 0
local_size = [0]
done_repos = set()
+ async = hasattr(urlgrabber.grabber, 'parallel_wait')
for po in remote_pkgs:
# Recheck if the file is there, works around a couple of weird
# edge cases.
@@ -2256,6 +2257,10 @@ class YumBase(depsolve.Depsolve):
if po in errors:
del errors[po]
+ kwargs = {}
+ if async and po.repo.async:
+ kwargs['failfunc'] = lambda obj, po=po: adderror(po, exception2msg(obj.exception))
+ kwargs['async'] = True
try:
if i == 1 and not local_size[0] and remote_size == po.size:
text = os.path.basename(po.relativepath)
@@ -2266,9 +2271,12 @@ class YumBase(depsolve.Depsolve):
checkfunc=checkfunc,
text=text,
cache=po.repo.http_caching != 'none',
+ **kwargs
)
except Errors.RepoError, e:
adderror(po, exception2msg(e))
+ if async:
+ urlgrabber.grabber.parallel_wait()
if hasattr(urlgrabber.progress, 'text_meter_total_size'):
urlgrabber.progress.text_meter_total_size(0)
commit ef033251187312535bfd1e5dcddfa444b1ac50d6
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date: Wed Jan 25 13:53:40 2012 +0100
downloadPkgs: add the 'success' code path to checkfunc
diff --git a/yum/__init__.py b/yum/__init__.py
index 12c552f..e8f28f6 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -2221,7 +2221,7 @@ class YumBase(depsolve.Depsolve):
urlgrabber.progress.text_meter_total_size(remote_size)
beg_download = time.time()
i = 0
- local_size = 0
+ local_size = [0]
done_repos = set()
for po in remote_pkgs:
# Recheck if the file is there, works around a couple of weird
@@ -2234,41 +2234,41 @@ class YumBase(depsolve.Depsolve):
remote_size -= po.size
if hasattr(urlgrabber.progress, 'text_meter_total_size'):
urlgrabber.progress.text_meter_total_size(remote_size,
- local_size)
+ local_size[0])
continue
if os.path.getsize(local) >= po.size:
os.unlink(local)
- checkfunc = (self.verifyPkg, (po, 1), {})
- try:
- if i == 1 and not local_size and remote_size == po.size:
- text = os.path.basename(po.relativepath)
- else:
- text = '(%s/%s): %s' % (i, len(remote_pkgs),
- os.path.basename(po.relativepath))
- mylocal = po.repo.getPackage(po,
- checkfunc=checkfunc,
- text=text,
- cache=po.repo.http_caching != 'none',
- )
- local_size += po.size
+ def checkfunc(obj, po=po):
+ self.verifyPkg(obj, po, 1)
+ local_size[0] += po.size
if hasattr(urlgrabber.progress, 'text_meter_total_size'):
urlgrabber.progress.text_meter_total_size(remote_size,
- local_size)
+ local_size[0])
if po.repoid not in done_repos:
+ done_repos.add(po.repoid)
# Check a single package per. repo. ... to give a hint to
# the user on big downloads.
result, errmsg = self.sigCheckPkg(po)
if result != 0:
self.verbose_logger.warn("%s", errmsg)
- done_repos.add(po.repoid)
+ po.localpath = obj.filename
+ if po in errors:
+ del errors[po]
+ try:
+ if i == 1 and not local_size[0] and remote_size == po.size:
+ text = os.path.basename(po.relativepath)
+ else:
+ text = '(%s/%s): %s' % (i, len(remote_pkgs),
+ os.path.basename(po.relativepath))
+ po.repo.getPackage(po,
+ checkfunc=checkfunc,
+ text=text,
+ cache=po.repo.http_caching != 'none',
+ )
except Errors.RepoError, e:
adderror(po, exception2msg(e))
- else:
- po.localpath = mylocal
- if po in errors:
- del errors[po]
if hasattr(urlgrabber.progress, 'text_meter_total_size'):
urlgrabber.progress.text_meter_total_size(0)
commit d3523081b32537262fb9d199b81fe6cfd06b0737
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date: Thu Jan 26 11:16:48 2012 +0100
getPackage, _getFile: kwargs pass-through
diff --git a/yum/yumRepo.py b/yum/yumRepo.py
index 8c68034..26b05fb 100644
--- a/yum/yumRepo.py
+++ b/yum/yumRepo.py
@@ -760,7 +760,7 @@ class YumRepository(Repository, config.RepoConf):
def _getFile(self, url=None, relative=None, local=None, start=None, end=None,
copy_local=None, checkfunc=None, text=None, reget='simple',
- cache=True, size=None):
+ cache=True, size=None, **kwargs):
"""retrieve file from the mirrorgroup for the repo
relative to local, optionally get range from
start to end, also optionally retrieve from a specific baseurl"""
@@ -854,7 +854,8 @@ Insufficient space in download directory %s
reget = reget,
checkfunc=checkfunc,
http_headers=headers,
- size=size
+ size=size,
+ **kwargs
)
except URLGrabError, e:
errstr = "failure: %s from %s: %s" % (relative, self.id, e)
@@ -866,7 +867,7 @@ Insufficient space in download directory %s
return result
__get = _getFile
- def getPackage(self, package, checkfunc=None, text=None, cache=True):
+ def getPackage(self, package, checkfunc=None, text=None, cache=True, **kwargs):
remote = package.relativepath
local = package.localPkg()
basepath = package.basepath
@@ -883,6 +884,7 @@ Insufficient space in download directory %s
text=text,
cache=cache,
size=package.size,
+ **kwargs
)
def getHeader(self, package, checkfunc = None, reget = 'simple',
commit b6e9c6553fbe5000bf1ca20f5a611d3c2ed612e0
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date: Fri Oct 21 15:25:29 2011 +0200
add 'max_connections' and 'async' options
If set in config, update the urlgrabber's default.
Copy max_connections parsed from the metalink to MG.
diff --git a/docs/yum.conf.5 b/docs/yum.conf.5
index 9eff4c3..90aece3 100644
--- a/docs/yum.conf.5
+++ b/docs/yum.conf.5
@@ -345,6 +345,13 @@ Determines how yum resolves host names.
`6' or `IPv6': resolve to IPv6 addresses only.
.IP
+\fBmax_connections \fR
+
+The maximum number of simultaneous connections. This overrides the urlgrabber
+default of 5 connections. Note that there are also per-mirror limits, and the
+downloader honors these too.
+
+.IP
\fBsslcacert \fR
Path to the directory containing the databases of the certificate authorities
yum should use to verify SSL certificates. Defaults to none - uses system
@@ -929,7 +936,11 @@ as greater/less than any other. defaults to 1000
If set to True yum will continue running if this repository cannot be
contacted for any reason. This should be set carefully as all repos are consulted
for any given command. Defaults to False.
+
.IP
+\fBasync \fR
+If set (the default) and urlgrabber supports it, yum will use parallel downloader
+for packages from this repo.
.SH "URL INCLUDE SYNTAX"
.LP
diff --git a/yum/__init__.py b/yum/__init__.py
index 40bb15b..12c552f 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -373,6 +373,11 @@ class YumBase(depsolve.Depsolve):
self._conf = config.readMainConfig(startupconf)
+ # update urlgrabber defaults
+ mc = self._conf.max_connections
+ if mc > 0:
+ default_grabber.opts.max_connections = mc
+
# We don't want people accessing/altering preconf after it becomes
# worthless. So we delete it, and thus. it'll raise AttributeError
del self.preconf
diff --git a/yum/config.py b/yum/config.py
index ee5876b..2bf4f45 100644
--- a/yum/config.py
+++ b/yum/config.py
@@ -788,6 +788,7 @@ class YumConf(StartupConf):
ip_resolve = CaselessSelectionOption(
allowed = ('ipv4', 'ipv6', 'whatever'),
mapper = {'4': 'ipv4', '6': 'ipv6'})
+ max_connections = IntOption(0)
http_caching = SelectionOption('all', ('none', 'packages', 'all'))
metadata_expire = SecondsOption(60 * 60 * 6) # Time in seconds (6h).
@@ -957,6 +958,7 @@ class RepoConf(BaseConfig):
ssl_check_cert_permissions = Inherit(YumConf.ssl_check_cert_permissions)
skip_if_unavailable = BoolOption(False)
+ async = BoolOption(True)
class VersionGroupConf(BaseConfig):
"""Option definitions for version groups."""
diff --git a/yum/metalink.py b/yum/metalink.py
index aaa4f25..51895fd 100755
--- a/yum/metalink.py
+++ b/yum/metalink.py
@@ -180,6 +180,7 @@ class MetaLinkRepoMD:
self.repomd = None
self.old_repomds = []
self.mirrors = []
+ self._host2mc = {}
if not os.path.exists(filename):
raise MetaLinkRepoErrorParseFail, "File %s does not exist" %filename
try:
@@ -225,8 +226,6 @@ class MetaLinkRepoMD:
# Get the hostname from a url, stripping away any usernames/passwords
# Borrowd from fastestmirror
url2host = lambda url: url.split('/')[2].split('@')[-1]
- hosts = set() # Don't want multiple urls for one host in plain mode
- # The list of URLs is sorted, so http is before ftp
for mirror in self.mirrors:
url = mirror.url
@@ -237,9 +236,9 @@ class MetaLinkRepoMD:
elif (url.startswith("http:") or url.startswith("ftp:") or
url.startswith("https:")):
host = url2host(url)
- if host in hosts:
+ if host in self._host2mc:
continue
- hosts.add(host)
+ self._host2mc[host] = mirror.max_connections
else:
continue
diff --git a/yum/yumRepo.py b/yum/yumRepo.py
index 655bbaa..8c68034 100644
--- a/yum/yumRepo.py
+++ b/yum/yumRepo.py
@@ -484,8 +484,20 @@ class YumRepository(Repository, config.RepoConf):
copy_local=self.copy_local,
reget='simple',
**ugopts)
+ def add_mc(url):
+ host = urlparse.urlsplit(url).netloc
+ mc = self.metalink_data._host2mc.get(host)
+ if mc > 0:
+ url = {
+ 'mirror': misc.to_utf8(url),
+ 'kwargs': { 'max_connections': mc },
+ }
+ return url
+ urls = self.urls
+ if self.metalink:
+ urls = map(add_mc, urls)
- self._grab = mgclass(self._grabfunc, self.urls,
+ self._grab = mgclass(self._grabfunc, urls,
failure_callback=self.mirror_failure_obj)
def _default_grabopts(self, cache=True):
More information about the Yum-commits
mailing list