[yum-commits] 12 commits - docs/yum.conf.5 yum/config.py yum/__init__.py yum/metalink.py yum/repos.py yum/yumRepo.py

zpavlas at osuosl.org zpavlas at osuosl.org
Thu May 3 11:34:54 UTC 2012


 docs/yum.conf.5 |   11 ++++++
 yum/__init__.py |   57 +++++++++++++++++++-------------
 yum/config.py   |    2 +
 yum/metalink.py |    7 +--
 yum/repos.py    |   42 +++++++++++++++++++++++
 yum/yumRepo.py  |   99 ++++++++++++++++++++++++++++++++++----------------------
 6 files changed, 154 insertions(+), 64 deletions(-)

New commits:
commit 02528bc021950e0dfea3fc32681dc44b32ac6a62
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date:   Tue Apr 17 09:45:15 2012 +0200

    smart evaluation of repo.async option
    
    False => don't use parallel downloader.
    True => use it "if possible".
    
    This is meant to automatically disable parallel
    downloading for repositories created by plugins,
    which would likely break.
    
    YumRepository instances must set 'self._async'
    explicitly to support parallel downloading.

diff --git a/docs/yum.conf.5 b/docs/yum.conf.5
index 14668f7..b456074 100644
--- a/docs/yum.conf.5
+++ b/docs/yum.conf.5
@@ -939,8 +939,8 @@ for any given command. Defaults to False.
 
 .IP
 \fBasync \fR
-If set (the default) and urlgrabber supports it, yum will use parallel downloader
-for packages from this repo.
+If set to True Yum will download packages and metadata from this repo in
+parallel, if possible.  Defaults to True.
 
 .SH "URL INCLUDE SYNTAX"
 .LP
diff --git a/yum/__init__.py b/yum/__init__.py
index b189d02..7e67d20 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -581,7 +581,8 @@ class YumBase(depsolve.Depsolve):
         repo.basecachedir = self.conf.cachedir
         repo.yumvar.update(self.conf.yumvar)
         repo.cfg = parser
-
+        # Enable parallel downloading
+        repo._async = repo.async
         return repo
 
     def disablePlugins(self):
@@ -2260,7 +2261,7 @@ class YumBase(depsolve.Depsolve):
 
             text = os.path.basename(po.relativepath)
             kwargs = {}
-            if async and po.repo.async:
+            if async and po.repo._async:
                 kwargs['failfunc'] = lambda obj, po=po: adderror(po, exception2msg(obj.exception))
                 kwargs['async'] = True
             elif not (i == 1 and not local_size[0] and remote_size == po.size):
diff --git a/yum/repos.py b/yum/repos.py
index 9ebf895..bd8f1a4 100644
--- a/yum/repos.py
+++ b/yum/repos.py
@@ -78,7 +78,7 @@ class RepoStorage:
 
         repos = []
         for repo in self.listEnabled():
-            if repo.async and repo._commonLoadRepoXML(repo):
+            if repo._async and repo._commonLoadRepoXML(repo):
                 mdtypes = repo._mdpolicy2mdtypes()
                 downloading = repo._commonRetrieveDataMD_list(mdtypes)
                 repos.append((repo, downloading, [False]))
@@ -325,7 +325,7 @@ class RepoStorage:
         if hasattr(urlgrabber.grabber, 'parallel_wait'):
             # download all metadata in parallel
             for repo in myrepos:
-                if repo.async:
+                if repo._async:
                     sack = repo.getPackageSack()
                     sack._retrieve_async(repo, data)
             urlgrabber.grabber.parallel_wait()
diff --git a/yum/yumRepo.py b/yum/yumRepo.py
index 124ad59..76e05d2 100644
--- a/yum/yumRepo.py
+++ b/yum/yumRepo.py
@@ -307,6 +307,7 @@ class YumRepository(Repository, config.RepoConf):
 
         self._grabfunc = None
         self._grab = None
+        self._async = False
 
     def __cmp__(self, other):
         """ Sort yum repos. by cost, and then by alphanumeric on their id. """
commit 2ce91e9b5e79bb9d1a4d643a73a5abedbafc8f85
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date:   Wed Mar 28 16:44:38 2012 +0200

    populateSack(): parallel metadata downloading

diff --git a/yum/repos.py b/yum/repos.py
index d7f7f09..9ebf895 100644
--- a/yum/repos.py
+++ b/yum/repos.py
@@ -322,6 +322,14 @@ class RepoStorage:
         else:
             data = [ mdtype ]
          
+        if hasattr(urlgrabber.grabber, 'parallel_wait'):
+            # download all metadata in parallel
+            for repo in myrepos:
+                if repo.async:
+                    sack = repo.getPackageSack()
+                    sack._retrieve_async(repo, data)
+            urlgrabber.grabber.parallel_wait()
+
         for repo in myrepos:
             sack = repo.getPackageSack()
             try:
diff --git a/yum/yumRepo.py b/yum/yumRepo.py
index d156c74..124ad59 100644
--- a/yum/yumRepo.py
+++ b/yum/yumRepo.py
@@ -124,6 +124,25 @@ class YumPackageSack(packageSack.PackageSack):
             # umm, wtf?
             pass
 
+    def _retrieve_async(self, repo, data):
+        """ Just schedule the metadata downloads """
+
+        for item in data:
+            if item in self.added.get(repo, []):
+                continue
+            if item == 'metadata':
+                mydbtype = 'primary_db'
+            elif item == 'filelists':
+                mydbtype = 'filelists_db'
+            elif item == 'otherdata':
+                mydbtype = 'other_db'
+            else:
+                continue
+
+            if self._check_db_version(repo, mydbtype):
+                if not self._check_uncompressed_db(repo, mydbtype):
+                    repo._retrieveMD(mydbtype, async=True, failfunc=None)
+
     def populate(self, repo, mdtype='metadata', callback=None, cacheonly=0):
         if mdtype == 'all':
             data = ['metadata', 'filelists', 'otherdata']
commit a522869c21768d53c3861be9c0a2394a3930ad66
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date:   Wed Mar 28 15:59:26 2012 +0200

    retrieveAllMD(): parallel metadata downloading
    
    Metadata are downloaded in parallel just before postreposetup.
    This implements the reverting behavior.

diff --git a/yum/repos.py b/yum/repos.py
index 3793bad..d7f7f09 100644
--- a/yum/repos.py
+++ b/yum/repos.py
@@ -22,6 +22,7 @@ import misc
 
 import Errors
 from packageSack import MetaSack
+import urlgrabber.grabber
 
 from weakref import proxy as weakref
 
@@ -67,6 +68,38 @@ class RepoStorage:
         self._cache_enabled_repos = []
         self.quick_enable_disable = {}
 
+    def retrieveAllMD(self):
+        """ Download metadata for all enabled repositories,
+            based on mdpolicy.
+        """
+
+        if not hasattr(urlgrabber.grabber, 'parallel_wait'):
+            return
+
+        repos = []
+        for repo in self.listEnabled():
+            if repo.async and repo._commonLoadRepoXML(repo):
+                mdtypes = repo._mdpolicy2mdtypes()
+                downloading = repo._commonRetrieveDataMD_list(mdtypes)
+                repos.append((repo, downloading, [False]))
+
+        # with sizes first, then without sizes..
+	for no_size in (False, True):
+            for repo, downloading, error in repos:
+                def failfunc(obj, error=error):
+                    error[0] = True
+                for (ndata, nmdtype) in downloading:
+                    if (ndata.size is None) == no_size:
+                        repo._retrieveMD(nmdtype, async=True, failfunc=failfunc)
+            urlgrabber.grabber.parallel_wait()
+
+        # done or revert
+        for repo, downloading, error in repos:
+            if error[0]: # some MD failed?
+                repo._revertOldRepoXML()
+            else:
+                repo._commonRetrieveDataMD_done(downloading)
+
     def doSetup(self, thisrepo = None):
         
         self.ayum.plugins.run('prereposetup')
@@ -89,6 +122,7 @@ class RepoStorage:
                 self.disableRepo(repo.id)
                 
         self._setup = True
+        self.retrieveAllMD()
         self.ayum.plugins.run('postreposetup')
         
     def __str__(self):
commit ae5055823318efb3b4bcefeb22bbe0a34f0b9a43
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date:   Wed Mar 28 12:56:36 2012 +0200

    _commonRetrieveDataMD(): split the code
    
    Add _commonRetrieveDataMD_list() that returns the list of metadata
    objects to be downloaded.
    
    Add _commonRetrieveDataMD_done() to unpack and commit new files
    after they have been succesfully downloaded.

diff --git a/yum/yumRepo.py b/yum/yumRepo.py
index 2592df1..d156c74 100644
--- a/yum/yumRepo.py
+++ b/yum/yumRepo.py
@@ -1326,6 +1326,17 @@ Insufficient space in download directory %s
             into the delete list, this means metadata can change filename
             without us leaking it. """
 
+        downloading = self._commonRetrieveDataMD_list(mdtypes)
+        for (ndata, nmdtype) in downloading:
+            if not self._retrieveMD(nmdtype, retrieve_can_fail=True):
+                self._revertOldRepoXML()
+                return False
+        self._commonRetrieveDataMD_done(downloading)
+        return True
+
+    def _commonRetrieveDataMD_list(self, mdtypes):
+        """ Return a list of metadata to be retrieved """
+
         def _mdtype_eq(omdtype, odata, nmdtype, ndata):
             """ Check if two returns from _get_mdtype_data() are equal. """
             if ndata is None:
@@ -1357,8 +1368,7 @@ Insufficient space in download directory %s
 
         # Inited twice atm. ... sue me
         self._oldRepoMDData['new_MD_files'] = []
-        downloading_with_size = []
-        downloading_no_size   = []
+        downloading = []
         for mdtype in all_mdtypes:
             (nmdtype, ndata) = self._get_mdtype_data(mdtype)
 
@@ -1395,43 +1405,20 @@ Insufficient space in download directory %s
             # No old repomd data, but we might still have uncompressed MD
             if self._groupCheckDataMDValid(ndata, nmdtype, mdtype):
                 continue
+            downloading.append((ndata, nmdtype))
+        return downloading
 
-            if ndata.size is None:
-                downloading_no_size.append((ndata, nmdtype))
-            else:
-                downloading_with_size.append((ndata, nmdtype))
-
-        if len(downloading_with_size) == 1:
-            downloading_no_size.extend(downloading_with_size)
-            downloading_with_size = []
-
-        remote_size = 0
-        local_size  = 0
-        for (ndata, nmdtype) in downloading_with_size: # Get total size...
-            remote_size += int(ndata.size)
+    def _commonRetrieveDataMD_done(self, downloading):
+        """ Uncompress the downloaded metadata """
 
-        for (ndata, nmdtype) in downloading_with_size:
-            urlgrabber.progress.text_meter_total_size(remote_size, local_size)
-            if not self._retrieveMD(nmdtype, retrieve_can_fail=True):
-                self._revertOldRepoXML()
-                return False
-            local_size += int(ndata.size)
-        urlgrabber.progress.text_meter_total_size(0)
-        for (ndata, nmdtype) in downloading_no_size:
-            if not self._retrieveMD(nmdtype, retrieve_can_fail=True):
-                self._revertOldRepoXML()
-                return False
-
-        for (ndata, nmdtype) in downloading_with_size + downloading_no_size:
+        for (ndata, nmdtype) in downloading:
             local = self._get_mdtype_fname(ndata, False)
             if nmdtype.endswith("_db"): # Uncompress any compressed files
                 dl_local = local
                 local = misc.decompress(dl_local)
                 misc.unlink_f(dl_local)
             self._oldRepoMDData['new_MD_files'].append(local)
-
         self._doneOldRepoXML()
-        return True
 
     def _groupLoadRepoXML(self, text=None, mdtypes=None):
         """ Retrieve the new repomd.xml from the repository, then check it
commit 9da352220ac6b9532ce9a43d654f74388d5c66bd
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date:   Mon Mar 19 17:19:10 2012 +0100

    _retrieveMD(): make it useable for async downloading.

diff --git a/yum/yumRepo.py b/yum/yumRepo.py
index 26b05fb..2592df1 100644
--- a/yum/yumRepo.py
+++ b/yum/yumRepo.py
@@ -1579,7 +1579,7 @@ Insufficient space in download directory %s
            mdtype can be 'primary', 'filelists', 'other' or 'group'."""
         return self._retrieveMD(mdtype)
 
-    def _retrieveMD(self, mdtype, retrieve_can_fail=False):
+    def _retrieveMD(self, mdtype, retrieve_can_fail=False, **kwargs):
         """ Internal function, use .retrieveMD() from outside yum. """
         #  Note that this can raise Errors.RepoMDError if mdtype doesn't exist
         # for this repo.
@@ -1617,7 +1617,9 @@ Insufficient space in download directory %s
                 return local # it's the same return the local one
 
         try:
-            checkfunc = (self.checkMD, (mdtype,), {})
+            def checkfunc(obj):
+                self.checkMD(obj, mdtype)
+                self.retrieved[mdtype] = 1
             text = "%s/%s" % (self.id, mdtype)
             if thisdata.size is None:
                 reget = None
@@ -1633,7 +1635,8 @@ Insufficient space in download directory %s
                                   checkfunc=checkfunc, 
                                   text=text,
                                   cache=self.http_caching == 'all',
-                                  size=thisdata.size)
+                                  size=thisdata.size,
+                                  **kwargs)
         except Errors.RepoError:
             if retrieve_can_fail:
                 return None
@@ -1644,7 +1647,6 @@ Insufficient space in download directory %s
             raise Errors.RepoError, \
                 "Could not retrieve %s matching remote checksum from %s" % (local, self)
         else:
-            self.retrieved[mdtype] = 1
             return local
 
 
commit 0282b38f1d5be2c07771b4570b91dc220b256d6c
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date:   Thu Feb 2 12:50:51 2012 +0100

    max_connections: man page update

diff --git a/docs/yum.conf.5 b/docs/yum.conf.5
index 90aece3..14668f7 100644
--- a/docs/yum.conf.5
+++ b/docs/yum.conf.5
@@ -348,8 +348,8 @@ Determines how yum resolves host names.
 \fBmax_connections \fR
 
 The maximum number of simultaneous connections.  This overrides the urlgrabber
-default of 5 connections.  Note that there are also per-mirror limits, and the
-downloader honors these too.
+default of 5 connections.  Note that there are also implicit per-mirror limits
+and the downloader honors these too.
 
 .IP
 \fBsslcacert \fR
commit 9fdc18d828565ebddf5ccd993b93fffd4be519a7
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date:   Thu Jan 26 16:25:27 2012 +0100

    enable timedhosts
    
    Q: Make it configurable?

diff --git a/yum/__init__.py b/yum/__init__.py
index b3cc4f5..b189d02 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -377,6 +377,7 @@ class YumBase(depsolve.Depsolve):
         mc = self._conf.max_connections
         if mc > 0:
             default_grabber.opts.max_connections = mc
+        default_grabber.opts.timedhosts = self._conf.cachedir + '/timedhosts'
 
         #  We don't want people accessing/altering preconf after it becomes
         # worthless. So we delete it, and thus. it'll raise AttributeError
commit 7e8c76173133fa3bb581fb0765c93c0497b1344a
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date:   Thu Jan 26 11:52:30 2012 +0100

    add prefix '(%s/%s): ' to blocking urlgrabs only

diff --git a/yum/__init__.py b/yum/__init__.py
index 84537ef..b3cc4f5 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -2257,16 +2257,14 @@ class YumBase(depsolve.Depsolve):
                 if po in errors:
                     del errors[po]
 
+            text = os.path.basename(po.relativepath)
             kwargs = {}
             if async and po.repo.async:
                 kwargs['failfunc'] = lambda obj, po=po: adderror(po, exception2msg(obj.exception))
                 kwargs['async'] = True
+            elif not (i == 1 and not local_size[0] and remote_size == po.size):
+                text = '(%s/%s): %s' % (i, len(remote_pkgs), text)
             try:
-                if i == 1 and not local_size[0] and remote_size == po.size:
-                    text = os.path.basename(po.relativepath)
-                else:
-                    text = '(%s/%s): %s' % (i, len(remote_pkgs),
-                                            os.path.basename(po.relativepath))
                 po.repo.getPackage(po,
                                    checkfunc=checkfunc,
                                    text=text,
commit 4857fd3a0a487912a84b65a11b560d83e6e5ec77
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date:   Thu Jan 26 11:32:50 2012 +0100

    downloadPkgs: use parallel downloader

diff --git a/yum/__init__.py b/yum/__init__.py
index e8f28f6..84537ef 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -2223,6 +2223,7 @@ class YumBase(depsolve.Depsolve):
         i = 0
         local_size = [0]
         done_repos = set()
+        async = hasattr(urlgrabber.grabber, 'parallel_wait')
         for po in remote_pkgs:
             #  Recheck if the file is there, works around a couple of weird
             # edge cases.
@@ -2256,6 +2257,10 @@ class YumBase(depsolve.Depsolve):
                 if po in errors:
                     del errors[po]
 
+            kwargs = {}
+            if async and po.repo.async:
+                kwargs['failfunc'] = lambda obj, po=po: adderror(po, exception2msg(obj.exception))
+                kwargs['async'] = True
             try:
                 if i == 1 and not local_size[0] and remote_size == po.size:
                     text = os.path.basename(po.relativepath)
@@ -2266,9 +2271,12 @@ class YumBase(depsolve.Depsolve):
                                    checkfunc=checkfunc,
                                    text=text,
                                    cache=po.repo.http_caching != 'none',
+                                   **kwargs
                                    )
             except Errors.RepoError, e:
                 adderror(po, exception2msg(e))
+        if async:
+            urlgrabber.grabber.parallel_wait()
 
         if hasattr(urlgrabber.progress, 'text_meter_total_size'):
             urlgrabber.progress.text_meter_total_size(0)
commit ef033251187312535bfd1e5dcddfa444b1ac50d6
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date:   Wed Jan 25 13:53:40 2012 +0100

    downloadPkgs: add the 'success' code path to checkfunc

diff --git a/yum/__init__.py b/yum/__init__.py
index 12c552f..e8f28f6 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -2221,7 +2221,7 @@ class YumBase(depsolve.Depsolve):
             urlgrabber.progress.text_meter_total_size(remote_size)
         beg_download = time.time()
         i = 0
-        local_size = 0
+        local_size = [0]
         done_repos = set()
         for po in remote_pkgs:
             #  Recheck if the file is there, works around a couple of weird
@@ -2234,41 +2234,41 @@ class YumBase(depsolve.Depsolve):
                     remote_size -= po.size
                     if hasattr(urlgrabber.progress, 'text_meter_total_size'):
                         urlgrabber.progress.text_meter_total_size(remote_size,
-                                                                  local_size)
+                                                                  local_size[0])
                     continue
                 if os.path.getsize(local) >= po.size:
                     os.unlink(local)
 
-            checkfunc = (self.verifyPkg, (po, 1), {})
-            try:
-                if i == 1 and not local_size and remote_size == po.size:
-                    text = os.path.basename(po.relativepath)
-                else:
-                    text = '(%s/%s): %s' % (i, len(remote_pkgs),
-                                            os.path.basename(po.relativepath))
-                mylocal = po.repo.getPackage(po,
-                                   checkfunc=checkfunc,
-                                   text=text,
-                                   cache=po.repo.http_caching != 'none',
-                                   )
-                local_size += po.size
+            def checkfunc(obj, po=po):
+                self.verifyPkg(obj, po, 1)
+                local_size[0] += po.size
                 if hasattr(urlgrabber.progress, 'text_meter_total_size'):
                     urlgrabber.progress.text_meter_total_size(remote_size,
-                                                              local_size)
+                                                              local_size[0])
                 if po.repoid not in done_repos:
+                    done_repos.add(po.repoid)
                     #  Check a single package per. repo. ... to give a hint to
                     # the user on big downloads.
                     result, errmsg = self.sigCheckPkg(po)
                     if result != 0:
                         self.verbose_logger.warn("%s", errmsg)
-                done_repos.add(po.repoid)
+                po.localpath = obj.filename
+                if po in errors:
+                    del errors[po]
 
+            try:
+                if i == 1 and not local_size[0] and remote_size == po.size:
+                    text = os.path.basename(po.relativepath)
+                else:
+                    text = '(%s/%s): %s' % (i, len(remote_pkgs),
+                                            os.path.basename(po.relativepath))
+                po.repo.getPackage(po,
+                                   checkfunc=checkfunc,
+                                   text=text,
+                                   cache=po.repo.http_caching != 'none',
+                                   )
             except Errors.RepoError, e:
                 adderror(po, exception2msg(e))
-            else:
-                po.localpath = mylocal
-                if po in errors:
-                    del errors[po]
 
         if hasattr(urlgrabber.progress, 'text_meter_total_size'):
             urlgrabber.progress.text_meter_total_size(0)
commit d3523081b32537262fb9d199b81fe6cfd06b0737
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date:   Thu Jan 26 11:16:48 2012 +0100

    getPackage, _getFile: kwargs pass-through

diff --git a/yum/yumRepo.py b/yum/yumRepo.py
index 8c68034..26b05fb 100644
--- a/yum/yumRepo.py
+++ b/yum/yumRepo.py
@@ -760,7 +760,7 @@ class YumRepository(Repository, config.RepoConf):
 
     def _getFile(self, url=None, relative=None, local=None, start=None, end=None,
             copy_local=None, checkfunc=None, text=None, reget='simple', 
-            cache=True, size=None):
+            cache=True, size=None, **kwargs):
         """retrieve file from the mirrorgroup for the repo
            relative to local, optionally get range from
            start to end, also optionally retrieve from a specific baseurl"""
@@ -854,7 +854,8 @@ Insufficient space in download directory %s
                                            reget = reget,
                                            checkfunc=checkfunc,
                                            http_headers=headers,
-                                           size=size
+                                           size=size,
+                                           **kwargs
                                            )
             except URLGrabError, e:
                 errstr = "failure: %s from %s: %s" % (relative, self.id, e)
@@ -866,7 +867,7 @@ Insufficient space in download directory %s
         return result
     __get = _getFile
 
-    def getPackage(self, package, checkfunc=None, text=None, cache=True):
+    def getPackage(self, package, checkfunc=None, text=None, cache=True, **kwargs):
         remote = package.relativepath
         local = package.localPkg()
         basepath = package.basepath
@@ -883,6 +884,7 @@ Insufficient space in download directory %s
                         text=text,
                         cache=cache,
                         size=package.size,
+                        **kwargs
                         )
 
     def getHeader(self, package, checkfunc = None, reget = 'simple',
commit b6e9c6553fbe5000bf1ca20f5a611d3c2ed612e0
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date:   Fri Oct 21 15:25:29 2011 +0200

    add 'max_connections' and 'async' options
    
    If set in config, update the urlgrabber's default.
    Copy max_connections parsed from the metalink to MG.

diff --git a/docs/yum.conf.5 b/docs/yum.conf.5
index 9eff4c3..90aece3 100644
--- a/docs/yum.conf.5
+++ b/docs/yum.conf.5
@@ -345,6 +345,13 @@ Determines how yum resolves host names.
 `6' or `IPv6': resolve to IPv6 addresses only.
 
 .IP
+\fBmax_connections \fR
+
+The maximum number of simultaneous connections.  This overrides the urlgrabber
+default of 5 connections.  Note that there are also per-mirror limits, and the
+downloader honors these too.
+
+.IP
 \fBsslcacert \fR
 Path to the directory containing the databases of the certificate authorities
 yum should use to verify SSL certificates. Defaults to none - uses system
@@ -929,7 +936,11 @@ as greater/less than any other. defaults to 1000
 If set to True yum will continue running if this repository cannot be 
 contacted for any reason. This should be set carefully as all repos are consulted
 for any given command. Defaults to False.
+
 .IP
+\fBasync \fR
+If set (the default) and urlgrabber supports it, yum will use parallel downloader
+for packages from this repo.
 
 .SH "URL INCLUDE SYNTAX"
 .LP
diff --git a/yum/__init__.py b/yum/__init__.py
index 40bb15b..12c552f 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -373,6 +373,11 @@ class YumBase(depsolve.Depsolve):
 
         self._conf = config.readMainConfig(startupconf)
 
+        # update urlgrabber defaults
+        mc = self._conf.max_connections
+        if mc > 0:
+            default_grabber.opts.max_connections = mc
+
         #  We don't want people accessing/altering preconf after it becomes
         # worthless. So we delete it, and thus. it'll raise AttributeError
         del self.preconf
diff --git a/yum/config.py b/yum/config.py
index ee5876b..2bf4f45 100644
--- a/yum/config.py
+++ b/yum/config.py
@@ -788,6 +788,7 @@ class YumConf(StartupConf):
     ip_resolve = CaselessSelectionOption(
             allowed = ('ipv4', 'ipv6', 'whatever'),
             mapper  = {'4': 'ipv4', '6': 'ipv6'})
+    max_connections = IntOption(0)
 
     http_caching = SelectionOption('all', ('none', 'packages', 'all'))
     metadata_expire = SecondsOption(60 * 60 * 6) # Time in seconds (6h).
@@ -957,6 +958,7 @@ class RepoConf(BaseConfig):
     ssl_check_cert_permissions = Inherit(YumConf.ssl_check_cert_permissions)
 
     skip_if_unavailable = BoolOption(False)
+    async = BoolOption(True)
     
 class VersionGroupConf(BaseConfig):
     """Option definitions for version groups."""
diff --git a/yum/metalink.py b/yum/metalink.py
index aaa4f25..51895fd 100755
--- a/yum/metalink.py
+++ b/yum/metalink.py
@@ -180,6 +180,7 @@ class MetaLinkRepoMD:
         self.repomd = None
         self.old_repomds = []
         self.mirrors = []
+        self._host2mc = {}
         if not os.path.exists(filename):
             raise MetaLinkRepoErrorParseFail, "File %s does not exist" %filename
         try:
@@ -225,8 +226,6 @@ class MetaLinkRepoMD:
         # Get the hostname from a url, stripping away any usernames/passwords
         # Borrowd from fastestmirror
         url2host = lambda url: url.split('/')[2].split('@')[-1]
-        hosts = set() # Don't want multiple urls for one host in plain mode
-                      # The list of URLs is sorted, so http is before ftp
 
         for mirror in self.mirrors:
             url = mirror.url
@@ -237,9 +236,9 @@ class MetaLinkRepoMD:
             elif (url.startswith("http:") or url.startswith("ftp:") or
                   url.startswith("https:")):
                 host = url2host(url)
-                if host in hosts:
+                if host in self._host2mc:
                     continue
-                hosts.add(host)
+                self._host2mc[host] = mirror.max_connections
             else:
                 continue
 
diff --git a/yum/yumRepo.py b/yum/yumRepo.py
index 655bbaa..8c68034 100644
--- a/yum/yumRepo.py
+++ b/yum/yumRepo.py
@@ -484,8 +484,20 @@ class YumRepository(Repository, config.RepoConf):
                                     copy_local=self.copy_local,
                                     reget='simple',
                                     **ugopts)
+        def add_mc(url):
+            host = urlparse.urlsplit(url).netloc
+            mc = self.metalink_data._host2mc.get(host)
+            if mc > 0:
+                url = {
+                    'mirror': misc.to_utf8(url),
+                    'kwargs': { 'max_connections': mc },
+                }
+            return url
+        urls = self.urls
+        if self.metalink:
+            urls = map(add_mc, urls)
 
-        self._grab = mgclass(self._grabfunc, self.urls,
+        self._grab = mgclass(self._grabfunc, urls,
                              failure_callback=self.mirror_failure_obj)
 
     def _default_grabopts(self, cache=True):


More information about the Yum-commits mailing list