[Yum-devel] [PATCH 1/6] add 'max_connections' and 'async' options

[Zdeněk Pavlas]

If set in config, update the urlgrabber's default.
Copy max_connections parsed from the metalink to MG.
---
 docs/yum.conf.5 |   11 +++++++++++
 yum/__init__.py |    4 ++++
 yum/config.py   |    2 ++
 yum/metalink.py |    7 +++----
 yum/yumRepo.py  |   16 ++++++++++++++--
 5 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/docs/yum.conf.5 b/docs/yum.conf.5
index d6fe824..d77436f 100644
--- a/docs/yum.conf.5
+++ b/docs/yum.conf.5
@@ -321,6 +321,13 @@ Determines how yum resolves host names.
 `6' or `IPv6': resolve to IPv6 addresses only.
 
 .IP
+\fBmax_connections \fR
+
+The maximum number of simultaneous connections.  This overrides the urlgrabber
+default of 5 connections.  Note that there are also per-mirror limits, and the
+downloader honors these too.
+
+.IP
 \fBsslcacert \fR
 Path to the directory containing the databases of the certificate authorities
 yum should use to verify SSL certificates. Defaults to none - uses system
@@ -860,7 +867,11 @@ as greater/less than any other. defaults to 1000
 If set to True yum will continue running if this repository cannot be 
 contacted for any reason. This should be set carefully as all repos are consulted
 for any given command. Defaults to False.
+
 .IP
+\fBasync \fR
+If set (the default) and urlgrabber supports it, yum will use parallel downloader
+for packages from this repo.
 
 .SH "URL INCLUDE SYNTAX"
 .LP
diff --git a/yum/__init__.py b/yum/__init__.py
index 2634670..509eea2 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -360,6 +360,10 @@ class YumBase(depsolve.Depsolve):
 
         self._conf = config.readMainConfig(startupconf)
 
+        # update urlgrabber defaults
+        mc = self._conf.max_connections
+        if mc > 0: default_grabber.opts.max_connections = mc
+
         #  We don't want people accessing/altering preconf after it becomes
         # worthless. So we delete it, and thus. it'll raise AttributeError
         del self.preconf
diff --git a/yum/config.py b/yum/config.py
index 6c09ee9..7cc734a 100644
--- a/yum/config.py
+++ b/yum/config.py
@@ -783,6 +783,7 @@ class YumConf(StartupConf):
     ip_resolve = CaselessSelectionOption(
             allowed = ('ipv4', 'ipv6', 'whatever'),
             mapper  = {'4': 'ipv4', '6': 'ipv6'})
+    max_connections = IntOption(0)
 
     http_caching = SelectionOption('all', ('none', 'packages', 'all'))
     metadata_expire = SecondsOption(60 * 60 * 6) # Time in seconds (6h).
@@ -944,6 +945,7 @@ class RepoConf(BaseConfig):
     sslclientkey = Inherit(YumConf.sslclientkey)
 
     skip_if_unavailable = BoolOption(False)
+    async = BoolOption(True)
     
 class VersionGroupConf(BaseConfig):
     """Option definitions for version groups."""
diff --git a/yum/metalink.py b/yum/metalink.py
index aaa4f25..51895fd 100755
--- a/yum/metalink.py
+++ b/yum/metalink.py
@@ -180,6 +180,7 @@ class MetaLinkRepoMD:
         self.repomd = None
         self.old_repomds = []
         self.mirrors = []
+        self._host2mc = {}
         if not os.path.exists(filename):
             raise MetaLinkRepoErrorParseFail, "File %s does not exist" %filename
         try:
@@ -225,8 +226,6 @@ class MetaLinkRepoMD:
         # Get the hostname from a url, stripping away any usernames/passwords
         # Borrowd from fastestmirror
         url2host = lambda url: url.split('/')[2].split('@')[-1]
-        hosts = set() # Don't want multiple urls for one host in plain mode
-                      # The list of URLs is sorted, so http is before ftp
 
         for mirror in self.mirrors:
             url = mirror.url
@@ -237,9 +236,9 @@ class MetaLinkRepoMD:
             elif (url.startswith("http:") or url.startswith("ftp:") or
                   url.startswith("https:")):
                 host = url2host(url)
-                if host in hosts:
+                if host in self._host2mc:
                     continue
-                hosts.add(host)
+                self._host2mc[host] = mirror.max_connections
             else:
                 continue
 
diff --git a/yum/yumRepo.py b/yum/yumRepo.py
index 62e53f8..9cd20b4 100644
--- a/yum/yumRepo.py
+++ b/yum/yumRepo.py
@@ -482,8 +482,20 @@ class YumRepository(Repository, config.RepoConf):
                                     copy_local=self.copy_local,
                                     reget='simple',
                                     **ugopts)
+        def add_mc(url):
+            host = urlparse.urlsplit(url).netloc
+            mc = self.metalink_data._host2mc.get(host)
+            if mc > 0:
+                url = {
+                    'mirror': misc.to_utf8(url),
+                    'kwargs': { 'max_connections': mc },
+                }
+            return url
+        urls = self.urls
+        if self.metalink:
+            urls = map(add_mc, urls)
 
-        self._grab = mgclass(self._grabfunc, self.urls,
+        self._grab = mgclass(self._grabfunc, urls,
                              failure_callback=self.mirror_failure_obj)
 
     def _default_grabopts(self, cache=True):
-- 
1.7.4.4