[Yum-devel] [PATCH 1/3] add 'max_connections' option

James Antill james at fedoraproject.org
Thu Dec 22 14:40:01 UTC 2011


On Wed, 2011-12-21 at 17:01 +0100, Zdeněk Pavlas wrote:
> This option sets the connection limit of the baseurl host.
> Mirrors use connection limits from metalink.xml.
> Also, the value of this option is sent to the MG instance
> and controls how many mirrors are used initially.
> ---
>  docs/yum.conf.5 |   15 ++++++++++++---
>  yum/config.py   |    2 ++
>  yum/metalink.py |    8 ++++----
>  yum/yumRepo.py  |   19 ++++++++++++++++---
>  4 files changed, 34 insertions(+), 10 deletions(-)
> 
> diff --git a/docs/yum.conf.5 b/docs/yum.conf.5
> index d6fe824..1a05659 100644
> --- a/docs/yum.conf.5
> +++ b/docs/yum.conf.5
> @@ -293,7 +293,7 @@ related issues. Try to at least use `packages' to minimize load on repository
>  servers.
>  
>  .IP
> -\fBthrottle \fR
> +\fBthrottle\fR
>  Enable bandwidth throttling for downloads. This option can be expressed as a
>  absolute data rate in bytes/sec. An SI prefix (k, M or G) may be appended to the
>  bandwidth value (eg. `5.5k' is 5.5 kilobytes/sec, `2M' is 2 Megabytes/sec).
> @@ -305,7 +305,7 @@ the maximum available bandwidth.
>  Set to `0' to disable bandwidth throttling. This is the default.
>  
>  .IP
> -\fBbandwidth \fR
> +\fBbandwidth\fR
>  Use to specify the maximum available network bandwidth in bytes/second.  Used
>  with the \fBthrottle\fR option (above). If \fBthrottle\fR is a percentage and
>  \fBbandwidth\fR is `0' then bandwidth throttling will be disabled. If
> @@ -313,7 +313,7 @@ with the \fBthrottle\fR option (above). If \fBthrottle\fR is a percentage and
>  ignored. Default is `0' (no bandwidth throttling). 
>  
>  .IP
> -\fBip_resolve \fR
> +\fBip_resolve\fR

 Why these space changes?

>  Determines how yum resolves host names.
>  
>  `4' or `IPv4': resolve to IPv4 addresses only.
> @@ -321,6 +321,15 @@ Determines how yum resolves host names.
>  `6' or `IPv6': resolve to IPv6 addresses only.
>  
>  .IP
> +\fBmax_connections\fR
> +
> +The number of concurrent connections yum should use when downloading packages.
> +A baseurl repo interprets this as the connection limit of the baseurl host.

 This implies that there is "the baseurl host", when baseurl can be (and
often is) a list.

> +Mirrorlists use separate per-host connection limits.  Yum tries to use enough
> +mirrors that their total connection limit is at least the value of this option.

 For something called max_connections it better be the "limit is at most
the value of this option" because that's what everyone will assume, and
they'll hurt one of us if not true ;).

> +.IP
>  \fBsslcacert \fR
>  Path to the directory containing the databases of the certificate authorities
>  yum should use to verify SSL certificates. Defaults to none - uses system
> diff --git a/yum/config.py b/yum/config.py
> index 6c09ee9..7dc68c5 100644
> --- a/yum/config.py
> +++ b/yum/config.py
> @@ -783,6 +783,7 @@ class YumConf(StartupConf):
>      ip_resolve = CaselessSelectionOption(
>              allowed = ('ipv4', 'ipv6', 'whatever'),
>              mapper  = {'4': 'ipv4', '6': 'ipv6'})
> +    max_connections = IntOption(5)
>  
>      http_caching = SelectionOption('all', ('none', 'packages', 'all'))
>      metadata_expire = SecondsOption(60 * 60 * 6) # Time in seconds (6h).
> @@ -928,6 +929,7 @@ class RepoConf(BaseConfig):
>      throttle = Inherit(YumConf.throttle)
>      timeout = Inherit(YumConf.timeout)
>      ip_resolve = Inherit(YumConf.ip_resolve)
> +    max_connections = Inherit(YumConf.max_connections)
>  
>      http_caching = Inherit(YumConf.http_caching)
>      metadata_expire = Inherit(YumConf.metadata_expire)
> diff --git a/yum/metalink.py b/yum/metalink.py
> index aaa4f25..867ae86 100755
> --- a/yum/metalink.py
> +++ b/yum/metalink.py
> @@ -180,6 +180,7 @@ class MetaLinkRepoMD:
>          self.repomd = None
>          self.old_repomds = []
>          self.mirrors = []
> +        self.host2con = {}

 Again, this feels too hacky to make it a public API and the name does
not immediately suggest "host to max connections". Again, we also need
to store the private attribute for mirrors somewhere ... having another
host2pri or something feels even worse.

>          if not os.path.exists(filename):
>              raise MetaLinkRepoErrorParseFail, "File %s does not exist" %filename
>          try:
> @@ -225,8 +226,6 @@ class MetaLinkRepoMD:
>          # Get the hostname from a url, stripping away any usernames/passwords
>          # Borrowd from fastestmirror
>          url2host = lambda url: url.split('/')[2].split('@')[-1]
> -        hosts = set() # Don't want multiple urls for one host in plain mode
> -                      # The list of URLs is sorted, so http is before ftp
>  
>          for mirror in self.mirrors:
>              url = mirror.url
> @@ -237,9 +236,10 @@ class MetaLinkRepoMD:
>              elif (url.startswith("http:") or url.startswith("ftp:") or
>                    url.startswith("https:")):
>                  host = url2host(url)
> -                if host in hosts:
> +                # Don't want multiple urls for one host in plain mode
> +                if host in self.host2con:
>                      continue
> -                hosts.add(host)
> +                self.host2con[host] = mirror.max_connections
>              else:
>                  continue
>  
> diff --git a/yum/yumRepo.py b/yum/yumRepo.py
> index f645a1a..d886187 100644
> --- a/yum/yumRepo.py
> +++ b/yum/yumRepo.py
> @@ -490,9 +490,22 @@ class YumRepository(Repository, config.RepoConf):
>                                      copy_local=self.copy_local,
>                                      reget='simple',
>                                      **ugopts)
> -
> -        self._grab = mgclass(self._grabfunc, self.urls,
> -                             failure_callback=self.mirror_failure_obj)
> +        def add_limit(url):
> +            limit = None
> +            if url in self._orig_baseurl:
> +                limit = self.max_connections
> +            elif self.metalink:
> +                host = urlparse.urlsplit(url).netloc
> +                limit = self.metalink_data.host2con.get(host)
> +            if limit: url = {
> +                'mirror': misc.to_utf8(url),
> +                'kwargs': { 'max_connections': limit },
> +            }
> +            return url
> +        urls = map(add_limit, self.urls)
> +        self._grab = mgclass(self._grabfunc, urls,
> +                             failure_callback=self.mirror_failure_obj,
> +                             max_connections=self.max_connections)
>  
>      def _default_grabopts(self, cache=True):
>          opts = { 'keepalive': self.keepalive,




More information about the Yum-devel mailing list