[Yum-devel] [PATCH 2/5] Integrate presto support to yum

Zdenek Pavlas zpavlas at redhat.com
Thu Feb 21 11:57:48 UTC 2013


Add the "presto" bool config option, document it.
Add hooks to downloadPkgs().
---
 docs/yum.conf.5 |  12 +++++
 yum/__init__.py |  18 ++++++++
 yum/config.py   |   2 +
 yum/presto.py   | 135 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 167 insertions(+)
 create mode 100644 yum/presto.py

diff --git a/docs/yum.conf.5 b/docs/yum.conf.5
index 62b76f8..93cb297 100644
--- a/docs/yum.conf.5
+++ b/docs/yum.conf.5
@@ -372,6 +372,13 @@ default of 5 connections.  Note that there are also implicit per-mirror limits
 and the downloader honors these too.
 
 .IP
+\fBpresto\fR
+
+Either `0' or `1'. Set this to `1' to use delta-RPM files, if available.
+This reduces the download size of updates significantly, but local rebuild
+is CPU intensive.  Default is `1' (on).
+
+.IP
 \fBsslcacert \fR
 Path to the directory containing the databases of the certificate authorities
 yum should use to verify SSL certificates. Defaults to none - uses system
@@ -930,6 +937,11 @@ repository.
 Overrides the \fBip_resolve\fR option from the [main] section for this
 repository.
 
+.IP
+\fBpresto\fR
+
+Overrides the \fBpresto\fR option from the [main] section for this
+repository.
 
 .IP
 \fBsslcacert \fR
diff --git a/yum/__init__.py b/yum/__init__.py
index 8766c95..c04b871 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -90,6 +90,7 @@ from packages import YumUrlPackage, YumNotFoundPackage
 from constants import *
 from yum.rpmtrans import RPMTransaction,SimpleCliCallBack
 from yum.i18n import to_unicode, to_str, exception2msg
+from yum.presto import Presto
 
 import string
 import StringIO
@@ -2236,6 +2237,7 @@ much more problems).
                 po.basepath # prefetch now; fails when repos are closed
             return False
 
+        pkgs = []
         for po in pkglist:
             if hasattr(po, 'pkgtype') and po.pkgtype == 'local':
                 continue
@@ -2243,8 +2245,21 @@ much more problems).
                 continue
             if errors:
                 return errors
+            pkgs.append(po)
+
+        # download presto metadata
+        presto = Presto(self, pkgs)
+        for po in pkgs:
+            if presto.to_drpm(po) and verify_local(po):
+                # there's .drpm already, use it
+                presto.rebuild(po, adderror)
+                continue
             remote_pkgs.append(po)
             remote_size += po.size
+        if presto.deltasize:
+            self.verbose_logger.info(_('Delta RPMs reduced %s of updates to %s (%d%% saved)'),
+                format_number(presto.rpmsize), format_number(presto.deltasize),
+                100 - presto.deltasize*100.0/presto.rpmsize)
 
         if downloadonly:
             # close DBs, unlock
@@ -2272,6 +2287,9 @@ much more problems).
                 if hasattr(urlgrabber.progress, 'text_meter_total_size'):
                     urlgrabber.progress.text_meter_total_size(remote_size,
                                                               local_size[0])
+                if po in presto.deltas:
+                    presto.rebuild(po, adderror)
+                    return
                 if po.repoid not in done_repos:
                     done_repos.add(po.repoid)
                     #  Check a single package per. repo. ... to give a hint to
diff --git a/yum/config.py b/yum/config.py
index 3b22e41..d279ab3 100644
--- a/yum/config.py
+++ b/yum/config.py
@@ -791,6 +791,7 @@ class YumConf(StartupConf):
             allowed = ('ipv4', 'ipv6', 'whatever'),
             mapper  = {'4': 'ipv4', '6': 'ipv6'})
     max_connections = IntOption(0)
+    presto = BoolOption(True)
 
     http_caching = SelectionOption('all', ('none', 'packages', 'all'))
     metadata_expire = SecondsOption(60 * 60 * 6) # Time in seconds (6h).
@@ -949,6 +950,7 @@ class RepoConf(BaseConfig):
     throttle = Inherit(YumConf.throttle)
     timeout = Inherit(YumConf.timeout)
     ip_resolve = Inherit(YumConf.ip_resolve)
+    presto = Inherit(YumConf.presto)
 
     http_caching = Inherit(YumConf.http_caching)
     metadata_expire = Inherit(YumConf.metadata_expire)
diff --git a/yum/presto.py b/yum/presto.py
new file mode 100644
index 0000000..08462f5
--- /dev/null
+++ b/yum/presto.py
@@ -0,0 +1,135 @@
+#  Integrated delta rpm support
+#  Copyright 2013 Zdenek Pavlas
+
+#   This library is free software; you can redistribute it and/or
+#   modify it under the terms of the GNU Lesser General Public
+#   License as published by the Free Software Foundation; either
+#   version 2.1 of the License, or (at your option) any later version.
+#
+#   This library is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#   Lesser General Public License for more details.
+#
+#   You should have received a copy of the GNU Lesser General Public
+#   License along with this library; if not, write to the
+#      Free Software Foundation, Inc.,
+#      59 Temple Place, Suite 330,
+#      Boston, MA  02111-1307  USA
+
+from yum.constants import TS_UPDATE
+from yum.Errors import RepoError
+from yum.i18n import exception2msg, _
+from urlgrabber import grabber
+async = hasattr(grabber, 'parallel_wait')
+from xml.etree.cElementTree import iterparse
+import os, gzip, subprocess
+
+class Presto:
+    def __init__(self, ayum, pkgs):
+        self.verbose_logger = ayum.verbose_logger
+        self.deltas = {}
+        self._rpmsave = {}
+        self.rpmsize = 0
+        self.deltasize = 0
+
+        # calculate update sizes
+        pinfo = {}
+        reposize = {}
+        for po in pkgs:
+            if not po.repo.presto:
+                continue
+            if po.state != TS_UPDATE and po.name not in ayum.conf.installonlypkgs:
+                continue
+            pinfo.setdefault(po.repo, {})[po.pkgtup] = po
+            reposize[po.repo] = reposize.get(po.repo, 0) + po.size
+
+        # download delta metadata
+        mdpath = {}
+        for repo in reposize:
+            for name in ('prestodelta', 'deltainfo'):
+                try: data = repo.repoXML.getData(name); break
+                except: pass
+            else:
+                self.verbose_logger.warn(_('No Presto metadata available for %s'), repo)
+                continue
+            path = repo.cachedir +'/'+ os.path.basename(data.location[1])
+            if not os.path.exists(path) and int(data.size) > reposize[repo]:
+                self.verbose_logger.info(_('Not downloading Presto metadata for %s'), repo)
+                continue
+
+            def failfunc(e, name=name, repo=repo):
+                mdpath.pop(repo, None)
+                if hasattr(e, 'exception'): e = e.exception
+                self.verbose_logger.warn(_('Failed to download %s for repository %s: %s'),
+                                         name, repo, exception2msg(e))
+            kwargs = {}
+            if async and repo._async:
+                kwargs['failfunc'] = failfunc
+                kwargs['async'] = True
+            try: mdpath[repo] = repo._retrieveMD(name, **kwargs)
+            except Errors.RepoError, e: failfunc(e)
+        if async:
+            grabber.parallel_wait()
+
+        # parse metadata, populate self.deltas
+        for repo, path in mdpath.items():
+            pinfo_repo = pinfo[repo]
+            if path.endswith('.gz'):
+                path = gzip.open(path)
+            for ev, el in iterparse(path):
+                if el.tag != 'newpackage': continue
+                new = el.get('name'), el.get('arch'), el.get('epoch'), el.get('version'), el.get('release')
+                po = pinfo_repo.get(new)
+                if po:
+                    best = po.size * 0.75 # make this configurable?
+                    have = ayum._up.installdict.get(new[:2], [])
+                    for el in el.findall('delta'):
+                        size = int(el.find('size').text)
+                        old = el.get('oldepoch'), el.get('oldversion'), el.get('oldrelease')
+                        if size >= best or old not in have:
+                            continue
+                        # the old version is installed, seq check should never fail. kill this?
+                        seq = el.find('sequence').text
+                        if subprocess.call(['/usr/bin/applydeltarpm', '-C', '-s', seq]) != 0:
+                            self.verbose_logger.warn(_('Deltarpm sequence check failed for %s'), seq)
+                            continue
+
+                        best = size
+                        csum = el.find('checksum')
+                        csum = csum.get('type'), csum.text
+                        self.deltas[po] = size, el.find('filename').text, csum
+                    if po not in self.deltas:
+                        self.verbose_logger.warn(_('No suitable drpm files for %s'), po)
+                el.clear()
+
+    def to_drpm(self, po):
+        try: size, remote, csum = self.deltas[po]
+        except KeyError: return False
+        self._rpmsave[po] = po.packagesize, po.relativepath, po.localpath
+
+        # update stats
+        self.rpmsize += po.packagesize
+        self.deltasize += size
+
+        # update size/path/checksum to drpm values
+        po.packagesize = size
+        po.relativepath = remote
+        po.localpath = os.path.dirname(po.localpath) +'/'+ os.path.basename(remote)
+        po.returnIdSum = lambda: csum
+        return True
+
+    def rebuild(self, po, adderror):
+        # restore rpm values
+        deltapath = po.localpath
+        po.packagesize, po.relativepath, po.localpath = self._rpmsave[po]
+        del po.returnIdSum
+
+        # rebuild it from drpm
+        if subprocess.call(['/usr/bin/applydeltarpm', deltapath, po.localpath]) != 0:
+            return adderror(po, _('Delta RPM rebuild failed'))
+        # source drpm was already checksummed.. is this necessary?
+        if not po.verifyLocalPkg():
+            return adderror(po, _('Checksum of the delta-rebuilt RPM failed'))
+        # no need to keep this
+        os.unlink(deltapath)
-- 
1.7.11.7



More information about the Yum-devel mailing list