[yum-commits] 13 commits - docs/yum.conf.5 yum/config.py yum/drpm.py yum/__init__.py

zpavlas at osuosl.org zpavlas at osuosl.org
Thu Feb 28 14:44:27 UTC 2013


 docs/yum.conf.5 |   11 ++
 yum/__init__.py |  244 +++++++++++++++++++++++++++++++-------------------------
 yum/config.py   |    2 
 yum/drpm.py     |  174 +++++++++++++++++++++++++++++++++++++++
 4 files changed, 325 insertions(+), 106 deletions(-)

New commits:
commit 559c996ec5d53d033537e54d2182a90f02eac406
Author: Zdenek Pavlas <zpavlas at redhat.com>
Date:   Thu Feb 28 12:21:05 2013 +0100

    drpm: repo_gen_decompress() prestodelta.xml

diff --git a/yum/drpm.py b/yum/drpm.py
index 108aa39..3339107 100644
--- a/yum/drpm.py
+++ b/yum/drpm.py
@@ -21,7 +21,7 @@ from yum.constants import TS_UPDATE
 from yum.Errors import RepoError
 from yum.i18n import exception2msg, _
 from yum.Errors import MiscError
-from misc import checksum
+from misc import checksum, repo_gen_decompress
 from urlgrabber import grabber
 async = hasattr(grabber, 'parallel_wait')
 from xml.etree.cElementTree import iterparse
@@ -124,10 +124,10 @@ class DeltaInfo:
                 for po in ayum.rpmdb.searchNevra(n, None, None, None, a)]
 
         # parse metadata, populate self.deltas
-        for repo, path in mdpath.items():
+        for repo, cpath in mdpath.items():
             pinfo_repo = pinfo[repo]
-            if path.endswith('.gz'):
-                path = gzip.open(path)
+            path = repo_gen_decompress(cpath, 'prestodelta.xml',
+                                       cached=repo.cache)
             for ev, el in iterparse(path):
                 if el.tag != 'newpackage': continue
                 new = el.get('name'), el.get('arch'), el.get('epoch'), el.get('version'), el.get('release')
commit c2074489142bd4ad2b067e8c84bb984961b12ab5
Author: Zdenek Pavlas <zpavlas at redhat.com>
Date:   Thu Feb 28 11:25:44 2013 +0100

    drpm: add rpmdb.searchNevra() fallback

diff --git a/yum/drpm.py b/yum/drpm.py
index 8620e37..108aa39 100644
--- a/yum/drpm.py
+++ b/yum/drpm.py
@@ -115,6 +115,14 @@ class DeltaInfo:
         if async:
             grabber.parallel_wait()
 
+        # use installdict or rpmdb
+        if ayum._up:
+            installed = ayum._up.installdict.get
+        else:
+            installed = lambda (n, a): [
+                (po.epoch, po.version, po.release)
+                for po in ayum.rpmdb.searchNevra(n, None, None, None, a)]
+
         # parse metadata, populate self.deltas
         for repo, path in mdpath.items():
             pinfo_repo = pinfo[repo]
@@ -127,7 +135,7 @@ class DeltaInfo:
                 if index is not None:
                     po = pkgs[index]
                     best = po.size * 0.75 # make this configurable?
-                    have = ayum._up.installdict.get(new[:2], [])
+                    have = installed(new[:2]) or []
                     for el in el.findall('delta'):
                         size = int(el.find('size').text)
                         old = el.get('oldepoch'), el.get('oldversion'), el.get('oldrelease')
commit 7e1a5524920978bfb898a6a87d438fef73b832ba
Author: Zdenek Pavlas <zpavlas at redhat.com>
Date:   Tue Feb 26 16:24:01 2013 +0100

    drop to_drpm() and to_rpm(), use DeltaPackage instances.

diff --git a/yum/__init__.py b/yum/__init__.py
index 8ef7989..fd54208 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -90,7 +90,7 @@ from packages import YumUrlPackage, YumNotFoundPackage
 from constants import *
 from yum.rpmtrans import RPMTransaction,SimpleCliCallBack
 from yum.i18n import to_unicode, to_str, exception2msg
-from yum.drpm import DeltaInfo
+from yum.drpm import DeltaInfo, DeltaPackage
 
 import string
 import StringIO
@@ -2191,8 +2191,8 @@ much more problems).
             b = bpo.getDiscNum()
             if a is None and b is None:
                 # deltas first to start rebuilding asap
-                return cmp(bpo in presto.deltas, apo in presto.deltas) \
-                    or cmp(apo, bpo)
+                return cmp(isinstance(bpo, DeltaPackage),
+                           isinstance(apo, DeltaPackage)) or cmp(apo, bpo)
             if a is None:
                 return -1
             if b is None:
@@ -2257,19 +2257,22 @@ much more problems).
                 return errors
             pkgs.append(po)
 
-        # download presto metadata
+        # download presto metadata and use drpms
         presto = DeltaInfo(self, pkgs)
+        deltasize = rpmsize = 0
         for po in pkgs:
-            if presto.to_drpm(po) and verify_local(po):
-                # there's .drpm already, use it
-                presto.rebuild(po, adderror)
-                continue
+            if isinstance(po, DeltaPackage):
+                if verify_local(po):
+                    # there's .drpm already, use it
+                    presto.rebuild(po, adderror)
+                    continue
+                deltasize += po.size
+                rpmsize = po.rpm.size
             remote_pkgs.append(po)
             remote_size += po.size
-        if presto.deltasize:
+        if deltasize:
             self.verbose_logger.info(_('Delta RPMs reduced %s of updates to %s (%d%% saved)'),
-                format_number(presto.rpmsize), format_number(presto.deltasize),
-                100 - presto.deltasize*100.0/presto.rpmsize)
+                format_number(rpmsize), format_number(deltasize), 100 - deltasize*100.0/rpmsize)
 
         if downloadonly:
             # close DBs, unlock
@@ -2298,7 +2301,7 @@ much more problems).
                     if hasattr(urlgrabber.progress, 'text_meter_total_size'):
                         urlgrabber.progress.text_meter_total_size(remote_size,
                                                                   local_size[0])
-                    if po in presto.deltas:
+                    if isinstance(po, DeltaPackage):
                         presto.rebuild(po, adderror)
                         return
                     if po.repoid not in done_repos:
@@ -2355,19 +2358,21 @@ much more problems).
                     
             fatal = False
             for po in errors:
-                if po not in presto.deltas:
-                    fatal = True; break
+                if not isinstance(po, DeltaPackage):
+                    fatal = True
+                    break
             if not errors or fatal:
                 break
 
             # there were drpm related errors *only*
-            remote_pkgs = errors.keys()
+            remote_pkgs = []
             remote_size = 0
-            for po in remote_pkgs:
-                presto.to_rpm(po) # needed, we don't rebuild() when DL fails
+            for po in errors:
+                po = po.rpm
+                remote_pkgs.append(po)
                 remote_size += po.size
+            errors.clear()
             self.verbose_logger.warn(_('Some delta RPMs failed to download or rebuild. Retrying..'))
-            presto.deltas.clear() # any error is now considered fatal
 
         if not downloadonly:
             # XXX: Run unlocked?  Skip this for now..
diff --git a/yum/drpm.py b/yum/drpm.py
index 81afe80..8620e37 100644
--- a/yum/drpm.py
+++ b/yum/drpm.py
@@ -20,6 +20,8 @@
 from yum.constants import TS_UPDATE
 from yum.Errors import RepoError
 from yum.i18n import exception2msg, _
+from yum.Errors import MiscError
+from misc import checksum
 from urlgrabber import grabber
 async = hasattr(grabber, 'parallel_wait')
 from xml.etree.cElementTree import iterparse
@@ -27,25 +29,56 @@ import os, gzip
 
 APPLYDELTA = '/usr/bin/applydeltarpm'
 
+class DeltaPackage:
+    def __init__(self, rpm, size, remote, csum):
+        # copy what needed
+        self.rpm = rpm
+        self.repo = rpm.repo
+        self.basepath = rpm.basepath
+        self.pkgtup = rpm.pkgtup
+
+        # set up drpm attributes
+        self.size = size
+        self.relativepath = remote
+        self.localpath = os.path.dirname(rpm.localpath) +'/'+ os.path.basename(remote)
+        self.csum = csum
+
+    def __str__(self):
+        return 'Delta RPM of %s' % self.rpm
+
+    def localPkg(self):
+        return self.localpath
+
+    def verifyLocalPkg(self):
+        # check file size first
+        try: fsize = os.path.getsize(self.localpath)
+        except OSError: return False
+        if fsize != self.size: return False
+
+        # checksum
+        ctype, csum = self.csum
+        try: fsum = checksum(ctype, self.localpath)
+        except MiscError: return False
+        if fsum != csum: return False
+
+        # hooray
+        return True
+
 class DeltaInfo:
     def __init__(self, ayum, pkgs):
         self.verbose_logger = ayum.verbose_logger
-        self.deltas = {}
-        self._rpmsave = {}
-        self.rpmsize = 0
-        self.deltasize = 0
         self.jobs = {}
         self.limit = ayum.conf.deltarpm
 
         # calculate update sizes
         pinfo = {}
         reposize = {}
-        for po in pkgs:
+        for index, po in enumerate(pkgs):
             if not po.repo.deltarpm:
                 continue
             if po.state != TS_UPDATE and po.name not in ayum.conf.installonlypkgs:
                 continue
-            pinfo.setdefault(po.repo, {})[po.pkgtup] = po
+            pinfo.setdefault(po.repo, {})[po.pkgtup] = index
             reposize[po.repo] = reposize.get(po.repo, 0) + po.size
 
         # don't use deltas when deltarpm not installed
@@ -90,8 +123,9 @@ class DeltaInfo:
             for ev, el in iterparse(path):
                 if el.tag != 'newpackage': continue
                 new = el.get('name'), el.get('arch'), el.get('epoch'), el.get('version'), el.get('release')
-                po = pinfo_repo.get(new)
-                if po:
+                index = pinfo_repo.get(new)
+                if index is not None:
+                    po = pkgs[index]
                     best = po.size * 0.75 # make this configurable?
                     have = ayum._up.installdict.get(new[:2], [])
                     for el in el.findall('delta'):
@@ -100,34 +134,12 @@ class DeltaInfo:
                         if size >= best or old not in have:
                             continue
                         best = size
+                        remote = el.find('filename').text
                         csum = el.find('checksum')
                         csum = csum.get('type'), csum.text
-                        self.deltas[po] = size, el.find('filename').text, csum
+                        pkgs[index] = DeltaPackage(po, size, remote, csum)
                 el.clear()
 
-    def to_drpm(self, po):
-        try: size, remote, csum = self.deltas[po]
-        except KeyError: return False
-        self._rpmsave[po] = po.packagesize, po.relativepath, po.localpath
-
-        # update stats
-        self.rpmsize += po.packagesize
-        self.deltasize += size
-
-        # update size/path/checksum to drpm values
-        po.packagesize = size
-        po.relativepath = remote
-        po.localpath = os.path.dirname(po.localpath) +'/'+ os.path.basename(remote)
-        po.returnIdSum = lambda: csum
-        return True
-
-    def to_rpm(self, po):
-        if po not in self._rpmsave:
-            return
-        # revert back to RPM
-        po.packagesize, po.relativepath, po.localpath = self._rpmsave.pop(po)
-        del po.returnIdSum
-
     def wait(self, limit = 1):
         # wait for some jobs, run callbacks
         while len(self.jobs) >= limit:
@@ -139,20 +151,16 @@ class DeltaInfo:
             callback(code)
 
     def rebuild(self, po, adderror):
-        # restore rpm values
-        deltapath = po.localpath
-        po.packagesize, po.relativepath, po.localpath = self._rpmsave.pop(po)
-        del po.returnIdSum
-
         # this runs when worker finishes
         def callback(code):
             if code != 0:
                 return adderror(po, _('Delta RPM rebuild failed'))
-            if not po.verifyLocalPkg():
+            if not po.rpm.verifyLocalPkg():
                 return adderror(po, _('Checksum of the delta-rebuilt RPM failed'))
-            os.unlink(deltapath)
+            os.unlink(po.localpath)
+            po.localpath = po.rpm.localpath # for --downloadonly
 
         # spawn a worker process
         self.wait(self.limit)
-        pid = os.spawnl(os.P_NOWAIT, APPLYDELTA, APPLYDELTA, deltapath, po.localpath)
+        pid = os.spawnl(os.P_NOWAIT, APPLYDELTA, APPLYDELTA, po.localpath, po.rpm.localpath)
         self.jobs[pid] = callback
commit 462d2bb54c1e500d464b697ca3fd105dd42ba218
Author: Zdenek Pavlas <zpavlas at redhat.com>
Date:   Tue Feb 26 14:12:49 2013 +0100

    rename option "presto" to "deltarpm"

diff --git a/docs/yum.conf.5 b/docs/yum.conf.5
index 25f0c19..f9a924a 100644
--- a/docs/yum.conf.5
+++ b/docs/yum.conf.5
@@ -372,7 +372,7 @@ default of 5 connections.  Note that there are also implicit per-mirror limits
 and the downloader honors these too.
 
 .IP
-\fBpresto\fR
+\fBdeltarpm\fR
 
 When non-zero, delta-RPM files are used if available.  The value specifies
 the maximum number of "applydeltarpm" processes Yum will spawn. (4 by default).
diff --git a/yum/__init__.py b/yum/__init__.py
index fca6d2d..8ef7989 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -90,7 +90,7 @@ from packages import YumUrlPackage, YumNotFoundPackage
 from constants import *
 from yum.rpmtrans import RPMTransaction,SimpleCliCallBack
 from yum.i18n import to_unicode, to_str, exception2msg
-from yum.presto import Presto
+from yum.drpm import DeltaInfo
 
 import string
 import StringIO
@@ -2258,7 +2258,7 @@ much more problems).
             pkgs.append(po)
 
         # download presto metadata
-        presto = Presto(self, pkgs)
+        presto = DeltaInfo(self, pkgs)
         for po in pkgs:
             if presto.to_drpm(po) and verify_local(po):
                 # there's .drpm already, use it
diff --git a/yum/config.py b/yum/config.py
index 4aae831..d2ca937 100644
--- a/yum/config.py
+++ b/yum/config.py
@@ -791,7 +791,7 @@ class YumConf(StartupConf):
             allowed = ('ipv4', 'ipv6', 'whatever'),
             mapper  = {'4': 'ipv4', '6': 'ipv6'})
     max_connections = IntOption(0)
-    presto = IntOption(4)
+    deltarpm = IntOption(4)
 
     http_caching = SelectionOption('all', ('none', 'packages', 'all'))
     metadata_expire = SecondsOption(60 * 60 * 6) # Time in seconds (6h).
@@ -950,7 +950,7 @@ class RepoConf(BaseConfig):
     throttle = Inherit(YumConf.throttle)
     timeout = Inherit(YumConf.timeout)
     ip_resolve = Inherit(YumConf.ip_resolve)
-    presto = Inherit(YumConf.presto)
+    deltarpm = Inherit(YumConf.deltarpm)
 
     http_caching = Inherit(YumConf.http_caching)
     metadata_expire = Inherit(YumConf.metadata_expire)
diff --git a/yum/drpm.py b/yum/drpm.py
new file mode 100644
index 0000000..81afe80
--- /dev/null
+++ b/yum/drpm.py
@@ -0,0 +1,158 @@
+#  Integrated delta rpm support
+#  Copyright 2013 Zdenek Pavlas
+
+#   This library is free software; you can redistribute it and/or
+#   modify it under the terms of the GNU Lesser General Public
+#   License as published by the Free Software Foundation; either
+#   version 2.1 of the License, or (at your option) any later version.
+#
+#   This library is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#   Lesser General Public License for more details.
+#
+#   You should have received a copy of the GNU Lesser General Public
+#   License along with this library; if not, write to the
+#      Free Software Foundation, Inc.,
+#      59 Temple Place, Suite 330,
+#      Boston, MA  02111-1307  USA
+
+from yum.constants import TS_UPDATE
+from yum.Errors import RepoError
+from yum.i18n import exception2msg, _
+from urlgrabber import grabber
+async = hasattr(grabber, 'parallel_wait')
+from xml.etree.cElementTree import iterparse
+import os, gzip
+
+APPLYDELTA = '/usr/bin/applydeltarpm'
+
+class DeltaInfo:
+    def __init__(self, ayum, pkgs):
+        self.verbose_logger = ayum.verbose_logger
+        self.deltas = {}
+        self._rpmsave = {}
+        self.rpmsize = 0
+        self.deltasize = 0
+        self.jobs = {}
+        self.limit = ayum.conf.deltarpm
+
+        # calculate update sizes
+        pinfo = {}
+        reposize = {}
+        for po in pkgs:
+            if not po.repo.deltarpm:
+                continue
+            if po.state != TS_UPDATE and po.name not in ayum.conf.installonlypkgs:
+                continue
+            pinfo.setdefault(po.repo, {})[po.pkgtup] = po
+            reposize[po.repo] = reposize.get(po.repo, 0) + po.size
+
+        # don't use deltas when deltarpm not installed
+        if reposize and not os.access(APPLYDELTA, os.X_OK):
+            self.verbose_logger.info(_('Delta RPMs disabled because %s not installed.'), APPLYDELTA)
+            return
+
+        # download delta metadata
+        mdpath = {}
+        for repo in reposize:
+            self.limit = max(self.limit, repo.deltarpm)
+            for name in ('prestodelta', 'deltainfo'):
+                try: data = repo.repoXML.getData(name); break
+                except: pass
+            else:
+                self.verbose_logger.warn(_('No Presto metadata available for %s'), repo)
+                continue
+            path = repo.cachedir +'/'+ os.path.basename(data.location[1])
+            if not os.path.exists(path) and int(data.size) > reposize[repo]:
+                self.verbose_logger.info(_('Not downloading Presto metadata for %s'), repo)
+                continue
+
+            def failfunc(e, name=name, repo=repo):
+                mdpath.pop(repo, None)
+                if hasattr(e, 'exception'): e = e.exception
+                self.verbose_logger.warn(_('Failed to download %s for repository %s: %s'),
+                                         name, repo, exception2msg(e))
+            kwargs = {}
+            if async and repo._async:
+                kwargs['failfunc'] = failfunc
+                kwargs['async'] = True
+            try: mdpath[repo] = repo._retrieveMD(name, **kwargs)
+            except Errors.RepoError, e: failfunc(e)
+        if async:
+            grabber.parallel_wait()
+
+        # parse metadata, populate self.deltas
+        for repo, path in mdpath.items():
+            pinfo_repo = pinfo[repo]
+            if path.endswith('.gz'):
+                path = gzip.open(path)
+            for ev, el in iterparse(path):
+                if el.tag != 'newpackage': continue
+                new = el.get('name'), el.get('arch'), el.get('epoch'), el.get('version'), el.get('release')
+                po = pinfo_repo.get(new)
+                if po:
+                    best = po.size * 0.75 # make this configurable?
+                    have = ayum._up.installdict.get(new[:2], [])
+                    for el in el.findall('delta'):
+                        size = int(el.find('size').text)
+                        old = el.get('oldepoch'), el.get('oldversion'), el.get('oldrelease')
+                        if size >= best or old not in have:
+                            continue
+                        best = size
+                        csum = el.find('checksum')
+                        csum = csum.get('type'), csum.text
+                        self.deltas[po] = size, el.find('filename').text, csum
+                el.clear()
+
+    def to_drpm(self, po):
+        try: size, remote, csum = self.deltas[po]
+        except KeyError: return False
+        self._rpmsave[po] = po.packagesize, po.relativepath, po.localpath
+
+        # update stats
+        self.rpmsize += po.packagesize
+        self.deltasize += size
+
+        # update size/path/checksum to drpm values
+        po.packagesize = size
+        po.relativepath = remote
+        po.localpath = os.path.dirname(po.localpath) +'/'+ os.path.basename(remote)
+        po.returnIdSum = lambda: csum
+        return True
+
+    def to_rpm(self, po):
+        if po not in self._rpmsave:
+            return
+        # revert back to RPM
+        po.packagesize, po.relativepath, po.localpath = self._rpmsave.pop(po)
+        del po.returnIdSum
+
+    def wait(self, limit = 1):
+        # wait for some jobs, run callbacks
+        while len(self.jobs) >= limit:
+            pid, code = os.wait()
+            # urlgrabber spawns child jobs, too.  But they exit synchronously,
+            # so we should never see an unknown pid here.
+            assert pid in self.jobs
+            callback = self.jobs.pop(pid)
+            callback(code)
+
+    def rebuild(self, po, adderror):
+        # restore rpm values
+        deltapath = po.localpath
+        po.packagesize, po.relativepath, po.localpath = self._rpmsave.pop(po)
+        del po.returnIdSum
+
+        # this runs when worker finishes
+        def callback(code):
+            if code != 0:
+                return adderror(po, _('Delta RPM rebuild failed'))
+            if not po.verifyLocalPkg():
+                return adderror(po, _('Checksum of the delta-rebuilt RPM failed'))
+            os.unlink(deltapath)
+
+        # spawn a worker process
+        self.wait(self.limit)
+        pid = os.spawnl(os.P_NOWAIT, APPLYDELTA, APPLYDELTA, deltapath, po.localpath)
+        self.jobs[pid] = callback
diff --git a/yum/presto.py b/yum/presto.py
deleted file mode 100644
index 002a82a..0000000
--- a/yum/presto.py
+++ /dev/null
@@ -1,158 +0,0 @@
-#  Integrated delta rpm support
-#  Copyright 2013 Zdenek Pavlas
-
-#   This library is free software; you can redistribute it and/or
-#   modify it under the terms of the GNU Lesser General Public
-#   License as published by the Free Software Foundation; either
-#   version 2.1 of the License, or (at your option) any later version.
-#
-#   This library is distributed in the hope that it will be useful,
-#   but WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   Lesser General Public License for more details.
-#
-#   You should have received a copy of the GNU Lesser General Public
-#   License along with this library; if not, write to the
-#      Free Software Foundation, Inc.,
-#      59 Temple Place, Suite 330,
-#      Boston, MA  02111-1307  USA
-
-from yum.constants import TS_UPDATE
-from yum.Errors import RepoError
-from yum.i18n import exception2msg, _
-from urlgrabber import grabber
-async = hasattr(grabber, 'parallel_wait')
-from xml.etree.cElementTree import iterparse
-import os, gzip
-
-APPLYDELTA = '/usr/bin/applydeltarpm'
-
-class Presto:
-    def __init__(self, ayum, pkgs):
-        self.verbose_logger = ayum.verbose_logger
-        self.deltas = {}
-        self._rpmsave = {}
-        self.rpmsize = 0
-        self.deltasize = 0
-        self.jobs = {}
-        self.limit = ayum.conf.presto
-
-        # calculate update sizes
-        pinfo = {}
-        reposize = {}
-        for po in pkgs:
-            if not po.repo.presto:
-                continue
-            if po.state != TS_UPDATE and po.name not in ayum.conf.installonlypkgs:
-                continue
-            self.limit = max(self.limit, po.repo.presto)
-            pinfo.setdefault(po.repo, {})[po.pkgtup] = po
-            reposize[po.repo] = reposize.get(po.repo, 0) + po.size
-
-        # don't use deltas when deltarpm not installed
-        if reposize and not os.access(APPLYDELTA, os.X_OK):
-            self.verbose_logger.info(_('Delta RPMs disabled because %s not installed.'), APPLYDELTA)
-            return
-
-        # download delta metadata
-        mdpath = {}
-        for repo in reposize:
-            for name in ('prestodelta', 'deltainfo'):
-                try: data = repo.repoXML.getData(name); break
-                except: pass
-            else:
-                self.verbose_logger.warn(_('No Presto metadata available for %s'), repo)
-                continue
-            path = repo.cachedir +'/'+ os.path.basename(data.location[1])
-            if not os.path.exists(path) and int(data.size) > reposize[repo]:
-                self.verbose_logger.info(_('Not downloading Presto metadata for %s'), repo)
-                continue
-
-            def failfunc(e, name=name, repo=repo):
-                mdpath.pop(repo, None)
-                if hasattr(e, 'exception'): e = e.exception
-                self.verbose_logger.warn(_('Failed to download %s for repository %s: %s'),
-                                         name, repo, exception2msg(e))
-            kwargs = {}
-            if async and repo._async:
-                kwargs['failfunc'] = failfunc
-                kwargs['async'] = True
-            try: mdpath[repo] = repo._retrieveMD(name, **kwargs)
-            except Errors.RepoError, e: failfunc(e)
-        if async:
-            grabber.parallel_wait()
-
-        # parse metadata, populate self.deltas
-        for repo, path in mdpath.items():
-            pinfo_repo = pinfo[repo]
-            if path.endswith('.gz'):
-                path = gzip.open(path)
-            for ev, el in iterparse(path):
-                if el.tag != 'newpackage': continue
-                new = el.get('name'), el.get('arch'), el.get('epoch'), el.get('version'), el.get('release')
-                po = pinfo_repo.get(new)
-                if po:
-                    best = po.size * 0.75 # make this configurable?
-                    have = ayum._up.installdict.get(new[:2], [])
-                    for el in el.findall('delta'):
-                        size = int(el.find('size').text)
-                        old = el.get('oldepoch'), el.get('oldversion'), el.get('oldrelease')
-                        if size >= best or old not in have:
-                            continue
-                        best = size
-                        csum = el.find('checksum')
-                        csum = csum.get('type'), csum.text
-                        self.deltas[po] = size, el.find('filename').text, csum
-                el.clear()
-
-    def to_drpm(self, po):
-        try: size, remote, csum = self.deltas[po]
-        except KeyError: return False
-        self._rpmsave[po] = po.packagesize, po.relativepath, po.localpath
-
-        # update stats
-        self.rpmsize += po.packagesize
-        self.deltasize += size
-
-        # update size/path/checksum to drpm values
-        po.packagesize = size
-        po.relativepath = remote
-        po.localpath = os.path.dirname(po.localpath) +'/'+ os.path.basename(remote)
-        po.returnIdSum = lambda: csum
-        return True
-
-    def to_rpm(self, po):
-        if po not in self._rpmsave:
-            return
-        # revert back to RPM
-        po.packagesize, po.relativepath, po.localpath = self._rpmsave.pop(po)
-        del po.returnIdSum
-
-    def wait(self, limit = 1):
-        # wait for some jobs, run callbacks
-        while len(self.jobs) >= limit:
-            pid, code = os.wait()
-            # urlgrabber spawns child jobs, too.  But they exit synchronously,
-            # so we should never see an unknown pid here.
-            assert pid in self.jobs
-            callback = self.jobs.pop(pid)
-            callback(code)
-
-    def rebuild(self, po, adderror):
-        # restore rpm values
-        deltapath = po.localpath
-        po.packagesize, po.relativepath, po.localpath = self._rpmsave.pop(po)
-        del po.returnIdSum
-
-        # this runs when worker finishes
-        def callback(code):
-            if code != 0:
-                return adderror(po, _('Delta RPM rebuild failed'))
-            if not po.verifyLocalPkg():
-                return adderror(po, _('Checksum of the delta-rebuilt RPM failed'))
-            os.unlink(deltapath)
-
-        # spawn a worker process
-        self.wait(self.limit)
-        pid = os.spawnl(os.P_NOWAIT, APPLYDELTA, APPLYDELTA, deltapath, po.localpath)
-        self.jobs[pid] = callback
commit f7a296a6e105b3014ae76da675a8cc6da5ca9999
Author: Zdenek Pavlas <zpavlas at redhat.com>
Date:   Tue Feb 26 09:53:17 2013 +0100

    Deactivate presto if deltarpm is not installed.  BZ 915426

diff --git a/yum/presto.py b/yum/presto.py
index cb33918..002a82a 100644
--- a/yum/presto.py
+++ b/yum/presto.py
@@ -25,6 +25,8 @@ async = hasattr(grabber, 'parallel_wait')
 from xml.etree.cElementTree import iterparse
 import os, gzip
 
+APPLYDELTA = '/usr/bin/applydeltarpm'
+
 class Presto:
     def __init__(self, ayum, pkgs):
         self.verbose_logger = ayum.verbose_logger
@@ -47,6 +49,11 @@ class Presto:
             pinfo.setdefault(po.repo, {})[po.pkgtup] = po
             reposize[po.repo] = reposize.get(po.repo, 0) + po.size
 
+        # don't use deltas when deltarpm not installed
+        if reposize and not os.access(APPLYDELTA, os.X_OK):
+            self.verbose_logger.info(_('Delta RPMs disabled because %s not installed.'), APPLYDELTA)
+            return
+
         # download delta metadata
         mdpath = {}
         for repo in reposize:
@@ -147,5 +154,5 @@ class Presto:
 
         # spawn a worker process
         self.wait(self.limit)
-        pid = os.spawnl(os.P_NOWAIT, '/usr/bin/applydeltarpm', 'applydeltarpm', deltapath, po.localpath)
+        pid = os.spawnl(os.P_NOWAIT, APPLYDELTA, APPLYDELTA, deltapath, po.localpath)
         self.jobs[pid] = callback
commit 659d3003b77aef6603b490bd2920638f4bbcab9e
Author: Zdenek Pavlas <zpavlas at redhat.com>
Date:   Tue Feb 26 09:52:02 2013 +0100

    Download drpms first, to overlap delta rebuild with rpm download.

diff --git a/yum/__init__.py b/yum/__init__.py
index 2128785..fca6d2d 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -2190,7 +2190,9 @@ much more problems).
             a = apo.getDiscNum()
             b = bpo.getDiscNum()
             if a is None and b is None:
-                return cmp(apo, bpo)
+                # deltas first to start rebuilding asap
+                return cmp(bpo in presto.deltas, apo in presto.deltas) \
+                    or cmp(apo, bpo)
             if a is None:
                 return -1
             if b is None:
commit 95ffe1d098895849923795a438ded3a189c8d2f6
Author: Zdenek Pavlas <zpavlas at redhat.com>
Date:   Thu Feb 21 17:16:28 2013 +0100

    Use multiple applydeltarpm workers

diff --git a/docs/yum.conf.5 b/docs/yum.conf.5
index ba3f8e1..25f0c19 100644
--- a/docs/yum.conf.5
+++ b/docs/yum.conf.5
@@ -374,9 +374,8 @@ and the downloader honors these too.
 .IP
 \fBpresto\fR
 
-Either `0' or `1'. Set this to `1' to use delta-RPM files, if available.
-This reduces the download size of updates significantly, but local rebuild
-is CPU intensive.  Default is `1' (on).
+When non-zero, delta-RPM files are used if available.  The value specifies
+the maximum number of "applydeltarpm" processes Yum will spawn. (4 by default).
 
 .IP
 \fBsslcacert \fR
diff --git a/yum/__init__.py b/yum/__init__.py
index d378080..2128785 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -2328,6 +2328,7 @@ much more problems).
                     adderror(po, exception2msg(e))
             if async:
                 urlgrabber.grabber.parallel_wait()
+            presto.wait()
 
             if hasattr(urlgrabber.progress, 'text_meter_total_size'):
                 urlgrabber.progress.text_meter_total_size(0)
diff --git a/yum/config.py b/yum/config.py
index d279ab3..4aae831 100644
--- a/yum/config.py
+++ b/yum/config.py
@@ -791,7 +791,7 @@ class YumConf(StartupConf):
             allowed = ('ipv4', 'ipv6', 'whatever'),
             mapper  = {'4': 'ipv4', '6': 'ipv6'})
     max_connections = IntOption(0)
-    presto = BoolOption(True)
+    presto = IntOption(4)
 
     http_caching = SelectionOption('all', ('none', 'packages', 'all'))
     metadata_expire = SecondsOption(60 * 60 * 6) # Time in seconds (6h).
diff --git a/yum/presto.py b/yum/presto.py
index 1805974..cb33918 100644
--- a/yum/presto.py
+++ b/yum/presto.py
@@ -23,7 +23,7 @@ from yum.i18n import exception2msg, _
 from urlgrabber import grabber
 async = hasattr(grabber, 'parallel_wait')
 from xml.etree.cElementTree import iterparse
-import os, gzip, subprocess
+import os, gzip
 
 class Presto:
     def __init__(self, ayum, pkgs):
@@ -32,6 +32,8 @@ class Presto:
         self._rpmsave = {}
         self.rpmsize = 0
         self.deltasize = 0
+        self.jobs = {}
+        self.limit = ayum.conf.presto
 
         # calculate update sizes
         pinfo = {}
@@ -41,6 +43,7 @@ class Presto:
                 continue
             if po.state != TS_UPDATE and po.name not in ayum.conf.installonlypkgs:
                 continue
+            self.limit = max(self.limit, po.repo.presto)
             pinfo.setdefault(po.repo, {})[po.pkgtup] = po
             reposize[po.repo] = reposize.get(po.repo, 0) + po.size
 
@@ -118,17 +121,31 @@ class Presto:
         po.packagesize, po.relativepath, po.localpath = self._rpmsave.pop(po)
         del po.returnIdSum
 
+    def wait(self, limit = 1):
+        # wait for some jobs, run callbacks
+        while len(self.jobs) >= limit:
+            pid, code = os.wait()
+            # urlgrabber spawns child jobs, too.  But they exit synchronously,
+            # so we should never see an unknown pid here.
+            assert pid in self.jobs
+            callback = self.jobs.pop(pid)
+            callback(code)
+
     def rebuild(self, po, adderror):
         # restore rpm values
         deltapath = po.localpath
         po.packagesize, po.relativepath, po.localpath = self._rpmsave.pop(po)
         del po.returnIdSum
 
-        # rebuild it from drpm
-        if subprocess.call(['/usr/bin/applydeltarpm', deltapath, po.localpath]) != 0:
-            return adderror(po, _('Delta RPM rebuild failed'))
-        # source drpm was already checksummed.. is this necessary?
-        if not po.verifyLocalPkg():
-            return adderror(po, _('Checksum of the delta-rebuilt RPM failed'))
-        # no need to keep this
-        os.unlink(deltapath)
+        # this runs when worker finishes
+        def callback(code):
+            if code != 0:
+                return adderror(po, _('Delta RPM rebuild failed'))
+            if not po.verifyLocalPkg():
+                return adderror(po, _('Checksum of the delta-rebuilt RPM failed'))
+            os.unlink(deltapath)
+
+        # spawn a worker process
+        self.wait(self.limit)
+        pid = os.spawnl(os.P_NOWAIT, '/usr/bin/applydeltarpm', 'applydeltarpm', deltapath, po.localpath)
+        self.jobs[pid] = callback
commit 738b2bc2073546a24a289a97d80353edd27e3a5f
Author: Zdenek Pavlas <zpavlas at redhat.com>
Date:   Thu Feb 21 15:40:34 2013 +0100

    downloadonly: don't update localpath on errors, so we retry correctly.

diff --git a/yum/__init__.py b/yum/__init__.py
index f70cdc0..d378080 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -2348,7 +2348,7 @@ much more problems).
                         # verifyPkg() didn't complain, so (potentially)
                         # overwriting another copy should not be a problem
                         os.rename(po.localpath, rpmfile)
-                    po.localpath = rpmfile
+                        po.localpath = rpmfile
                     
             fatal = False
             for po in errors:
commit 4fa5ceefa1f63f774a56340a50735877e2e74eb4
Author: Zdenek Pavlas <zpavlas at redhat.com>
Date:   Thu Feb 21 12:19:21 2013 +0100

    remove the sequence check
    
    The only case where this makes a difference is a broken old
    package install.  Delta rebuild fails, and we retry downloading
    a full rpm file instead.  Cost and probability of this is low.
    OTOH, the check is quite slow, and avoiding it is a big win.

diff --git a/yum/presto.py b/yum/presto.py
index 3bcd391..1805974 100644
--- a/yum/presto.py
+++ b/yum/presto.py
@@ -89,12 +89,6 @@ class Presto:
                         old = el.get('oldepoch'), el.get('oldversion'), el.get('oldrelease')
                         if size >= best or old not in have:
                             continue
-                        # the old version is installed, seq check should never fail. kill this?
-                        seq = el.find('sequence').text
-                        if subprocess.call(['/usr/bin/applydeltarpm', '-C', '-s', seq]) != 0:
-                            self.verbose_logger.warn(_('Deltarpm sequence check failed for %s'), seq)
-                            continue
-
                         best = size
                         csum = el.find('checksum')
                         csum = csum.get('type'), csum.text
commit 76a5787a799de1e15ebc6bf89fc52346b75d1075
Author: Zdenek Pavlas <zpavlas at redhat.com>
Date:   Wed Feb 20 18:17:25 2013 +0100

    Add DRPM => RPM retries
    
    It's ok to try downloading rpms when delta rebuild fails,
    but when we can't DL a drpm, rpm probably fails, too.
    To handle possibly obsolete presto MD, retry anyway..

diff --git a/yum/__init__.py b/yum/__init__.py
index bb4d4a7..f70cdc0 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -2275,7 +2275,7 @@ much more problems).
             self.closeRpmDB()
             self.doUnlock()
 
-        if 1:
+        while True:
             remote_pkgs.sort(mediasort)
             #  This is kind of a hack and does nothing in non-Fedora versions,
             # we'll fix it one way or anther soon.
@@ -2350,6 +2350,22 @@ much more problems).
                         os.rename(po.localpath, rpmfile)
                     po.localpath = rpmfile
                     
+            fatal = False
+            for po in errors:
+                if po not in presto.deltas:
+                    fatal = True; break
+            if not errors or fatal:
+                break
+
+            # there were drpm related errors *only*
+            remote_pkgs = errors.keys()
+            remote_size = 0
+            for po in remote_pkgs:
+                presto.to_rpm(po) # needed, we don't rebuild() when DL fails
+                remote_size += po.size
+            self.verbose_logger.warn(_('Some delta RPMs failed to download or rebuild. Retrying..'))
+            presto.deltas.clear() # any error is now considered fatal
+
         if not downloadonly:
             # XXX: Run unlocked?  Skip this for now..
             self.plugins.run('postdownload', pkglist=pkglist, errors=errors)
diff --git a/yum/presto.py b/yum/presto.py
index 08462f5..3bcd391 100644
--- a/yum/presto.py
+++ b/yum/presto.py
@@ -99,8 +99,6 @@ class Presto:
                         csum = el.find('checksum')
                         csum = csum.get('type'), csum.text
                         self.deltas[po] = size, el.find('filename').text, csum
-                    if po not in self.deltas:
-                        self.verbose_logger.warn(_('No suitable drpm files for %s'), po)
                 el.clear()
 
     def to_drpm(self, po):
@@ -119,10 +117,17 @@ class Presto:
         po.returnIdSum = lambda: csum
         return True
 
+    def to_rpm(self, po):
+        if po not in self._rpmsave:
+            return
+        # revert back to RPM
+        po.packagesize, po.relativepath, po.localpath = self._rpmsave.pop(po)
+        del po.returnIdSum
+
     def rebuild(self, po, adderror):
         # restore rpm values
         deltapath = po.localpath
-        po.packagesize, po.relativepath, po.localpath = self._rpmsave[po]
+        po.packagesize, po.relativepath, po.localpath = self._rpmsave.pop(po)
         del po.returnIdSum
 
         # rebuild it from drpm
commit 931828c457175376f8bbd8d94129c060b7252b1a
Author: Zdenek Pavlas <zpavlas at redhat.com>
Date:   Wed Feb 20 18:09:07 2013 +0100

    indent only, to make next patch easier to read

diff --git a/yum/__init__.py b/yum/__init__.py
index c202d36..bb4d4a7 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -2275,80 +2275,82 @@ much more problems).
             self.closeRpmDB()
             self.doUnlock()
 
-        remote_pkgs.sort(mediasort)
-        #  This is kind of a hack and does nothing in non-Fedora versions,
-        # we'll fix it one way or anther soon.
-        if (hasattr(urlgrabber.progress, 'text_meter_total_size') and
-            len(remote_pkgs) > 1):
-            urlgrabber.progress.text_meter_total_size(remote_size)
-        beg_download = time.time()
-        i = 0
-        local_size = [0]
-        done_repos = set()
-        async = hasattr(urlgrabber.grabber, 'parallel_wait')
-        for po in remote_pkgs:
-            i += 1
-
-            def checkfunc(obj, po=po):
-                self.verifyPkg(obj, po, 1)
-                local_size[0] += po.size
-                if hasattr(urlgrabber.progress, 'text_meter_total_size'):
-                    urlgrabber.progress.text_meter_total_size(remote_size,
-                                                              local_size[0])
-                if po in presto.deltas:
-                    presto.rebuild(po, adderror)
-                    return
-                if po.repoid not in done_repos:
-                    done_repos.add(po.repoid)
-                    #  Check a single package per. repo. ... to give a hint to
-                    # the user on big downloads.
-                    result, errmsg = self.sigCheckPkg(po)
-                    if result != 0:
-                        self.verbose_logger.warn("%s", errmsg)
-                po.localpath = obj.filename
-                if po in errors:
-                    del errors[po]
-
-            text = os.path.basename(po.relativepath)
-            kwargs = {}
-            if async and po.repo._async:
-                kwargs['failfunc'] = lambda obj, po=po: adderror(po, exception2msg(obj.exception))
-                kwargs['async'] = True
-            elif not (i == 1 and not local_size[0] and remote_size == po.size):
-                text = '(%s/%s): %s' % (i, len(remote_pkgs), text)
-            try:
-                po.repo.getPackage(po,
-                                   checkfunc=checkfunc,
-                                   text=text,
-                                   cache=po.repo.http_caching != 'none',
-                                   **kwargs
-                                   )
-            except Errors.RepoError, e:
-                adderror(po, exception2msg(e))
-        if async:
-            urlgrabber.grabber.parallel_wait()
-
-        if hasattr(urlgrabber.progress, 'text_meter_total_size'):
-            urlgrabber.progress.text_meter_total_size(0)
-        if callback_total is not None and not errors:
-            callback_total(remote_pkgs, remote_size, beg_download)
-
-        if downloadonly:
+        if 1:
+            remote_pkgs.sort(mediasort)
+            #  This is kind of a hack and does nothing in non-Fedora versions,
+            # we'll fix it one way or anther soon.
+            if (hasattr(urlgrabber.progress, 'text_meter_total_size') and
+                len(remote_pkgs) > 1):
+                urlgrabber.progress.text_meter_total_size(remote_size)
+            beg_download = time.time()
+            i = 0
+            local_size = [0]
+            done_repos = set()
+            async = hasattr(urlgrabber.grabber, 'parallel_wait')
             for po in remote_pkgs:
-                rpmfile = po.localpath.rsplit('.', 2)[0]
-                if po in errors:
-                    # we may throw away partial file here- but we don't lock,
-                    # so can't rename tempfile to rpmfile safely
-                    misc.unlink_f(po.localpath)
-
-                #  Note that for file:// repos. urlgrabber won't "download"
-                # so we have to check that po.localpath exists.
-                elif os.path.exists(po.localpath):
-                    # verifyPkg() didn't complain, so (potentially)
-                    # overwriting another copy should not be a problem
-                    os.rename(po.localpath, rpmfile)
-                po.localpath = rpmfile
-        else:
+                i += 1
+
+                def checkfunc(obj, po=po):
+                    self.verifyPkg(obj, po, 1)
+                    local_size[0] += po.size
+                    if hasattr(urlgrabber.progress, 'text_meter_total_size'):
+                        urlgrabber.progress.text_meter_total_size(remote_size,
+                                                                  local_size[0])
+                    if po in presto.deltas:
+                        presto.rebuild(po, adderror)
+                        return
+                    if po.repoid not in done_repos:
+                        done_repos.add(po.repoid)
+                        #  Check a single package per. repo. ... to give a hint to
+                        # the user on big downloads.
+                        result, errmsg = self.sigCheckPkg(po)
+                        if result != 0:
+                            self.verbose_logger.warn("%s", errmsg)
+                    po.localpath = obj.filename
+                    if po in errors:
+                        del errors[po]
+
+                text = os.path.basename(po.relativepath)
+                kwargs = {}
+                if async and po.repo._async:
+                    kwargs['failfunc'] = lambda obj, po=po: adderror(po, exception2msg(obj.exception))
+                    kwargs['async'] = True
+                elif not (i == 1 and not local_size[0] and remote_size == po.size):
+                    text = '(%s/%s): %s' % (i, len(remote_pkgs), text)
+                try:
+                    po.repo.getPackage(po,
+                                       checkfunc=checkfunc,
+                                       text=text,
+                                       cache=po.repo.http_caching != 'none',
+                                       **kwargs
+                                       )
+                except Errors.RepoError, e:
+                    adderror(po, exception2msg(e))
+            if async:
+                urlgrabber.grabber.parallel_wait()
+
+            if hasattr(urlgrabber.progress, 'text_meter_total_size'):
+                urlgrabber.progress.text_meter_total_size(0)
+            if callback_total is not None and not errors:
+                callback_total(remote_pkgs, remote_size, beg_download)
+
+            if downloadonly:
+                for po in remote_pkgs:
+                    rpmfile = po.localpath.rsplit('.', 2)[0]
+                    if po in errors:
+                        # we may throw away partial file here- but we don't lock,
+                        # so can't rename tempfile to rpmfile safely
+                        misc.unlink_f(po.localpath)
+
+                    #  Note that for file:// repos. urlgrabber won't "download"
+                    # so we have to check that po.localpath exists.
+                    elif os.path.exists(po.localpath):
+                        # verifyPkg() didn't complain, so (potentially)
+                        # overwriting another copy should not be a problem
+                        os.rename(po.localpath, rpmfile)
+                    po.localpath = rpmfile
+                    
+        if not downloadonly:
             # XXX: Run unlocked?  Skip this for now..
             self.plugins.run('postdownload', pkglist=pkglist, errors=errors)
 
commit 68d94e39bfa0d4b33ab72321318e9e5435d609af
Author: Zdenek Pavlas <zpavlas at redhat.com>
Date:   Wed Feb 20 12:43:53 2013 +0100

    Integrate presto support to yum
    
    Add the "presto" bool config option, document it.
    Add hooks to downloadPkgs().

diff --git a/docs/yum.conf.5 b/docs/yum.conf.5
index 6f400cf..ba3f8e1 100644
--- a/docs/yum.conf.5
+++ b/docs/yum.conf.5
@@ -372,6 +372,13 @@ default of 5 connections.  Note that there are also implicit per-mirror limits
 and the downloader honors these too.
 
 .IP
+\fBpresto\fR
+
+Either `0' or `1'. Set this to `1' to use delta-RPM files, if available.
+This reduces the download size of updates significantly, but local rebuild
+is CPU intensive.  Default is `1' (on).
+
+.IP
 \fBsslcacert \fR
 Path to the directory containing the databases of the certificate authorities
 yum should use to verify SSL certificates. Defaults to none - uses system
@@ -930,6 +937,11 @@ repository.
 Overrides the \fBip_resolve\fR option from the [main] section for this
 repository.
 
+.IP
+\fBpresto\fR
+
+Overrides the \fBpresto\fR option from the [main] section for this
+repository.
 
 .IP
 \fBsslcacert \fR
diff --git a/yum/__init__.py b/yum/__init__.py
index b445831..c202d36 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -90,6 +90,7 @@ from packages import YumUrlPackage, YumNotFoundPackage
 from constants import *
 from yum.rpmtrans import RPMTransaction,SimpleCliCallBack
 from yum.i18n import to_unicode, to_str, exception2msg
+from yum.presto import Presto
 
 import string
 import StringIO
@@ -2244,6 +2245,7 @@ much more problems).
                 po.basepath # prefetch now; fails when repos are closed
             return False
 
+        pkgs = []
         for po in pkglist:
             if hasattr(po, 'pkgtype') and po.pkgtype == 'local':
                 continue
@@ -2251,8 +2253,21 @@ much more problems).
                 continue
             if errors:
                 return errors
+            pkgs.append(po)
+
+        # download presto metadata
+        presto = Presto(self, pkgs)
+        for po in pkgs:
+            if presto.to_drpm(po) and verify_local(po):
+                # there's .drpm already, use it
+                presto.rebuild(po, adderror)
+                continue
             remote_pkgs.append(po)
             remote_size += po.size
+        if presto.deltasize:
+            self.verbose_logger.info(_('Delta RPMs reduced %s of updates to %s (%d%% saved)'),
+                format_number(presto.rpmsize), format_number(presto.deltasize),
+                100 - presto.deltasize*100.0/presto.rpmsize)
 
         if downloadonly:
             # close DBs, unlock
@@ -2280,6 +2295,9 @@ much more problems).
                 if hasattr(urlgrabber.progress, 'text_meter_total_size'):
                     urlgrabber.progress.text_meter_total_size(remote_size,
                                                               local_size[0])
+                if po in presto.deltas:
+                    presto.rebuild(po, adderror)
+                    return
                 if po.repoid not in done_repos:
                     done_repos.add(po.repoid)
                     #  Check a single package per. repo. ... to give a hint to
diff --git a/yum/config.py b/yum/config.py
index 3b22e41..d279ab3 100644
--- a/yum/config.py
+++ b/yum/config.py
@@ -791,6 +791,7 @@ class YumConf(StartupConf):
             allowed = ('ipv4', 'ipv6', 'whatever'),
             mapper  = {'4': 'ipv4', '6': 'ipv6'})
     max_connections = IntOption(0)
+    presto = BoolOption(True)
 
     http_caching = SelectionOption('all', ('none', 'packages', 'all'))
     metadata_expire = SecondsOption(60 * 60 * 6) # Time in seconds (6h).
@@ -949,6 +950,7 @@ class RepoConf(BaseConfig):
     throttle = Inherit(YumConf.throttle)
     timeout = Inherit(YumConf.timeout)
     ip_resolve = Inherit(YumConf.ip_resolve)
+    presto = Inherit(YumConf.presto)
 
     http_caching = Inherit(YumConf.http_caching)
     metadata_expire = Inherit(YumConf.metadata_expire)
diff --git a/yum/presto.py b/yum/presto.py
new file mode 100644
index 0000000..08462f5
--- /dev/null
+++ b/yum/presto.py
@@ -0,0 +1,135 @@
+#  Integrated delta rpm support
+#  Copyright 2013 Zdenek Pavlas
+
+#   This library is free software; you can redistribute it and/or
+#   modify it under the terms of the GNU Lesser General Public
+#   License as published by the Free Software Foundation; either
+#   version 2.1 of the License, or (at your option) any later version.
+#
+#   This library is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#   Lesser General Public License for more details.
+#
+#   You should have received a copy of the GNU Lesser General Public
+#   License along with this library; if not, write to the
+#      Free Software Foundation, Inc.,
+#      59 Temple Place, Suite 330,
+#      Boston, MA  02111-1307  USA
+
+from yum.constants import TS_UPDATE
+from yum.Errors import RepoError
+from yum.i18n import exception2msg, _
+from urlgrabber import grabber
+async = hasattr(grabber, 'parallel_wait')
+from xml.etree.cElementTree import iterparse
+import os, gzip, subprocess
+
+class Presto:
+    def __init__(self, ayum, pkgs):
+        self.verbose_logger = ayum.verbose_logger
+        self.deltas = {}
+        self._rpmsave = {}
+        self.rpmsize = 0
+        self.deltasize = 0
+
+        # calculate update sizes
+        pinfo = {}
+        reposize = {}
+        for po in pkgs:
+            if not po.repo.presto:
+                continue
+            if po.state != TS_UPDATE and po.name not in ayum.conf.installonlypkgs:
+                continue
+            pinfo.setdefault(po.repo, {})[po.pkgtup] = po
+            reposize[po.repo] = reposize.get(po.repo, 0) + po.size
+
+        # download delta metadata
+        mdpath = {}
+        for repo in reposize:
+            for name in ('prestodelta', 'deltainfo'):
+                try: data = repo.repoXML.getData(name); break
+                except: pass
+            else:
+                self.verbose_logger.warn(_('No Presto metadata available for %s'), repo)
+                continue
+            path = repo.cachedir +'/'+ os.path.basename(data.location[1])
+            if not os.path.exists(path) and int(data.size) > reposize[repo]:
+                self.verbose_logger.info(_('Not downloading Presto metadata for %s'), repo)
+                continue
+
+            def failfunc(e, name=name, repo=repo):
+                mdpath.pop(repo, None)
+                if hasattr(e, 'exception'): e = e.exception
+                self.verbose_logger.warn(_('Failed to download %s for repository %s: %s'),
+                                         name, repo, exception2msg(e))
+            kwargs = {}
+            if async and repo._async:
+                kwargs['failfunc'] = failfunc
+                kwargs['async'] = True
+            try: mdpath[repo] = repo._retrieveMD(name, **kwargs)
+            except Errors.RepoError, e: failfunc(e)
+        if async:
+            grabber.parallel_wait()
+
+        # parse metadata, populate self.deltas
+        for repo, path in mdpath.items():
+            pinfo_repo = pinfo[repo]
+            if path.endswith('.gz'):
+                path = gzip.open(path)
+            for ev, el in iterparse(path):
+                if el.tag != 'newpackage': continue
+                new = el.get('name'), el.get('arch'), el.get('epoch'), el.get('version'), el.get('release')
+                po = pinfo_repo.get(new)
+                if po:
+                    best = po.size * 0.75 # make this configurable?
+                    have = ayum._up.installdict.get(new[:2], [])
+                    for el in el.findall('delta'):
+                        size = int(el.find('size').text)
+                        old = el.get('oldepoch'), el.get('oldversion'), el.get('oldrelease')
+                        if size >= best or old not in have:
+                            continue
+                        # the old version is installed, seq check should never fail. kill this?
+                        seq = el.find('sequence').text
+                        if subprocess.call(['/usr/bin/applydeltarpm', '-C', '-s', seq]) != 0:
+                            self.verbose_logger.warn(_('Deltarpm sequence check failed for %s'), seq)
+                            continue
+
+                        best = size
+                        csum = el.find('checksum')
+                        csum = csum.get('type'), csum.text
+                        self.deltas[po] = size, el.find('filename').text, csum
+                    if po not in self.deltas:
+                        self.verbose_logger.warn(_('No suitable drpm files for %s'), po)
+                el.clear()
+
+    def to_drpm(self, po):
+        try: size, remote, csum = self.deltas[po]
+        except KeyError: return False
+        self._rpmsave[po] = po.packagesize, po.relativepath, po.localpath
+
+        # update stats
+        self.rpmsize += po.packagesize
+        self.deltasize += size
+
+        # update size/path/checksum to drpm values
+        po.packagesize = size
+        po.relativepath = remote
+        po.localpath = os.path.dirname(po.localpath) +'/'+ os.path.basename(remote)
+        po.returnIdSum = lambda: csum
+        return True
+
+    def rebuild(self, po, adderror):
+        # restore rpm values
+        deltapath = po.localpath
+        po.packagesize, po.relativepath, po.localpath = self._rpmsave[po]
+        del po.returnIdSum
+
+        # rebuild it from drpm
+        if subprocess.call(['/usr/bin/applydeltarpm', deltapath, po.localpath]) != 0:
+            return adderror(po, _('Delta RPM rebuild failed'))
+        # source drpm was already checksummed.. is this necessary?
+        if not po.verifyLocalPkg():
+            return adderror(po, _('Checksum of the delta-rebuilt RPM failed'))
+        # no need to keep this
+        os.unlink(deltapath)
commit e3e092ea326d1038631414097e88c8d0e7f86b94
Author: Zdenek Pavlas <zpavlas at redhat.com>
Date:   Fri Feb 8 15:20:30 2013 +0100

    downloadPkgs() cleanup
    
    add check_local() function.  Kill the "Recheck if the file is there"
    code, add a duplicated package check and warning instead.

diff --git a/yum/__init__.py b/yum/__init__.py
index d485d05..b445831 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -2200,9 +2200,6 @@ much more problems).
                 return 1
             return 0
         
-        """download list of package objects handed to you, output based on
-           callback, raise yum.Errors.YumBaseError on problems"""
-
         errors = {}
         def adderror(po, msg):
             errors.setdefault(po, []).append(msg)
@@ -2218,41 +2215,45 @@ much more problems).
         self.history.close()
 
         self.plugins.run('predownload', pkglist=pkglist)
+        beenthere = set() # only once, please. BZ 468401
         downloadonly = getattr(self.conf, 'downloadonly', False)
-        repo_cached = False
         remote_pkgs = []
         remote_size = 0
-        for po in pkglist:
-            if hasattr(po, 'pkgtype') and po.pkgtype == 'local':
-                continue
-                    
+
+        def verify_local(po):
             local = po.localPkg()
+            if local in beenthere:
+                # This is definitely a depsolver bug.  Make it fatal?
+                self.verbose_logger.warn(_("ignoring a dupe of %s") % po)
+                return True
+            beenthere.add(local)
             if os.path.exists(local):
-                if not self.verifyPkg(local, po, False):
-                    if po.repo.cache:
-                        repo_cached = True
-                        adderror(po, _('package fails checksum but caching is '
-                            'enabled for %s') % po.repo.id)
-                else:
-                    self.verbose_logger.debug(_("using local copy of %s") %(po,))
-                    continue
-                        
+                if self.verifyPkg(local, po, False):
+                    self.verbose_logger.debug(_("using local copy of %s") % po)
+                    return True
+                if po.repo.cache:
+                    adderror(po, _('package fails checksum but caching is '
+                        'enabled for %s') % po.repo.id)
+                    return False
+                if os.path.getsize(local) >= po.size:
+                    os.unlink(local)
             if downloadonly:
-                # download to temp file
-                rpmfile = po.localpath
                 po.localpath += '.%d.tmp' % os.getpid()
-                try: os.rename(rpmfile, po.localpath)
+                try: os.rename(local, po.localpath)
                 except OSError: pass
                 po.basepath # prefetch now; fails when repos are closed
+            return False
 
+        for po in pkglist:
+            if hasattr(po, 'pkgtype') and po.pkgtype == 'local':
+                continue
+            if verify_local(po):
+                continue
+            if errors:
+                return errors
             remote_pkgs.append(po)
             remote_size += po.size
-            
-            # caching is enabled and the package 
-            # just failed to check out there's no 
-            # way to save this, report the error and return
-            if (self.conf.cache or repo_cached) and errors:
-                return errors
+
         if downloadonly:
             # close DBs, unlock
             self.repos.close()
@@ -2271,20 +2272,7 @@ much more problems).
         done_repos = set()
         async = hasattr(urlgrabber.grabber, 'parallel_wait')
         for po in remote_pkgs:
-            #  Recheck if the file is there, works around a couple of weird
-            # edge cases.
-            local = po.localPkg()
             i += 1
-            if os.path.exists(local):
-                if self.verifyPkg(local, po, False):
-                    self.verbose_logger.debug(_("using local copy of %s") %(po,))
-                    remote_size -= po.size
-                    if hasattr(urlgrabber.progress, 'text_meter_total_size'):
-                        urlgrabber.progress.text_meter_total_size(remote_size,
-                                                                  local_size[0])
-                    continue
-                if os.path.getsize(local) >= po.size:
-                    os.unlink(local)
 
             def checkfunc(obj, po=po):
                 self.verifyPkg(obj, po, 1)
@@ -2337,7 +2325,7 @@ much more problems).
 
                 #  Note that for file:// repos. urlgrabber won't "download"
                 # so we have to check that po.localpath exists.
-                if po not in errors and os.path.exists(po.localpath):
+                elif os.path.exists(po.localpath):
                     # verifyPkg() didn't complain, so (potentially)
                     # overwriting another copy should not be a problem
                     os.rename(po.localpath, rpmfile)


More information about the Yum-commits mailing list