[yum-commits] 2 commits - yumcommands.py yum/drpm.py

zpavlas at osuosl.org zpavlas at osuosl.org
Wed Mar 6 16:16:55 UTC 2013


 yum/drpm.py    |   60 ++++++++++++++++++++++++++++++++++++++-------------------
 yumcommands.py |    1 
 2 files changed, 42 insertions(+), 19 deletions(-)

New commits:
commit bcae68208a17e5a5ba7d6d6e94235e7518685df9
Author: Zdenek Pavlas <zpavlas at redhat.com>
Date:   Wed Mar 6 17:03:07 2013 +0100

    drpm: smarter selection of delta candidates
    
    We'd like to avoid downloading prestodelta.xml unconditionally.
    Trigger it only if the requested package:
    
    1) is an update
    2) is installonly pkg
    3) have .rpm with same name-arch
    
    Also unpack prestodelta.xml on 'yum makecache'.

diff --git a/yum/drpm.py b/yum/drpm.py
index d3284e9..9f0e109 100644
--- a/yum/drpm.py
+++ b/yum/drpm.py
@@ -25,7 +25,7 @@ from misc import checksum, repo_gen_decompress
 from urlgrabber import grabber
 async = hasattr(grabber, 'parallel_wait')
 from xml.etree.cElementTree import iterparse
-import os
+import os, re
 
 APPLYDELTA = '/usr/bin/applydeltarpm'
 
@@ -75,11 +75,26 @@ class DeltaInfo:
         self.limit = ayum.conf.deltarpm
 
         # calculate update sizes
+        oldrpms = {}
         pinfo = {}
         reposize = {}
         for index, po in enumerate(pkgs):
             if not po.repo.deltarpm:
                 continue
+            if po.state == TS_UPDATE: pass
+            elif po.name in ayum.conf.installonlypkgs: pass
+            else:
+                names = oldrpms.get(po.repo)
+                if names is None:
+                    # load all locally cached rpms
+                    names = oldrpms[po.repo] = {}
+                    for rpmfn in os.listdir(po.repo.pkgdir):
+                        m = re.match('^(.+)-(.+)-(.+)\.(.+)\.rpm$', rpmfn)
+                        if m:
+                            n, v, r, a = m.groups()
+                            names.setdefault((n, a), set()).add((v, r))
+                if (po.name, po.arch) not in names:
+                    continue
             pinfo.setdefault(po.repo, {})[po.pkgtup] = index
             reposize[po.repo] = reposize.get(po.repo, 0) + po.size
 
@@ -131,6 +146,7 @@ class DeltaInfo:
                 if index is not None:
                     po = pkgs[index]
                     best = po.size * (repo.deltarpm_percentage / 100.0)
+                    have = oldrpms.get(repo, {}).get((name, arch), {})
                     for el in el.findall('delta'):
                         size = int(el.find('size').text)
                         if size >= best:
@@ -140,12 +156,12 @@ class DeltaInfo:
                         epoch = el.get('oldepoch')
                         ver = el.get('oldversion')
                         rel = el.get('oldrelease')
-                        if ayum.rpmdb.searchNevra(name, epoch, ver, rel, arch):
-                            oldrpm = None
-                        else:
+                        if (ver, rel) in have:
                             oldrpm = '%s/%s-%s-%s.%s.rpm' % (repo.pkgdir, name, ver, rel, arch)
-                            if not os.access(oldrpm, os.R_OK):
+                        else:
+                            if not ayum.rpmdb.searchNevra(name, epoch, ver, rel, arch):
                                 continue
+                            oldrpm = None
 
                         best = size
                         remote = el.find('filename').text
diff --git a/yumcommands.py b/yumcommands.py
index 153a0af..2368ba3 100644
--- a/yumcommands.py
+++ b/yumcommands.py
@@ -1303,6 +1303,7 @@ class MakeCacheCommand(YumCommand):
             fname_map = {'group_gz'   : 'groups.xml',
                          'pkgtags'    : 'pkgtags.sqlite',
                          'updateinfo' : 'updateinfo.xml',
+                         'prestodelta': 'prestodelta.xml',
                          }
             for repo in base.repos.listEnabled():
                 for MD in repo.repoXML.fileTypes():
commit cbd21db1335e36ecb49292cd2a73cbcd56493b35
Author: Zdenek Pavlas <zpavlas at redhat.com>
Date:   Tue Mar 5 15:40:38 2013 +0100

    drpm: use locally cached .rpm files, too
    
    There's a problem that we can no longer use drpms for updates only.
    As a result, we get 'delta rpm metadata not available' message for
    almost every repository used.

diff --git a/yum/drpm.py b/yum/drpm.py
index 8536e4b..d3284e9 100644
--- a/yum/drpm.py
+++ b/yum/drpm.py
@@ -30,7 +30,7 @@ import os
 APPLYDELTA = '/usr/bin/applydeltarpm'
 
 class DeltaPackage:
-    def __init__(self, rpm, size, remote, csum):
+    def __init__(self, rpm, size, remote, csum, oldrpm):
         # copy what needed
         self.rpm = rpm
         self.repo = rpm.repo
@@ -42,6 +42,7 @@ class DeltaPackage:
         self.relativepath = remote
         self.localpath = os.path.dirname(rpm.localpath) +'/'+ os.path.basename(remote)
         self.csum = csum
+        self.oldrpm = oldrpm
 
     def __str__(self):
         return 'Delta RPM of %s' % self.rpm
@@ -79,8 +80,6 @@ class DeltaInfo:
         for index, po in enumerate(pkgs):
             if not po.repo.deltarpm:
                 continue
-            if po.state != TS_UPDATE and po.name not in ayum.conf.installonlypkgs:
-                continue
             pinfo.setdefault(po.repo, {})[po.pkgtup] = index
             reposize[po.repo] = reposize.get(po.repo, 0) + po.size
 
@@ -97,7 +96,7 @@ class DeltaInfo:
                 try: data = repo.repoXML.getData(name); break
                 except: pass
             else:
-                self.verbose_logger.warn(_('No Presto metadata available for %s'), repo)
+                self.verbose_logger.info(_('No Presto metadata available for %s'), repo)
                 continue
             path = repo.cachedir +'/'+ os.path.basename(data.location[1])
             if not os.path.exists(path) and int(data.size) > reposize[repo]:
@@ -118,14 +117,6 @@ class DeltaInfo:
         if async:
             grabber.parallel_wait()
 
-        # use installdict or rpmdb
-        if ayum._up:
-            installed = ayum._up.installdict.get
-        else:
-            installed = lambda (n, a): [
-                (po.epoch, po.version, po.release)
-                for po in ayum.rpmdb.searchNevra(n, None, None, None, a)]
-
         # parse metadata, create DeltaPackage instances
         for repo, cpath in mdpath.items():
             pinfo_repo = pinfo[repo]
@@ -133,22 +124,34 @@ class DeltaInfo:
                                        cached=repo.cache)
             for ev, el in iterparse(path):
                 if el.tag != 'newpackage': continue
-                new = el.get('name'), el.get('arch'), el.get('epoch'), el.get('version'), el.get('release')
+                name = el.get('name')
+                arch = el.get('arch')
+                new = name, arch, el.get('epoch'), el.get('version'), el.get('release')
                 index = pinfo_repo.get(new)
                 if index is not None:
                     po = pkgs[index]
                     best = po.size * (repo.deltarpm_percentage / 100.0)
-                    have = installed(new[:2]) or []
                     for el in el.findall('delta'):
                         size = int(el.find('size').text)
-                        old = el.get('oldepoch'), el.get('oldversion'), el.get('oldrelease')
-                        if size >= best or old not in have:
+                        if size >= best:
                             continue
+
+                        # can we use this delta?
+                        epoch = el.get('oldepoch')
+                        ver = el.get('oldversion')
+                        rel = el.get('oldrelease')
+                        if ayum.rpmdb.searchNevra(name, epoch, ver, rel, arch):
+                            oldrpm = None
+                        else:
+                            oldrpm = '%s/%s-%s-%s.%s.rpm' % (repo.pkgdir, name, ver, rel, arch)
+                            if not os.access(oldrpm, os.R_OK):
+                                continue
+
                         best = size
                         remote = el.find('filename').text
                         csum = el.find('checksum')
                         csum = csum.get('type'), csum.text
-                        pkgs[index] = DeltaPackage(po, size, remote, csum)
+                        pkgs[index] = DeltaPackage(po, size, remote, csum, oldrpm)
                 el.clear()
 
     def wait(self, limit = 1):
@@ -175,5 +178,8 @@ class DeltaInfo:
 
         # spawn a worker process
         self.wait(self.limit)
-        pid = os.spawnl(os.P_NOWAIT, APPLYDELTA, APPLYDELTA, po.localpath, po.rpm.localpath)
+        args = ()
+        if po.oldrpm: args += '-r', po.oldrpm
+        args += po.localpath, po.rpm.localpath
+        pid = os.spawnl(os.P_NOWAIT, APPLYDELTA, APPLYDELTA, *args)
         self.jobs[pid] = callback


More information about the Yum-commits mailing list