[yum-commits] 6 commits - docs/yum.conf.5 yum/config.py yum/drpm.py yum/__init__.py yum/rpmsack.py

James Antill james at osuosl.org
Fri Mar 8 20:53:07 UTC 2013


 docs/yum.conf.5 |    7 +++-
 yum/__init__.py |    4 ++
 yum/config.py   |    2 -
 yum/drpm.py     |   98 +++++++++++++++++++++++++++++++++++++++++++++++++++++---
 yum/rpmsack.py  |   18 ----------
 5 files changed, 105 insertions(+), 24 deletions(-)

New commits:
commit 9283a1973d884eb7ef7401ffd6030b70fbfcca6c
Author: James Antill <james at and.org>
Date:   Fri Mar 8 09:20:56 2013 -0500

    Minor comment so we don't repeat obsoletes provides checking.

diff --git a/yum/rpmsack.py b/yum/rpmsack.py
index 4e26d51..a4e5336 100644
--- a/yum/rpmsack.py
+++ b/yum/rpmsack.py
@@ -1492,6 +1492,8 @@ class RPMDBPackageSack(PackageSackBase):
                 prob = RPMDBProblemDependency(pkg, "conflicts", found=found,
                                               conflicts=res)
                 problems.append(prob)
+
+            # Note that obsoletes are checked seperately, and are name only.
         return problems
 
     def _iter_two_pkgs(self, ignore_provides):
commit e2a9cd5429b0987e2c09b94bccb9f437de0ab01d
Author: James Antill <james at and.org>
Date:   Fri Mar 8 09:16:51 2013 -0500

    Force the deltarpm workers default to 2, 8+ by default crushes laptop IO.

diff --git a/docs/yum.conf.5 b/docs/yum.conf.5
index 9d9a261..e07d38c 100644
--- a/docs/yum.conf.5
+++ b/docs/yum.conf.5
@@ -377,7 +377,12 @@ and the downloader honors these too.
 When non-zero, delta-RPM files are used if available.  The value specifies
 the maximum number of "applydeltarpm" processes Yum will spawn, if the value
 is negative then yum works out how many cores you have and multiplies that
-by the value (cores=2, deltarpm=-2; 4 processes). (-2 by default).
+by the value (cores=2, deltarpm=-2; 4 processes). (2 by default).
+
+Note that the "applydeltarpm" process uses a significant amount of disk IO,
+so running too many instances can significantly slow down all disk IO including
+the downloads that yum is doing (thus. a too high value can make everything
+slower).
 
 .IP
 \fBdeltarpm_percentage\fR
diff --git a/yum/config.py b/yum/config.py
index f658f5b..438cb88 100644
--- a/yum/config.py
+++ b/yum/config.py
@@ -791,7 +791,7 @@ class YumConf(StartupConf):
             allowed = ('ipv4', 'ipv6', 'whatever'),
             mapper  = {'4': 'ipv4', '6': 'ipv6'})
     max_connections = IntOption(0, range_min=0)
-    deltarpm = IntOption(-2, range_min=-16, range_max=128)
+    deltarpm = IntOption(2, range_min=-16, range_max=128)
     deltarpm_percentage = IntOption(75, range_min=1, range_max=100)
 
     http_caching = SelectionOption('all', ('none', 'packages', 'all'))
commit 7455cc601cf7307ebc4dd21b41aa268aecf8e610
Author: James Antill <james at and.org>
Date:   Fri Mar 8 09:10:18 2013 -0500

    Revert "Do the rpmdb checks on obsoletes as well as conflicts."
    
    This reverts commit bd0f9dde0443aa90f90cf97b9b497e2d3b54fde8.
    
    Obsoletes don't use provides ... uh.

diff --git a/yum/rpmsack.py b/yum/rpmsack.py
index 03f4646..4e26d51 100644
--- a/yum/rpmsack.py
+++ b/yum/rpmsack.py
@@ -159,11 +159,6 @@ class RPMDBProblemDependency(RPMDBProblem):
             return "%s %s %s" % (self.pkg, _('has missing requires of'),
                                  self.missing)
 
-        if self.problem == 'obsoletes':
-            return "%s %s %s: %s" % (self.pkg, _('has installed obsoletes'),
-                                     self.found,', '.join(map(str,
-                                                              self.obsoletes)))
-
         return "%s %s %s: %s" % (self.pkg, _('has installed conflicts'),
                                  self.found,', '.join(map(str, self.conflicts)))
 
@@ -1497,19 +1492,6 @@ class RPMDBPackageSack(PackageSackBase):
                 prob = RPMDBProblemDependency(pkg, "conflicts", found=found,
                                               conflicts=res)
                 problems.append(prob)
-
-            for creq in pkg.obsoletes:
-                if creq[0].startswith('rpmlib'): continue
-
-                (req, flags, ver) = creq
-                res = self.getProvides(req, flags, ver)
-                if not res:
-                    continue
-                flags = yum.depsolve.flags.get(flags, flags)
-                found = miscutils.formatRequire(req, ver, flags)
-                prob = RPMDBProblemDependency(pkg, "obsoletes", found=found,
-                                              obsoletes=res)
-                problems.append(prob)
         return problems
 
     def _iter_two_pkgs(self, ignore_provides):
commit b06d4d50f18d177878835e4900fdea885c092284
Author: James Antill <james at and.org>
Date:   Thu Mar 7 09:53:58 2013 -0500

    Add dequeue_max() and use that instead of dequeue() to service the queue.

diff --git a/yum/__init__.py b/yum/__init__.py
index 8aec608..801815a 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -2305,8 +2305,7 @@ much more problems).
                         presto.rebuild(po, adderror)
                         return
                     else:
-                        while presto.dequeue(block=False):
-                            pass
+                        presto.dequeue_max()
 
                     if po.repoid not in done_repos:
                         done_repos.add(po.repoid)
diff --git a/yum/drpm.py b/yum/drpm.py
index 9317d51..41dab0e 100644
--- a/yum/drpm.py
+++ b/yum/drpm.py
@@ -211,12 +211,14 @@ class DeltaInfo:
                 el.clear()
 
     def wait(self, num=None):
+        """ Wait for "num" number of jobs to finish, or all of them. Blocks. """
         if num is None:
             num = len(self.jobs)
 
         # wait for some jobs, run callbacks
         while num > 0:
-            assert self.jobs
+            if not self.jobs: # This is probably broken logic, which is bad.
+                return
             num -= self._wait(block=True)
 
     def _wait(self, block=False):
@@ -239,6 +241,8 @@ class DeltaInfo:
         return num
 
     def rebuild(self, po, adderror):
+        """ Turn a drpm into an rpm, by adding it to the queue and trying to
+            service the queue. """
         # this runs when worker finishes
         def callback(code):
             if code != 0:
@@ -255,7 +259,7 @@ class DeltaInfo:
         args += po.localpath, po.rpm.localpath
 
         self.queue(args, callback)
-        self.dequeue(block=False)
+        self.dequeue_max()
 
     def queue(self, args, callback):
         """ Queue a delta rebuild up. """
@@ -267,6 +271,19 @@ class DeltaInfo:
         while self._future_jobs:
             self.dequeue()
 
+    def dequeue_max(self):
+        """ De-Queue all delta rebuilds we can and spawn the rebuild
+            processes. """
+
+        if not self._future_jobs:
+            # Just trim the zombies...
+            self._wait()
+            return
+
+        while self._future_jobs:
+            if not self.dequeue(block=False):
+                break
+
     def dequeue(self, block=True):
         """ Try to De-Queue a delta rebuild and spawn the rebuild process. """
         # Do this here, just to keep the zombies at bay...
commit d384581006357fa99bd4e62209c6427396ec41cc
Author: James Antill <james at and.org>
Date:   Thu Mar 7 01:20:01 2013 -0500

    Add a __cmp__ and __hash__ to DeltaPackage, also add .name/etc.

diff --git a/yum/drpm.py b/yum/drpm.py
index 9ea5f1d..9317d51 100644
--- a/yum/drpm.py
+++ b/yum/drpm.py
@@ -36,6 +36,9 @@ class DeltaPackage:
         self.repo = rpm.repo
         self.basepath = rpm.basepath
         self.pkgtup = rpm.pkgtup
+        (self.name, self.arch, self.epoch,
+         self.version, self.release) = self.pkgtup
+        self._hash = None
 
         # set up drpm attributes
         self.size = size
@@ -47,6 +50,24 @@ class DeltaPackage:
     def __str__(self):
         return 'Delta RPM of %s' % self.rpm
 
+    def __cmp__(self, other):
+        if other is None:
+            return 1
+
+        #  Not a PackageObject(), so do this ourselves the bad way:
+        return (cmp(self.name, other.name) or
+                cmp(self.epoch, other.epoch) or
+                cmp(self.version, other.version) or
+                cmp(self.release, other.release) or
+                cmp(self.arch, other.arch))
+
+    def __hash__(self): # C&P from PackageObject...
+        if self._hash is None:
+            mystr = '%s - %s:%s-%s-%s.%s' % (self.repo.id, self.epoch, self.name,
+                                         self.version, self.release, self.arch)
+            self._hash = hash(mystr)
+        return self._hash
+
     def localPkg(self):
         return self.localpath
 
commit 60046227ec7acce55487067f387bbc288ad8631f
Author: James Antill <james at and.org>
Date:   Thu Mar 7 00:58:28 2013 -0500

     Add queue to presto delta rebuilds, and change .wait() API. So we can
    be non-blocking for downloads.
     Poll qeueue in checkfunc callback.
    
     Test via. yum up --downloadonly --setopt=deltarpm=1

diff --git a/yum/__init__.py b/yum/__init__.py
index 4a491a6..8aec608 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -2304,6 +2304,10 @@ much more problems).
                     if isinstance(po, DeltaPackage):
                         presto.rebuild(po, adderror)
                         return
+                    else:
+                        while presto.dequeue(block=False):
+                            pass
+
                     if po.repoid not in done_repos:
                         done_repos.add(po.repoid)
                         #  Check a single package per. repo. ... to give a hint to
@@ -2333,6 +2337,7 @@ much more problems).
                     adderror(po, exception2msg(e))
             if async:
                 urlgrabber.grabber.parallel_wait()
+            presto.dequeue_all()
             presto.wait()
 
             if hasattr(urlgrabber.progress, 'text_meter_total_size'):
diff --git a/yum/drpm.py b/yum/drpm.py
index 8008618..9ea5f1d 100644
--- a/yum/drpm.py
+++ b/yum/drpm.py
@@ -88,6 +88,7 @@ class DeltaInfo:
     def __init__(self, ayum, pkgs):
         self.verbose_logger = ayum.verbose_logger
         self.jobs = {}
+        self._future_jobs = []
         self.limit = ayum.conf.deltarpm
         if self.limit < 0:
             nprocs = _num_cpus_online()
@@ -188,15 +189,33 @@ class DeltaInfo:
                         pkgs[index] = DeltaPackage(po, size, remote, csum, oldrpm)
                 el.clear()
 
-    def wait(self, limit = 1):
+    def wait(self, num=None):
+        if num is None:
+            num = len(self.jobs)
+
         # wait for some jobs, run callbacks
-        while len(self.jobs) >= limit:
-            pid, code = os.wait()
+        while num > 0:
+            assert self.jobs
+            num -= self._wait(block=True)
+
+    def _wait(self, block=False):
+        num = 0
+
+        while self.jobs:
+            if block:
+                pid, code = os.wait()
+            else:
+                pid, code = os.waitpid(-1, os.WNOHANG)
+                if not pid:
+                    break
+
             # urlgrabber spawns child jobs, too.  But they exit synchronously,
             # so we should never see an unknown pid here.
             assert pid in self.jobs
             callback = self.jobs.pop(pid)
             callback(code)
+            num += 1
+        return num
 
     def rebuild(self, po, adderror):
         # this runs when worker finishes
@@ -210,10 +229,41 @@ class DeltaInfo:
             os.unlink(po.localpath)
             po.localpath = po.rpm.localpath # for --downloadonly
 
-        # spawn a worker process
-        self.wait(self.limit)
         args = ()
         if po.oldrpm: args += '-r', po.oldrpm
         args += po.localpath, po.rpm.localpath
+
+        self.queue(args, callback)
+        self.dequeue(block=False)
+
+    def queue(self, args, callback):
+        """ Queue a delta rebuild up. """
+        self._future_jobs.append((args, callback))
+
+    def dequeue_all(self):
+        """ De-Queue all delta rebuilds and spawn the rebuild processes. """
+
+        while self._future_jobs:
+            self.dequeue()
+
+    def dequeue(self, block=True):
+        """ Try to De-Queue a delta rebuild and spawn the rebuild process. """
+        # Do this here, just to keep the zombies at bay...
+        self._wait()
+
+        if not self._future_jobs:
+            return False
+
+        if self.limit <= len(self.jobs):
+            if not block:
+                return False
+            self.wait((self.limit - len(self.jobs)) + 1)
+
+        args, callback = self._future_jobs.pop(0)
+        self._spawn(args, callback)
+
+        return True
+
+    def _spawn(self, args, callback):
         pid = os.spawnl(os.P_NOWAIT, APPLYDELTA, APPLYDELTA, *args)
         self.jobs[pid] = callback


More information about the Yum-commits mailing list