[yum-commits] Branch 'yum-3_2_X' - 3 commits - yum/misc.py yum/rpmsack.py

[James Antill]

 yum/misc.py    |    9 +++++++++
 yum/rpmsack.py |   28 ++++++++++++++++++++++++----
 2 files changed, 33 insertions(+), 4 deletions(-)

New commits:
commit 7ed105eccfed4483245b21a44520ef92806561e0
Author: James Antill <james at and.org>
Date:   Thu Jul 22 15:18:21 2010 -0400

    Be cleverer about looking in the yumdb cache

diff --git a/yum/rpmsack.py b/yum/rpmsack.py
index 4501470..8e32fe0 100644
--- a/yum/rpmsack.py
+++ b/yum/rpmsack.py
@@ -1533,7 +1533,7 @@ class RPMDBAdditionalDataPackage(object):
         if info is None:
             raise AttributeError, "%s has no attribute %s" % (self, attr)
 
-        if self._yumdb_cache is not None:
+        if info.st_nlink > 1 and self._yumdb_cache is not None:
             key = (info.st_dev, info.st_ino)
             if key in self._yumdb_cache:
                 self._read_cached_data[attr] = self._yumdb_cache[key]
@@ -1544,7 +1544,7 @@ class RPMDBAdditionalDataPackage(object):
         fo.close()
         del fo
 
-        if self._yumdb_cache is not None:
+        if info.st_nlink > 1 and self._yumdb_cache is not None:
             self._yumdb_cache[key] = self._read_cached_data[attr]
 
         return self._read_cached_data[attr]
commit 789c4633a2c7c4c1a7322a50ffd8820a3051171e
Author: James Antill <james at and.org>
Date:   Thu Jul 22 15:10:06 2010 -0400

    Add a cache for common yumdb data, that is hardlinked.

diff --git a/yum/rpmsack.py b/yum/rpmsack.py
index f59abf9..4501470 100644
--- a/yum/rpmsack.py
+++ b/yum/rpmsack.py
@@ -1440,7 +1440,8 @@ class RPMDBAdditionalData(object):
                 self.conf.writable = True
         #  Don't call _load_all_package_paths to preload, as it's expensive
         # if the dirs. aren't in cache.
-                
+        self.yumdb_cache = {}
+
     def _load_all_package_paths(self):
         # glob the path and get a dict of pkgs to their subdir
         glb = '%s/*/*/' % self.conf.db_path
@@ -1468,7 +1469,8 @@ class RPMDBAdditionalData(object):
         else:
             raise ValueError,"Pass something to RPMDBAdditionalData.get_package"
         
-        return RPMDBAdditionalDataPackage(self.conf, thisdir)
+        return RPMDBAdditionalDataPackage(self.conf, thisdir,
+                                          yumdb_cache=self.yumdb_cache)
 
     def sync_with_rpmdb(self, rpmdbobj):
         """populate out the dirs and remove all the items no longer in the rpmd
@@ -1482,12 +1484,19 @@ class RPMDBAdditionalData(object):
         pass
 
 class RPMDBAdditionalDataPackage(object):
-    def __init__(self, conf, pkgdir):
+    def __init__(self, conf, pkgdir, yumdb_cache=None):
         self._conf = conf
         self._mydir = pkgdir
         # FIXME needs some intelligent caching beyond the FS cache
         self._read_cached_data = {}
 
+        #  'from_repo' is the most often requested piece of data, and is often
+        # the same for a huge number of packages. So we use hardlinks to share
+        # data, and try to optimize for that.
+        #  It's useful for other keys too (installed_by/changed_by/reason/etc.)
+        # so we make it generic.
+        self._yumdb_cache = yumdb_cache
+
     def _write(self, attr, value):
         # check for self._conf.writable before going on?
         if not os.path.exists(self._mydir):
@@ -1520,13 +1529,24 @@ class RPMDBAdditionalDataPackage(object):
             return self._read_cached_data[attr]
 
         fn = self._mydir + '/' + attr
-        if not os.path.exists(fn):
+        info = misc.stat_f(fn)
+        if info is None:
             raise AttributeError, "%s has no attribute %s" % (self, attr)
 
+        if self._yumdb_cache is not None:
+            key = (info.st_dev, info.st_ino)
+            if key in self._yumdb_cache:
+                self._read_cached_data[attr] = self._yumdb_cache[key]
+                return self._read_cached_data[attr]
+
         fo = open(fn, 'r')
         self._read_cached_data[attr] = fo.read()
         fo.close()
         del fo
+
+        if self._yumdb_cache is not None:
+            self._yumdb_cache[key] = self._read_cached_data[attr]
+
         return self._read_cached_data[attr]
     
     def _delete(self, attr):
commit 49a66fc6e5b9574673335a1c6ca447b5018d9f06
Author: James Antill <james at and.org>
Date:   Thu Jul 22 15:09:10 2010 -0400

    Add a non-exceptional stat wrapper

diff --git a/yum/misc.py b/yum/misc.py
index 9d3180b..0f80d7d 100644
--- a/yum/misc.py
+++ b/yum/misc.py
@@ -901,6 +901,15 @@ def unlink_f(filename):
         if e.errno != errno.ENOENT:
             raise
 
+def stat_f(filename):
+    """ Call os.stat(), but don't die if the file isn't there. Returns None. """
+    try:
+        return os.stat(filename)
+    except OSError, e:
+        if e.errno != errno.ENOENT:
+            raise
+        return None
+
 def _getloginuid():
     """ Get the audit-uid/login-uid, if available. None is returned if there
         was a problem. Note that no caching is done here. """