[yum-commits] Branch 'yum-3_2_X' - 5 commits - yum/__init__.py yum/misc.py yum/pkgtag_db.py yum/repos.py yum.spec

skvidal at osuosl.org skvidal at osuosl.org
Sat Feb 6 16:31:47 UTC 2010


 yum.spec         |    1 
 yum/__init__.py  |   74 +++++++++++++++++++++++++++++
 yum/misc.py      |   25 ++++++++++
 yum/pkgtag_db.py |  136 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 yum/repos.py     |    3 -
 5 files changed, 236 insertions(+), 3 deletions(-)

New commits:
commit 9e63a85a734ef1e8220ecff45d243bc8a5ab7fdf
Author: Seth Vidal <skvidal at fedoraproject.org>
Date:   Fri Feb 5 18:31:01 2010 -0500

    actually add the pkgtag classes

diff --git a/yum/pkgtag_db.py b/yum/pkgtag_db.py
new file mode 100644
index 0000000..eddf175
--- /dev/null
+++ b/yum/pkgtag_db.py
@@ -0,0 +1,136 @@
+#!/usr/bin/python -tt
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+# Copyright 2009 Red Hat, Inc
+# written by seth vidal
+
+# parse sqlite tag database
+# return pkgnames and tag that was matched
+import sqlite3 as sqlite
+from sqlutils import executeSQL
+from Errors import PkgTagsError
+import sqlutils
+import sys
+import misc
+
+def catchSqliteException(func):
+    """This decorator converts sqlite exceptions into PkgTagsError"""
+    def newFunc(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except sqlutils.sqlite.Error, e:
+            # 2.4.x requires this, but 2.6.x complains about even hasattr()
+            # of e.message ... *sigh*
+            if sys.hexversion < 0x02050000:
+                if hasattr(e,'message'):
+                    raise PkgTagsError, str(e.message)
+                else:
+                    raise PkgTagsError, str(e)
+            raise PkgTagsError, str(e)
+
+    newFunc.__name__ = func.__name__
+    newFunc.__doc__ = func.__doc__
+    newFunc.__dict__.update(func.__dict__)
+    return newFunc
+
+
+
+class PackageTagDB(object):
+    def __init__(self, repoid, sqlite_file):
+        self.sqlite_file = sqlite_file
+        self.repoid = repoid
+        # take a path to the sqlite tag db
+        # open it and leave a cursor in place for the db
+        self._conn = sqlite.connect(sqlite_file)
+        self.cur = self._conn.cursor()
+        self.count = self._sql_exec("select count(*) from packagetags",)
+        
+        
+    @catchSqliteException
+    def _sql_exec(self, sql, *args):
+        """ Exec SQL against an MD of the repo, return a cursor. """
+        
+        executeSQL(self.cur, sql, *args)
+        return self.cur
+    
+    def search_tags(self, tag):
+        """Search by tag name/glob
+           Return dict of dict[packagename] = [stringmatched, stringmatched, ...]"""
+        res = {}
+        tag = '%%%s%%' % tag
+        query = "SELECT name, tag, score FROM packagetags where tag like ?"
+        rows = self._sql_exec(query, (tag,))
+        for (name, tag, score) in rows:
+            if name not in res:
+                res[name] = []
+            res[name].append(tag)
+            
+        return res
+        
+    def search_names(self, name):
+        """Search by package name/glob.
+           Return dict of dict[packagename] = [tag1, tag2, tag3, tag4, ...]"""
+        res = {}
+        name = '%%%s%%' % name
+        query = "SELECT name, tag, score FROM packagetags where name like ?"
+        rows = self._sql_exec(query, (name,))
+        for (name, tag, score) in rows:
+            if name not in res:
+                res[name] = []
+            res[name].append(tag)
+
+        return res
+
+class PackageTags(object):
+    def __init__(self):
+        self.db_objs = {}
+        
+    def add(self, repoid, sqlite_file):
+        if repoid in self.db_objs:
+            raise PkgTagsError, "Already added tags from %s" % repoid
+            
+        dbobj = PackageTagDB(repoid, sqlite_file)
+        self.db_objs[repoid] = dbobj
+
+    def remove(self, repoid):
+        if repoid in self.db_objs:
+            del self.db_objs[repoid]
+        else:
+            raise PkgTagsError, "No tag db for %s" % repoid
+    
+    def search_names(self, name):
+        res = {}
+        for ptd in self.db_objs.values():
+            for (name, taglist) in ptd.search_names(name).items():
+                if not name in res:
+                    res[name] = []
+                res[name].extend(taglist)
+        
+        out = {}
+        for (name, taglist) in res.items():
+            out[name] = misc.unique(taglist)
+        return out
+
+    def search_tags(self, tagname):
+        res = {}
+        for ptd in self.db_objs.values():
+            for (name, taglist) in ptd.search_tags(tagname).items():
+                if not name in res:
+                    res[name] = []
+                res[name].extend(taglist)
+        out = {}
+        for (name, taglist) in res.items():
+            out[name] = misc.unique(taglist)
+        return out
+        
commit a56aa4be830ec4fc2a699371c354299ae652aa32
Author: Seth Vidal <skvidal at fedoraproject.org>
Date:   Fri Feb 5 17:39:56 2010 -0500

    pkgtags db - searching
    
    this patch adds the pkgtags metatadata and the integrates it into our
    search mechanism. Completely optional repodata format.
    
    Also adds a decompress() function from yum.misc that allows for us
    to pass files to it and not care what compression mechanism they are using
    it's a bit simplistic at the moment but can easily be enhanced.

diff --git a/yum/__init__.py b/yum/__init__.py
index 083b445..8b69788 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -44,6 +44,7 @@ import rpmUtils.updates
 from rpmUtils.arch import canCoinstall, ArchStorage, isMultiLibArch
 import rpmUtils.transaction
 import comps
+import pkgtag_db
 from repos import RepoStorage
 import misc
 from parser import ConfigPreProcessor, varReplace
@@ -144,6 +145,7 @@ class YumBase(depsolve.Depsolve):
         self._history = None
         self._pkgSack = None
         self._lockfile = None
+        self._tags = None
         self.skipped_packages = []   # packages skip by the skip-broken code
         self.logger = logging.getLogger("yum.YumBase")
         self.verbose_logger = logging.getLogger("yum.verbose.YumBase")
@@ -719,6 +721,38 @@ class YumBase(depsolve.Depsolve):
         self.verbose_logger.debug('group time: %0.3f' % (time.time() - group_st))                
         return self._comps
 
+    def _getTags(self):
+        """ create the tags object used to search/report from the pkgtags 
+            metadata"""
+        
+        tag_st = time.time()
+        self.verbose_logger.log(logginglevels.DEBUG_4,
+                                _('Getting pkgtags metadata'))
+        
+        if self._tags is None:
+            self._tags = yum.pkgtag_db.PackageTags()
+           
+            for repo in self.repos.listEnabled():
+                if 'pkgtags' not in repo.repoXML.fileTypes():
+                    continue
+
+                self.verbose_logger.log(logginglevels.DEBUG_4,
+                    _('Adding tags from repository: %s'), repo)
+                
+                # fetch the sqlite tagdb
+                try:
+                    tag_md = repo.retrieveMD('pkgtags')
+                    tag_sqlite  = yum.misc.decompress(tag_md)
+                    # feed it into _tags.add()
+                    self._tags.add(repo.id, tag_sqlite)
+                except (Errors.RepoError, Errors.PkgTagsError), e:
+                    msg = _('Failed to add Pkg Tags for repository: %s - %s') % (repo, str(e))
+                    self.logger.critical(msg)
+                    
+                
+        self.verbose_logger.debug('tags time: %0.3f' % (time.time() - tag_st))
+        return self._tags
+        
     def _getHistory(self):
         """auto create the history object that to access/append the transaction
            history information. """
@@ -764,6 +798,11 @@ class YumBase(depsolve.Depsolve):
                        fset=lambda self, value: setattr(self, "_history",value),
                        fdel=lambda self: setattr(self, "_history", None),
                        doc="Yum History Object")
+
+    pkgtags = property(fget=lambda self: self._getTags(),
+                       fset=lambda self, value: setattr(self, "_tags",value),
+                       fdel=lambda self: setattr(self, "_tags", None),
+                       doc="Yum Package Tags Object")
     
     
     def doSackFilelistPopulate(self):
@@ -2076,6 +2115,22 @@ class YumBase(depsolve.Depsolve):
         results2sorted_lists(tmpres, sorted_lists)
         del tmpres
 
+        tmpres = self.searchPackageTags(real_crit_lower)
+        for pkg in tmpres:
+            count = 0
+            matchkeys = []
+            tagresults = []
+            for (match, taglist) in tmpres[pkg]:
+                count += len(taglist)
+                matchkeys.append(rcl2c[match])
+                tagresults.extend(taglist)
+
+
+            if count not in sorted_lists: sorted_lists[count] = []
+            sorted_lists[count].append((pkg, matchkeys, tagresults))
+        
+        del tmpres
+        
         # By default just sort using package sorting
         sort_func = operator.itemgetter(0)
         if keys:
@@ -2097,7 +2152,22 @@ class YumBase(depsolve.Depsolve):
                 if not showdups:
                     yielded[(po.name, po.arch)] = 1
 
-
+    def searchPackageTags(self, criteria):
+        results = {} # name = [(criteria, taglist)]
+        for c in criteria:
+            c = c.lower()
+            res = self.pkgtags.search_tags(c)
+            for (name, taglist) in res.items():
+                pkgs = self.pkgSack.searchNevra(name=name)
+                if not pkgs:
+                    continue
+                pkg = pkgs[0]
+                if pkg not in results:
+                    results[pkg] = []
+                results[pkg].append((c, taglist))
+        
+        return results
+        
     def searchPackages(self, fields, criteria, callback=None):
         """Search specified fields for matches to criteria
            optional callback specified to print out results
diff --git a/yum/misc.py b/yum/misc.py
index 4b0226b..d63993b 100644
--- a/yum/misc.py
+++ b/yum/misc.py
@@ -17,6 +17,7 @@ import glob
 import pwd
 import fnmatch
 import bz2
+import gzip
 from stat import *
 try:
     import gpgme
@@ -945,3 +946,27 @@ def get_uuid(savepath):
         
         return myid
         
+def decompress(filename):
+    """take a filename and decompress it into the same relative location.
+       if the file is not compressed just return the file"""
+    out = filename
+    if filename.endswith('.gz'):
+        out = filename.replace('.gz', '')
+        decom = gzip.open(filename)
+        fo = open(out, 'w')
+        fo.write(decom.read())
+        fo.flush()
+        fo.close()
+        decom.close() 
+    elif filename.endswith('.bz') or filename.endswith('.bz2'):
+        if filename.endswith('.bz'):
+            out = filename.replace('.bz','')
+        else:
+            out = filename.replace('.bz2', '')
+        bunzipFile(filename, out)
+
+    #add magical lzma/xz trick here
+    
+    return out
+    
+    
diff --git a/yum/repos.py b/yum/repos.py
index c6ebf8f..cd477ba 100644
--- a/yum/repos.py
+++ b/yum/repos.py
@@ -206,7 +206,7 @@ class RepoStorage:
     def setCacheDir(self, cachedir):
         """sets the cachedir value in all repos"""
         
-        self.repos._cachedir = cachedir
+        self._cachedir = cachedir
         for repo in self.repos.values():
             repo.old_base_cache_dir = repo.basecachedir
             repo.basecachedir = cachedir
commit 27b42ef05055cd73c70f185673d6d145384e111e
Merge: b8871b7... c866d70...
Author: Seth Vidal <skvidal at fedoraproject.org>
Date:   Fri Feb 5 17:39:54 2010 -0500

    Merge branch 'yum-3_2_X' of ssh://yum.baseurl.org/srv/projects/yum/git/yum into yum-3_2_X
    
    * 'yum-3_2_X' of ssh://yum.baseurl.org/srv/projects/yum/git/yum:
      Add a fast mode to pkg.verify(), don't verify the digest if others pass

commit b8871b7decf9b86e33cd15502a212b4e96211742
Author: Seth Vidal <skvidal at fedoraproject.org>
Date:   Fri Feb 5 12:02:34 2010 -0500

    when we run setCacheDir() we should, you know, actually SET the cachedir

diff --git a/yum/__init__.py b/yum/__init__.py
index 968362c..083b445 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -4263,7 +4263,7 @@ class YumBase(depsolve.Depsolve):
         cachedir += varReplace(suffix, self.conf.yumvar)
         self.repos.setCacheDir(cachedir)
         self.rpmdb.setCacheDir(cachedir)
-
+        self.conf.cachedir = cachedir
         return True # We got a new cache dir
 
     def _does_this_update(self, pkg1, pkg2):
diff --git a/yum/repos.py b/yum/repos.py
index 248753e..c6ebf8f 100644
--- a/yum/repos.py
+++ b/yum/repos.py
@@ -205,7 +205,8 @@ class RepoStorage:
 
     def setCacheDir(self, cachedir):
         """sets the cachedir value in all repos"""
-
+        
+        self.repos._cachedir = cachedir
         for repo in self.repos.values():
             repo.old_base_cache_dir = repo.basecachedir
             repo.basecachedir = cachedir
commit c8507142722baa6769f07ffcacb5f975fe718056
Author: Seth Vidal <skvidal at fedoraproject.org>
Date:   Thu Feb 4 16:53:58 2010 -0500

    conflict with rpm5 b/c the rpm-pythons are diverging enough

diff --git a/yum.spec b/yum.spec
index 8a9fc6a..44018ba 100644
--- a/yum.spec
+++ b/yum.spec
@@ -21,6 +21,7 @@ Requires: python-iniparse
 Requires: pygpgme
 Prereq: /sbin/chkconfig, /sbin/service, coreutils
 Conflicts: yum-skip-broken
+Conflicts: rpm >= 5-0
 Obsoletes: yum-basearchonly
 Obsoletes: yum-allow-downgrade < 1.1.20-0
 Obsoletes: yum-plugin-allow-downgrade < 1.1.22-0


More information about the Yum-commits mailing list