[yum-commits] Branch 'yum-3_2_X' - yum/packages.py yum/sqlitesack.py yum/transactioninfo.py
James Antill
james at osuosl.org
Mon Nov 2 19:02:47 UTC 2009
yum/packages.py | 46 +++++++++++++++-------------------------
yum/sqlitesack.py | 13 ++++++++---
yum/transactioninfo.py | 55 +++++++++++++++++++++++++++++++++++++++++++++++--
3 files changed, 80 insertions(+), 34 deletions(-)
New commits:
commit ef160b81debc7f3000bab4d04642016fe5ee8204
Author: James Antill <james at and.org>
Date: Fri Oct 30 15:18:33 2009 -0400
Big speed boost for depsolving "large" transactions.
We keep a PackageSack() of what was added to the transaction, so we
don't have to query everything in all pkgSacks for every
getProvides()/getRequires().
So that we don't have to download filelists, we create our own
PackageSack() and use .simpleFiles() ... however in sqlitesack
.simpleFiles() doesn't have an index, so we have to create one or perf.
goes out the window (simple patch to y-m-p to fix that though).
This are the differences on my box:
before
------
yum up tzdata = 4.132
yum upD glibc = 5.889
yum upD = 25.964
yum upD --disablerepo=fedora,updates = 18.628
yum upT = 7.326
after
-----
yum up tzdata = 3.996 = ~2.5%
yum upD glibc = 5.590 = ~5%
yum upD = 19.550 = ~25%
yum upD --disablerepo=fedora,updates = 16.830 = ~10%
yum upT = 6.537 = ~10%
Currently keeping the old code as default, until pkgfiles index makes
it out there.
diff --git a/yum/packages.py b/yum/packages.py
index 3c62f2a..b3fa332 100644
--- a/yum/packages.py
+++ b/yum/packages.py
@@ -443,11 +443,17 @@ class RpmBase(object):
"""return changelog entries"""
return self._changelog
- def returnFileEntries(self, ftype='file'):
- """return list of files based on type"""
- # fixme - maybe should die - use direct access to attribute
+ def returnFileEntries(self, ftype='file', primary_only=False):
+ """return list of files based on type, you can pass primary_only=True
+ to limit to those files in the primary repodata"""
if self.files:
if self.files.has_key(ftype):
+ if primary_only:
+ if ftype == 'dir':
+ match = misc.re_primary_dirname
+ else:
+ match = misc.re_primary_filename
+ return [fn for fn in self.files[ftype] if match(fn)]
return self.files[ftype]
return []
@@ -468,6 +474,9 @@ class RpmBase(object):
return self.provides_names
def simpleFiles(self, ftype='files'):
+ warnings.warn('simpleFiles() will go away in a future version of Yum.'
+ 'Use returnFileEntries(primary_only=True)\n',
+ Errors.YumDeprecationWarning, stacklevel=2)
if self.files and self.files.has_key(ftype):
return self.files[ftype]
return []
@@ -945,27 +954,6 @@ class YumAvailablePackage(PackageObject, RpmBase):
if mylist: msg += " </rpm:%s>" % pcotype
return msg
- def _return_primary_files(self, list_of_files=None):
- returns = {}
- if list_of_files is None:
- list_of_files = self.returnFileEntries('file')
- for item in list_of_files:
- if item is None:
- continue
- if misc.re_primary_filename(item):
- returns[item] = 1
- return returns.keys()
-
- def _return_primary_dirs(self):
- returns = {}
- for item in self.returnFileEntries('dir'):
- if item is None:
- continue
- if misc.re_primary_dirname(item):
- returns[item] = 1
- return returns.keys()
-
-
def _dump_files(self, primary=False):
msg =""
if not primary:
@@ -973,9 +961,9 @@ class YumAvailablePackage(PackageObject, RpmBase):
dirs = self.returnFileEntries('dir')
ghosts = self.returnFileEntries('ghost')
else:
- files = self._return_primary_files()
- ghosts = self._return_primary_files(list_of_files = self.returnFileEntries('ghost'))
- dirs = self._return_primary_dirs()
+ files = self.returnFileEntries('file', primary_only=True)
+ dirs = self.returnFileEntries('dir', primary_only=True)
+ ghosts = self.returnFileEntries('ghost', primary_only=True)
for fn in files:
msg += """ <file>%s</file>\n""" % misc.to_xml(fn)
@@ -1201,10 +1189,10 @@ class YumHeaderPackage(YumAvailablePackage):
self._loadedfiles = True
- def returnFileEntries(self, ftype='file'):
+ def returnFileEntries(self, ftype='file', primary_only=False):
"""return list of files based on type"""
self._loadFiles()
- return YumAvailablePackage.returnFileEntries(self,ftype)
+ return YumAvailablePackage.returnFileEntries(self,ftype,primary_only)
def returnChangelog(self):
# note - if we think it is worth keeping changelogs in memory
diff --git a/yum/sqlitesack.py b/yum/sqlitesack.py
index 21b5c43..0cbd2c4 100644
--- a/yum/sqlitesack.py
+++ b/yum/sqlitesack.py
@@ -336,7 +336,6 @@ class YumAvailablePackageSqlite(YumAvailablePackage, PackageObject, RpmBase):
result.append((c_date, _share_data(c_author), c_log))
self._changelog = result
return
-
def returnIdSum(self):
return (self.checksum_type, self.pkgId)
@@ -345,15 +344,23 @@ class YumAvailablePackageSqlite(YumAvailablePackage, PackageObject, RpmBase):
self._loadChangelog()
return self._changelog
- def returnFileEntries(self, ftype='file'):
+ def returnFileEntries(self, ftype='file', primary_only=False):
+ if primary_only and not self._loadedfiles:
+ sql = "SELECT name as fname FROM files WHERE pkgKey = ? and type = ?"
+ cur = self._sql_MD('primary', sql, (self.pkgKey, ftype))
+ return map(lambda x: x['fname'], cur)
+
self._loadFiles()
- return RpmBase.returnFileEntries(self,ftype)
+ return RpmBase.returnFileEntries(self,ftype,primary_only)
def returnFileTypes(self):
self._loadFiles()
return RpmBase.returnFileTypes(self)
def simpleFiles(self, ftype='file'):
+ warnings.warn('simpleFiles() will go away in a future version of Yum.'
+ 'Use returnFileEntries(primary_only=True)\n',
+ Errors.YumDeprecationWarning, stacklevel=2)
sql = "SELECT name as fname FROM files WHERE pkgKey = ? and type = ?"
cur = self._sql_MD('primary', sql, (self.pkgKey, ftype))
return map(lambda x: x['fname'], cur)
diff --git a/yum/transactioninfo.py b/yum/transactioninfo.py
index bd7bf80..bade3e4 100644
--- a/yum/transactioninfo.py
+++ b/yum/transactioninfo.py
@@ -31,6 +31,43 @@ from packages import YumInstalledPackage
from sqlitesack import YumAvailablePackageSqlite
import Errors
import warnings
+import misc
+
+class GetProvReqOnlyPackageSack(PackageSack):
+ def __init__(self, need_files=False):
+ PackageSack.__init__(self)
+ self._need_index_files = need_files
+
+ def __addPackageToIndex_primary_files(self, obj):
+ for ftype in obj.returnFileTypes():
+ for file in obj.returnFileEntries(ftype, primary_only=True):
+ self._addToDictAsList(self.filenames, file, obj)
+ def __addPackageToIndex_files(self, obj):
+ for ftype in obj.returnFileTypes():
+ for file in obj.returnFileEntries(ftype):
+ self._addToDictAsList(self.filenames, file, obj)
+ def _addPackageToIndex(self, obj):
+ for (n, fl, (e,v,r)) in obj.returnPrco('provides'):
+ self._addToDictAsList(self.provides, n, obj)
+ for (n, fl, (e,v,r)) in obj.returnPrco('requires'):
+ self._addToDictAsList(self.requires, n, obj)
+ if self._need_index_files:
+ self.__addPackageToIndex_files(obj)
+ else:
+ self.__addPackageToIndex_primary_files(obj)
+
+ def __buildFileIndexes(self):
+ for repoid in self.pkgsByRepo:
+ for obj in self.pkgsByRepo[repoid]:
+ self.__addPackageToIndex_files(obj)
+ def searchFiles(self, name):
+ if not self._need_index_files and not misc.re_primary_filename(name):
+ self._need_index_files = True
+ if self.indexesBuilt:
+ self.filenames = {}
+ self.__buildFileIndexes()
+
+ return PackageSack.searchFiles(self, name)
class TransactionData:
"""Data Structure designed to hold information on a yum Transaction Set"""
@@ -52,6 +89,9 @@ class TransactionData:
self.pkgSack = None
self.pkgSackPackages = 0
self.localSack = PackageSack()
+ # FIXME: This is turned off atm. ... it'll be turned on when
+ # the new yum-metadata-parser with the "pkgfiles" index is std.
+ self._inSack = None # GetProvReqOnlyPackageSack()
# lists of txmbrs in their states - just placeholders
self.instgroups = []
@@ -198,6 +238,8 @@ class TransactionData:
self.localSack.addPackage(txmember.po)
elif isinstance(txmember.po, YumAvailablePackageSqlite):
self.pkgSackPackages += 1
+ if self._inSack is not None and txmember.output_state in TS_INSTALL_STATES:
+ self._inSack.addPackage(txmember.po)
if self.conditionals.has_key(txmember.name):
for pkg in self.conditionals[txmember.name]:
@@ -219,6 +261,8 @@ class TransactionData:
self.localSack.delPackage(txmbr.po)
elif isinstance(txmbr.po, YumAvailablePackageSqlite):
self.pkgSackPackages -= 1
+ if self._inSack is not None and txmbr.output_state in TS_INSTALL_STATES:
+ self._inSack.delPackage(txmbr.po)
self._namedict[txmbr.name].remove(txmbr)
self._unresolvedMembers.add(txmbr)
@@ -454,10 +498,13 @@ class TransactionData:
"""return dict { packages -> list of matching provides }
searches in packages to be installed"""
result = { }
- if self.pkgSackPackages:
+ if self._inSack is None:
for pkg, hits in self.pkgSack.getProvides(name, flag, version).iteritems():
if self.getMembersWithState(pkg.pkgtup, TS_INSTALL_STATES):
result[pkg] = hits
+ else:
+ for pkg, hits in self._inSack.getProvides(name, flag, version).iteritems():
+ result[pkg] = hits
result.update(self.localSack.getProvides(name, flag, version))
return result
@@ -480,10 +527,14 @@ class TransactionData:
"""return dict { packages -> list of matching provides }
searches in packages to be installed"""
result = { }
- if self.pkgSackPackages:
+ if self._inSack is None:
for pkg, hits in self.pkgSack.getRequires(name, flag, version).iteritems():
if self.getMembersWithState(pkg.pkgtup, TS_INSTALL_STATES):
result[pkg] = hits
+ else:
+ for pkg, hits in self._inSack.getRequires(name, flag, version).iteritems():
+ result[pkg] = hits
+
result.update(self.localSack.getRequires(name, flag, version))
return result
More information about the Yum-commits
mailing list