[yum-git] Branch 'yum-3_2_X' - 4 commits - yum/constants.py yum/__init__.py yum/rpmsack.py yum/sqlitesack.py
James Antill
james at linux.duke.edu
Fri Sep 12 06:30:36 UTC 2008
yum/__init__.py | 16 ++++++----------
yum/constants.py | 11 +++++++++--
yum/rpmsack.py | 9 +++++++--
yum/sqlitesack.py | 18 ++++++++++--------
4 files changed, 32 insertions(+), 22 deletions(-)
New commits:
commit 3ac146836b184b8ca825790d6bb69d27e398c583
Author: James Antill <james at and.org>
Date: Fri Sep 12 02:30:28 2008 -0400
Speedup for includepkgs, use a set for main in test
diff --git a/yum/__init__.py b/yum/__init__.py
index 54ea989..29c57c2 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -918,9 +918,10 @@ class YumBase(depsolve.Depsolve):
self.verbose_logger.log(logginglevels.INFO_2,
_('Reducing %s to included packages only'), repo.name)
rmlist = []
+ keeplist = set(exactmatch + matched)
for po in pkglist:
- if po in exactmatch + matched:
+ if po in keeplist:
self.verbose_logger.debug(_('Keeping included package %s'), po)
continue
else:
commit b4674bddab0d6b273061184a55ddc8c4030893ff
Author: James Antill <james at and.org>
Date: Fri Sep 12 02:28:13 2008 -0400
Don't use parsePacakges where just returnPackages() dtrt. now
diff --git a/yum/__init__.py b/yum/__init__.py
index f30f4bd..54ea989 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -892,11 +892,10 @@ class YumBase(depsolve.Depsolve):
self.verbose_logger.log(logginglevels.INFO_2, _('Excluding Packages from %s'),
repo.name)
- pkgs = self._pkgSack.returnPackages(repoid, patterns=excludelist)
- exactmatch, matched, unmatched = \
- parsePackages(pkgs, excludelist, casematch=1, unique='repo-pkgkey')
+ pkgs = self._pkgSack.returnPackages(repoid, patterns=excludelist,
+ ignore_case=False)
- for po in exactmatch + matched:
+ for po in pkgs:
self.verbose_logger.debug('Excluding %s', po)
po.repo.sack.delPackage(po)
@@ -2362,11 +2361,7 @@ class YumBase(depsolve.Depsolve):
# Do we still want to return errors here?
# We don't in the cases below, so I didn't here...
if 'pattern' in kwargs:
- pats = [kwargs['pattern']]
- pkgs = self.rpmdb.returnPackages(patterns=pats)
- exactmatch, matched, unmatched = parsePackages(pkgs, pats,
- casematch=1)
- pkgs = exactmatch + matched
+ pkgs = self.rpmdb.returnPackages(patterns=[kwargs['pattern']])
if 'name' in kwargs:
pkgs = self.rpmdb.searchNevra(name=kwargs['name'])
for pkg in pkgs:
commit cb6d608ab72a93583ef9e188d39144181a25fa3c
Author: James Antill <james at and.org>
Date: Fri Sep 12 02:22:00 2008 -0400
Make rpmdb.returnPackages dtrt. like sqlitesack, always doing the minimization
diff --git a/yum/rpmsack.py b/yum/rpmsack.py
index 0fd5b8c..31ae142 100644
--- a/yum/rpmsack.py
+++ b/yum/rpmsack.py
@@ -21,7 +21,7 @@ from rpmUtils import miscutils
from rpmUtils.transaction import initReadOnlyTransaction
import misc
import Errors
-from packages import YumInstalledPackage
+from packages import YumInstalledPackage, parsePackages
from packageSack import PackageSackBase
# For returnPackages(patterns=)
@@ -296,7 +296,12 @@ class RPMDBPackageSack(PackageSackBase):
if self._match_repattern(rpats, hdr):
self._makePackageObject(hdr, idx)
self._completely_loaded = patterns is None
- return self._idx2pkg.values()
+
+ pkgobjlist = self._idx2pkg.values()
+ if patterns:
+ pkgobjlist = parsePackages(pkgobjlist, patterns, not ignore_case)
+ pkgobjlist = pkgobjlist[0] + pkgobjlist[1]
+ return pkgobjlist
@staticmethod
def _find_search_fields(fields, searchstrings, hdr):
commit d0108a32e297b1377cabb3fd1c3cb36b510e00c5
Author: James Antill <james at and.org>
Date: Thu Sep 11 22:13:18 2008 -0400
Patterns max constant change, adding PATTERNS_INDEXED_MAX and comments.
Also dtrt. wrt. large returnPackages() pattern lists (using parsePackages
instead of returning everything).
Testing done with:
time yum ls $(perl -e 'print "python-urlgrabber " x 8')
time yum ls $(perl -e 'print "ustr " x 128')
...the last two are stupidly fast still, due to the index on name so they
get a different limit.
Also tested:
time yum search $(perl -e 'print "ustr " x 14')
...but that doesn't seem affected, as it grows linearly but takes even longer
in python. And:
time yum groupinfo -v \*
...but that seems to be just as fast when PATTERNS_*MAX is low and thus.
requires multiple SQL queries.
diff --git a/yum/constants.py b/yum/constants.py
index 00a2bba..c6de596 100644
--- a/yum/constants.py
+++ b/yum/constants.py
@@ -98,5 +98,12 @@ RPM_TO_SQLITE = { 'packagesize' : 'size_package',
'license' : 'rpm_license'
}
-# Cut over for when we should just give up and load everything
-PATTERNS_MAX = 512
+# Cut over for when we should just give up and load everything.
+# The main problem here is not so much SQLite dying (although that happens
+# at large values: http://sqlite.org/limits.html#max_variable_number) but that
+# but SQLite going really slow when it gets medium sized values (much slower
+# than just loading everything and filtering it in python).
+PATTERNS_MAX = 8
+# We have another value here because name is indexed and sqlite is _much_
+# faster even at large numbers of patterns.
+PATTERNS_INDEXED_MAX = 128
diff --git a/yum/sqlitesack.py b/yum/sqlitesack.py
index 7b8d11f..520ca2e 100644
--- a/yum/sqlitesack.py
+++ b/yum/sqlitesack.py
@@ -24,7 +24,7 @@ import os.path
import fnmatch
import yumRepo
-from packages import PackageObject, RpmBase, YumAvailablePackage
+from packages import PackageObject, RpmBase, YumAvailablePackage, parsePackages
import Errors
import misc
@@ -870,7 +870,7 @@ class YumSqlitePackageSack(yumRepo.YumPackageSack):
return []
returnList = []
- max_entries = constants.PATTERNS_MAX
+ max_entries = constants.PATTERNS_INDEXED_MAX
if len(names) > max_entries:
for names in seq_max_split(names, max_entries):
returnList.extend(self.searchNames(names))
@@ -1104,12 +1104,6 @@ class YumSqlitePackageSack(yumRepo.YumPackageSack):
patterns = []
returnList = []
- max_entries = constants.PATTERNS_MAX / 7
- if len(patterns) > max_entries:
- for patterns in seq_max_split(patterns, max_entries):
- returnList.extend(self._buildPkgObjList(repoid, patterns,
- ignore_case))
- return returnList
fields = ['name', 'sql_nameArch', 'sql_nameVerRelArch',
'sql_nameVer', 'sql_nameVerRel',
@@ -1120,8 +1114,12 @@ class YumSqlitePackageSack(yumRepo.YumPackageSack):
need_full = True
break
+ pat_max = constants.PATTERNS_MAX
if not need_full:
fields = ['name']
+ pat_max = constants.PATTERNS_INDEXED_MAX
+ if len(patterns) > pat_max:
+ patterns = []
for (repo,cache) in self.primarydb.items():
if (repoid == None or repoid == repo.id):
@@ -1170,6 +1168,10 @@ class YumSqlitePackageSack(yumRepo.YumPackageSack):
else:
pkgobjlist = self._buildPkgObjList(repoid, patterns, ignore_case)
+ if hasattr(self, 'pkgobjlist') and patterns:
+ pkgobjlist = parsePackages(pkgobjlist, patterns, not ignore_case)
+ pkgobjlist = pkgobjlist[0] + pkgobjlist[1]
+
returnList = []
for po in pkgobjlist:
if self._pkgExcluded(po):
More information about the Yum-cvs-commits
mailing list