[yum-git] Branch 'yum-3_2_X' - 4 commits - yum/config.py yum/misc.py yum/sqlitesack.py yum/yumRepo.py
James Antill
james at linux.duke.edu
Mon Aug 18 05:00:54 UTC 2008
yum/config.py | 10 +++++--
yum/misc.py | 9 ++++++
yum/sqlitesack.py | 74 ++++++++++++++++++++++++++++++++++++++++--------------
yum/yumRepo.py | 15 ++++++++++
4 files changed, 87 insertions(+), 21 deletions(-)
New commits:
commit 84c5d9394ea5ca93bf1e24bf44e0eb82e6c8df72
Author: James Antill <james at and.org>
Date: Mon Aug 18 01:00:00 2008 -0400
Skip checksumming old downloaded .sqlite files, we still check when we
download them. Note that this _used_ to be needed before atomic MD
because we might have an old DBMD file lying around ... that cannot be true
anymore, unless the user does it via. cp. And it's costing is ~25% for
simple operations.
Note because of the historical behaviour I added skip_old_DBMD_check to
turn this on/off, if we are super paranoid we _might_ want to have it off
by default but on in yumcli?
diff --git a/yum/yumRepo.py b/yum/yumRepo.py
index 376f7db..ba32f69 100644
--- a/yum/yumRepo.py
+++ b/yum/yumRepo.py
@@ -44,6 +44,16 @@ import glob
import shutil
import stat
+# If you want yum to _always_ check the MD .sqlite files then set this to
+# False (this doesn't affect .xml files or .sqilte files derived from them).
+# With this as True yum will only check when a new repomd.xml or
+# new MD is downloaded.
+# Note that with atomic MD, we can't have old MD lying around anymore so
+# the only way we need this check is if someone does something like:
+# cp primary.sqlite /var/cache/yum/blah
+# ...at which point you lose.
+skip_old_DBMD_check = True
+
warnings.simplefilter("ignore", Errors.YumFutureDeprecationWarning)
logger = logging.getLogger("yum.Repos")
@@ -189,6 +199,9 @@ class YumPackageSack(packageSack.PackageSack):
result = None
if os.path.exists(db_un_fn):
+ if skip_old_DBMD_check and repo._using_old_MD:
+ return db_un_fn
+
try:
repo.checkMD(db_un_fn, mdtype, openchecksum=True)
except URLGrabError:
@@ -220,6 +233,7 @@ class YumRepository(Repository, config.RepoConf):
# eventually want
self.repoMDFile = 'repodata/repomd.xml'
self._repoXML = None
+ self._using_old_MD = None
self._oldRepoMDData = {}
self.cache = 0
self.mirrorlistparsed = 0
@@ -930,6 +944,7 @@ class YumRepository(Repository, config.RepoConf):
self._revertOldRepoXML()
return False
+ self._using_old_MD = caching
if caching:
return False # Skip any work.
commit 14abe04c21e2f6d3050c8545b37d4447d45f631a
Author: James Antill <james at and.org>
Date: Mon Aug 18 00:33:26 2008 -0400
If we can get away with searching just the name field, do that as it's
indexed so is basically as fast as searchNames().
This "fails" on terms like "python-*" ... but so what.
Saves about 0.1 seconds on my machine (which is ~10% now).
diff --git a/yum/misc.py b/yum/misc.py
index a60c17b..2b96a68 100644
--- a/yum/misc.py
+++ b/yum/misc.py
@@ -87,6 +87,15 @@ def re_primary_filename(filename):
return True
return False
+_re_compiled_full_match = None
+def re_full_search_needed(s):
+ """ Tests if a string needs a full nevra match, instead of just name. """
+ # re.match('[\*\?].$', name)
+ global _re_compiled_full_match
+ if _re_compiled_full_match is None:
+ _re_compiled_full_match = re.compile('[-*?].*.$')
+ return _re_compiled_full_match.match(s)
+
###########
# Title: Remove duplicates from a sequence
# Submitter: Tim Peters
diff --git a/yum/sqlitesack.py b/yum/sqlitesack.py
index 53e2b9d..1590fe1 100644
--- a/yum/sqlitesack.py
+++ b/yum/sqlitesack.py
@@ -1093,6 +1093,18 @@ class YumSqlitePackageSack(yumRepo.YumPackageSack):
ignore_case))
return returnList
+ fields = ['name', 'sql_nameArch', 'sql_nameVerRelArch',
+ 'sql_nameVer', 'sql_nameVerRel',
+ 'sql_envra', 'sql_nevra']
+ need_full = False
+ for pat in patterns:
+ if misc.re_full_search_needed(pat):
+ need_full = True
+ break
+
+ if not need_full:
+ fields = ['name']
+
for (repo,cache) in self.primarydb.items():
if (repoid == None or repoid == repo.id):
cur = cache.cursor()
@@ -1104,9 +1116,7 @@ class YumSqlitePackageSack(yumRepo.YumPackageSack):
pat_data = []
for pattern in patterns:
done = False
- for field in ['name', 'sql_nameArch', 'sql_nameVerRelArch',
- 'sql_nameVer', 'sql_nameVerRel',
- 'sql_envra', 'sql_nevra']:
+ for field in fields:
if ignore_case:
if not done:
done = True
commit 8674e45949452bf88353f28cae93f090d9d4baf8
Author: James Antill <james at and.org>
Date: Mon Aug 18 00:29:02 2008 -0400
We don't want to do a single "search for all packages with a bad arch" SQL
search because "normally" there are no bad arches.
So we move to looking at the arch. at pkg load time, a la exclude by pkgKey.
This saves about .2 seconds on my machine.
diff --git a/yum/sqlitesack.py b/yum/sqlitesack.py
index 2c8dd20..53e2b9d 100644
--- a/yum/sqlitesack.py
+++ b/yum/sqlitesack.py
@@ -389,8 +389,15 @@ class YumSqlitePackageSack(yumRepo.YumPackageSack):
return (repo, pkgKey) in self._excludes
+ def _pkgArchExcluded(self, pkgarch):
+ """ Test the arch for a package against the archlist we were passed. """
+ if pkgarch not in self._arch_allowed:
+ return True
+ return False
+
def _pkgExcluded(self, po):
- return self._pkgKeyExcluded(po.repo, po.pkgKey)
+ return (self._pkgKeyExcluded(po.repo, po.pkgKey) or
+ self._pkgArchExcluded(po.arch))
def _packageByKey(self, repo, pkgKey):
""" Lookup a pkg by it's pkgKey, if we don't have it load it """
@@ -402,10 +409,14 @@ class YumSqlitePackageSack(yumRepo.YumPackageSack):
cur = self._sql_MD('primary', repo, sql, (pkgKey,))
po = self.pc(repo, cur.fetchone())
self._key2pkg[repo][pkgKey] = po
+ if self._pkgArchExcluded(self._key2pkg[repo][pkgKey].arch):
+ return None
return self._key2pkg[repo][pkgKey]
def _packageByKeyData(self, repo, pkgKey, data):
""" Like _packageByKey() but we already have the data for .pc() """
+ if self._pkgArchExcluded(data['arch']):
+ return None
if data['pkgKey'] not in self._key2pkg.get(repo, {}):
po = self.pc(repo, data)
self._key2pkg.setdefault(repo, {})[pkgKey] = po
@@ -451,9 +462,12 @@ class YumSqlitePackageSack(yumRepo.YumPackageSack):
if self._pkgKeyExcluded(repo, ob['pkgKey']):
continue
if have_data:
- pkgs.append(self._packageByKeyData(repo, ob['pkgKey'], ob))
+ pkg = self._packageByKeyData(repo, ob['pkgKey'], ob)
else:
- pkgs.append(self._packageByKey(repo, ob['pkgKey']))
+ pkg = self._packageByKey(repo, ob['pkgKey'])
+ if pkg is None:
+ continue
+ pkgs.append(pkg)
return pkgs
@staticmethod
@@ -627,7 +641,10 @@ class YumSqlitePackageSack(yumRepo.YumPackageSack):
for ob in cur:
if self._pkgKeyExcluded(rep, ob['pkgKey']):
continue
- result.append((self._packageByKey(rep, ob['pkgKey']), ob['total']))
+ pkg = self._packageByKey(rep, ob['pkgKey'])
+ if pkg is None:
+ continue
+ result.append((pkg, ob['total']))
return result
@catchSqliteException
@@ -761,7 +778,10 @@ class YumSqlitePackageSack(yumRepo.YumPackageSack):
for pkgKey, hits in tmp.iteritems():
if self._pkgKeyExcluded(rep, pkgKey):
continue
- result[self._packageByKey(rep, pkgKey)] = hits
+ pkg = self._packageByKey(rep, pkgKey)
+ if pkg is None:
+ continue
+ result[pkg] = hits
for (rep,cache) in primarydb_items:
if rep in self._all_excludes:
@@ -781,7 +801,10 @@ class YumSqlitePackageSack(yumRepo.YumPackageSack):
for pkgKey, hits in tmp.iteritems():
if self._pkgKeyExcluded(rep, pkgKey):
continue
- result[self._packageByKey(rep, pkgKey)] = hits
+ pkg = self._packageByKey(rep, pkgKey)
+ if pkg is None:
+ continue
+ result[pkg] = hits
if prcotype != 'provides' or name[0] != '/':
if not preload:
@@ -807,7 +830,10 @@ class YumSqlitePackageSack(yumRepo.YumPackageSack):
for ob in cur:
if self._pkgKeyExcluded(rep, ob['pkgKey']):
continue
- result[self._packageByKey(rep, ob['pkgKey'])] = [(name, None, None)]
+ pkg = self._packageByKey(rep, ob['pkgKey'])
+ if pkg is None:
+ continue
+ result[pkg] = [(name, None, None)]
self._search_cache[prcotype][req] = result
return result
@@ -1094,8 +1120,12 @@ class YumSqlitePackageSack(yumRepo.YumPackageSack):
if pat_sqls:
qsql = _FULL_PARSE_QUERY_BEG + " OR ".join(pat_sqls)
executeSQL(cur, qsql, pat_data)
+ # Note: Not using _sql_pkgKey2po() so that we can "un-exclude"
+ # things later on ... if that matters.
for x in cur:
po = self._packageByKeyData(repo, x['pkgKey'], x)
+ if po is None: # Arch exclude is done here.
+ continue
returnList.append(po)
if not patterns:
self.pkgobjlist = returnList
@@ -1159,14 +1189,18 @@ class YumSqlitePackageSack(yumRepo.YumPackageSack):
def excludeArchs(self, archlist):
"""excludes incompatible arches - archlist is a list of compat arches"""
+ self._arch_allowed = set(archlist)
sarchlist = map(lambda x: "'%s'" % x , archlist)
arch_query = ",".join(sarchlist)
for (rep, cache) in self.primarydb.items():
cur = cache.cursor()
- # First of all, make sure this isn't a *-source repo or something
- # where we'll be excluding everything.
+ # This is a minor hack opt. for source repos. ... if they are
+ # enabled normally, we don't want to exclude each package so we
+ # check it and exclude the entire thing.
+ if not rep.id.endswith("-source") or 'src' in self._arch_allowed:
+ continue
has_arch = False
executeSQL(cur, "SELECT DISTINCT arch FROM packages")
for row in cur:
@@ -1176,12 +1210,6 @@ class YumSqlitePackageSack(yumRepo.YumPackageSack):
if not has_arch:
self._delAllPackages(rep)
return
-
- myq = "select pkgId, pkgKey from packages where arch not in (%s)" % arch_query
- executeSQL(cur, myq)
- for row in cur:
- obj = self.pc(rep, row)
- self.delPackage(obj)
# Simple helper functions
commit 63fe9c6b75dd8c5c1f99242bc76fd51bcaddbb0a
Author: James Antill <james at and.org>
Date: Mon Aug 18 00:07:46 2008 -0400
Don't walk each config. option in populate(), .1 - .2 opt.
diff --git a/yum/config.py b/yum/config.py
index a924935..fb34c06 100644
--- a/yum/config.py
+++ b/yum/config.py
@@ -474,12 +474,16 @@ class BaseConfig(object):
self.cfg = parser
self._section = section
+ if parser.has_section(section):
+ opts = set(parser.options(section))
+ else:
+ opts = set()
for name in self.iterkeys():
option = self.optionobj(name)
value = None
- try:
- value = parser.get(section, name)
- except (NoSectionError, NoOptionError):
+ if name in opts:
+ value = parser.data[section][name]
+ else:
# No matching option in this section, try inheriting
if parent and option.inherit:
value = getattr(parent, name)
More information about the Yum-cvs-commits
mailing list