[yum-cvs] 2 commits - yum/sqlitecache.py yum/storagefactory.py yum/yumRepo.py
James Bowes
jbowes at linux.duke.edu
Fri Aug 3 15:03:09 UTC 2007
yum/sqlitecache.py | 479 --------------------------------------------------
yum/storagefactory.py | 88 ---------
yum/yumRepo.py | 14 -
3 files changed, 8 insertions(+), 573 deletions(-)
New commits:
commit 90828274f8be41b752a4c617d6d4baa5e59c2744
Author: James Bowes <jbowes at redhat.com>
Date: Thu Aug 2 22:05:10 2007 -0400
Remove storagefactory and reference sqlitecachec directly
diff --git a/yum/storagefactory.py b/yum/storagefactory.py
deleted file mode 100644
index 4edb669..0000000
--- a/yum/storagefactory.py
+++ /dev/null
@@ -1,61 +0,0 @@
-#!/usr/bin/python -tt
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Library General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-# Copyright 2004 Duke University
-
-import Errors
-
-class Storage:
-
- def __init__(self):
- pass
-
- def Name(self):
- pass
-
- def GetCacheHandler(self, storedir, repoid, callback):
- pass
-
- def GetPackageSack(self):
- pass
-
-
-class StorageSqliteC(Storage):
- def __init__(self):
- import sqlitecachec
- import sqlitesack
-
- self.sqlitecache = sqlitecachec
- self.sqlitesack = sqlitesack
-
- def Name(self):
- return "sqlite-c"
-
- def GetCacheHandler(self, storedir, repoid, callback):
- return self.sqlitecache.RepodataParserSqlite(storedir, repoid, callback)
-
- def GetPackageSack(self):
- return self.sqlitesack.YumSqlitePackageSack(self.sqlitesack.YumAvailablePackageSqlite)
-
-
-def GetStorage():
- storage = None
-
- try:
- storage = StorageSqliteC()
- return storage
- except ImportError:
- pass
-
- raise Errors.YumBaseError, 'Could not find any working storages.'
diff --git a/yum/yumRepo.py b/yum/yumRepo.py
index 70dd73f..c45fb85 100644
--- a/yum/yumRepo.py
+++ b/yum/yumRepo.py
@@ -27,7 +27,8 @@ import repoMDObject
import packageSack
from repos import Repository
import parser
-import storagefactory
+import sqlitecachec
+import sqlitesack
from yum import config
from yum import misc
from constants import *
@@ -105,7 +106,7 @@ class YumPackageSack(packageSack.PackageSack):
data = [ mdtype ]
if not hasattr(repo, 'cacheHandler'):
- repo.cacheHandler = repo.storage.GetCacheHandler(
+ repo.cacheHandler = sqlitecachec.RepodataParserSqlite(
storedir=repo.cachedir,
repoid=repo.id,
callback=callback,
@@ -245,8 +246,8 @@ class YumRepository(Repository, config.RepoConf):
# callback function for handling media
self.mediafunc = None
- self.storage = storagefactory.GetStorage()
- self.sack = self.storage.GetPackageSack()
+ self.sack = sqlitesack.YumSqlitePackageSack(
+ sqlitesack.YumAvailablePackageSqlite)
self._grabfunc = None
self._grab = None
@@ -256,8 +257,9 @@ class YumRepository(Repository, config.RepoConf):
Repository.close(self)
def _resetSack(self):
- self.sack = self.storage.GetPackageSack()
-
+ self.sack = sqlitesack.YumSqlitePackageSack(
+ sqlitesack.YumAvailablePackageSqlite)
+
def __getProxyDict(self):
self.doProxyDict()
if self._proxy_dict:
commit 4296f08fac571f4e739c672dda5de95b8564e379
Author: James Bowes <jbowes at redhat.com>
Date: Thu Aug 2 22:05:09 2007 -0400
Remove the internal metadata parser
diff --git a/yum/sqlitecache.py b/yum/sqlitecache.py
deleted file mode 100644
index ca5016a..0000000
--- a/yum/sqlitecache.py
+++ /dev/null
@@ -1,479 +0,0 @@
-#!/usr/bin/python -tt
-
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Library General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-# Copyright 2005 Duke University
-
-# TODO
-# - Add support for multiple checksums per rpm (not required)
-
-import os
-try:
- import sqlite3 as sqlite
-except ImportError:
- import sqlite
-import time
-import mdparser
-import logging
-import logginglevels
-from sqlitesack import encodefiletypelist,encodefilenamelist
-from constants import *
-
-
-from sqlutils import executeSQL
-
-
-class RepodataParserSqlite:
- def __init__(self, storedir, repoid, callback=None):
- self.storedir = storedir
- self.callback = callback
- self.repodata = {
- 'metadata': {},
- 'filelists': {},
- 'otherdata': {}
- }
- self.repoid = repoid
- self.debug = 0
- self.logger = logging.getLogger("yum.RepodataParserSqlite")
- self.verbose_logger = \
- logging.getLogger("yum.verbose.RepodataParserSqlite")
-
- def loadCache(self,filename):
- """Load cache from filename, check if it is valid and that dbversion
- matches the required dbversion"""
- db = sqlite.connect(filename)
- if sqlite.version_info[0] > 1:
- db.row_factory = sqlite.Row
- cur = db.cursor()
- executeSQL(cur, "select * from db_info")
- info = cur.fetchone()
- # If info is not in there this is an incompelete cache file
- # (this could happen when the user hits ctrl-c or kills yum
- # when the cache is being generated or updated)
- if (not info):
- raise sqlite.DatabaseError, "Incomplete database cache file"
-
- # Now check the database version
- if (info['dbversion'] != DBVERSION):
- self.verbose_logger.log(logginglevels.INFO_2, "Warning: cache file is version %s, we need %s, will regenerate", info['dbversion'], DBVERSION)
- raise sqlite.DatabaseError, "Older version of yum sqlite"
-
- # This appears to be a valid database, return checksum value and
- # database object
- return (info['checksum'],db)
-
- def getFilename(self,location):
- return location + '.sqlite'
-
- def open_database(self, filename):
- if not filename:
- return None
- (csum,db) = self.loadCache(filename)
- return db
-
- def getDatabase(self, location, cachetype):
- filename = self.getFilename(location)
- dbchecksum = ''
- # First try to open an existing database
- try:
- (dbchecksum,db) = self.loadCache(filename)
- except (IOError,sqlite.DatabaseError,KeyError):
- # If it doesn't exist, create it
- db = self.makeSqliteCacheFile(filename,cachetype)
- return (db,dbchecksum)
-
- def _getbase(self, location, checksum, metadatatype):
- (db, dbchecksum) = self.getDatabase(location, metadatatype)
- # db should now contain a valid database object, check if it is
- # up to date
- if (checksum != dbchecksum):
- self.verbose_logger.debug("%s sqlite cache needs updating, reading in metadata",
- metadatatype)
- parser = mdparser.MDParser(location)
- self.updateSqliteCache(db, parser, checksum, metadatatype)
- db.commit()
- return db
-
- def getPrimary(self, location, checksum):
- """Load primary.xml.gz from an sqlite cache and update it
- if required"""
- return self._getbase(location, checksum, 'primary')
-
- def getFilelists(self, location, checksum):
- """Load filelist.xml.gz from an sqlite cache and update it if
- required"""
- return self._getbase(location, checksum, 'filelists')
-
- def getOtherdata(self, location, checksum):
- """Load other.xml.gz from an sqlite cache and update it if required"""
- return self._getbase(location, checksum, 'other')
-
- def createTablesFilelists(self,db):
- """Create the required tables for filelists metadata in the sqlite
- database"""
- cur = db.cursor()
- self.createDbInfo(cur)
- # This table is needed to match pkgKeys to pkgIds
- executeSQL(cur, """CREATE TABLE packages(
- pkgKey INTEGER PRIMARY KEY,
- pkgId TEXT)
- """)
- executeSQL(cur, """CREATE TABLE filelist(
- pkgKey INTEGER,
- dirname TEXT,
- filenames TEXT,
- filetypes TEXT)
- """)
- executeSQL(cur, "CREATE INDEX keyfile ON filelist (pkgKey)")
- executeSQL(cur, "CREATE INDEX pkgId ON packages (pkgId)")
-
- def createTablesOther(self,db):
- """Create the required tables for other.xml.gz metadata in the sqlite
- database"""
- cur = db.cursor()
- self.createDbInfo(cur)
- # This table is needed to match pkgKeys to pkgIds
- executeSQL(cur, """CREATE TABLE packages(
- pkgKey INTEGER PRIMARY KEY,
- pkgId TEXT)
- """)
- executeSQL(cur, """CREATE TABLE changelog(
- pkgKey INTEGER,
- author TEXT,
- date TEXT,
- changelog TEXT)
- """)
- executeSQL(cur, "CREATE INDEX keychange ON changelog (pkgKey)")
- executeSQL(cur, "CREATE INDEX pkgId ON packages (pkgId)")
-
- def createTablesPrimary(self,db):
- """Create the required tables for primary metadata in the sqlite
- database"""
-
- cur = db.cursor()
- self.createDbInfo(cur)
- # The packages table contains most of the information in primary.xml.gz
-
- q = 'CREATE TABLE packages(\n' \
- 'pkgKey INTEGER PRIMARY KEY,\n'
-
- cols = []
- for col in PackageToDBAdapter.COLUMNS:
- cols.append('%s TEXT' % col)
- q += ',\n'.join(cols) + ')'
-
- executeSQL(cur, q)
-
- # Create requires, provides, conflicts and obsoletes tables
- # to store prco data
- for t in ('requires','provides','conflicts','obsoletes'):
- extraCol = ""
- if t == 'requires':
- extraCol= ", pre BOOL DEFAULT FALSE"
- executeSQL(cur, """CREATE TABLE %s (
- name TEXT,
- flags TEXT,
- epoch TEXT,
- version TEXT,
- release TEXT,
- pkgKey TEXT %s)
- """ % (t, extraCol))
- executeSQL(cur, "CREATE INDEX IF NOT EXISTS pkg%s on %s (pkgKey)"
- % (t, t))
- # Create the files table to hold all the file information
- executeSQL(cur, """CREATE TABLE files (
- name TEXT,
- type TEXT,
- pkgKey TEXT)
- """)
- # Create indexes for faster searching
- executeSQL(cur, "CREATE INDEX packagename ON packages (name)")
- executeSQL(cur, "CREATE INDEX providesname ON provides (name)")
- executeSQL(cur, "CREATE INDEX packageId ON packages (pkgId)")
- db.commit()
-
- def createDbInfo(self,cur):
- # Create the db_info table, this contains sqlite cache metadata
- executeSQL(cur, """CREATE TABLE db_info (
- dbversion TEXT,
- checksum TEXT)
- """)
-
- def insertHash(self,table,hash,cursor):
- """Insert the key value pairs in hash into a database table"""
-
- keys = hash.keys()
- values = hash.values()
- query = "INSERT INTO %s (" % (table)
- query += ",".join(keys)
- query += ") VALUES ("
- # Quote all values by replacing None with NULL and ' by ''
- for x in values:
- if (x == None):
- query += "NULL,"
- else:
- try:
- query += "'%s'," % (x.replace("'","''"))
- except AttributeError:
- query += "'%s'," % x
- # Remove the last , from query
- query = query[:-1]
- # And replace it with )
- query += ")"
- cursor.execute(query.encode('utf8'))
- return cursor.lastrowid
-
- def makeSqliteCacheFile(self, filename, cachetype):
- """Create an initial database in the given filename"""
-
- # If it exists, remove it as we were asked to create a new one
- if (os.path.exists(filename)):
- self.verbose_logger.debug("Warning: cache already exists, removing old version")
- try:
- os.unlink(filename)
- except OSError:
- pass
-
- # Try to create the databse in filename, or use in memory when
- # this fails
- try:
- db = sqlite.connect(filename)
- if sqlite.version_info[0] > 1:
- db.row_factory = sqlite.Row
- except IOError:
- self.verbose_logger.log(logginglevels.INFO_1, "Warning could not create sqlite cache file, using in memory cache instead")
- db = sqlite.connect(":memory:")
- if sqlite.version_info[0] > 1:
- db.row_factory = sqlite.Row
-
- # The file has been created, now create the tables and indexes
- if (cachetype == 'primary'):
- self.createTablesPrimary(db)
- elif (cachetype == 'filelists'):
- self.createTablesFilelists(db)
- elif (cachetype == 'other'):
- self.createTablesOther(db)
- else:
- raise sqlite.DatabaseError, "Sorry don't know how to store %s" % (cachetype)
- return db
-
- def addPrimary(self, pkgId, package, cur):
- """Add a package to the primary cache"""
- # Store the package info into the packages table
- pkgKey = self.insertHash('packages', PackageToDBAdapter(package), cur)
-
- # Now store all prco data
- for ptype in package.prco:
- for entry in package.prco[ptype]:
- data = {
- 'pkgKey': pkgKey,
- 'name': entry.get('name'),
- 'flags': entry.get('flags'),
- 'epoch': entry.get('epoch'),
- 'version': entry.get('ver'),
- 'release': entry.get('rel'),
- }
- if ptype == 'requires' and entry.has_key('pre'):
- if entry.get('pre'):
- data['pre'] = True
- self.insertHash(ptype,data,cur)
-
- # Now store all file information
- for f in package.files:
- data = {
- 'name': f,
- 'type': package.files[f],
- 'pkgKey': pkgKey,
- }
- self.insertHash('files',data,cur)
-
- def addFilelists(self, pkgId, package,cur):
- """Add a package to the filelists cache"""
- pkginfo = {'pkgId': pkgId}
- pkgKey = self.insertHash('packages',pkginfo, cur)
- dirs = {}
- for (filename,filetype) in package.files.iteritems():
- (dirname,filename) = (os.path.split(filename))
- if (dirs.has_key(dirname)):
- dirs[dirname]['files'].append(filename)
- dirs[dirname]['types'].append(filetype)
- else:
- dirs[dirname] = {}
- dirs[dirname]['files'] = [filename]
- dirs[dirname]['types'] = [filetype]
-
- for (dirname,dir) in dirs.items():
- data = {
- 'pkgKey': pkgKey,
- 'dirname': dirname,
- 'filenames': encodefilenamelist(dir['files']),
- 'filetypes': encodefiletypelist(dir['types'])
- }
- self.insertHash('filelist',data,cur)
-
- def addOther(self, pkgId, package,cur):
- pkginfo = {'pkgId': pkgId}
- pkgKey = self.insertHash('packages', pkginfo, cur)
- for entry in package['changelog']:
- data = {
- 'pkgKey': pkgKey,
- 'author': entry.get('author'),
- 'date': entry.get('date'),
- 'changelog': entry.get('value'),
- }
- self.insertHash('changelog', data, cur)
-
- def updateSqliteCache(self, db, parser, checksum, cachetype):
- """Update the sqlite cache by making it fit the packages described
- in dobj (info that has been read from primary.xml metadata) afterwards
- update the checksum of the database to checksum"""
-
- t = time.time()
- delcount = 0
- newcount = 0
-
- # We start be removing the old db_info, as it is no longer valid
- cur = db.cursor()
- executeSQL(cur, "DELETE FROM db_info")
-
- # First create a list of all pkgIds that are in the database now
- executeSQL(cur, "SELECT pkgId, pkgKey from packages")
- currentpkgs = {}
- for pkg in cur:
- currentpkgs[pkg['pkgId']] = pkg['pkgKey']
-
- if (cachetype == 'primary'):
- deltables = ("packages","files","provides","requires",
- "conflicts","obsoletes")
- elif (cachetype == 'filelists'):
- deltables = ("packages","filelist")
- elif (cachetype == 'other'):
- deltables = ("packages","changelog")
- else:
- raise sqlite.DatabaseError,"Unknown type %s" % (cachetype)
-
- # Add packages that are not in the database yet and build up a list of
- # all pkgids in the current metadata
-
- all_pkgIds = {}
- for package in parser:
-
- if self.callback is not None:
- self.callback.progressbar(parser.count, parser.total, self.repoid)
-
- pkgId = package['pkgId']
- all_pkgIds[pkgId] = 1
-
- # This package is already in the database, skip it now
- if (currentpkgs.has_key(pkgId)):
- continue
-
- # This is a new package, lets insert it
- newcount += 1
- if cachetype == 'primary':
- self.addPrimary(pkgId, package, cur)
- elif cachetype == 'filelists':
- self.addFilelists(pkgId, package, cur)
- elif cachetype == 'other':
- self.addOther(pkgId, package, cur)
-
- # Remove those which are not in dobj
- delpkgs = []
- for (pkgId, pkgKey) in currentpkgs.items():
- if not all_pkgIds.has_key(pkgId):
- delcount += 1
- delpkgs.append(str(pkgKey))
- if len(delpkgs) > 0:
- delpkgs = "("+",".join(delpkgs)+")"
- for table in deltables:
- executeSQL(cur, "DELETE FROM "+table+ " where pkgKey in %s" %(delpkgs,))
-
- executeSQL(cur, "INSERT into db_info (dbversion,checksum) VALUES (?,?)",
- (DBVERSION,checksum))
- db.commit()
- self.verbose_logger.log(logginglevels.INFO_2, "Added %s new packages, deleted %s old in %.2f seconds",
- newcount, delcount, time.time()-t)
- return db
-
- def log(self, level, msg):
- '''Log to callback (if set)
- '''
- if self.callback:
- self.callback.log(level, msg)
-
-class PackageToDBAdapter:
-
- '''
- Adapt a PrimaryEntry instance to suit the sqlite database.
-
- This hides certain attributes and converts some column names in order to
- decouple the parser implementation from the sqlite database schema.
- '''
-
- NAME_MAPS = {
- 'rpm_package': 'package',
- 'version': 'ver',
- 'release': 'rel',
- 'rpm_license': 'license',
- 'rpm_vendor': 'vendor',
- 'rpm_group': 'group',
- 'rpm_buildhost': 'buildhost',
- 'rpm_sourcerpm': 'sourcerpm',
- 'rpm_packager': 'packager',
- }
-
- COLUMNS = (
- 'pkgId',
- 'name',
- 'arch',
- 'version',
- 'epoch',
- 'release',
- 'summary',
- 'description',
- 'url',
- 'time_file',
- 'time_build',
- 'rpm_license',
- 'rpm_vendor',
- 'rpm_group',
- 'rpm_buildhost',
- 'rpm_sourcerpm',
- 'rpm_header_start',
- 'rpm_header_end',
- 'rpm_packager',
- 'size_package',
- 'size_installed',
- 'size_archive',
- 'location_href',
- 'location_base',
- 'checksum_type',
- 'checksum_value',
- )
-
- def __init__(self, package):
- self._pkg = package
-
- def __getitem__(self, k):
- return self._pkg[self.NAME_MAPS.get(k, k)]
-
- def keys(self):
- return self.COLUMNS
-
- def values(self):
- out = []
- for k in self.keys():
- out.append(self[k])
- return out
-
diff --git a/yum/storagefactory.py b/yum/storagefactory.py
index 0dc895f..4edb669 100644
--- a/yum/storagefactory.py
+++ b/yum/storagefactory.py
@@ -30,24 +30,6 @@ class Storage:
def GetPackageSack(self):
pass
-class StorageSqlite(Storage):
-
- def __init__(self):
- import sqlitecache
- import sqlitesack
-
- self.sqlitecache = sqlitecache
- self.sqlitesack = sqlitesack
-
- def Name(self):
- return "sqlite"
-
- def GetCacheHandler(self, storedir, repoid, callback):
- return self.sqlitecache.RepodataParserSqlite(storedir, repoid, callback)
-
- def GetPackageSack(self):
- return self.sqlitesack.YumSqlitePackageSack(self.sqlitesack.YumAvailablePackageSqlite)
-
class StorageSqliteC(Storage):
def __init__(self):
@@ -70,19 +52,10 @@ class StorageSqliteC(Storage):
def GetStorage():
storage = None
- # Try to load storages, prefering the fastest one.
-
try:
storage = StorageSqliteC()
return storage
except ImportError:
pass
- try:
- storage = StorageSqlite()
- return storage
- except ImportError:
- pass
-
raise Errors.YumBaseError, 'Could not find any working storages.'
-
More information about the Yum-cvs-commits
mailing list