[yum-commits] Branch 'yum-3_2_X' - 3 commits - yum/__init__.py yum/misc.py yum/update_md.py yum/yumRepo.py
skvidal at osuosl.org
skvidal at osuosl.org
Fri May 28 12:21:05 UTC 2010
yum/__init__.py | 8 ++---
yum/misc.py | 76 ++++++++++++++++++++++++++++++++++++++++---------------
yum/update_md.py | 10 ++++---
yum/yumRepo.py | 27 +++++++++----------
4 files changed, 79 insertions(+), 42 deletions(-)
New commits:
commit 28823e8cee46059faed6ef5e98ba583f55da99ef
Merge: 0612a88... 0389ee6...
Author: Seth Vidal <skvidal at fedoraproject.org>
Date: Fri May 28 08:21:54 2010 -0400
Merge branch 'yum-3_2_X' of ssh://yum.baseurl.org/srv/projects/yum/git/yum into yum-3_2_X
* 'yum-3_2_X' of ssh://yum.baseurl.org/srv/projects/yum/git/yum:
l10n: Updates to Finnish (fi) translation
commit 0612a88cd98ff59eab52a816099c0abaf78d9764
Author: Seth Vidal <skvidal at fedoraproject.org>
Date: Thu May 27 13:16:06 2010 -0400
- go through all of our code and change our decompressors to use misc.decompress()
- this should mean we can access metadata compressed using any supported compression library
provided it is NAMED accordingly
- add fn_only option to misc.decompress() so we can get a returned name w/o actually
having the file - useful for finding a decompressed file if we only have the compressed one
diff --git a/yum/__init__.py b/yum/__init__.py
index 8a2fbf4..1c398fd 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -30,7 +30,7 @@ import fnmatch
import logging
import logging.config
import operator
-import gzip
+
import yum.i18n
_ = yum.i18n._
@@ -716,9 +716,9 @@ class YumBase(depsolve.Depsolve):
self.verbose_logger.log(logginglevels.DEBUG_4,
_('Adding group file from repository: %s'), repo)
groupfile = repo.getGroups()
- # open it up as a file object so iterparse can cope with our gz file
- if groupfile is not None and groupfile.endswith('.gz'):
- groupfile = gzip.open(groupfile)
+ # open it up as a file object so iterparse can cope with our compressed file
+ if groupfile:
+ groupfile = misc.decompress(groupfile)
try:
self._comps.add(groupfile)
diff --git a/yum/misc.py b/yum/misc.py
index 92e9127..7793807 100644
--- a/yum/misc.py
+++ b/yum/misc.py
@@ -1005,7 +1005,7 @@ def get_uuid(savepath):
return myid
-def decompress(filename, dest=None):
+def decompress(filename, dest=None, fn_only=False):
"""take a filename and decompress it into the same relative location.
if the file is not compressed just return the file"""
@@ -1035,7 +1035,7 @@ def decompress(filename, dest=None):
out = filename # returning the same file since it is not compressed
ztype = None
- if ztype:
+ if ztype and not fn_only:
_decompress_chunked(filename, out, ztype)
return out
diff --git a/yum/update_md.py b/yum/update_md.py
index 90c5582..cf8136a 100644
--- a/yum/update_md.py
+++ b/yum/update_md.py
@@ -22,12 +22,11 @@ Update metadata (updateinfo.xml) parsing.
"""
import sys
-import gzip
from yum.i18n import utf8_text_wrap, to_utf8
from yum.yumRepo import YumRepository
from yum.packages import FakeRepository
-from yum.misc import to_xml
+from yum.misc import to_xml, decompress
import Errors
import rpmUtils.miscutils
@@ -382,14 +381,17 @@ class UpdateMetadata(object):
if not obj:
raise UpdateNoticeException
if type(obj) in (type(''), type(u'')):
- infile = obj.endswith('.gz') and gzip.open(obj) or open(obj, 'rt')
+ unfile = decompress(obj)
+ infile = open(unfile, 'rt')
+
elif isinstance(obj, YumRepository):
if obj.id not in self._repos:
self._repos.append(obj.id)
md = obj.retrieveMD(mdtype)
if not md:
raise UpdateNoticeException()
- infile = gzip.open(md)
+ unfile = decompress(md)
+ infile = open(unfile, 'rt')
elif isinstance(obj, FakeRepository):
raise Errors.RepoMDError, "No updateinfo for local pkg"
else: # obj is a file object
diff --git a/yum/yumRepo.py b/yum/yumRepo.py
index 3a695d8..ac6c7c0 100644
--- a/yum/yumRepo.py
+++ b/yum/yumRepo.py
@@ -19,7 +19,6 @@ import time
import types
import urlparse
urlparse.uses_fragment.append("media")
-import gzip
import Errors
from urlgrabber.grabber import URLGrabber
@@ -163,7 +162,7 @@ class YumPackageSack(packageSack.PackageSack):
if self._check_db_version(repo, mydbtype):
# see if we have the uncompressed db and check it's checksum vs the openchecksum
- # if not download the bz2 file
+ # if not download the compressed file
# decompress it
# unlink it
@@ -171,9 +170,8 @@ class YumPackageSack(packageSack.PackageSack):
if not db_un_fn:
db_fn = repo._retrieveMD(mydbtype)
if db_fn:
- db_un_fn = db_fn.replace('.bz2', '')
if not repo.cache:
- misc.bunzipFile(db_fn, db_un_fn)
+ db_un_fn = misc.decompress(db_fn)
misc.unlink_f(db_fn)
db_un_fn = self._check_uncompressed_db(repo, mydbtype)
@@ -199,8 +197,8 @@ class YumPackageSack(packageSack.PackageSack):
mydbdata = repo.repoXML.getData(mdtype)
(r_base, remote) = mydbdata.location
fname = os.path.basename(remote)
- bz2_fn = repo.cachedir + '/' + fname
- db_un_fn = bz2_fn.replace('.bz2', '')
+ compressed_fn = repo.cachedir + '/' + fname
+ db_un_fn = misc.decompress(compressed_fn, fn_only=True)
result = None
@@ -1089,7 +1087,7 @@ class YumRepository(Repository, config.RepoConf):
local = self.cachedir + '/' + os.path.basename(remote)
if compressed: # DB file, we need the uncompressed version
- local = local.replace('.bz2', '')
+ local = misc.decompress(local, fn_only=True)
return local
def _groupCheckDataMDNewer(self):
@@ -1244,7 +1242,7 @@ class YumRepository(Repository, config.RepoConf):
compressed = False
local = self._get_mdtype_fname(data, False)
if not os.path.exists(local):
- local = local.replace('.bz2', '')
+ local = misc.decompress(local, fn_only=True)
compressed = True
# If we can, make a copy of the system-wide-cache version of this file,
# note that we often don't get here. So we also do this in
@@ -1351,10 +1349,9 @@ class YumRepository(Repository, config.RepoConf):
for (ndata, nmdtype) in downloading_with_size + downloading_no_size:
local = self._get_mdtype_fname(ndata, False)
- if nmdtype.endswith("_db"): # Uncompress any .sqlite.bz2 files
+ if nmdtype.endswith("_db"): # Uncompress any compressed files
dl_local = local
- local = local.replace('.bz2', '')
- misc.bunzipFile(dl_local, local)
+ local = misc.decompress(dl_local)
misc.unlink_f(dl_local)
self._oldRepoMDData['new_MD_files'].append(local)
@@ -1516,6 +1513,8 @@ class YumRepository(Repository, config.RepoConf):
""" Internal function, use .retrieveMD() from outside yum. """
# Note that this can raise Errors.RepoMDError if mdtype doesn't exist
# for this repo.
+ # FIXME - maybe retrieveMD should call decompress() after we've checked
+ # the checksum by default? since we're never acting on compressed MD
thisdata = self.repoXML.getData(mdtype)
(r_base, remote) = thisdata.location
@@ -1768,9 +1767,9 @@ class YumRepository(Repository, config.RepoConf):
grpfile = self.getGroups()
- # open it up as a file object so iterparse can cope with our gz file
- if grpfile is not None and grpfile.endswith('.gz'):
- grpfile = gzip.open(grpfile)
+ # open it up as a file object so iterparse can cope with our compressed file
+ if grpfile is not None:
+ grpfile = misc.decompress(grpfile)
try:
c = comps.Comps()
c.add(grpfile)
commit fa9a82967408b0789564a6be9cc19ea09e0e98f0
Author: Seth Vidal <skvidal at fedoraproject.org>
Date: Thu May 27 12:01:05 2010 -0400
- cleanup decompress some
- keep stub bunzipFile
- add lzma support, if available
diff --git a/yum/misc.py b/yum/misc.py
index ebce8e1..92e9127 100644
--- a/yum/misc.py
+++ b/yum/misc.py
@@ -19,6 +19,13 @@ import pwd
import fnmatch
import bz2
import gzip
+_available_compression = ['gz', 'bz2']
+try:
+ import lzma
+ _available_compression.append('xz')
+except ImportError:
+ lzma = None
+
from rpmUtils.miscutils import stringToVersion, flagToString
from stat import *
try:
@@ -675,10 +682,22 @@ def refineSearchPattern(arg):
restring = re.escape(arg)
return restring
+
+
+def _decompress_chunked(source, dest, ztype):
+
+ if ztype not in _available_compression:
+ msg = "%s compression not available" % ztype
+ raise Errors.MiscError, msg
+
+ if ztype == 'bz2':
+ s_fn = bz2.BZ2File(source, 'r')
+ elif ztype == 'xz':
+ s_fn = lzma.LZMAFile(source, 'r')
+ elif ztype == 'gz':
+ s_fn = gzip.GzipFile(source, 'r')
+
-def bunzipFile(source,dest):
- """ Extract the bzipped contents of source to dest. """
- s_fn = bz2.BZ2File(source, 'r')
destination = open(dest, 'w')
while True:
@@ -697,7 +716,11 @@ def bunzipFile(source,dest):
destination.close()
s_fn.close()
-
+
+def bunzipFile(source,dest):
+ """ Extract the bzipped contents of source to dest. """
+ _decompress_chunked(source, dest, ztype='bz2')
+
def get_running_kernel_pkgtup(ts):
"""This takes the output of uname and figures out the pkgtup of the running
kernel (name, arch, epoch, version, release)."""
@@ -982,29 +1005,42 @@ def get_uuid(savepath):
return myid
-def decompress(filename):
+def decompress(filename, dest=None):
"""take a filename and decompress it into the same relative location.
if the file is not compressed just return the file"""
- out = filename
+
+ out = dest
+ if not dest:
+ out = filename
+
if filename.endswith('.gz'):
- out = filename.replace('.gz', '')
- decom = gzip.open(filename)
- fo = open(out, 'w')
- fo.write(decom.read())
- fo.flush()
- fo.close()
- decom.close()
- elif filename.endswith('.bz') or filename.endswith('.bz2'):
- if filename.endswith('.bz'):
- out = filename.replace('.bz','')
- else:
- out = filename.replace('.bz2', '')
- bunzipFile(filename, out)
+ ztype='gz'
+ if not dest:
+ out = filename.replace('.gz', '')
- #add magical lzma/xz trick here
+ elif filename.endswith('.bz') or filename.endswith('.bz2'):
+ ztype='bz2'
+ if not dest:
+ if filename.endswith('.bz'):
+ out = filename.replace('.bz','')
+ else:
+ out = filename.replace('.bz2', '')
+
+ elif filename.endswith('.xz'):
+ ztype='xz'
+ if not dest:
+ out = filename.replace('.xz', '')
+
+ else:
+ out = filename # returning the same file since it is not compressed
+ ztype = None
+ if ztype:
+ _decompress_chunked(filename, out, ztype)
+
return out
+
def read_in_items_from_dot_dir(thisglob, line_as_list=True):
"""takes a glob of a dir (like /etc/foo.d/*.foo)
returns a list of all the lines in all the files matching
More information about the Yum-commits
mailing list