[yum-commits] Branch 'yum-3_2_X' - 3 commits - yum/__init__.py yum/misc.py yum/update_md.py yum/yumRepo.py

skvidal at osuosl.org skvidal at osuosl.org
Fri May 28 12:21:05 UTC 2010


 yum/__init__.py  |    8 ++---
 yum/misc.py      |   76 ++++++++++++++++++++++++++++++++++++++++---------------
 yum/update_md.py |   10 ++++---
 yum/yumRepo.py   |   27 +++++++++----------
 4 files changed, 79 insertions(+), 42 deletions(-)

New commits:
commit 28823e8cee46059faed6ef5e98ba583f55da99ef
Merge: 0612a88... 0389ee6...
Author: Seth Vidal <skvidal at fedoraproject.org>
Date:   Fri May 28 08:21:54 2010 -0400

    Merge branch 'yum-3_2_X' of ssh://yum.baseurl.org/srv/projects/yum/git/yum into yum-3_2_X
    
    * 'yum-3_2_X' of ssh://yum.baseurl.org/srv/projects/yum/git/yum:
      l10n: Updates to Finnish (fi) translation

commit 0612a88cd98ff59eab52a816099c0abaf78d9764
Author: Seth Vidal <skvidal at fedoraproject.org>
Date:   Thu May 27 13:16:06 2010 -0400

    - go through all of our code and change our decompressors to use misc.decompress()
    - this should mean we can access metadata compressed using any supported compression library
      provided it is NAMED accordingly
    - add fn_only option to misc.decompress() so we can get a returned name w/o actually
      having the file - useful for finding a decompressed file if we only have the compressed one

diff --git a/yum/__init__.py b/yum/__init__.py
index 8a2fbf4..1c398fd 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -30,7 +30,7 @@ import fnmatch
 import logging
 import logging.config
 import operator
-import gzip
+
 
 import yum.i18n
 _ = yum.i18n._
@@ -716,9 +716,9 @@ class YumBase(depsolve.Depsolve):
             self.verbose_logger.log(logginglevels.DEBUG_4,
                 _('Adding group file from repository: %s'), repo)
             groupfile = repo.getGroups()
-            # open it up as a file object so iterparse can cope with our gz file
-            if groupfile is not None and groupfile.endswith('.gz'):
-                groupfile = gzip.open(groupfile)
+            # open it up as a file object so iterparse can cope with our compressed file
+            if groupfile:
+                groupfile = misc.decompress(groupfile)
                 
             try:
                 self._comps.add(groupfile)
diff --git a/yum/misc.py b/yum/misc.py
index 92e9127..7793807 100644
--- a/yum/misc.py
+++ b/yum/misc.py
@@ -1005,7 +1005,7 @@ def get_uuid(savepath):
         
         return myid
         
-def decompress(filename, dest=None):
+def decompress(filename, dest=None, fn_only=False):
     """take a filename and decompress it into the same relative location.
        if the file is not compressed just return the file"""
     
@@ -1035,7 +1035,7 @@ def decompress(filename, dest=None):
         out = filename # returning the same file since it is not compressed
         ztype = None
     
-    if ztype:
+    if ztype and not fn_only:
         _decompress_chunked(filename, out, ztype)
         
     return out
diff --git a/yum/update_md.py b/yum/update_md.py
index 90c5582..cf8136a 100644
--- a/yum/update_md.py
+++ b/yum/update_md.py
@@ -22,12 +22,11 @@ Update metadata (updateinfo.xml) parsing.
 """
 
 import sys
-import gzip
 
 from yum.i18n import utf8_text_wrap, to_utf8
 from yum.yumRepo import YumRepository
 from yum.packages import FakeRepository
-from yum.misc import to_xml
+from yum.misc import to_xml, decompress
 import Errors
 
 import rpmUtils.miscutils
@@ -382,14 +381,17 @@ class UpdateMetadata(object):
         if not obj:
             raise UpdateNoticeException
         if type(obj) in (type(''), type(u'')):
-            infile = obj.endswith('.gz') and gzip.open(obj) or open(obj, 'rt')
+            unfile = decompress(obj)
+            infile = open(unfile, 'rt')
+
         elif isinstance(obj, YumRepository):
             if obj.id not in self._repos:
                 self._repos.append(obj.id)
                 md = obj.retrieveMD(mdtype)
                 if not md:
                     raise UpdateNoticeException()
-                infile = gzip.open(md)
+                unfile = decompress(md)
+                infile = open(unfile, 'rt')
         elif isinstance(obj, FakeRepository):
             raise Errors.RepoMDError, "No updateinfo for local pkg"
         else:   # obj is a file object
diff --git a/yum/yumRepo.py b/yum/yumRepo.py
index 3a695d8..ac6c7c0 100644
--- a/yum/yumRepo.py
+++ b/yum/yumRepo.py
@@ -19,7 +19,6 @@ import time
 import types
 import urlparse
 urlparse.uses_fragment.append("media")
-import gzip
 
 import Errors
 from urlgrabber.grabber import URLGrabber
@@ -163,7 +162,7 @@ class YumPackageSack(packageSack.PackageSack):
 
             if self._check_db_version(repo, mydbtype):
                 # see if we have the uncompressed db and check it's checksum vs the openchecksum
-                # if not download the bz2 file
+                # if not download the compressed file
                 # decompress it
                 # unlink it
 
@@ -171,9 +170,8 @@ class YumPackageSack(packageSack.PackageSack):
                 if not db_un_fn:
                     db_fn = repo._retrieveMD(mydbtype)
                     if db_fn:
-                        db_un_fn = db_fn.replace('.bz2', '')
                         if not repo.cache:
-                            misc.bunzipFile(db_fn, db_un_fn)
+                            db_un_fn = misc.decompress(db_fn)
                             misc.unlink_f(db_fn)
                             db_un_fn = self._check_uncompressed_db(repo, mydbtype)
 
@@ -199,8 +197,8 @@ class YumPackageSack(packageSack.PackageSack):
         mydbdata = repo.repoXML.getData(mdtype)
         (r_base, remote) = mydbdata.location
         fname = os.path.basename(remote)
-        bz2_fn = repo.cachedir + '/' + fname
-        db_un_fn = bz2_fn.replace('.bz2', '')
+        compressed_fn = repo.cachedir + '/' + fname
+        db_un_fn = misc.decompress(compressed_fn, fn_only=True)
 
         result = None
 
@@ -1089,7 +1087,7 @@ class YumRepository(Repository, config.RepoConf):
         local = self.cachedir + '/' + os.path.basename(remote)
 
         if compressed: # DB file, we need the uncompressed version
-            local = local.replace('.bz2', '')
+            local = misc.decompress(local, fn_only=True)
         return local
 
     def _groupCheckDataMDNewer(self):
@@ -1244,7 +1242,7 @@ class YumRepository(Repository, config.RepoConf):
             compressed = False
             local = self._get_mdtype_fname(data, False)
             if not os.path.exists(local):
-                local = local.replace('.bz2', '')
+                local = misc.decompress(local, fn_only=True)
                 compressed = True
         #  If we can, make a copy of the system-wide-cache version of this file,
         # note that we often don't get here. So we also do this in
@@ -1351,10 +1349,9 @@ class YumRepository(Repository, config.RepoConf):
 
         for (ndata, nmdtype) in downloading_with_size + downloading_no_size:
             local = self._get_mdtype_fname(ndata, False)
-            if nmdtype.endswith("_db"): # Uncompress any .sqlite.bz2 files
+            if nmdtype.endswith("_db"): # Uncompress any compressed files
                 dl_local = local
-                local = local.replace('.bz2', '')
-                misc.bunzipFile(dl_local, local)
+                local = misc.decompress(dl_local)
                 misc.unlink_f(dl_local)
             self._oldRepoMDData['new_MD_files'].append(local)
 
@@ -1516,6 +1513,8 @@ class YumRepository(Repository, config.RepoConf):
         """ Internal function, use .retrieveMD() from outside yum. """
         #  Note that this can raise Errors.RepoMDError if mdtype doesn't exist
         # for this repo.
+        # FIXME - maybe retrieveMD should call decompress() after we've checked
+        # the checksum by default? since we're never acting on compressed MD
         thisdata = self.repoXML.getData(mdtype)
 
         (r_base, remote) = thisdata.location
@@ -1768,9 +1767,9 @@ class YumRepository(Repository, config.RepoConf):
 
         grpfile = self.getGroups()
 
-        # open it up as a file object so iterparse can cope with our gz file
-        if grpfile is not None and grpfile.endswith('.gz'):
-            grpfile = gzip.open(grpfile)
+        # open it up as a file object so iterparse can cope with our compressed file
+        if grpfile is not None:
+            grpfile = misc.decompress(grpfile)
         try:
             c = comps.Comps()
             c.add(grpfile)
commit fa9a82967408b0789564a6be9cc19ea09e0e98f0
Author: Seth Vidal <skvidal at fedoraproject.org>
Date:   Thu May 27 12:01:05 2010 -0400

    - cleanup decompress some
    - keep stub bunzipFile
    - add lzma support, if available

diff --git a/yum/misc.py b/yum/misc.py
index ebce8e1..92e9127 100644
--- a/yum/misc.py
+++ b/yum/misc.py
@@ -19,6 +19,13 @@ import pwd
 import fnmatch
 import bz2
 import gzip
+_available_compression = ['gz', 'bz2']
+try:
+    import lzma
+    _available_compression.append('xz')
+except ImportError:
+    lzma = None
+
 from rpmUtils.miscutils import stringToVersion, flagToString
 from stat import *
 try:
@@ -675,10 +682,22 @@ def refineSearchPattern(arg):
         restring = re.escape(arg)
         
     return restring
+
+
+def _decompress_chunked(source, dest, ztype):
+
+    if ztype not in _available_compression:
+        msg = "%s compression not available" % ztype
+        raise Errors.MiscError, msg
+    
+    if ztype == 'bz2':
+        s_fn = bz2.BZ2File(source, 'r')
+    elif ztype == 'xz':
+        s_fn = lzma.LZMAFile(source, 'r')
+    elif ztype == 'gz':
+        s_fn = gzip.GzipFile(source, 'r')
+    
     
-def bunzipFile(source,dest):
-    """ Extract the bzipped contents of source to dest. """
-    s_fn = bz2.BZ2File(source, 'r')
     destination = open(dest, 'w')
 
     while True:
@@ -697,7 +716,11 @@ def bunzipFile(source,dest):
     
     destination.close()
     s_fn.close()
-
+    
+def bunzipFile(source,dest):
+    """ Extract the bzipped contents of source to dest. """
+    _decompress_chunked(source, dest, ztype='bz2')
+    
 def get_running_kernel_pkgtup(ts):
     """This takes the output of uname and figures out the pkgtup of the running
        kernel (name, arch, epoch, version, release)."""
@@ -982,29 +1005,42 @@ def get_uuid(savepath):
         
         return myid
         
-def decompress(filename):
+def decompress(filename, dest=None):
     """take a filename and decompress it into the same relative location.
        if the file is not compressed just return the file"""
-    out = filename
+    
+    out = dest
+    if not dest:
+        out = filename
+        
     if filename.endswith('.gz'):
-        out = filename.replace('.gz', '')
-        decom = gzip.open(filename)
-        fo = open(out, 'w')
-        fo.write(decom.read())
-        fo.flush()
-        fo.close()
-        decom.close() 
-    elif filename.endswith('.bz') or filename.endswith('.bz2'):
-        if filename.endswith('.bz'):
-            out = filename.replace('.bz','')
-        else:
-            out = filename.replace('.bz2', '')
-        bunzipFile(filename, out)
+        ztype='gz'
+        if not dest: 
+            out = filename.replace('.gz', '')
 
-    #add magical lzma/xz trick here
+    elif filename.endswith('.bz') or filename.endswith('.bz2'):
+        ztype='bz2'
+        if not dest:
+            if filename.endswith('.bz'):
+                out = filename.replace('.bz','')
+            else:
+                out = filename.replace('.bz2', '')
+    
+    elif filename.endswith('.xz'):
+        ztype='xz'
+        if not dest:
+            out = filename.replace('.xz', '')
+        
+    else:
+        out = filename # returning the same file since it is not compressed
+        ztype = None
     
+    if ztype:
+        _decompress_chunked(filename, out, ztype)
+        
     return out
     
+    
 def read_in_items_from_dot_dir(thisglob, line_as_list=True):
     """takes a glob of a dir (like /etc/foo.d/*.foo)
        returns a list of all the lines in all the files matching


More information about the Yum-commits mailing list