[Rpm-metadata] 8 commits - createrepo/__init__.py docs/modifyrepo.1 modifyrepo.py
valentina at osuosl.org
valentina at osuosl.org
Fri Mar 24 14:32:06 UTC 2017
createrepo/__init__.py | 79 ++++++++++++++++++++++++++++---------------------
docs/modifyrepo.1 | 2 -
modifyrepo.py | 3 +
3 files changed, 50 insertions(+), 34 deletions(-)
New commits:
commit 22e266964b977f796c5096f9658f00b3bfee5454
Merge: 6b76c6d 9dd33d9
Author: Valentina Mukhamedzhanova <umirra at gmail.com>
Date: Fri Mar 24 15:30:30 2017 +0100
Merge pull request #5 from dmnks/bz1287714
modifyrepo: docs: remove compat compress type. BZ 1287714
commit 9dd33d94185b9f78de865af227ec682f81ecab21
Author: Michal Domonkos <mdomonko at redhat.com>
Date: Fri Mar 24 13:35:16 2017 +0100
modifyrepo: docs: remove compat compress type. BZ 1287714
We never supported "compat" here as it only makes sense in the
createrepo domain (it means use a different method for the xml files
(gz) and for the sqlite files (bzip2) -- we don't generate sqlite files
in modifyrepo).
diff --git a/docs/modifyrepo.1 b/docs/modifyrepo.1
index ae2cea5..abd5357 100644
--- a/docs/modifyrepo.1
+++ b/docs/modifyrepo.1
@@ -24,7 +24,7 @@ Compress the new repodata before adding it to the repo. This is used by default.
Do not compress the new repodata before adding it to the repo.
.IP "\fB\-\-compress-type <compress-type>\fP"
-Specify which compression type to use: compat (default), xz (may not be available), gz, bz2.
+Specify which compression type to use: gz, xz (may not be available), bz2.
.IP "\fB\-s, \-\-checksum <sumtype>\fP"
Specify the checksum type to use.
commit 6b76c6dba6cf87771fa537aca226698cee275620
Merge: 51c00ac c626c54
Author: Valentina Mukhamedzhanova <umirra at gmail.com>
Date: Thu Mar 23 17:17:37 2017 +0100
Merge pull request #4 from dmnks/bz1287685
modifyrepo: handle empty file with LZMA. BZ 1287685
commit c626c54074b36edb090254fe6f4985bc20893436
Author: Michal Domonkos <mdomonko at redhat.com>
Date: Thu Mar 23 16:05:16 2017 +0100
modifyrepo: handle empty file with LZMA. BZ 1287685
When trying to compress an empty string with LZMA, we will get the
unfriendly "LZMA.error: unknown error!". Let's handle this case
ourselves and raise a more user-friendly error instead.
diff --git a/modifyrepo.py b/modifyrepo.py
index 34b0902..ade5607 100755
--- a/modifyrepo.py
+++ b/modifyrepo.py
@@ -125,6 +125,9 @@ class RepoMetadata:
else:
raise MDError, 'invalid metadata type'
+ if not md and self.compress_type == 'xz':
+ raise MDError, 'LZMA does not support compressing empty files'
+
## Compress the metadata and move it into the repodata
mdtype = self._get_mdtype(mdname, mdtype)
destmd = os.path.join(self.repodir, mdname)
commit 51c00ac3fcc725698e0e43fdd95ee1466b786584
Merge: 659d74b 2f83221
Author: Valentina Mukhamedzhanova <umirra at gmail.com>
Date: Thu Mar 9 16:56:27 2017 +0100
Merge pull request #2 from dmnks/bz1125437
createrepo: allow xz and bz2 for xml files. BZ 1125437
commit 659d74b825c18d0d78509e192c9554671eda83bf
Merge: 364dfa5 19711d9
Author: Valentina Mukhamedzhanova <umirra at gmail.com>
Date: Wed Feb 22 12:32:26 2017 +0100
Merge pull request #3 from dmnks/bz1406418-fixup
Add safety check for oldData attribute
commit 19711d9fd87e39a8bb8ca2c4caf99ec7aaf192d1
Author: Michal Domonkos <mdomonko at redhat.com>
Date: Mon Feb 20 14:49:17 2017 +0100
Add safety check for oldData attribute
We may not have it set up yet at the point of calling the cleanup
method.
diff --git a/createrepo/__init__.py b/createrepo/__init__.py
index 9e89afc..8f021e0 100644
--- a/createrepo/__init__.py
+++ b/createrepo/__init__.py
@@ -1484,7 +1484,7 @@ class MetaDataGenerator:
def _cleanup_update_tmp_dir(self):
- if self.conf.update:
+ if self.conf.update and hasattr(self, 'oldData'):
self.oldData.cleanup()
commit 2f832212a0ba5fc8d2ac448b0053ae34b134352d
Author: Michal Domonkos <mdomonko at redhat.com>
Date: Fri Nov 25 18:39:35 2016 +0100
createrepo: allow xz and bz2 for xml files. BZ 1125437
Currently, we don't honor --compress-type for the
primary/filelists/other.xml files and always force gz because libxml2
(used by yum-metadata-parser) didn't use to support anything other than
gz.
This has been worked around in yum since then by decompressing the files
first before passing them to y-m-p (commit cfe43e8). If we do the same
in createrepo (which uses y-m-p to generate the sqlite files), we can
enable these additional compress types for primary/filelists/other.xml
-- and that's what this commit does.
Note that libxml2 also natively supports xz in addition to gz so we only
need to do the decompression for bz2.
diff --git a/createrepo/__init__.py b/createrepo/__init__.py
index 9e89afc..6d56ff8 100644
--- a/createrepo/__init__.py
+++ b/createrepo/__init__.py
@@ -447,11 +447,10 @@ class MetaDataGenerator:
def _setupPrimary(self):
# setup the primary metadata file
- # FIXME - make this be conf.compress_type once y-m-p is fixed
- fpz = self.conf.primaryfile + '.' + 'gz'
+ fpz = self.conf.primaryfile + '.' + self.conf.compress_type
primaryfilepath = os.path.join(self.conf.outputdir, self.conf.tempdir,
fpz)
- fo = compressOpen(primaryfilepath, 'w', 'gz')
+ fo = compressOpen(primaryfilepath, 'w', self.conf.compress_type)
fo.write('<?xml version="1.0" encoding="UTF-8"?>\n')
fo.write('<metadata xmlns="http://linux.duke.edu/metadata/common"' \
' xmlns:rpm="http://linux.duke.edu/metadata/rpm" packages="%s">' %
@@ -460,11 +459,10 @@ class MetaDataGenerator:
def _setupFilelists(self):
# setup the filelist file
- # FIXME - make this be conf.compress_type once y-m-p is fixed
- fpz = self.conf.filelistsfile + '.' + 'gz'
+ fpz = self.conf.filelistsfile + '.' + self.conf.compress_type
filelistpath = os.path.join(self.conf.outputdir, self.conf.tempdir,
fpz)
- fo = compressOpen(filelistpath, 'w', 'gz')
+ fo = compressOpen(filelistpath, 'w', self.conf.compress_type)
fo.write('<?xml version="1.0" encoding="UTF-8"?>\n')
fo.write('<filelists xmlns="http://linux.duke.edu/metadata/filelists"' \
' packages="%s">' % self.pkgcount)
@@ -472,11 +470,10 @@ class MetaDataGenerator:
def _setupOther(self):
# setup the other file
- # FIXME - make this be conf.compress_type once y-m-p is fixed
- fpz = self.conf.otherfile + '.' + 'gz'
+ fpz = self.conf.otherfile + '.' + self.conf.compress_type
otherfilepath = os.path.join(self.conf.outputdir, self.conf.tempdir,
fpz)
- fo = compressOpen(otherfilepath, 'w', 'gz')
+ fo = compressOpen(otherfilepath, 'w', self.conf.compress_type)
fo.write('<?xml version="1.0" encoding="UTF-8"?>\n')
fo.write('<otherdata xmlns="http://linux.duke.edu/metadata/other"' \
' packages="%s">' %
@@ -1217,21 +1214,34 @@ class MetaDataGenerator:
rp = sqlitecachec.RepodataParserSqlite(repopath, repomd.repoid, None)
for (rpm_file, ftype) in workfiles:
- # when we fix y-m-p and non-gzipped xml files - then we can make this just add
- # self.conf.compress_type
- if ftype in ('other', 'filelists', 'primary'):
- rpm_file = rpm_file + '.' + 'gz'
- elif rpm_file.find('.') != -1 and rpm_file.split('.')[-1] not in _available_compression:
+ unpath = os.path.join(repopath, rpm_file)
+ if (ftype in ('other', 'filelists', 'primary')
+ or (rpm_file.find('.') != -1 and rpm_file.split('.')[-1]
+ not in _available_compression)):
rpm_file = rpm_file + '.' + self.conf.compress_type
complete_path = os.path.join(repopath, rpm_file)
zfo = compressOpen(complete_path)
+ dfo = None
+ if (self.conf.compress_type == 'bz2' and self.conf.database and
+ ftype in ('other', 'filelists', 'primary')):
+ # yum-metadata-parser doesn't understand bz2 so let's write the
+ # decompressed data to a file and pass that via gen_func
+ # instead of the compressed version
+ dfo = open(unpath, 'w')
# This is misc.checksum() done locally so we can get the size too.
data = misc.Checksums([sumtype])
- while data.read(zfo, 2**16):
- pass
+ while True:
+ chunk = data.read(zfo, 2**16)
+ if not chunk:
+ break
+ if dfo is not None:
+ dfo.write(chunk)
uncsum = data.hexdigest(sumtype)
unsize = len(data)
zfo.close()
+ if dfo is not None:
+ dfo.close()
+
csum = misc.checksum(sumtype, complete_path)
timestamp = os.stat(complete_path)[8]
@@ -1244,21 +1254,29 @@ class MetaDataGenerator:
self.callback.log("Starting %s db creation: %s" % (ftype,
time.ctime()))
+ gen_func = None
if ftype == 'primary':
- #FIXME - in theory some sort of try/except here
- # TypeError appears to be raised, sometimes :(
- rp.getPrimary(complete_path, csum)
-
+ gen_func = rp.getPrimary
elif ftype == 'filelists':
- #FIXME and here
- rp.getFilelists(complete_path, csum)
-
+ gen_func = rp.getFilelists
elif ftype == 'other':
- #FIXME and here
- rp.getOtherdata(complete_path, csum)
+ gen_func = rp.getOtherdata
+ if gen_func is not None:
+ if dfo is None:
+ #FIXME - in theory some sort of try/except here
+ # TypeError appears to be raised, sometimes :(
+ gen_func(complete_path, csum)
+ else:
+ #FIXME and here
+ gen_func(unpath, uncsum)
+ os.unlink(unpath)
if ftype in ['primary', 'filelists', 'other']:
- tmp_result_name = '%s.xml.gz.sqlite' % ftype
+ if dfo is None:
+ compress_ext = '.%s' % self.conf.compress_type
+ else:
+ compress_ext = ''
+ tmp_result_name = '%s.xml%s.sqlite' % (ftype, compress_ext)
tmp_result_path = os.path.join(repopath, tmp_result_name)
good_name = '%s.sqlite' % ftype
resultpath = os.path.join(repopath, good_name)
@@ -1323,13 +1341,8 @@ class MetaDataGenerator:
data.openchecksum = (sumtype, uncsum)
if self.conf.unique_md_filenames:
- if ftype in ('primary', 'filelists', 'other'):
- compress = 'gz'
- else:
- compress = self.conf.compress_type
-
main_name = '.'.join(rpm_file.split('.')[:-1])
- res_file = '%s-%s.%s' % (csum, main_name, compress)
+ res_file = '%s-%s.%s' % (csum, main_name, self.conf.compress_type)
orig_file = os.path.join(repopath, rpm_file)
dest_file = os.path.join(repopath, res_file)
os.rename(orig_file, dest_file)
More information about the Rpm-metadata
mailing list