[Rpm-metadata] createrepo/__init__.py createrepo/utils.py createrepo/yumbased.py genpkgmetadata.py
Seth Vidal
skvidal at linux.duke.edu
Wed Jan 16 17:16:01 UTC 2008
createrepo/__init__.py | 169 +++++++++++++++++++++++++++++++++++++++++--------
createrepo/utils.py | 6 -
createrepo/yumbased.py | 15 ++--
genpkgmetadata.py | 147 ++++--------------------------------------
4 files changed, 170 insertions(+), 167 deletions(-)
New commits:
commit 77d30d7581dea9adfa5966088eddcf48efacb5c5
Author: Seth Vidal <skvidal at fedoraproject.org>
Date: Wed Jan 16 12:12:44 2008 -0500
clean up api to simplify all of the code calling it
probably severely break --split for the moment, though
diff --git a/createrepo/__init__.py b/createrepo/__init__.py
index 0421e1e..210d6fe 100644
--- a/createrepo/__init__.py
+++ b/createrepo/__init__.py
@@ -37,7 +37,7 @@ except ImportError:
pass
-from utils import _gzipOpen, bzipFile
+from utils import _gzipOpen, bzipFile, checkAndMakeDir
__version__ = '0.9.1'
@@ -98,10 +98,7 @@ class SimpleMDCallBack(object):
sys.stdout.write("\r%d/%d - %s" % (current, total, item))
sys.stdout.flush()
-#FIXME = make it so you pass in a dir to doPkgMetadata() and it
-# parses out basedir and directory for relative dir from there
-# it creates the .repodata directory in the output location, etc
-
+
class MetaDataGenerator:
def __init__(self, config_obj=None, callback=None):
self.conf = config_obj
@@ -116,17 +113,69 @@ class MetaDataGenerator:
self.ts = rpmUtils.transaction.initReadOnlyTransaction()
self.pkgcount = 0
self.files = []
+
+ if not self.conf.directory and not self.conf.directories:
+ raise MDError, "No directory given on which to run."
+
+ # this does the dir setup we need done
+ self._parse_directory(self.conf.directory)
+ self._test_setup_dirs()
- def _setup_and_check_repo_dir(self, direc):
+ def _parse_directory(self, direc):
if os.path.isabs(direc):
self.conf.basedir = os.path.dirname(direc)
- self.conf.directory = os.path.basename(direc)
+ self.conf.relative_dir = os.path.basename(direc)
else:
self.conf.basedir = os.path.realpath(self.conf.basedir)
+ self.conf.relative_dir = direc
+
+ self.package_dir = os.path.join(self.conf.basedir, self.conf.relative_dir)
+
+ if not self.conf.outputdir:
+ self.conf.outputdir = os.path.join(self.conf.basedir, self.conf.relative_dir)
+
+
+ def _test_setup_dirs(self):
+ testdir = os.path.realpath(os.path.join(self.conf.basedir, self.conf.relative_dir))
+ # start the sanity/stupidity checks
+ if not os.path.exists(testdir):
+ raise MDError, _('Directory %s must exist') % self.conf.directory
+
+ if not os.path.isdir(testdir):
+ raise MDError, _('%s must be a directory') % self.conf.directory
+
+ if not os.access(self.conf.outputdir, os.W_OK):
+ raise MDError, _('Directory %s must be writable.') % self.conf.outputdir
+
+# SPLIT STUFF?
+# if self.conf.split:
+# oldbase = self.conf.basedir
+# self.conf.basedir = os.path.join(self.conf.basedir, self.conf.directory)
- if not self.conf.opts.outputdir:
- self.conf.outputdir = os.path.join(self.conf.basedir, direc)
+ temp_output = os.path.join(self.conf.outputdir, self.conf.tempdir)
+ if not checkAndMakeDir(temp_output):
+ raise MDError, _('Cannot create/verify %s') % temp_output
+ temp_final = os.path.join(self.conf.outputdir, self.conf.finaldir)
+ if not checkAndMakeDir(temp_final):
+ raise MDError, _('Cannot create/verify %s') % temp_final
+
+ if os.path.exists(os.path.join(self.conf.outputdir, self.conf.olddir)):
+ raise MDError, _('Old data directory exists, please remove: %s') % self.conf.olddir
+
+ # make sure we can write to where we want to write to:
+ # and pickup the mdtimestamps while we're at it
+ for direc in ['tempdir', 'finaldir']:
+ for f in ['primaryfile', 'filelistsfile', 'otherfile', 'repomdfile']:
+ filepath = os.path.join(self.conf.outputdir, direc, f)
+ if os.path.exists(filepath):
+ if not os.access(filepath, os.W_OK):
+ raise MDError, _('error in must be able to write to metadata files:\n -> %s') % filepath
+
+ if conf.checkts:
+ timestamp = os.path.getctime(filepath)
+ if timestamp > self.conf.mdtimestamp:
+ self.conf.mdtimestamp = timestamp
def _os_path_walk(self, top, func, arg):
"""Directory tree walk with callback function.
@@ -143,7 +192,7 @@ class MetaDataGenerator:
if os.path.isdir(name):
self._os_path_walk(name, func, arg)
# module
- def getFileList(self, basepath, directory, ext):
+ def getFileList(self, directory, ext):
"""Return all files in path matching ext, store them in filelist,
recurse dirs. Returns a list object"""
@@ -161,7 +210,7 @@ class MetaDataGenerator:
filelist.append(os.path.join(relativepath,fn))
filelist = []
- startdir = os.path.join(basepath, directory) + '/'
+ startdir = directory + '/'
self._os_path_walk(startdir, extension_visitor, filelist)
return filelist
@@ -198,11 +247,9 @@ class MetaDataGenerator:
files.remove(file)
return files
- def doPkgMetadata(self, directory=None):
+ def doPkgMetadata(self):
"""all the heavy lifting for the package metadata"""
- if not directory:
- directory = self.conf.directory
-
+
# rpms we're going to be dealing with
if self.conf.update:
#build the paths
@@ -211,7 +258,7 @@ class MetaDataGenerator:
otherfile = os.path.join(self.conf.outputdir, self.conf.finaldir, self.conf.otherfile)
opts = {
'verbose' : self.conf.verbose,
- 'pkgdir' : os.path.normpath(os.path.join(self.conf.basedir, directory))
+ 'pkgdir' : os.path.normpath(self.package_dir)
}
if self.conf.skip_stat:
opts['do_stat'] = False
@@ -222,12 +269,12 @@ class MetaDataGenerator:
if self.conf.pkglist:
packages = self.conf.pkglist
else:
- packages = self.getFileList(self.conf.basedir, directory, '.rpm')
+ packages = self.getFileList(self.package_dir, '.rpm')
packages = self.trimRpms(packages)
self.pkgcount = len(packages)
self.openMetadataDocs()
- self.writeMetadataDocs(packages, directory)
+ self.writeMetadataDocs(packages)
self.closeMetadataDocs()
# module
@@ -264,19 +311,23 @@ class MetaDataGenerator:
return fo
- def read_in_package(self, directory, rpmfile):
- # directory is stupid - just make it part of the class
- rpmfile = '%s/%s/%s' % (self.conf.basedir, directory, rpmfile)
+ def read_in_package(self, rpmfile):
+ """rpmfile == relative path to file from self.packge_dir"""
+
+ rpmfile = '%s/%s' % (self.package_dir, rpmfile)
try:
po = yumbased.CreateRepoPackage(self.ts, rpmfile)
except Errors.MiscError, e:
raise MDError, "Unable to open package: %s" % e
return po
- def writeMetadataDocs(self, pkglist, directory, current=0):
- # FIXME
- # directory is unused, kill it, pkglist should come from self
- # I don't see why current needs to be this way at all
+ def writeMetadataDocs(self, pkglist=[], directory=None, current=0):
+
+ if not pkglist:
+ pkglist = self.conf.pkglist
+ if not directory:
+ directory=self.conf.directory
+
for pkg in pkglist:
current+=1
recycled = False
@@ -294,7 +345,7 @@ class MetaDataGenerator:
if not recycled:
#scan rpm files
try:
- po = self.read_in_package(directory, pkg)
+ po = self.read_in_package(pkg)
except MDError, e:
# need to say something here
self.callback.errorlog("\nError %s: %s\n" % (pkg, e))
@@ -486,6 +537,72 @@ class MetaDataGenerator:
del repodoc
+
+ def doFinalMove(self):
+ """move the just-created repodata from .repodata to repodata
+ also make sure to preserve any files we didn't mess with in the
+ metadata dir"""
+
+ output_final_dir = os.path.join(self.conf.outputdir, self.conf.finaldir)
+ output_old_dir = os.path.join(self.conf.outputdir, self.conf.olddir)
+
+ if os.path.exists(output_final_dir):
+ try:
+ os.rename(output_final_dir, output_old_dir)
+ except:
+ raise MDError, _('Error moving final %s to old dir %s' % (output_final_dir,
+ output_old_dir))
+
+ output_temp_dir = os.path.join(self.conf.outputdir, self.conf.tempdir)
+
+ try:
+ os.rename(output_temp_dir, output_final_dir)
+ except:
+ # put the old stuff back
+ os.rename(output_old_dir, output_final_dir)
+ raise MDError, _('Error moving final metadata into place')
+
+ for f in ['primaryfile', 'filelistsfile', 'otherfile', 'repomdfile', 'groupfile']:
+ if getattr(self.conf, f):
+ fn = os.path.basename(getattr(self.conf, f))
+ else:
+ continue
+ oldfile = os.path.join(output_old_dir, fn)
+
+ if os.path.exists(oldfile):
+ try:
+ os.remove(oldfile)
+ except OSError, e:
+ raise MDError, _('Could not remove old metadata file: %s: %s') % (oldfile, e)
+
+ # Move everything else back from olddir (eg. repoview files)
+ for f in os.listdir(output_old_dir):
+ oldfile = os.path.join(output_old_dir, f)
+ finalfile = os.path.join(output_final_dir, f)
+ if os.path.exists(finalfile):
+ # Hmph? Just leave it alone, then.
+ try:
+ if os.path.isdir(oldfile):
+ shutil.rmtree(oldfile)
+ else:
+ os.remove(oldfile)
+ except OSError, e:
+ raise MDError, _('Could not remove old metadata file: %s: %s') % (oldfile, e)
+ else:
+ try:
+ os.rename(oldfile, finalfile)
+ except OSError, e:
+ msg = _('Could not restore old non-metadata file: %s -> %s') % (oldfile, finalfile)
+ msg += _('Error was %s') % e
+ raise MDError, msg
+
+ try:
+ os.rmdir(output_old_dir)
+ except OSError, e:
+ self.errorlog(_('Could not remove old metadata dir: %s') % self.conf.olddir)
+ self.errorlog(_('Error was %s') % e)
+ self.errorlog(_('Please clean up this directory manually.'))
+
class SplitMetaDataGenerator(MetaDataGenerator):
def __init__(self, config_obj=None, callback=None):
diff --git a/createrepo/utils.py b/createrepo/utils.py
index 51e56cd..13d1d67 100644
--- a/createrepo/utils.py
+++ b/createrepo/utils.py
@@ -104,11 +104,11 @@ def checkAndMakeDir(dir):
"""
if os.path.exists(dir):
if not os.path.isdir(dir):
- errorprint(_('%s is not a dir') % dir)
+ #errorprint(_('%s is not a dir') % dir)
result = False
else:
if not os.access(dir, os.W_OK):
- errorprint(_('%s is not writable') % dir)
+ #errorprint(_('%s is not writable') % dir)
result = False
else:
result = True
@@ -116,7 +116,7 @@ def checkAndMakeDir(dir):
try:
os.mkdir(dir)
except OSError, e:
- errorprint(_('Error creating dir %s: %s') % (dir, e))
+ #errorprint(_('Error creating dir %s: %s') % (dir, e))
result = False
else:
result = True
diff --git a/createrepo/yumbased.py b/createrepo/yumbased.py
index 88fcb0c..e705f6e 100644
--- a/createrepo/yumbased.py
+++ b/createrepo/yumbased.py
@@ -64,6 +64,7 @@ class CreateRepoPackage(YumLocalPackage):
def _xml(self, item):
item = utils.utf8String(item)
+ item = item.rstrip()
return xml.sax.saxutils.escape(item)
def _do_checksum(self):
@@ -154,7 +155,7 @@ class CreateRepoPackage(YumLocalPackage):
<packager>%s</packager>
<url>%s</url>
<time file="%s" build="%s"/>
- <size package="%s" installed="%s" archive="%s"/>""" % (self.name,
+ <size package="%s" installed="%s" archive="%s"/>\n""" % (self.name,
self.arch, self.epoch, self.ver, self.rel, self.checksum,
self._xml(self.summary), self._xml(self.description), packager,
url, self.filetime, self.buildtime, self.packagesize, self.size,
@@ -162,9 +163,9 @@ class CreateRepoPackage(YumLocalPackage):
if baseurl:
- msg += """<location xml:base="%s" href="%s"/>""" % (self._xml(baseurl), relpath)
+ msg += """<location xml:base="%s" href="%s"/>\n""" % (self._xml(baseurl), relpath)
else:
- msg += """<location href="%s"/>""" % relpath
+ msg += """<location href="%s"/>\n""" % relpath
return msg
@@ -173,22 +174,22 @@ class CreateRepoPackage(YumLocalPackage):
if self.license:
msg += """ <rpm:license>%s</rpm:license>\n""" % self._xml(self.license)
else:
- msg += """ <rpm:license/>"""
+ msg += """ <rpm:license/>\n"""
if self.vendor:
msg += """ <rpm:vendor>%s</rpm:vendor>\n""" % self._xml(self.vendor)
else:
- msg += """ <rpm:vendor/>"""
+ msg += """ <rpm:vendor/>\n"""
if self.group:
msg += """ <rpm:group>%s</rpm:group>\n""" % self._xml(self.group)
else:
- msg += """ <rpm:group/>"""
+ msg += """ <rpm:group/>\n"""
if self.buildhost:
msg += """ <rpm:buildhost>%s</rpm:buildhost>\n""" % self._xml(self.buildhost)
else:
- msg += """ <rpm:buildhost/>"""
+ msg += """ <rpm:buildhost/>\n"""
if self.sourcerpm:
msg += """ <rpm:sourcerpm>%s</rpm:sourcerpm>\n""" % self._xml(self.sourcerpm)
diff --git a/genpkgmetadata.py b/genpkgmetadata.py
index 38cd151..0449258 100755
--- a/genpkgmetadata.py
+++ b/genpkgmetadata.py
@@ -106,16 +106,10 @@ def parseArgs(args, conf):
setattr(conf, opt.dest, getattr(opts, opt.dest))
directory = directories[0]
- directory = os.path.normpath(directory)
- if conf.split:
- pass
- elif os.path.isabs(directory):
- conf.basedir = os.path.dirname(directory)
- directory = os.path.basename(directory)
- else:
- conf.basedir = os.path.realpath(conf.basedir)
+ conf.directory = directory
+ conf.directories = directories
-
+ # FIXME - I think this is unnecessary
if not opts.outputdir:
conf.outputdir = os.path.join(conf.basedir, directory)
if conf.groupfile:
@@ -124,10 +118,12 @@ def parseArgs(args, conf):
a = os.path.join(conf.basedir, directory, conf.groupfile)
elif not os.path.isabs(a):
a = os.path.join(conf.basedir, directory, conf.groupfile)
+ # FIXME, move this test most likely inside the class
if not os.path.exists(a):
errorprint(_('Error: groupfile %s cannot be found.' % a))
usage()
conf.groupfile = a
+ # FIXME - move this one inside the class, too
if conf.cachedir:
conf.cache = True
a = conf.cachedir
@@ -148,12 +144,6 @@ def parseArgs(args, conf):
conf.pkglist = lst
- #setup some defaults
-
- # Fixup first directory
- directories[0] = directory
- conf.directory = directory
- conf.directories = directories
return conf
@@ -172,130 +162,25 @@ class MDCallBack(object):
def main(args):
conf = createrepo.MetaDataConfig()
conf = parseArgs(args, conf)
- # FIXME - some of these should be moved into the module and out of the cli routines
- testdir = os.path.realpath(os.path.join(conf.basedir, conf.directory))
- # start the sanity/stupidity checks
- if not os.path.exists(testdir):
- errorprint(_('Directory %s must exist') % (conf.directory,))
- sys.exit(1)
- if not os.path.isdir(testdir):
- errorprint(_('%s - must be a directory')
- % (conf.directory,))
- sys.exit(1)
-
- if not os.access(conf.outputdir, os.W_OK):
- errorprint(_('Directory %s must be writable.') % (conf.outputdir,))
- sys.exit(1)
-
- if conf.split:
- oldbase = conf.basedir
- conf.basedir = os.path.join(conf.basedir, conf.directory)
- if not checkAndMakeDir(os.path.join(conf.outputdir, conf.tempdir)):
- sys.exit(1)
-
- if not checkAndMakeDir(os.path.join(conf.outputdir, conf.finaldir)):
- sys.exit(1)
-
- if os.path.exists(os.path.join(conf.outputdir, conf.olddir)):
- errorprint(_('Old data directory exists, please remove: %s') % conf.olddir)
- sys.exit(1)
-
- # make sure we can write to where we want to write to:
- for direc in ['tempdir', 'finaldir']:
- for f in ['primaryfile', 'filelistsfile', 'otherfile', 'repomdfile']:
- filepath = os.path.join(conf.outputdir, direc, f)
- if os.path.exists(filepath):
- if not os.access(filepath, os.W_OK):
- errorprint(_('error in must be able to write to metadata files:\n -> %s') % filepath)
- usage()
- if conf.checkts:
- timestamp = os.path.getctime(filepath)
- if timestamp > conf.mdtimestamp:
- conf.mdtimestamp = timestamp
-
- if conf.split:
- conf.basedir = oldbase
- mdgen = createrepo.SplitMetaDataGenerator(config_obj=conf, callback=MDCallBack())
- else:
- mdgen = createrepo.MetaDataGenerator(config_obj=conf, callback=MDCallBack())
- if mdgen.checkTimeStamps():
- if mdgen.conf.verbose:
- print _('repo is up to date')
- sys.exit(0)
try:
+ if conf.split:
+ mdgen = createrepo.SplitMetaDataGenerator(config_obj=conf, callback=MDCallBack())
+ else:
+ mdgen = createrepo.MetaDataGenerator(config_obj=conf, callback=MDCallBack())
+ if mdgen.checkTimeStamps():
+ if mdgen.conf.verbose:
+ print _('repo is up to date')
+ sys.exit(0)
+
mdgen.doPkgMetadata()
mdgen.doRepoMetadata()
+ mdgen.doFinalMove()
+
except MDError, e:
errorprint(_('%s') % e)
sys.exit(1)
- output_final_dir = os.path.join(mdgen.conf.outputdir, mdgen.conf.finaldir)
- output_old_dir = os.path.join(mdgen.conf.outputdir, mdgen.conf.olddir)
-
- if os.path.exists(output_final_dir):
- try:
- os.rename(output_final_dir, output_old_dir)
- except:
- errorprint(_('Error moving final %s to old dir %s' % (output_final_dir,
- output_old_dir)))
- sys.exit(1)
-
- output_temp_dir =os.path.join(mdgen.conf.outputdir, mdgen.conf.tempdir)
-
- try:
- os.rename(output_temp_dir, output_final_dir)
- except:
- errorprint(_('Error moving final metadata into place'))
- # put the old stuff back
- os.rename(output_old_dir, output_final_dir)
- sys.exit(1)
-
- for f in ['primaryfile', 'filelistsfile', 'otherfile', 'repomdfile', 'groupfile']:
- if getattr(mdgen.conf, f):
- fn = os.path.basename(getattr(mdgen.conf, f))
- else:
- continue
- oldfile = os.path.join(output_old_dir, fn)
-
- if os.path.exists(oldfile):
- try:
- os.remove(oldfile)
- except OSError, e:
- errorprint(_('Could not remove old metadata file: %s') % oldfile)
- errorprint(_('Error was %s') % e)
- sys.exit(1)
-
- # Move everything else back from olddir (eg. repoview files)
- for f in os.listdir(output_old_dir):
- oldfile = os.path.join(output_old_dir, f)
- finalfile = os.path.join(output_final_dir, f)
- if os.path.exists(finalfile):
- # Hmph? Just leave it alone, then.
- try:
- if os.path.isdir(oldfile):
- shutil.rmtree(oldfile)
- else:
- os.remove(oldfile)
- except OSError, e:
- errorprint(_('Could not remove old non-metadata file: %s') % oldfile)
- errorprint(_('Error was %s') % e)
- sys.exit(1)
- else:
- try:
- os.rename(oldfile, finalfile)
- except OSError, e:
- errorprint(_('Could not restore old non-metadata file: %s -> %s') % (oldfile, finalfile))
- errorprint(_('Error was %s') % e)
- sys.exit(1)
-
-
- try:
- os.rmdir(output_old_dir)
- except OSError, e:
- errorprint(_('Could not remove old metadata dir: %s') % mdgen.conf.olddir)
- errorprint(_('Error was %s') % e)
- errorprint(_('Please clean up this directory manually.'))
if __name__ == "__main__":
if len(sys.argv) > 1:
More information about the Rpm-metadata
mailing list