[Rpm-metadata] createrepo/__init__.py createrepo/yumbased.py genpkgmetadata.py
Seth Vidal
skvidal at linux.duke.edu
Wed Feb 20 20:27:00 UTC 2008
createrepo/__init__.py | 81 ++++++++++++++++++++++++++++++++++---------------
createrepo/yumbased.py | 18 ++++------
genpkgmetadata.py | 5 ++-
3 files changed, 68 insertions(+), 36 deletions(-)
New commits:
commit bb9a6387107b3ecd356ab995079735fe15bc8ad5
Author: Seth Vidal <skvidal at fedoraproject.org>
Date: Wed Feb 20 15:22:49 2008 -0500
allow --pkglist or self.conf.pkglist to be a list of arbitrary
urls to packages. createrepo will download the remote urls to
a tempdir, read them in and add them to the metadata.
diff --git a/createrepo/__init__.py b/createrepo/__init__.py
index 2e5fa6e..e91cf85 100644
--- a/createrepo/__init__.py
+++ b/createrepo/__init__.py
@@ -21,6 +21,8 @@ import fnmatch
import time
import yumbased
import shutil
+from urlgrabber import grabber
+import tempfile
from yum import misc, Errors
import rpmUtils.transaction
@@ -72,6 +74,7 @@ class MetaDataConfig(object):
self.directories = []
self.changelog_limit = None # needs to be an int or None
self.unique_md_filenames = False
+
class SimpleMDCallBack(object):
@@ -278,8 +281,6 @@ class MetaDataGenerator:
# function for the first dir passed to --split, not all of them
# this needs to be fixed by some magic in readMetadata.py
# using opts.pkgdirs as a list, I think.
- # FIXME - this needs to read the old repomd.xml to figure out where
- # the files WERE to pass in the right fns.
if self.conf.update:
#build the paths
opts = {
@@ -288,10 +289,19 @@ class MetaDataGenerator:
}
if self.conf.skip_stat:
opts['do_stat'] = False
-
+
#and scan the old repo
self.oldData = readMetadata.MetadataIndex(self.conf.outputdir, opts)
-
+
+ def _setup_grabber(self):
+ if not hasattr(self, '_grabber'):
+ self._grabber = grabber.URLGrabber()
+
+ return self._grabber
+
+ grabber = property(fget = lambda self: self._setup_grabber())
+
+
def doPkgMetadata(self):
"""all the heavy lifting for the package metadata"""
if self.conf.update:
@@ -342,18 +352,38 @@ class MetaDataGenerator:
return fo
- def read_in_package(self, rpmfile, pkgpath=None):
+ def read_in_package(self, rpmfile, pkgpath=None, reldir=None):
"""rpmfile == relative path to file from self.packge_dir"""
- # TODO/FIXME
- # consider adding a routine to download the package from a remote location
- # to a tempdir, operate on it, then use that location as a the baseurl
- # for the package. That would make it possible to have repos entirely
- # comprised of remote packages.
+ remote_package = False
+ baseurl = self.conf.baseurl
if not pkgpath:
pkgpath = self.package_dir
- rpmfile = '%s/%s' % (pkgpath, rpmfile)
+ if not rpmfile.strip():
+ raise MDError, "Blank filename passed in, skipping"
+
+ if rpmfile.find("://") != -1:
+ remote_package = True
+
+ if not hasattr(self, 'tempdir'):
+ self.tempdir = tempfile.mkdtemp()
+
+ pkgname = os.path.basename(rpmfile)
+ baseurl = os.path.dirname(rpmfile)
+ reldir = self.tempdir
+ dest = os.path.join(self.tempdir, pkgname)
+ if not self.conf.quiet:
+ self.callback.log('\nDownloading %s' % rpmfile)
+ try:
+ rpmfile = self.grabber.urlgrab(rpmfile, dest)
+ except grabber.URLGrabError, e:
+ raise MDError, "Unable to retrieve remote package %s: %s" %(rpmfile, e)
+
+
+ else:
+ rpmfile = '%s/%s' % (pkgpath, rpmfile)
+
try:
po = yumbased.CreateRepoPackage(self.ts, rpmfile)
except Errors.MiscError, e:
@@ -362,11 +392,8 @@ class MetaDataGenerator:
# you can do it
po.crp_changelog_limit = self.conf.changelog_limit
po.crp_cachedir = self.conf.cachedir
-
- # FIXME if we wanted to put in a baseurl-per-package here is where
- # we should do it
- # it would be easy to have a lookup dict in the MetaDataConfig object
- # and work down from there for the baseurl
+ po.crp_baseurl = baseurl
+ po.crp_reldir = reldir
if po.checksum in (None, ""):
raise MDError, "No Package ID found for package %s, not going to add it" % e
@@ -377,6 +404,7 @@ class MetaDataGenerator:
if not pkglist:
pkglist = self.conf.pkglist
+
if not pkgpath:
directory=self.conf.directory
else:
@@ -385,13 +413,16 @@ class MetaDataGenerator:
for pkg in pkglist:
current+=1
recycled = False
-
+
# look to see if we can get the data from the old repodata
# if so write this one out that way
if self.conf.update:
#see if we can pull the nodes from the old repo
#print self.oldData.basenodes.keys()
- nodes = self.oldData.getNodes(pkg)
+ old_pkg = pkg
+ if pkg.find("://") != -1:
+ old_pkg = os.path.basename(pkg)
+ nodes = self.oldData.getNodes(old_pkg)
if nodes is not None:
recycled = True
@@ -399,17 +430,19 @@ class MetaDataGenerator:
# otherwise do it individually
if not recycled:
#scan rpm files
+ if not pkgpath:
+ reldir = os.path.join(self.conf.basedir, directory)
+ else:
+ reldir = pkgpath
+
try:
- po = self.read_in_package(pkg, pkgpath=pkgpath)
+ po = self.read_in_package(pkg, pkgpath=pkgpath, reldir=reldir)
except MDError, e:
# need to say something here
self.callback.errorlog("\nError %s: %s\n" % (pkg, e))
continue
- if not pkgpath:
- reldir = os.path.join(self.conf.basedir, directory)
- else:
- reldir = pkgpath
- self.primaryfile.write(po.do_primary_xml_dump(reldir, baseurl=self.conf.baseurl))
+
+ self.primaryfile.write(po.do_primary_xml_dump())
self.flfile.write(po.do_filelists_xml_dump())
self.otherfile.write(po.do_other_xml_dump())
else:
diff --git a/createrepo/yumbased.py b/createrepo/yumbased.py
index 8f424fd..17af4e6 100644
--- a/createrepo/yumbased.py
+++ b/createrepo/yumbased.py
@@ -167,15 +167,11 @@ class CreateRepoPackage(YumLocalPackage):
hdrend = property(fget=lambda self: self._get_header_byte_range()[1])
hdrstart = property(fget=lambda self: self._get_header_byte_range()[0])
- def _dump_base_items(self, basedir, baseurl=None):
- """Takes an optional baseurl and required basedir.
- basedir is the relative path to remove from the location
- baseurl is whether or not this package already has a
- baseurl rather than just '.'"""
+ def _dump_base_items(self):
# if we start seeing fullpaths in the location tag - this is the culprit
- if self.localpath.startswith(basedir):
- relpath = self.localpath.replace(basedir, '')
+ if self.crp_reldir and self.localpath.startswith(self.crp_reldir):
+ relpath = self.localpath.replace(self.crp_reldir, '')
if relpath[0] == '/': relpath = relpath[1:]
else:
relpath = self.localpath
@@ -204,8 +200,8 @@ class CreateRepoPackage(YumLocalPackage):
self.archivesize)
- if baseurl:
- msg += """<location xml:base="%s" href="%s"/>\n""" % (self._xml(baseurl), relpath)
+ if self.crp_baseurl:
+ msg += """<location xml:base="%s" href="%s"/>\n""" % (self._xml(self.crp_baseurl), relpath)
else:
msg += """<location href="%s"/>\n""" % relpath
@@ -379,9 +375,9 @@ class CreateRepoPackage(YumLocalPackage):
del c
return msg
- def do_primary_xml_dump(self, basedir, baseurl=None):
+ def do_primary_xml_dump(self):
msg = """\n<package type="rpm">"""
- msg += self._dump_base_items(basedir, baseurl)
+ msg += self._dump_base_items()
msg += self._dump_format_items()
msg += """\n</package>"""
return msg
diff --git a/genpkgmetadata.py b/genpkgmetadata.py
index ed369da..e9de6b3 100755
--- a/genpkgmetadata.py
+++ b/genpkgmetadata.py
@@ -20,6 +20,7 @@
import os
import sys
+import re
from optparse import OptionParser
import createrepo
@@ -118,7 +119,9 @@ def parseArgs(args, conf):
if conf.pkglist:
pfo = open(conf.pkglist, 'r')
for line in pfo.readlines():
- line = line.replace('\n', '')
+ line = line.strip()
+ if re.match('^\s*\#.*', line) or re.match('^\s*$', line):
+ continue
lst.append(line)
pfo.close()
More information about the Rpm-metadata
mailing list