[Rpm-metadata] 2 commits - createrepo/__init__.py genpkgmetadata.py

[skvidal at osuosl.org]

 createrepo/__init__.py |   40 +++++++++++++++++++++++++++++++++-------
 genpkgmetadata.py      |    2 ++
 2 files changed, 35 insertions(+), 7 deletions(-)

New commits:
commit 79b7871aaba432e267a8e15a4070118a9168f9e4
Author: Seth Vidal <skvidal at fedoraproject.org>
Date:   Wed Jul 27 18:07:35 2011 -0400

    reset the worker cmd location, whoops

diff --git a/createrepo/__init__.py b/createrepo/__init__.py
index 769defa..8cce31a 100644
--- a/createrepo/__init__.py
+++ b/createrepo/__init__.py
@@ -107,9 +107,9 @@ class MetaDataConfig(object):
                                    # read in this run of createrepo
         self.collapse_glibc_requires = True
         self.workers = 1 # number of workers to fork off to grab metadata from the pkgs
-        #self.worker_cmd = '/usr/share/createrepo/worker.py'
+        self.worker_cmd = '/usr/share/createrepo/worker.py'
         
-        self.worker_cmd = './worker.py' # helpful when testing
+        #self.worker_cmd = './worker.py' # helpful when testing
         self.retain_old_md = 0
         
 class SimpleMDCallBack(object):
commit 180e3042bf7337edba2b80dd867f3f7953622725
Author: Seth Vidal <skvidal at fedoraproject.org>
Date:   Wed Jul 27 18:05:17 2011 -0400

    add retain_old_md config value and --retain-old-md cli option.
    
    This adds the ability for createrepo to keep around the latest N old,
    uniquely named, primary, filelist and otherdata xml and sqlite files
    if they exist. This is handy for sites which have mirrors which are
    not always in sync. It means if a user has an old repomd.xml they
    can still get the xml or sqlite file referred to in that from
    the normal mirrors

diff --git a/createrepo/__init__.py b/createrepo/__init__.py
index 8549188..769defa 100644
--- a/createrepo/__init__.py
+++ b/createrepo/__init__.py
@@ -107,9 +107,10 @@ class MetaDataConfig(object):
                                    # read in this run of createrepo
         self.collapse_glibc_requires = True
         self.workers = 1 # number of workers to fork off to grab metadata from the pkgs
-        self.worker_cmd = '/usr/share/createrepo/worker.py'
+        #self.worker_cmd = '/usr/share/createrepo/worker.py'
         
-        #self.worker_cmd = './worker.py' # helpful when testing
+        self.worker_cmd = './worker.py' # helpful when testing
+        self.retain_old_md = 0
         
 class SimpleMDCallBack(object):
     def errorlog(self, thing):
@@ -1094,17 +1095,42 @@ class MetaDataGenerator:
                     raise MDError, _(
                     'Could not remove old metadata file: %s: %s') % (oldfile, e)
 
+        old_to_remove = []
+        old_pr = []
+        old_fl = []
+        old_ot = []
+        old_pr_db = []
+        old_fl_db = []
+        old_ot_db = []
         for f in os.listdir(output_old_dir):
             oldfile = os.path.join(output_old_dir, f)
             finalfile = os.path.join(output_final_dir, f)
-            if f.find('-') != -1 and f.split('-')[1] in ('primary.sqlite.bz2',
-                    'filelists.sqlite.bz2', 'primary.xml.gz','other.sqlite.bz2',
-                    'other.xml.gz','filelists.xml.gz'):
-                os.remove(oldfile) # kill off the old ones
-                continue
+
+            for (end,lst) in (('-primary.sqlite.bz2', old_pr_db), ('-primary.xml.gz', old_pr),
+                           ('-filelists.sqlite.bz2', old_fl_db), ('-filelists.xml.gz', old_fl),
+                           ('-other.sqlite.bz2', old_ot_db), ('-other.xml.gz', old_ot)):
+                if f.endswith(end):
+                    lst.append(oldfile)
+                    break
+
+        # make a list of the old metadata files we don't want to remove.
+        for lst in (old_pr, old_fl, old_ot, old_pr_db, old_fl_db, old_ot_db):
+            sortlst = sorted(lst, key=lambda x: os.path.getmtime(x),
+                             reverse=True)
+            for thisf in sortlst[self.conf.retain_old_md:]:
+                old_to_remove.append(thisf)
+
+        for f in os.listdir(output_old_dir):
+            oldfile = os.path.join(output_old_dir, f)
+            finalfile = os.path.join(output_final_dir, f)
+                
             if f in ('filelists.sqlite.bz2', 'other.sqlite.bz2',
-                     'primary.sqlite.bz2'):
-                os.remove(oldfile)
+                     'primary.sqlite.bz2') or oldfile in old_to_remove:
+                try:
+                    os.remove(oldfile)
+                except (OSError, IOError), e:
+                    raise MDError, _(
+                    'Could not remove old metadata file: %s: %s') % (oldfile, e)
                 continue
 
             if os.path.exists(finalfile):
diff --git a/genpkgmetadata.py b/genpkgmetadata.py
index 8c98191..6c9217a 100755
--- a/genpkgmetadata.py
+++ b/genpkgmetadata.py
@@ -100,6 +100,8 @@ def parse_args(args, conf):
     parser.add_option("--simple-md-filenames", dest="simple_md_filenames",
         help="do not include the file's checksum in the filename, helps with proxies",
         default=False, action="store_true")
+    parser.add_option("--retain-old-md", default=0, type='int', dest='retain_old_md',
+        help="keep around the latest (by timestamp) N copies of the old repodata")
     parser.add_option("--distro", default=[], action="append",
         help="distro tag and optional cpeid: --distro" "'cpeid,textname'")
     parser.add_option("--content", default=[], dest='content_tags',