[Yum-devel] [PATCH] add support for saving out depsolved transactions and reloading them.

Seth Vidal skvidal at fedoraproject.org
Wed Nov 3 19:09:01 UTC 2010


This saves a considerable amount of time if you find you have to deal with
gpg keys, between updates. It is also flexible enough that it could allow users
with systems in relative, or precise, lockstep, to pass around a .ts file and
have the deps pre-solved - cutting out a considerable amount of time searching
for providers.

this adds the load-ts/load-transaction command to yum-cli which lets you load them from a file
---
 cli.py                   |    1 +
 test/misc-tests.py       |    1 +
 test/skipbroken-tests.py |    1 +
 test/testbase.py         |    7 +-
 yum/__init__.py          |  228 +++++++++++++++++++++++++++++++++++++++++++++-
 yum/config.py            |    3 +
 yum/depsolve.py          |    1 +
 yum/transactioninfo.py   |   51 ++++++++---
 yumcommands.py           |   27 ++++++
 yummain.py               |    2 +
 10 files changed, 306 insertions(+), 16 deletions(-)

diff --git a/cli.py b/cli.py
index d5a85c4..bf7a6a7 100644
--- a/cli.py
+++ b/cli.py
@@ -101,6 +101,7 @@ class YumBaseCli(yum.YumBase, output.YumOutput):
         self.registerCommand(yumcommands.HistoryCommand())
         self.registerCommand(yumcommands.CheckRpmdbCommand())
         self.registerCommand(yumcommands.DistroSyncCommand())
+        self.registerCommand(yumcommands.LoadTransactionCommand())
 
     def registerCommand(self, command):
         for name in command.getNames():
diff --git a/test/misc-tests.py b/test/misc-tests.py
index d6e49d8..d34c161 100644
--- a/test/misc-tests.py
+++ b/test/misc-tests.py
@@ -93,6 +93,7 @@ class MiscTests(DepsolveTests):
     
     def resolveCode(self,skip = False):
         solver = YumBase()
+        solver.save_ts = save_ts
         solver.conf = FakeConf()
         solver.arch.setup_arch('x86_64')
         solver.conf.skip_broken = skip
diff --git a/test/skipbroken-tests.py b/test/skipbroken-tests.py
index 43b0726..c1d0c1f 100644
--- a/test/skipbroken-tests.py
+++ b/test/skipbroken-tests.py
@@ -626,6 +626,7 @@ class SkipBrokenTests(DepsolveTests):
     
     def resolveCode(self,skip = False):
         solver = YumBase()
+        solver.save_ts  =  save_ts
         solver.arch.setup_arch('x86_64')
         solver.conf = FakeConf()
         solver.conf.skip_broken = skip
diff --git a/test/testbase.py b/test/testbase.py
index 30cf177..81591ad 100644
--- a/test/testbase.py
+++ b/test/testbase.py
@@ -27,6 +27,10 @@ from rpmUtils.transaction import initReadOnlyTransaction
 def _(msg):
     return msg
 
+# dummy save_ts to avoid lots of errors
+def save_ts(*args, **kwargs):
+    pass
+
 class FakeConf(object):
 
     def __init__(self):
@@ -358,9 +362,9 @@ class DepsolveTests(_DepsolveTestsBase):
     def resetTsInfo(self):
         self.tsInfo = transactioninfo.TransactionData()
         
-
     def resolveCode(self):
         solver = YumBase()
+        solver.save_ts  = save_ts
         solver.conf = FakeConf()
         solver.arch.setup_arch('x86_64')
         solver.tsInfo = solver._tsInfo = self.tsInfo
@@ -409,6 +413,7 @@ class OperationsTests(_DepsolveTestsBase):
         requirements from.
         """
         depsolver = YumBaseCli()
+        depsolver.save_ts = save_ts
         depsolver.arch.setup_arch('x86_64')
         self.rpmdb = depsolver.rpmdb = FakeRpmDb()
         self.xsack = depsolver._pkgSack  = packageSack.PackageSack()
diff --git a/yum/__init__.py b/yum/__init__.py
index 6279cda..05039a4 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -43,7 +43,7 @@ import fnmatch
 import logging
 import logging.config
 import operator
-
+import tempfile
 
 import yum.i18n
 _ = yum.i18n._
@@ -178,6 +178,7 @@ class YumBase(depsolve.Depsolve):
         self._pkgSack = None
         self._lockfile = None
         self._tags = None
+        self._ts_save_file = None
         self.skipped_packages = []   # packages skip by the skip-broken code
         self.logger = logging.getLogger("yum.YumBase")
         self.verbose_logger = logging.getLogger("yum.verbose.YumBase")
@@ -414,7 +415,7 @@ class YumBase(depsolve.Depsolve):
             else:
                 thisrepo.repo_config_age = repo_age
                 thisrepo.repofile = repofn
-
+                
             if thisrepo.id in self.repo_setopts:
                 for opt in self.repo_setopts[thisrepo.id].items:
                     setattr(thisrepo, opt, getattr(self.repo_setopts[thisrepo.id], opt))
@@ -1000,6 +1001,8 @@ class YumBase(depsolve.Depsolve):
                 restring.append(_('Trying to remove "%s", which is protected') %
                                 pkgname)
 
+        if rescode == 2:
+            self.save_ts(auto=True)
         self.verbose_logger.debug('Depsolve time: %0.3f' % (time.time() - ds_st))
         return rescode, restring
 
@@ -1372,6 +1375,13 @@ class YumBase(depsolve.Depsolve):
         # invalid cache).
         self.rpmdb.transactionResultVersion(frpmdbv)
 
+        # transaction has started - all bets are off on our saved ts file
+        try:
+            os.unlink(self._ts_save_file)
+        except (IOError, OSError), e:
+            pass
+        self._ts_save_file = None
+        
         errors = self.ts.run(cb.callback, '')
         # ts.run() exit codes are, hmm, "creative": None means all ok, empty 
         # list means some errors happened in the transaction and non-empty 
@@ -1411,6 +1421,7 @@ class YumBase(depsolve.Depsolve):
                 except (IOError, OSError), e:
                     self.logger.critical(_('Failed to remove transaction file %s') % fn)
 
+        
         # drop out the rpm cache so we don't step on bad hdr indexes
         self.rpmdb.dropCachedDataPostTransaction(list(self.tsInfo))
         self.plugins.run('posttrans')
@@ -2928,7 +2939,7 @@ class YumBase(depsolve.Depsolve):
                 if len(dep_split) == 3:
                     depname, flagsymbol, depver = dep_split
                     if not flagsymbol in SYMBOLFLAGS:
-                        raise Errors.YumBaseError, _('Invalid version flag')
+                        raise Errors.YumBaseError, _('Invalid version flag from: %s') % str(depstring)
                     depflags = SYMBOLFLAGS[flagsymbol]
 
         return self.rpmdb.getProvides(depname, depflags, depver).keys()
@@ -4682,6 +4693,7 @@ class YumBase(depsolve.Depsolve):
         newrepo = yumRepo.YumRepository(repoid)
         newrepo.name = repoid
         newrepo.basecachedir = self.conf.cachedir
+
         var_convert = kwargs.get('variable_convert', True)
         
         if baseurls:
@@ -4778,3 +4790,213 @@ class YumBase(depsolve.Depsolve):
         self.plugins.run('verify_package', verify_package=verify_package)
         return verify_package
 
+    def save_ts(self, filename=None, auto=False):
+        """saves out a transaction to .yumts file to be loaded later"""
+        
+        if self.tsInfo._unresolvedMembers:
+            if auto:
+                self.logger.critical(_("Dependencies not solved. Will not save unresolved transaction."))
+                return
+            raise Errors.YumBaseError(_("Dependencies not solved. Will not save unresolved transaction."))
+        
+        if not filename:
+            prefix = 'yum_save_ts-%s' % time.strftime('%Y-%m-%d-%H-%M')
+            fd,filename = tempfile.mkstemp(suffix='.ts', prefix=prefix)
+            f = os.fdopen(fd, 'w')
+        else:
+            f = open(filename, 'w')
+        
+        self._ts_save_file = filename
+        
+        msg = "%s\n" % self.rpmdb.simpleVersion(main_only=True)[0]
+        msg += "%s\n" % self.ts.getTsFlags()
+        msg += "%s\n" % len(self.repos.listEnabled())
+        for r in self.repos.listEnabled():
+            msg += "%s:%s:%s\n" % (r.id, len(r.sack), r.repoXML.revision)
+        msg += "%s\n" % len(self.tsInfo.getMembers())
+        for txmbr in self.tsInfo.getMembers():
+            msg += txmbr._dump()
+        try:
+            f.write(msg)
+            f.close()
+        except (IOError, OSError), e:
+            self._ts_save_file = None
+            if auto:
+                self.logger.critical(_("Could not save transaction file %s: %s") % (filename, str(e)))
+            else:
+                raise Errors.YumBaseError(_("Could not save transaction file %s: %s") % (filename, str(e)))
+
+        
+    def load_ts(self, filename, ignorerpm=None, ignoremissing=None):
+        """loads a transaction from a .yumts file"""
+        # check rpmversion - if not match throw a fit
+        # check repoversions  (and repos)- if not match throw a fit
+        # load each txmbr - if pkgs being updated don't exist, bail w/error
+        # setup any ts flags
+        # setup cmds for history/yumdb to know about
+        try:
+            data = open(filename, 'r').readlines()
+        except (IOError, OSError), e:
+            raise Errors.YumBaseError(_("Could not access/read saved transaction %s : %s") % (filename, str(e)))
+            
+
+        if ignorerpm is None:
+            ignorerpm = self.conf.loadts_ignorerpm
+        if ignoremissing is None:
+            ignoremissing = self.conf.loadts_ignoremissing
+            
+        # data format
+        # 0 == rpmdb version
+        # 1 == tsflags
+        # 2 == numrepos
+        # 3:numrepos = repos
+        # 3+numrepos = num pkgs
+        # 3+numrepos+1 -> EOF= txmembers
+        
+        # rpm db ver
+        rpmv = data[0].strip()
+        if rpmv != str(self.rpmdb.simpleVersion(main_only=True)[0]):
+            msg = _("rpmdb ver mismatched saved transaction version, ")
+            if ignorerpm:
+                msg += _(" ignoring, as requested.")
+                self.logger.critical(_(msg))
+            else:
+                msg += _(" aborting.")
+                raise Errors.YumBaseError(msg)
+        
+        # tsflags
+        # FIXME - probably should let other tsflags play nicely together
+        #         so someone can add --nogpgcheck or --nodocs or --nodiskspace or some nonsense and have it work
+        try:
+            tsflags = int(data[1].strip())
+        except (ValueError, IndexError), e:
+            msg = _("cannot find tsflags or tsflags not integer.")
+            raise Errors.YumBaseError(msg)
+
+        self.ts.setFlags(tsflags)
+        
+        # repos
+        numrepos = int(data[2].strip())
+        repos = []
+        rindex=3+numrepos
+        for r in data[3:rindex]:
+            repos.append(r.strip().split(':'))
+
+        # pkgs/txmbrs
+        numpkgs = int(data[rindex].strip())
+
+        pkgstart = rindex + 1
+        
+        pkgcount = 0
+        pkgprob = False
+        curpkg = None
+        for l in data[pkgstart:]:
+            l = l.rstrip()
+            # our main txmbrs
+            if l.startswith('mbr:'):
+                if curpkg:
+                    self.tsInfo.add(curpkg)
+                    if curpkg in self.tsInfo._unresolvedMembers and not missingany:
+                        self.tsInfo._unresolvedMembers.remove(curpkg)
+
+                missingany = False
+                pkgtup, current_state = l.split(':')[1].strip().split(' ')
+                current_state = int(current_state.strip())
+                pkgtup = tuple(pkgtup.strip().split(','))
+                try:
+                    if current_state == TS_INSTALL:
+                        po = self.getInstalledPackageObject(pkgtup)
+                    elif current_state == TS_AVAILABLE:
+                        po = self.getPackageObject(pkgtup)
+                    else:
+                        msg = _("Found txmbr in unknown current state: %s" % current_state)
+                        raise Errors.YumBaseError(msg)
+                except Errors.YumBaseError, e:
+                    missingany = True
+                    msg = _("Could not find txmbr: %s in state %s" % (str(pkgtup), current_state))
+                    if not ignoremissing:
+                        raise Errors.YumBaseError(msg)
+                    else:
+                        self.logger.critical(msg)
+                else:
+                    pkgcount += 1
+                    curpkg = transactioninfo.TransactionMember(po)
+                    curpkg.current_state = current_state
+                    continue
+
+            l = l.strip()
+            k,v = l.split(':', 1)
+            v = v.lstrip()
+            # attributes of our txmbrs
+            if k in ('isDep', 'reinstall'):
+                v = v.strip().lower()
+                if v == 'false':
+                    setattr(curpkg, k, False)
+                elif v == 'true':
+                    setattr(curpkg, k, True)
+            elif k in ('output_state'):
+                setattr(curpkg, k, int(v.strip()))
+            elif k in ('groups'):
+                curpkg.groups.extend(v.split(' '))
+            # the relationships to our main txmbrs
+            elif k in ('updated_by', 'obsoleted_by', 'downgraded_by', 
+                       'downgrades', 'updates', 'obsoletes', 'depends_on'):
+                for pkgspec in v.strip().split(' '):
+                    pkgtup, origin  = pkgspec.split('@')
+                    try:
+                        if origin == 'i':
+                            po = self.getInstalledPackageObject(tuple(pkgtup.split(',')))
+                        else:
+                            po = self.getPackageObject(tuple(pkgtup.split(',')))
+                    except Errors.YumBaseError, e:
+                        msg = _("Could not find txmbr: %s from origin: %s" % (str(pkgtup), origin))
+                        self.logger.critical(msg)
+                        missingany = True
+                    else:
+                        curlist = getattr(curpkg, k)
+                        curlist.append(po)
+                        setattr(curpkg, k, curlist)
+            elif k in ('relatedto'):
+                for item in v.split(' '):
+                    pkgspec, rel = item.split(':')
+                    pkgtup,origin = pkgspec.split('@')
+                    try:
+                        if origin == 'i':
+                            po = self.getInstalledPackageObject(tuple(pkgtup.split(',')))
+                        else:
+                            po = self.getPackageObject(tuple(pkgtup.split(',')))
+                    except Errors.YumBaseError, e:
+                        msg = _("Could not find txmbr: %s from origin: %s" % (str(pkgtup), origin))
+                        self.logger.critical(msg)
+                        missingany = True
+                    else:
+                        curlist = getattr(curpkg, k)
+                        curlist.append((po,rel))
+                        setattr(curpkg, k, curlist)
+                        
+            # the plain strings
+            else: #ts_state, reason
+                setattr(curpkg, k, v.strip())
+            
+            if missingany:
+                pkgprob = True
+                
+        # make sure we get the last one in!
+        self.tsInfo.add(curpkg)
+        if curpkg in self.tsInfo._unresolvedMembers:
+            self.tsInfo._unresolvedMembers.remove(curpkg)
+
+            
+        if numpkgs != pkgcount:
+            pkgprob = True
+            
+        if pkgprob:
+            msg = _("Transaction members, relations are missing or ts has been modified,")
+            if ignoremissing:
+                msg += _(" ignoring, as requested. You must redepsolve!")
+                self.logger.critical(msg)
+            else:
+                msg += _(" aborting.")
+                raise Errors.YumBaseError(msg)
+            
+
diff --git a/yum/config.py b/yum/config.py
index 2118792..2f133ef 100644
--- a/yum/config.py
+++ b/yum/config.py
@@ -740,6 +740,9 @@ class YumConf(StartupConf):
                                     parse_default=True)
     exit_on_lock = BoolOption(False)
     
+    loadts_ignoremissing = BoolOption(False)
+    loadts_ignorerpm = BoolOption(False)
+    
     _reposlist = []
 
     def dump(self):
diff --git a/yum/depsolve.py b/yum/depsolve.py
index cba0fc5..3cc7a7f 100644
--- a/yum/depsolve.py
+++ b/yum/depsolve.py
@@ -805,6 +805,7 @@ class Depsolve(object):
 
         if not len(self.tsInfo):
             return (0, [_('Success - empty transaction')])
+        
         return (2, [_('Success - deps resolved')])
 
     def _resolveRequires(self, errors):
diff --git a/yum/transactioninfo.py b/yum/transactioninfo.py
index 4c59301..b0c7ddd 100644
--- a/yum/transactioninfo.py
+++ b/yum/transactioninfo.py
@@ -670,7 +670,6 @@ class TransactionData:
                     txmbr.output_state = TS_OBSOLETING
                     txmbr.po.state = TS_OBSOLETING
 
-
 class ConditionalTransactionData(TransactionData):
     """A transaction data implementing conditional package addition"""
     def __init__(self):
@@ -741,7 +740,7 @@ class TransactionMember:
         self.output_state = None # what state to list if printing it
         self.isDep = 0
         self.reason = 'user' # reason for it to be in the transaction set
-        self.process = None # 
+        self.process = None #  I think this is used nowhere by nothing - skv 2010/11/03
         self.relatedto = [] # ([relatedpkg, relationship)]
         self.depends_on = []
         self.obsoletes = []
@@ -788,14 +787,42 @@ class TransactionMember:
 
     def __repr__(self):
         return "<%s : %s (%s)>" % (self.__class__.__name__, str(self),hex(id(self))) 
-        
-    # This is the tricky part - how do we nicely setup all this data w/o going insane
-    # we could make the txmember object be created from a YumPackage base object
-    # we still may need to pass in 'groups', 'ts_state', 'output_state', 'reason', 'current_state'
-    # and any related packages. A world of fun that will be, you betcha
     
-    
-    # definitions
-    # current and output states are defined in constants
-    # relationships are defined in constants
-    # ts states are: u, i, e
+    def _dump(self):
+        msg = "mbr: %s,%s,%s,%s,%s %s\n" % (self.name, self.arch, self.epoch, 
+                     self.version, self.release, self.current_state)
+        msg += "  repo: %s\n" % self.po.repo.id
+        msg += "  ts_state: %s\n" % self.ts_state
+        msg += "  output_state: %s\n" %  self.output_state
+        msg += "  isDep: %s\n" %  bool(self.isDep)
+        msg += "  reason: %s\n" % self.reason
+        #msg += "  process: %s\n" % self.process
+        msg += "  reinstall: %s\n" % bool(self.reinstall)
+        
+        if self.relatedto:
+            msg += "  relatedto:"
+            for (po, rel) in self.relatedto:
+                pkgorigin = 'a'
+                if isinstance(po, YumInstalledPackage):
+                    pkgorigin = 'i'
+                msg += " %s,%s,%s,%s,%s@%s:%s" % (po.name, po.arch, po.epoch, 
+                      po.version, po.release, pkgorigin, rel)
+            msg += "\n"
+            
+        for lst in ['depends_on', 'obsoletes', 'obsoleted_by', 'downgrades',
+                    'downgraded_by', 'updates', 'updated_by']:
+            thislist = getattr(self, lst)
+            if thislist:
+                msg += "  %s:" % lst
+                for po in thislist:
+                    pkgorigin = 'a'
+                    if isinstance(po, YumInstalledPackage):
+                        pkgorigin = 'i'
+                    msg += " %s,%s,%s,%s,%s@%s" % (po.name, po.arch, po.epoch, 
+                        po.version, po.release, pkgorigin)
+                msg += "\n"
+                
+        if self.groups:
+            msg += "  groups: %s\n" % ' '.join(self.groups)
+
+        return msg
diff --git a/yumcommands.py b/yumcommands.py
index 3c009f8..67ba2ce 100644
--- a/yumcommands.py
+++ b/yumcommands.py
@@ -1440,3 +1440,30 @@ class CheckRpmdbCommand(YumCommand):
     def needTs(self, base, basecmd, extcmds):
         return False
 
+class LoadTransactionCommand(YumCommand):
+    def getNames(self):
+        return ['load-transaction', 'load-ts']
+
+    def getUsage(self):
+        return "filename"
+
+    def getSummary(self):
+        return _("load a saved transaction from filename")
+
+    def doCommand(self, base, basecmd, extcmds):
+        if not extcmds:
+            base.logger.critical(_("No saved transaction file specified."))
+            raise cli.CliError
+        
+        load_file = extcmds[0]
+        self.doneCommand(base, _("loading transaction from %s") % load_file)
+        
+        try:
+            base.load_ts(load_file)
+        except yum.Errors.YumBaseError, e:
+            return 1, [to_unicode(e)]
+        return 2, [_('Transaction loaded from %s with %s members') % (load_file, len(base.tsInfo))]
+
+    def needTs(self, base, basecmd, extcmds):
+        return False
+
diff --git a/yummain.py b/yummain.py
index 2892c34..0241684 100755
--- a/yummain.py
+++ b/yummain.py
@@ -213,6 +213,8 @@ def main(args):
         if not base._rpmdb_warn_checks(out=verbose_logger.info, warn=False):
             verbose_logger.info(_(" You could try running: rpm -Va --nofiles --nodigest"))
         return_code = result
+        if base._ts_save_file:
+            verbose_logger.info(_("Your transaction was saved, rerun it with: yum load-transaction %s") % base._ts_save_file)
     else:
         verbose_logger.log(logginglevels.INFO_2, _('Complete!'))
 
-- 
1.7.2.1



More information about the Yum-devel mailing list