[yum-commits] 11 commits - cli.py docs/yum.8 docs/yum.conf.5 etc/Makefile yumcommands.py yum/config.py yum-cron/yum-cron.py yum/fssnapshots.py yum/__init__.py yum/plugins.py yum/repos.py yum.spec yum/update_md.py
James Antill
james at osuosl.org
Fri Jun 14 21:44:45 UTC 2013
cli.py | 12 +
docs/yum.8 | 29 ++++
docs/yum.conf.5 | 26 +++
etc/Makefile | 1
yum-cron/yum-cron.py | 2
yum.spec | 1
yum/__init__.py | 75 ++++++++++-
yum/config.py | 8 +
yum/fssnapshots.py | 338 +++++++++++++++++++++++++++++++++++++++++++++++++++
yum/plugins.py | 1
yum/repos.py | 34 ++++-
yum/update_md.py | 45 +++++-
yumcommands.py | 150 ++++++++++++++++++++--
13 files changed, 687 insertions(+), 35 deletions(-)
New commits:
commit 4f7a41dcef039a7ecd095f7be3900140fe052e60
Author: James Antill <james at and.org>
Date: Fri Jun 14 16:26:15 2013 -0400
Add fssnapshot to documentation and specfile.
diff --git a/docs/yum.8 b/docs/yum.8
index 1c6a27b..3ea88f7 100644
--- a/docs/yum.8
+++ b/docs/yum.8
@@ -84,6 +84,8 @@ gnome\-packagekit application\&.
.br
.I \fR * updateinfo [summary | list | info | remove-pkgs-ts | exclude-updates | exclude-all | check-running-kernel]
.br
+.I \fR * fssnapshot [summary | list | have-space | create | delete]
+.br
.I \fR * check
.br
.I \fR * help [command]
@@ -661,7 +663,32 @@ updateinfo data:
.I \fR yum updateinfo check-running-kernel
.br
-.PP
+.IP
+.IP "\fBfssnapshot\fP"
+This command has a few sub-commands to act on the LVM data of the host, to list
+snapshots and the create and remove them. The simplest commands, to display
+information about the configured LVM snapshotable devices, are:
+
+.br
+.I \fR yum fssnapshot [summary]
+.br
+.I \fR yum fssnapshot list
+.br
+.I \fR yum fssnapshot have-space
+.br
+
+then you can create and delete snapshots using:
+
+.br
+.I \fR yum fssnap create
+.br
+.I \fR yum fssnap delete <device(s)>
+.br
+
+.br
+Configuration Options: \fBfssnap_automatic_pre\fP, \fBfssnap_automatic_post\fP, \fBfssnap_automatic_keep\fP, \fBfssnap_percentage\fP, \fBfssnap_devices\fP
+
+.IP
.IP "\fBcheck\fP"
Checks the local rpmdb and produces information on any problems it finds. You
can pass the check command the arguments "dependencies" or "duplicates", to
diff --git a/docs/yum.conf.5 b/docs/yum.conf.5
index d8d60a5..0ae7f5c 100644
--- a/docs/yum.conf.5
+++ b/docs/yum.conf.5
@@ -814,6 +814,32 @@ current machine.
Note that if loadts_ignorerpm is True, this option does nothing.
Boolean (1, 0, True, False, yes, no) Defaults to False
+.IP
+\fBfssnap_automatic_pre\fR
+Should yum try to automatically create a snapshot before it runs a transaction.
+Boolean (1, 0, True, False, yes, no) Defaults to False
+
+.IP
+\fBfssnap_automatic_post\fR
+Should yum try to automatically create a snapshot after it runs a transaction.
+Boolean (1, 0, True, False, yes, no) Defaults to False
+
+.IP
+\fBfssnap_automatic_keep\fR
+How many old snapshots should yum keep when trying to automatically create a
+new snapshot. Setting to 0 disables this feature. Default is '0'.
+
+.IP
+\fBfssnap_automatic_percentage\fR
+The size of new snaphosts, expressed as a percentage of the old origin device.
+Any number between 1 and 100. Default is '100'.
+
+.IP
+\fBfssnap_automatic_devices\fR
+The origin LVM devices to use for snapshots. Wildcards and negation are allowed,
+first match (positive or negative) wins.
+Default is: !*/swap !*/lv_swap glob:/etc/yum/fssnap.d/*.conf
+
.SH "[repository] OPTIONS"
.LP
diff --git a/etc/Makefile b/etc/Makefile
index 3e14af8..49f1d81 100644
--- a/etc/Makefile
+++ b/etc/Makefile
@@ -10,6 +10,7 @@ clean:
install:
mkdir -p $(DESTDIR)/etc/yum/
+ mkdir -p $(DESTDIR)/etc/yum/fssnap.d
mkdir -p $(DESTDIR)/etc/yum/protected.d
mkdir -p $(DESTDIR)/etc/yum/repos.d
mkdir -p $(DESTDIR)/etc/yum/vars
diff --git a/yum.spec b/yum.spec
index 455c3ae..c234291 100644
--- a/yum.spec
+++ b/yum.spec
@@ -333,6 +333,7 @@ exit 0
%config(noreplace) %{_sysconfdir}/yum/version-groups.conf
%dir %{_sysconfdir}/yum
%dir %{_sysconfdir}/yum/protected.d
+%dir %{_sysconfdir}/yum/fssnap.d
%dir %{_sysconfdir}/yum/vars
%config(noreplace) %{_sysconfdir}/logrotate.d/%{name}
%(dirname %{compdir})
commit 52a42d1797bb4d3792c829f868d512365bfff4e3
Author: James Antill <james at and.org>
Date: Mon Apr 29 17:08:07 2013 -0400
Add first version of snapshots command, and API.
diff --git a/cli.py b/cli.py
index 97685c4..8b6b983 100755
--- a/cli.py
+++ b/cli.py
@@ -111,6 +111,7 @@ class YumBaseCli(yum.YumBase, output.YumOutput):
self.registerCommand(yumcommands.RepoPkgsCommand())
self.registerCommand(yumcommands.UpdateinfoCommand())
self.registerCommand(yumcommands.UpdateMinimalCommand())
+ self.registerCommand(yumcommands.FSSnapshotCommand())
def registerCommand(self, command):
"""Register a :class:`yumcommands.YumCommand` so that it can be called by
diff --git a/yum/__init__.py b/yum/__init__.py
index 983b0d7..26ef1f9 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -79,6 +79,7 @@ import logginglevels
import yumRepo
import callbacks
import yum.history
+import yum.fssnapshots
import yum.igroups
import update_md
@@ -196,6 +197,7 @@ class YumBase(depsolve.Depsolve):
self._lockfile = None
self._tags = None
self._upinfo = None
+ self._fssnap = None
self._ts_save_file = None
self.skipped_packages = [] # packages skip by the skip-broken code
self._not_found_a = {}
@@ -1022,6 +1024,15 @@ class YumBase(depsolve.Depsolve):
releasever=self.conf.yumvar['releasever'])
return self._history
+ def _getFSsnap(self):
+ """ create the fssnap object used to query/create snapshots. """
+ if self._fssnap is None:
+ devices = self.conf.fssnap_devices
+ self._fssnap = yum.fssnapshots._FSSnap(root=self.conf.installroot,
+ devices=devices)
+
+ return self._fssnap
+
def _getIGroups(self):
"""auto create the installed groups object that to access/change the
installed groups information. """
@@ -1081,7 +1092,11 @@ class YumBase(depsolve.Depsolve):
fset=lambda self, value: setattr(self, "_upinfo", value),
fdel=lambda self: setattr(self, "_upinfo", None),
doc="Yum Update Info Object")
-
+
+ fssnap = property(fget=lambda self: self._getFSsnap(),
+ fset=lambda self, value: setattr(self, "_fssnap",value),
+ fdel=lambda self: setattr(self, "_fssnap", None),
+ doc="Yum FS snapshot Object")
def doSackFilelistPopulate(self):
"""Convenience function to populate the repositories with the
@@ -1677,6 +1692,38 @@ much more problems).
:raises: :class:`yum.Errors.YumRPMTransError` if there is a
transaction cannot be completed
"""
+ if ((self.conf.fssnap_automatic_pre or
+ self.conf.fssnap_automatic_post) and
+ self.conf.fssnap_automatic_keep):
+ # Automatically kill old snapshots...
+ snaps = self.fssnap.old_snapshots()
+ snaps = sorted(snaps, key=lambda x: (x['ctime'], x['origin_dev']),
+ reverse=True)
+ last = '<n/a>'
+ num = 0
+ todel = []
+ for snap in snaps:
+ num += 1
+
+ if last != snap['origin_dev']:
+ last = snap['origin_dev']
+ num = 1
+ continue
+
+ if num > self.conf.fssnap_automatic_keep:
+ todel.append(snap['dev'])
+ # Display something to the user?
+ self.fssnap.del_snapshots(devices=todel)
+
+ if (not self.ts.isTsFlagSet(rpm.RPMTRANS_FLAG_TEST) and
+ self.conf.fssnap_automatic_pre):
+ if not self.fssnap.has_space(self.conf.fssnap_percentage):
+ msg = _("Not enough space to create pre. FS snapshot, aborting transaction.")
+ raise Errors.YumRPMTransError(msg=msg, errors=[])
+ else:
+ tags = {'*': ['reason=automatic']} # FIXME: pre. tags
+ self.fssnap.snapshot(self.conf.fssnap_percentage, tags=tags)
+
self.plugins.run('pretrans')
# We may want to put this other places, eventually, but for now it's
@@ -1807,6 +1854,16 @@ much more problems).
self.verifyTransaction(resultobject, vTcb)
if self.conf.group_command == 'objects':
self.igroups.save()
+
+ if (not self.ts.isTsFlagSet(rpm.RPMTRANS_FLAG_TEST) and
+ self.conf.fssnap_automatic_post):
+ if not self.fssnap.has_space(self.conf.fssnap_percentage):
+ msg = _("Not enough space to create post trans FS snapshot.")
+ self.logger.critical(msg)
+ else:
+ tags = {'*': ['reason=automatic']} # FIXME: post tags
+ self.fssnap.snapshot(self.conf.fssnap_percentage, tags=tags)
+
return resultobject
def verifyTransaction(self, resultobject=None, txmbr_cb=None):
diff --git a/yum/config.py b/yum/config.py
index 1ca5cdc..6fcfb3e 100644
--- a/yum/config.py
+++ b/yum/config.py
@@ -886,6 +886,14 @@ class YumConf(StartupConf):
recheck_installed_requires = BoolOption(False)
+ fssnap_automatic_pre = BoolOption(False)
+ fssnap_automatic_post = BoolOption(False)
+ fssnap_automatic_keep = IntOption(1)
+ fssnap_percentage = IntOption(100, range_min=1, range_max=100)
+ fssnap_devices = ListOption("!*/swap !*/lv_swap "
+ "glob:/etc/yum/fssnap.d/*.conf",
+ parse_default=True)
+
_reposlist = []
def dump(self):
diff --git a/yum/fssnapshots.py b/yum/fssnapshots.py
new file mode 100644
index 0000000..16e2595
--- /dev/null
+++ b/yum/fssnapshots.py
@@ -0,0 +1,338 @@
+
+
+import os
+import fnmatch
+import time
+
+import subprocess
+
+try:
+ import lvm
+
+ # Check that lvm2 is at least 2.2.99... In theory hacked versions of
+ # .98 work, but meh.
+
+ _ver = lvm.getVersion()
+ # Looks liks: 2.02.84(2) (2011-02-09)
+ _ver = _ver.split()[0]
+ _ver = _ver.split('(')[0]
+ _ver = tuple(map(int, _ver.split('.')))
+ if _ver < (2, 2, 99):
+ lvm = None
+except:
+ lvm = None
+ _ver = None
+
+
+def _is_origin(lv):
+ snap = lv.getProperty("lv_attr")
+ # snap=(<value>, <is settable>)
+ if not snap[0]: # Broken??
+ return None
+ return snap[0][0] in ('o', 'O')
+
+def _is_snap(lv):
+ snap = lv.getProperty("lv_attr")
+ # snap=(<value>, <is settable>)
+ if not snap[0]: # Broken??
+ return None
+ return snap[0][0] in ('s', 'S')
+
+def _is_virt(lv):
+ snap = lv.getProperty("lv_attr")
+ # snap=(<value>, <is settable>)
+ if not snap[0]: # Broken??
+ return None
+ return snap[0][0] == 'v'
+
+def _vg_name2lv(vg, lvname):
+ try:
+ return vg.lvFromName(lvname)
+ except:
+ return None
+
+def _list_vg_names():
+ names = lvm.listVgNames()
+
+ if not names: # Could be just broken...
+ p = subprocess.Popen(["/sbin/lvm", "vgs", "-o", "vg_name"],
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ err = p.wait()
+ if err:
+ return [] # Meh.
+
+ output = p.communicate()[0]
+ output = output.split('\n')
+ if not output:
+ return []
+ header = output[0].strip()
+ if header != 'VG':
+ return []
+ names = []
+ for name in output[1:]:
+ if not name:
+ break
+ names.append(name.strip())
+
+ return names
+
+def _z_off(z, ctime=0):
+ if len(z) == 5: # +0000 / -0130 / etc.
+ off = int(z[1:3]) * 60
+ off += int(z[3:5])
+ off *= 60
+ if z[0] == '+':
+ ctime -= off
+ if z[0] == '-':
+ ctime += off
+ return ctime
+
+def _lv_ctime2utc(ctime):
+ try: # Welcome to insanity ...
+ d,t,z = ctime.split()
+ ctime = time.strptime(d + ' ' + t, "%Y-%m-%d %H:%M:%S")
+
+ ctime = time.mktime(ctime)
+
+ if False: # Ignore the offset atm. ... we using this to delete older.
+ cur_z = time.strftime("%z")
+ if cur_z != z: # lol ...
+ cur_z = _z_off(cur_z)
+ z = _z_off(z)
+ ctime += (cur_z - z)
+
+ except:
+ ctime = 0
+
+ return ctime
+
+def _lv_data(vg, lv):
+ vgname = vg.getName()
+ lvname = lv.getName()
+
+ size = lv.getSize()
+ origin = lv.getProperty("origin")[0]
+ tags = lv.getTags()
+
+ ctime = _lv_ctime2utc(lv.getProperty("lv_time")[0])
+
+ used = lv.getProperty("snap_percent")[0]
+ used = float(used)
+ used = used / (1 * 1000 * 1000)
+
+ data = {'dev' : "%s/%s" % (vgname, lvname),
+ 'ctime' : ctime,
+ 'origin' : origin,
+ 'origin_dev' : "%s/%s" % (vgname, origin),
+ 'free' : vg.getFreeSize(),
+ 'tags' : tags,
+ 'size' : size,
+ 'used' : used}
+
+ return data
+
+
+class _FSSnap(object):
+
+ # Old style was: vg/lv_root vg/lv_swap
+ # New style is: fedora/root fedora/swap
+ # New style is: redhat/root redhat/swap
+ def __init__(self, root="/", lookup_mounts=True,
+ devices=('!*/swap', '!*/lv_swap')):
+ if not lvm or os.geteuid():
+ devices = []
+
+ self.version = _ver
+ self.postfix_static = "_yum_"
+ self._postfix = None
+ self._root = root
+ self._devs = devices
+ self._vgnames = []
+
+ if not self._devs:
+ return
+
+ self._vgnames = _list_vg_names()
+
+ def _use_dev(self, vgname, lv=None):
+
+ if lv is not None:
+ if _is_snap(lv) or _is_virt(lv): # Don't look at these.
+ return False
+
+ found_neg = False
+
+ for dev in self._devs:
+ if '/' not in dev: # Bad...
+ continue
+
+ neg = False
+ if dev[0] == '!':
+ found_neg = True
+ neg = True
+ dev = dev[1:]
+
+ vgn,lvn = dev.split('/', 1)
+ if '/' in lvn:
+ continue
+
+ if not fnmatch.fnmatch(vgname, vgn):
+ continue
+
+ if lvn == '*':
+ return not neg
+
+ if lv is None:
+ return None
+ lvname = lv.getName()
+
+ if not fnmatch.fnmatch(lvname, lvn):
+ continue
+
+ return not neg
+
+ return found_neg
+
+ def has_space(self, percentage=100):
+ """ See if we have enough space to try a snapshot. """
+
+ for vgname in self._vgnames:
+ use = self._use_dev(vgname)
+ if use is not None and not use:
+ continue
+
+ vg = lvm.vgOpen(vgname, 'r')
+ if not vg:
+ return False
+
+ vgfsize = vg.getFreeSize()
+ lvssize = 0
+
+ for lv in vg.listLVs():
+ if not self._use_dev(vgname, lv):
+ continue
+
+ lvssize += lv.getSize()
+
+ vg.close()
+
+ if (lvssize * percentage) > (100*vgfsize):
+ return False
+
+ return True
+
+ def _get_postfix(self):
+ if self._postfix is None:
+ self._postfix = self.postfix_static
+ self._postfix += time.strftime("%Y%m%d%H%M%S")
+ return self._postfix
+
+ postfix = property(fget=lambda self: self._get_postfix(),
+ fset=lambda self, value: setattr(self, "_postfix",value),
+ fdel=lambda self: setattr(self, "_postfix", None),
+ doc="postfix for snapshots")
+
+ def snapshot(self, percentage=100, prefix='', postfix=None, tags={}):
+ """ Attempt to take a snapshot, note that errors can happen after
+ this function succeeds. """
+
+ if postfix is None:
+ postfix = self.postfix
+
+ ret = []
+ for vgname in self._vgnames:
+ use = self._use_dev(vgname)
+ if use is not None and not use:
+ continue
+
+ vg = lvm.vgOpen(vgname, 'w')
+ if not vg:
+ return False
+
+ for lv in vg.listLVs():
+ lvname = lv.getName()
+
+ if not self._use_dev(vgname, lv):
+ continue
+
+ nlvname = "%s%s%s" % (prefix, lvname, postfix)
+ nlv = lv.snapshot(nlvname, (lv.getSize() * percentage) / 100)
+ if not nlv: # Failed here ... continuing seems bad.
+ vg.close()
+ return None
+
+ odev = "%s/%s" % (vgname, lvname)
+ ndev = "%s/%s" % (vgname, nlvname)
+
+ # FIXME: yum_fssnapshot_pre_lv_name=<blah>
+ eq_tags = set()
+ for val in (ndev, odev, '*'):
+ for tag in tags.get(val, []):
+ if '=' in tag:
+ eq_tag_key,eq_tag_val = tag.split('=', 1)
+ if eq_tag_key in eq_tags:
+ continue
+ eq_tags.add(eq_tag_key)
+
+ nlv.addTag(tag)
+
+ ret.append((odev, ndev))
+
+ vg.close()
+
+ return ret
+
+ def old_snapshots(self):
+ """ List data for old snapshots. """
+
+ ret = []
+ for vgname in self._vgnames:
+ # We could filter out the VGs using _use_dev() but this way we'll
+ # see stuff after changing config. options.
+
+ vg = lvm.vgOpen(vgname, 'w')
+
+ for lv in vg.listLVs():
+
+ if not _is_snap(lv): # No snapshot means, we don't care.
+ continue
+
+ ret.append(_lv_data(vg, lv))
+ vg.close()
+
+ return ret
+
+ def del_snapshots(self, devices=[]):
+ """ Remove snapshots. """
+
+ if not lvm:
+ return []
+
+ ret = []
+
+ togo = {}
+ for dev in devices:
+ vgname,lvname = dev.split('/')
+ if vgname not in togo:
+ togo[vgname] = set([lvname])
+ else:
+ togo[vgname].add(lvname)
+
+ for vgname in togo:
+ vg = lvm.vgOpen(vgname, 'w')
+
+ for lvname in togo[vgname]:
+ lv = _vg_name2lv(vg, lvname)
+ if not lv:
+ continue
+
+ if not _is_snap(lv): # No snapshot means don't try to delete!
+ continue
+
+ ret.append(_lv_data(vg, lv))
+
+ lv.remove()
+
+ vg.close()
+
+ return ret
diff --git a/yumcommands.py b/yumcommands.py
index 93c6278..e2968b5 100644
--- a/yumcommands.py
+++ b/yumcommands.py
@@ -4077,3 +4077,126 @@ class UpdateMinimalCommand(YumCommand):
return 2, [msg]
else:
return 0, ['No Packages marked for minimal Update']
+
+
+class FSSnapshotCommand(YumCommand):
+ def getNames(self):
+ return ['fssnapshot', 'fssnap']
+
+ def getUsage(self):
+ return "[]"
+
+ def getSummary(self):
+ return _("Creates filesystem snapshots, or lists/deletes current snapshots.")
+
+ def doCheck(self, base, basecmd, extcmds):
+ """Verify that conditions are met so that this command can run.
+ These include that the program is being run by the root user,
+ that there are enabled repositories with gpg keys, and that
+ this command is called with appropriate arguments.
+
+ :param base: a :class:`yum.Yumbase` object
+ :param basecmd: the name of the command
+ :param extcmds: the command line arguments passed to *basecmd*
+ """
+ checkRootUID(base)
+
+ @staticmethod
+ def _li_snaps(base, snaps):
+ snaps = sorted(snaps, key=lambda x: x['dev'])
+
+ max_dev = utf8_width(_('Snapshot'))
+ max_ori = utf8_width(_('Origin'))
+ for data in snaps:
+ max_dev = max(max_dev, len(data['dev']))
+ max_ori = max(max_ori, len(data['origin']))
+
+ done = False
+ for data in snaps:
+ if not done:
+ print ("%s %s %s %s %s %s" %
+ (utf8_width_fill(_('Snapshot'), max_dev),
+ utf8_width_fill(_('Size'), 6, left=False),
+ utf8_width_fill(_('Used'), 6, left=False),
+ utf8_width_fill(_('Free'), 6, left=False),
+ utf8_width_fill(_('Origin'), max_ori), _('Tags')))
+ done = True
+ print ("%*s %6s %5.1f%% %6s %*s %s" %
+ (max_dev, data['dev'], base.format_number(data['size']),
+ data['used'],
+ base.format_number(data['free']),
+ max_ori, data['origin'], ",".join(data['tags'])))
+
+ def doCommand(self, base, basecmd, extcmds):
+ """Execute this command.
+
+ :param base: a :class:`yum.Yumbase` object
+ :param basecmd: the name of the command
+ :param extcmds: the command line arguments passed to *basecmd*
+ :return: (exit_code, [ errors ])
+
+ exit_code is::
+
+ 0 = we're done, exit
+ 1 = we've errored, exit with error string
+ 2 = we've got work yet to do, onto the next stage
+ """
+ if extcmds and extcmds[0] in ('list', 'delete', 'create', 'summary',
+ 'have-space', 'has-space'):
+ subcommand = extcmds[0]
+ extcmds = extcmds[1:]
+ else:
+ subcommand = 'summary'
+
+ if subcommand == 'list':
+ snaps = base.fssnap.old_snapshots()
+ print _("List of %u snapshosts:") % len(snaps)
+ self._li_snaps(base, snaps)
+
+ if subcommand == 'delete':
+ snaps = base.fssnap.old_snapshots()
+ devs = [x['dev'] for x in snaps]
+ snaps = set()
+ for dev in devs:
+ if dev in snaps:
+ continue
+
+ for extcmd in extcmds:
+ if dev == extcmd or fnmatch.fnmatch(dev, extcmd):
+ snaps.add(dev)
+ break
+ snaps = base.fssnap.del_snapshots(devices=snaps)
+ print _("Deleted %u snapshosts:") % len(snaps)
+ self._li_snaps(base, snaps)
+
+ if subcommand in ('have-space', 'has-space'):
+ pc = base.conf.fssnap_percentage
+ if base.fssnap.has_space(pc):
+ print _("Space available to take a snapshot.")
+ else:
+ print _("Not enough space available to take a snapshot.")
+
+ if subcommand == 'create':
+ tags = {'*': ['reason=manual']}
+ pc = base.conf.fssnap_percentage
+ for (odev, ndev) in base.fssnap.snapshot(pc, tags=tags):
+ print _("Created snapshot from %s, results is: %s") %(odev,ndev)
+ else:
+ print _("Failed to create snapshots")
+
+ if subcommand == 'summary':
+ snaps = base.fssnap.old_snapshots()
+ if not snaps:
+ print _("No snapshots")
+ return 0, [basecmd + ' ' + subcommand + ' done']
+
+ used = 0
+ dev_oris = set()
+ for snap in snaps:
+ used += snap['used']
+ dev_oris.add(snap['origin_dev'])
+
+ msg = _("Have %u snapshots, using %s space, from %u origins.")
+ print msg % (len(snaps), base.format_number(used), len(dev_oris))
+
+ return 0, [basecmd + ' ' + subcommand + ' done']
commit 7e217766a45ff562646a2bacc9d0386f61e8f251
Author: James Antill <james at and.org>
Date: Fri Jun 14 15:36:55 2013 -0400
Add logging to updateinfo parsing, and tell people to complain to repos.
diff --git a/yum/__init__.py b/yum/__init__.py
index 1945485..983b0d7 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -990,7 +990,10 @@ class YumBase(depsolve.Depsolve):
_('Getting updateinfo metadata'))
if self._upinfo is None:
- self._upinfo = update_md.UpdateMetadata()
+ logger = logging.getLogger("yum.update_md")
+ vlogger = logging.getLogger("yum.verbose.update_md")
+ self._upinfo = update_md.UpdateMetadata(logger=logger,
+ vlogger=vlogger)
for repo in self.repos.listEnabled():
if 'updateinfo' not in repo.repoXML.fileTypes():
diff --git a/yum/update_md.py b/yum/update_md.py
index 714de89..66cd93b 100644
--- a/yum/update_md.py
+++ b/yum/update_md.py
@@ -23,23 +23,28 @@ Update metadata (updateinfo.xml) parsing.
import sys
-from yum.i18n import utf8_text_wrap, to_utf8, to_unicode
+from yum.i18n import utf8_text_wrap, to_utf8, to_unicode, _
from yum.yumRepo import YumRepository
from yum.packages import FakeRepository
from yum.misc import to_xml, decompress, repo_gen_decompress
from yum.misc import cElementTree_iterparse as iterparse
import Errors
+import logginglevels
+
import rpmUtils.miscutils
-def safe_iterparse(filename):
+def safe_iterparse(filename, logger=None):
""" Works like iterparse, but hides XML errors (prints a warning). """
try:
for event, elem in iterparse(filename):
yield event, elem
except SyntaxError: # Bad XML
- print >> sys.stderr, "File is not valid XML:", filename
+ if logger:
+ logger.critical(_("Updateinfo file is not valid XML: %s"), filename)
+ else:
+ print >> sys.stderr, "Updateinfo file is not valid XML:", filename
class UpdateNoticeException(Exception):
""" An exception thrown for bad UpdateNotice data. """
@@ -404,11 +409,15 @@ class UpdateMetadata(object):
The root update metadata object.
"""
- def __init__(self, repos=[]):
+ def __init__(self, repos=[], logger=None, vlogger=None):
self._notices = {}
self._cache = {} # a pkg nvr => notice cache for quick lookups
self._no_cache = {} # a pkg name only => notice list
self._repos = [] # list of repo ids that we've parsed
+
+ self._logger = logger
+ self._vlogger = vlogger
+
for repo in repos:
try: # attempt to grab the updateinfo.xml.gz from the repodata
self.add(repo)
@@ -516,6 +525,12 @@ class UpdateMetadata(object):
def add(self, obj, mdtype='updateinfo'):
""" Parse a metadata from a given YumRepository, file, or filename. """
+
+ def _rid(repoid, fmt=_(' (from %s)')):
+ if not repoid:
+ return ''
+ return fmt % repoid
+
if not obj:
raise UpdateNoticeException
repoid = None
@@ -537,20 +552,28 @@ class UpdateMetadata(object):
else: # obj is a file object
infile = obj
- for event, elem in safe_iterparse(infile):
+ have_dup = False
+ for event, elem in safe_iterparse(infile, logger=self._logger):
if elem.tag == 'update':
try:
un = UpdateNotice(elem)
except UpdateNoticeException, e:
- print >> sys.stderr, "An update notice is broken, skipping."
- # what else should we do?
+ msg = _("An update notice%s is broken, skipping.") % _rid(repoid)
+ if self._vlogger:
+ self._vlogger.log(logginglevels.DEBUG_1, "%s", msg)
+ else:
+ print >> sys.stderr, msg
continue
+
if not self.add_notice(un):
- if repoid is None:
- upid = un['update_id']
+ msg = _("Update notice %s%s is broken, or a bad duplicate, skipping.") % (un['update_id'], _rid(repoid))
+ if not have_dup:
+ msg += _('\nYou should report this problem to the owner of the %srepository.') % _rid(repoid, "%s ")
+ have_dup = True
+ if self._vlogger:
+ self._vlogger.warn("%s", msg)
else:
- upid = "%s/%s" % (repoid, un['update_id'])
- print >> sys.stderr, "An update notice is broken, or duplicate, skipping:", upid
+ print >> sys.stderr, msg
def __unicode__(self):
ret = u''
commit e00c8d654817af557b0f9df3e19a41c933f9b529
Author: James Antill <james at and.org>
Date: Mon Jun 10 15:27:01 2013 -0400
Solve weird edge case with comps blanking (mainly yumdb).
diff --git a/yum/__init__.py b/yum/__init__.py
index c6f9a9d..1945485 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -875,7 +875,12 @@ class YumBase(depsolve.Depsolve):
# if we unset the comps object, we need to undo which repos have
# been added to the group file as well
if self._repos:
- for repo in self._repos.listGroupsEnabled():
+ # Used to do listGroupsEnabled(), which seems fine but requires
+ # calling .listEnalbed() ... which doesn't work on __del__ path
+ # if we haven't already called that (due to
+ # "prelistenabledrepos" plugins). So just blank it for
+ # all repos.
+ for repo in self._repos.sort():
repo.groups_added = False
self._comps = val
commit 543eea7f0de2fdcc38f4cb19bbc52e383b4c2aa4
Author: James Antill <james at and.org>
Date: Mon Jun 10 14:57:57 2013 -0400
Remove repos.findRepos("*"), for repos.sort().
diff --git a/yum-cron/yum-cron.py b/yum-cron/yum-cron.py
index 721bdcb..26526b7 100755
--- a/yum-cron/yum-cron.py
+++ b/yum-cron/yum-cron.py
@@ -825,7 +825,7 @@ class YumCronBase(yum.YumBase):
def populateUpdateMetadata(self):
"""Populate the metadata for the packages in the update."""
- for repo in self.repos.findRepos('*'):
+ for repo in self.repos.sort():
repo.metadata_expire = 0
self.upinfo
diff --git a/yumcommands.py b/yumcommands.py
index b6cb7f6..93c6278 100644
--- a/yumcommands.py
+++ b/yumcommands.py
@@ -153,8 +153,8 @@ def checkRepoPackageArg(base, basecmd, extcmds):
if not repos[0].isEnabled():
# Might as well just fix this...
base.repos.enableRepo(repos[0].id)
- base.logger.critical(
- _('Repo %s has been automatically enabled') % repos[0].ui_id)
+ base.verbose_logger.info(
+ _('Repo %s has been automatically enabled.') % repos[0].ui_id)
def checkItemArg(base, basecmd, extcmds):
@@ -1322,7 +1322,7 @@ class MakeCacheCommand(YumCommand):
fast = True
if True: # Try, YumBase...
- for repo in base.repos.findRepos('*'):
+ for repo in base.repos.sort():
repo.metadata_expire = 0
if not fast:
repo.mdpolicy = "group:all"
commit 5453a388f4000c1b571882805f9fac4199a4b6d0
Author: James Antill <james at and.org>
Date: Mon Jun 10 14:56:37 2013 -0400
Do cacheRequirement() tests before doCheck(), and do it for all repos.
diff --git a/cli.py b/cli.py
index ba98058..97685c4 100755
--- a/cli.py
+++ b/cli.py
@@ -389,6 +389,13 @@ class YumBaseCli(yum.YumBase, output.YumOutput):
self.basecmd, sys.argv[0])
raise CliError
+ cmd = self.yum_cli_commands[self.basecmd]
+ cacheReq = 'write'
+ if hasattr(cmd, 'cacheRequirement'):
+ cacheReq = cmd.cacheRequirement(self, self.basecmd, self.extcmds)
+ for repo in self.repos.sort():
+ repo._metadata_cache_req = cacheReq
+
self.yum_cli_commands[self.basecmd].doCheck(self, self.basecmd, self.extcmds)
def _shell_history_write(self):
@@ -513,10 +520,12 @@ class YumBaseCli(yum.YumBase, output.YumOutput):
except yum.Errors.YumBaseError, e:
return 1, [exception2msg(e)]
+ # This should already have been done at doCheck() time, but just in
+ # case repos. got added or something do it again.
cacheReq = 'write'
if hasattr(cmd, 'cacheRequirement'):
cacheReq = cmd.cacheRequirement(self, self.basecmd, self.extcmds)
- for repo in self.repos.listEnabled():
+ for repo in self.repos.sort():
repo._metadata_cache_req = cacheReq
return self.yum_cli_commands[self.basecmd].doCommand(self, self.basecmd, self.extcmds)
commit a4e85bfdc5b4fce1b9e386ef9b36a1c9d93c6a3f
Author: James Antill <james at and.org>
Date: Mon Jun 10 14:39:08 2013 -0400
Auto. enable disabled repos. in repo-pkgs.
diff --git a/yumcommands.py b/yumcommands.py
index 3300e6c..b6cb7f6 100644
--- a/yumcommands.py
+++ b/yumcommands.py
@@ -145,16 +145,16 @@ def checkRepoPackageArg(base, basecmd, extcmds):
repos = [r for r in repos if r.isEnabled()]
if len(repos) > 1:
- repos = ", ".join([r.id for r in repos])
+ repos = ", ".join([r.ui_id for r in repos])
base.logger.critical(
_('Error: Need to pass only a single valid repoid. to %s, passed: %s') % (basecmd, repos))
_err_mini_usage(base, basecmd)
raise cli.CliError
if not repos[0].isEnabled():
+ # Might as well just fix this...
+ base.repos.enableRepo(repos[0].id)
base.logger.critical(
- _('Error: Repo %s is not enabled') % extcmds[0])
- _err_mini_usage(base, basecmd)
- raise cli.CliError
+ _('Repo %s has been automatically enabled') % repos[0].ui_id)
def checkItemArg(base, basecmd, extcmds):
commit b14de5258f1cd050d1b42d12bc98dd36b2449976
Author: James Antill <james at and.org>
Date: Mon Jun 10 14:37:04 2013 -0400
Add simple ui_id to base Repository class.
diff --git a/yum/repos.py b/yum/repos.py
index b5e34d4..f1c24a9 100644
--- a/yum/repos.py
+++ b/yum/repos.py
@@ -422,6 +422,11 @@ class Repository:
def __del__(self):
self.close()
+ def _ui_id(self):
+ """ Show self.id, so we can use it and override it. """
+ return self.id
+ ui_id = property(_ui_id)
+
def close(self):
pass
commit 65d1cd21418b140e7ab74e568b16067a1b01d2c9
Author: James Antill <james at and.org>
Date: Fri Jun 7 11:37:32 2013 -0400
Extend findRepos() so it can work like repolist. BZ 971599.
diff --git a/yum/repos.py b/yum/repos.py
index e1eb581..b5e34d4 100644
--- a/yum/repos.py
+++ b/yum/repos.py
@@ -202,17 +202,28 @@ class RepoStorage:
raise Errors.RepoError, \
'Error getting repository data for %s, repository not found' % (repoid)
- def findRepos(self,pattern):
- """find all repositories matching fnmatch `pattern`"""
+ def findRepos(self, pattern, name_match=False, ignore_case=False):
+ """ Find all repositories matching fnmatch `pattern` on the repo.id,
+ can also do case insensitive searches and/or search on the name."""
+
+ if pattern in self.repos: # Minor opt. as we do this a lot...
+ return [self.repos[pattern]]
result = []
for item in pattern.split(','):
item = item.strip()
- match = re.compile(fnmatch.translate(item)).match
+ if ignore_case:
+ match = re.compile(fnmatch.translate(item), re.I).match
+ else:
+ match = re.compile(fnmatch.translate(item)).match
for name,repo in self.repos.items():
+ assert name == repo.id
if match(name):
result.append(repo)
+ elif name_match and match(repo.name):
+ result.append(repo)
+
return result
def disableRepo(self, repoid):
diff --git a/yumcommands.py b/yumcommands.py
index 8e0d7d1..3300e6c 100644
--- a/yumcommands.py
+++ b/yumcommands.py
@@ -134,16 +134,20 @@ def checkRepoPackageArg(base, basecmd, extcmds):
_err_mini_usage(base, basecmd)
raise cli.CliError
- repos = base.repos.findRepos(extcmds[0])
+ repos = base.repos.findRepos(extcmds[0], name_match=True, ignore_case=True)
if not repos:
base.logger.critical(
_('Error: Need to pass a single valid repoid. to %s') % basecmd)
_err_mini_usage(base, basecmd)
raise cli.CliError
- if len(repos) != 1 or repos[0].id != extcmds[0]:
+ if len(repos) > 1:
+ repos = [r for r in repos if r.isEnabled()]
+
+ if len(repos) > 1:
+ repos = ", ".join([r.id for r in repos])
base.logger.critical(
- _('Error: Need to pass a single valid repoid. to %s') % basecmd)
+ _('Error: Need to pass only a single valid repoid. to %s, passed: %s') % (basecmd, repos))
_err_mini_usage(base, basecmd)
raise cli.CliError
if not repos[0].isEnabled():
@@ -2068,12 +2072,9 @@ class RepoListCommand(YumCommand):
return base.format_number(ret)
def _repo_match(repo, patterns):
- rid = repo.id.lower()
- rnm = repo.name.lower()
for pat in patterns:
- if fnmatch.fnmatch(rid, pat):
- return True
- if fnmatch.fnmatch(rnm, pat):
+ if repo in base.repos.findRepos(pat, name_match=True,
+ ignore_case=True):
return True
return False
commit 6bab1c69685fdf21d0aecca4ba871f29917a278d
Author: James Antill <james at and.org>
Date: Wed Jun 5 16:38:56 2013 -0400
Add a prelistenabledrepos plugin point, so tmprepo. etc. can add repos.
diff --git a/yum/plugins.py b/yum/plugins.py
index 9ddcae6..a10e4fb 100644
--- a/yum/plugins.py
+++ b/yum/plugins.py
@@ -84,6 +84,7 @@ SLOT_TO_CONDUIT = {
'args': 'ArgsPluginConduit',
'predownload': 'DownloadPluginConduit',
'postdownload': 'DownloadPluginConduit',
+ 'prelistenabledrepos': 'PreRepoSetupPluginConduit',
'prereposetup': 'PreRepoSetupPluginConduit',
'postreposetup': 'PostRepoSetupPluginConduit',
'close': 'PluginConduit',
diff --git a/yum/repos.py b/yum/repos.py
index e519bc1..e1eb581 100644
--- a/yum/repos.py
+++ b/yum/repos.py
@@ -69,6 +69,10 @@ class RepoStorage:
self._cache_enabled_repos = []
self.quick_enable_disable = {}
+ # This allows plugins to setup a repo. just before the first
+ # listEnabled() call.
+ self._list_enabled_hasrun = False
+
def retrieveAllMD(self):
""" Download metadata for all enabled repositories,
based on mdpolicy.
@@ -112,6 +116,10 @@ class RepoStorage:
def doSetup(self, thisrepo = None):
+ if thisrepo is None:
+ # Just in case the prelistenabledrepos plugin point hasn't run.
+ self.listEnabled()
+
self.ayum.plugins.run('prereposetup')
if thisrepo is None:
@@ -246,6 +254,10 @@ class RepoStorage:
def listEnabled(self):
"""return list of enabled repo objects"""
+ if not self._list_enabled_hasrun:
+ self.ayum.plugins.run('prelistenabledrepos')
+ self._list_enabled_hasrun = True
+
if (self._cache_enabled_repos is not None and
not self.quick_enable_disable):
return self._cache_enabled_repos
commit 7f06b684e8ce681b8c21685677a37906acf046bd
Author: James Antill <james at and.org>
Date: Wed Jun 5 15:34:30 2013 -0400
Remove typo'd trainling commas.
diff --git a/yum/__init__.py b/yum/__init__.py
index 08792f4..c6f9a9d 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -3732,7 +3732,7 @@ much more problems).
if (pkg not in igroup_data or
igroup_data[pkg].startswith('blacklisted')):
# (upgrade and igroup_data[pkg] == 'available')):
- msg = _('Skipping package %s from group %s'),
+ msg = _('Skipping package %s from group %s')
self.verbose_logger.log(logginglevels.DEBUG_2,
msg, pkg, thisgroup.groupid)
continue
@@ -3904,7 +3904,7 @@ much more problems).
for grpid in evgrp.groups:
if (grpid not in igroup_data or
igroup_data[grpid].startswith('blacklisted')):
- msg = _('Skipping group %s from environment %s'),
+ msg = _('Skipping group %s from environment %s')
self.verbose_logger.log(logginglevels.DEBUG_2,
msg, grpid, evgrp.environmentid)
continue
More information about the Yum-commits
mailing list