[yum-commits] 6 commits - yumcommands.py yummain.py yum.spec yum/yumRepo.py
James Antill
james at osuosl.org
Tue Feb 19 22:09:41 UTC 2013
yum.spec | 25 +++++++++++++--
yum/yumRepo.py | 90 ++++++++++++++++++++++++++++++++++++++-------------------
yumcommands.py | 37 +++++++++++++++++++++++
yummain.py | 15 +++++++++
4 files changed, 134 insertions(+), 33 deletions(-)
New commits:
commit 87fd258d42ded7424ab171c3ceb2388c3c28a43d
Merge: 2d8c54b f643341
Author: James Antill <james at and.org>
Date: Tue Feb 19 17:09:34 2013 -0500
Merge branch 'master' of ssh://yum.baseurl.org/srv/projects/yum/git/yum
* 'master' of ssh://yum.baseurl.org/srv/projects/yum/git/yum:
Add "search all" completion.
commit 2d8c54bf23ea3dd718172b2f73cf1f0cd0370a63
Author: James Antill <james at and.org>
Date: Tue Feb 19 16:59:05 2013 -0500
Add cache req. "past" to clean and check-rpmdb.
diff --git a/yumcommands.py b/yumcommands.py
index ba85775..ad529cd 100644
--- a/yumcommands.py
+++ b/yumcommands.py
@@ -1394,6 +1394,17 @@ class CleanCommand(YumCommand):
"""
return False
+ def cacheRequirement(self, base, basecmd, extcmds):
+ """Return the cache requirements for the remote repos.
+
+ :param base: a :class:`yum.Yumbase` object
+ :param basecmd: the name of the command
+ :param extcmds: a list of arguments passed to *basecmd*
+ :return: Type of requirement: read-only:past, read-only:present, read-only:future, write
+ """
+ return 'read-only:past'
+
+
class ProvidesCommand(YumCommand):
"""A class containing methods needed by the cli to execute the
provides command.
@@ -3078,6 +3089,16 @@ class CheckRpmdbCommand(YumCommand):
"""
return False
+ def cacheRequirement(self, base, basecmd, extcmds):
+ """Return the cache requirements for the remote repos.
+
+ :param base: a :class:`yum.Yumbase` object
+ :param basecmd: the name of the command
+ :param extcmds: a list of arguments passed to *basecmd*
+ :return: Type of requirement: read-only:past, read-only:present, read-only:future, write
+ """
+ return 'read-only:past'
+
class LoadTransactionCommand(YumCommand):
"""A class containing methods needed by the cli to execute the
commit b947fe057bebd115e11cd6bc9f20b67fa7995422
Author: James Antill <james at and.org>
Date: Tue Feb 19 15:25:00 2013 -0500
Add cache req. and needTs opts. for listing the transactions.
diff --git a/yumcommands.py b/yumcommands.py
index 1cc4fdf..ba85775 100644
--- a/yumcommands.py
+++ b/yumcommands.py
@@ -3217,8 +3217,24 @@ class LoadTransactionCommand(YumCommand):
:param extcmds: a list of arguments passed to *basecmd*
:return: True if a transaction set is needed, False otherwise
"""
+ if not extcmds or os.path.isdir(extcmds[0]):
+ return False
+
return True
+ def cacheRequirement(self, base, basecmd, extcmds):
+ """Return the cache requirements for the remote repos.
+
+ :param base: a :class:`yum.Yumbase` object
+ :param basecmd: the name of the command
+ :param extcmds: a list of arguments passed to *basecmd*
+ :return: Type of requirement: read-only:past, read-only:present, read-only:future, write
+ """
+ if not extcmds or os.path.isdir(extcmds[0]):
+ return 'read-only:past'
+
+ return 'write'
+
class SwapCommand(YumCommand):
"""A class containing methods needed by the cli to execute the
commit 310eac803c07ea89f63ccc294b14b613e44969ff
Author: James Antill <james at and.org>
Date: Mon Feb 18 16:30:18 2013 -0500
Auto expire caches on repo errors.
diff --git a/yum/yumRepo.py b/yum/yumRepo.py
index 8c38093..342ed42 100644
--- a/yum/yumRepo.py
+++ b/yum/yumRepo.py
@@ -980,9 +980,6 @@ Insufficient space in download directory %s
except URLGrabError, e:
self._del_dl_file(local, size)
errstr = "failed to retrieve %s from %s\nerror was %s" % (relative, self, e)
- if self.mirrorurls:
- errstr +="\n You could try running: yum clean expire-cache"
- errstr +="\n To get a new set of mirrors."
raise Errors.RepoError, errstr
diff --git a/yummain.py b/yummain.py
index a9f001b..ac94f65 100755
--- a/yummain.py
+++ b/yummain.py
@@ -126,6 +126,11 @@ def main(args):
result, resultmsgs = base.doCommands()
except plugins.PluginYumExit, e:
return exPluginExit(e)
+ except Errors.RepoError, e:
+ result = 1
+ resultmsgs = [exception2msg(e)]
+ # For RepoErrors ... help out by forcing new repodata next time.
+ base.cleanExpireCache()
except Errors.YumBaseError, e:
result = 1
resultmsgs = [exception2msg(e)]
@@ -167,6 +172,11 @@ def main(args):
(result, resultmsgs) = base.buildTransaction()
except plugins.PluginYumExit, e:
return exPluginExit(e)
+ except Errors.RepoError, e:
+ result = 1
+ resultmsgs = [exception2msg(e)]
+ # For RepoErrors ... help out by forcing new repodata next time.
+ base.cleanExpireCache()
except Errors.YumBaseError, e:
result = 1
resultmsgs = [exception2msg(e)]
@@ -209,6 +219,11 @@ def main(args):
return_code = base.doTransaction()
except plugins.PluginYumExit, e:
return exPluginExit(e)
+ except Errors.RepoError, e:
+ result = 1
+ resultmsgs = [exception2msg(e)]
+ # For RepoErrors ... help out by forcing new repodata next time.
+ base.cleanExpireCache()
except Errors.YumBaseError, e:
return exFatal(e)
except KeyboardInterrupt:
commit c148eb10b798270b3d15087433c8efb2a79a69d0
Author: James Antill <james at and.org>
Date: Mon Feb 18 16:17:06 2013 -0500
Use xattr data as well as file size for "fast checksumming".
diff --git a/yum/yumRepo.py b/yum/yumRepo.py
index efbc42a..8c38093 100644
--- a/yum/yumRepo.py
+++ b/yum/yumRepo.py
@@ -52,15 +52,54 @@ import stat
import errno
import tempfile
-# If you want yum to _always_ check the MD .sqlite files then set this to
-# False (this doesn't affect .xml files or .sqilte files derived from them).
-# With this as True yum will only check when a new repomd.xml or
-# new MD is downloaded.
-# Note that with atomic MD, we can't have old MD lying around anymore so
-# the only way we need this check is if someone does something like:
-# cp primary.sqlite /var/cache/yum/blah
-# ...at which point you lose.
-skip_old_DBMD_check = True
+# This is unused now, probably nothing uses it but it was global/public.
+skip_old_DBMD_check = False
+
+try:
+ import xattr
+ if not hasattr(xattr, 'get') or not hasattr(xattr, 'set'):
+ xattr = None # This is a "newer" API.
+except ImportError:
+ xattr = None
+
+# The problem we are trying to solve here is that:
+#
+# 1. We rarely want to be downloading MD/pkgs/etc.
+# 2. We want to check those files are valid (match checksums) when we do
+# download them.
+# 3. We _really_ don't want to checksum all the files everytime we
+# run (100s of MBs).
+# 4. We can continue to download files from bad mirrors, or retry files due to
+# C-c etc.
+#
+# ...we used to solve this by just checking the file size, and assuming the
+# files had been downloaded and checksumed as correct if that matched. But that
+# was error prone on bad mirrors, so now we store the checksum in an
+# xattr ... this does mean that if you can't store xattrs (Eg. NFS) you will
+# rechecksum everything constantly.
+
+def _xattr_get_chksum(filename, chktype):
+ if not xattr:
+ return None
+
+ try:
+ ret = xattr.get(filename, 'user.yum.checksum.' + chktype)
+ except: # Documented to be "EnvironmentError", but make sure
+ return None
+
+ return ret
+
+def _xattr_set_chksum(filename, chktype, chksum):
+ if not xattr:
+ return None
+
+ try:
+ xattr.set(filename, 'user.yum.checksum.' + chktype, chksum)
+ except:
+ return False # Data too long. = IOError ... ignore everything.
+
+ return True
+
warnings.simplefilter("ignore", Errors.YumFutureDeprecationWarning)
@@ -228,7 +267,7 @@ class YumPackageSack(packageSack.PackageSack):
# get rid of all this stuff we don't need now
del repo.cacheHandler
- def _check_uncompressed_db_gen(self, repo, mdtype, fast=True):
+ def _check_uncompressed_db_gen(self, repo, mdtype):
"""return file name of db in gen/ dir if good, None if not"""
mydbdata = repo.repoXML.getData(mdtype)
@@ -238,7 +277,7 @@ class YumPackageSack(packageSack.PackageSack):
db_un_fn = mdtype + '.sqlite'
if not repo._checkMD(compressed_fn, mdtype, data=mydbdata,
- check_can_fail=fast, fast=fast):
+ check_can_fail=True):
return None
ret = misc.repo_gen_decompress(compressed_fn, db_un_fn,
@@ -261,8 +300,7 @@ class YumPackageSack(packageSack.PackageSack):
result = None
if os.path.exists(db_un_fn):
- if skip_old_DBMD_check and repo._using_old_MD:
- return db_un_fn
+
try:
repo.checkMD(db_un_fn, mdtype, openchecksum=True)
@@ -296,7 +334,6 @@ class YumRepository(Repository, config.RepoConf):
# eventually want
self.repoMDFile = 'repodata/repomd.xml'
self._repoXML = None
- self._using_old_MD = None
self._oldRepoMDData = {}
self.cache = 0
self.mirrorlistparsed = 0
@@ -1407,7 +1444,6 @@ Insufficient space in download directory %s
self._revertOldRepoXML()
return False
- self._using_old_MD = caching
if caching:
return False # Skip any work.
@@ -1673,7 +1709,7 @@ Insufficient space in download directory %s
return self._checkMD(fn, mdtype, openchecksum)
def _checkMD(self, fn, mdtype, openchecksum=False,
- data=None, check_can_fail=False, fast=False):
+ data=None, check_can_fail=False):
""" Internal function, use .checkMD() from outside yum. """
thisdata = data # So the argument name is nicer
@@ -1696,17 +1732,15 @@ Insufficient space in download directory %s
if size is not None:
size = int(size)
- if fast and skip_old_DBMD_check:
+ l_csum = _xattr_get_chksum(file, r_ctype)
+ if l_csum:
fsize = misc.stat_f(file)
- if fsize is None: # File doesn't exist...
- return None
- if size is None:
- return 1
- if size == fsize.st_size:
- return 1
- if check_can_fail:
- return None
- raise URLGrabError(-1, 'Metadata file does not match size')
+ if fsize is not None: # We just got an xattr, so it should be there
+ if size is None and l_csum == r_csum:
+ return 1
+ if size == fsize.st_size and l_csum == r_csum:
+ return 1
+ # Anything goes wrong, run the checksums as normal...
try: # get the local checksum
l_csum = self._checksum(r_ctype, file, datasize=size)
@@ -1716,6 +1750,7 @@ Insufficient space in download directory %s
raise URLGrabError(-3, 'Error performing checksum')
if l_csum == r_csum:
+ _xattr_set_chksum(file, r_ctype, l_csum)
return 1
else:
if check_can_fail:
commit 64e88c95794f1e881b1c478f19965e3f00e9ba79
Author: James Antill <james at and.org>
Date: Mon Feb 18 13:25:50 2013 -0500
Turn off cron for rhel-6, so we can build test versions of yum again.
diff --git a/yum.spec b/yum.spec
index f18cce8..734f0a7 100644
--- a/yum.spec
+++ b/yum.spec
@@ -2,6 +2,12 @@
%define auto_sitelib 1
%define yum_updatesd 0
%define disable_check 0
+%define yum_cron 1
+
+%if 0%{?rhel} == 6
+# rhel-6 doesn't have the systemd stuff, so won't build...
+%define yum_cron 0
+%endif
%if %{auto_sitelib}
@@ -111,7 +117,7 @@ Requires(postun): /sbin/service
yum-updatesd provides a daemon which checks for available updates and
can notify you when they are available via email, syslog or dbus.
-
+%if %{yum_cron}
%package cron
Summary: Files needed to run yum updates as a cron job
Group: System Environment/Base
@@ -124,7 +130,7 @@ Requires(postun): systemd
%description cron
These are the files needed to run yum updates as a cron job.
Install this package if you want auto yum updates nightly via cron.
-
+%endif
%prep
@@ -179,6 +185,15 @@ chmod +x $RPM_BUILD_ROOT/%{python_sitelib}/rpmUtils/*.py
%find_lang %name
+%if ! %{yum_cron}
+# Remove the yum-cron stuff to make rpmbuild happy..
+rm -f $RPM_BUILD_ROOT/%{_unitdir}/yum-cron.service
+rm -f $RPM_BUILD_ROOT/%{_sysconfdir}/cron.daily/0yum-update.cron
+rm -f $RPM_BUILD_ROOT/%{_sysconfdir}/yum/yum-cron.conf
+rm -f $RPM_BUILD_ROOT/%{_sbindir}/yum-cron
+rm -f $RPM_BUILD_ROOT/%{_mandir}/man*/yum-cron.*
+%endif
+
%clean
[ "$RPM_BUILD_ROOT" != "/" ] && rm -rf $RPM_BUILD_ROOT
@@ -197,7 +212,7 @@ fi
exit 0
%endif
-
+%if %{yum_cron}
%post cron
#systemd_post yum-cron.service
@@ -222,7 +237,7 @@ fi
%postun cron
%systemd_postun_with_restart yum-cron.service
-
+%endif
%files -f %{name}.lang
@@ -263,6 +278,7 @@ fi
%dir %{yum_pluginslib}
%dir %{yum_pluginsshare}
+%if %{yum_cron}
%files cron
%defattr(-,root,root)
%doc COPYING
@@ -271,6 +287,7 @@ fi
%{_unitdir}/yum-cron.service
%{_sbindir}/yum-cron
%{_mandir}/man*/yum-cron.*
+%endif
%if %{yum_updatesd}
%files updatesd
More information about the Yum-commits
mailing list