[yum-commits] 13 commits - docs/yum.conf.5 yum/config.py yum/drpm.py yum/__init__.py
zpavlas at osuosl.org
zpavlas at osuosl.org
Thu Feb 28 14:44:27 UTC 2013
docs/yum.conf.5 | 11 ++
yum/__init__.py | 244 +++++++++++++++++++++++++++++++-------------------------
yum/config.py | 2
yum/drpm.py | 174 +++++++++++++++++++++++++++++++++++++++
4 files changed, 325 insertions(+), 106 deletions(-)
New commits:
commit 559c996ec5d53d033537e54d2182a90f02eac406
Author: Zdenek Pavlas <zpavlas at redhat.com>
Date: Thu Feb 28 12:21:05 2013 +0100
drpm: repo_gen_decompress() prestodelta.xml
diff --git a/yum/drpm.py b/yum/drpm.py
index 108aa39..3339107 100644
--- a/yum/drpm.py
+++ b/yum/drpm.py
@@ -21,7 +21,7 @@ from yum.constants import TS_UPDATE
from yum.Errors import RepoError
from yum.i18n import exception2msg, _
from yum.Errors import MiscError
-from misc import checksum
+from misc import checksum, repo_gen_decompress
from urlgrabber import grabber
async = hasattr(grabber, 'parallel_wait')
from xml.etree.cElementTree import iterparse
@@ -124,10 +124,10 @@ class DeltaInfo:
for po in ayum.rpmdb.searchNevra(n, None, None, None, a)]
# parse metadata, populate self.deltas
- for repo, path in mdpath.items():
+ for repo, cpath in mdpath.items():
pinfo_repo = pinfo[repo]
- if path.endswith('.gz'):
- path = gzip.open(path)
+ path = repo_gen_decompress(cpath, 'prestodelta.xml',
+ cached=repo.cache)
for ev, el in iterparse(path):
if el.tag != 'newpackage': continue
new = el.get('name'), el.get('arch'), el.get('epoch'), el.get('version'), el.get('release')
commit c2074489142bd4ad2b067e8c84bb984961b12ab5
Author: Zdenek Pavlas <zpavlas at redhat.com>
Date: Thu Feb 28 11:25:44 2013 +0100
drpm: add rpmdb.searchNevra() fallback
diff --git a/yum/drpm.py b/yum/drpm.py
index 8620e37..108aa39 100644
--- a/yum/drpm.py
+++ b/yum/drpm.py
@@ -115,6 +115,14 @@ class DeltaInfo:
if async:
grabber.parallel_wait()
+ # use installdict or rpmdb
+ if ayum._up:
+ installed = ayum._up.installdict.get
+ else:
+ installed = lambda (n, a): [
+ (po.epoch, po.version, po.release)
+ for po in ayum.rpmdb.searchNevra(n, None, None, None, a)]
+
# parse metadata, populate self.deltas
for repo, path in mdpath.items():
pinfo_repo = pinfo[repo]
@@ -127,7 +135,7 @@ class DeltaInfo:
if index is not None:
po = pkgs[index]
best = po.size * 0.75 # make this configurable?
- have = ayum._up.installdict.get(new[:2], [])
+ have = installed(new[:2]) or []
for el in el.findall('delta'):
size = int(el.find('size').text)
old = el.get('oldepoch'), el.get('oldversion'), el.get('oldrelease')
commit 7e1a5524920978bfb898a6a87d438fef73b832ba
Author: Zdenek Pavlas <zpavlas at redhat.com>
Date: Tue Feb 26 16:24:01 2013 +0100
drop to_drpm() and to_rpm(), use DeltaPackage instances.
diff --git a/yum/__init__.py b/yum/__init__.py
index 8ef7989..fd54208 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -90,7 +90,7 @@ from packages import YumUrlPackage, YumNotFoundPackage
from constants import *
from yum.rpmtrans import RPMTransaction,SimpleCliCallBack
from yum.i18n import to_unicode, to_str, exception2msg
-from yum.drpm import DeltaInfo
+from yum.drpm import DeltaInfo, DeltaPackage
import string
import StringIO
@@ -2191,8 +2191,8 @@ much more problems).
b = bpo.getDiscNum()
if a is None and b is None:
# deltas first to start rebuilding asap
- return cmp(bpo in presto.deltas, apo in presto.deltas) \
- or cmp(apo, bpo)
+ return cmp(isinstance(bpo, DeltaPackage),
+ isinstance(apo, DeltaPackage)) or cmp(apo, bpo)
if a is None:
return -1
if b is None:
@@ -2257,19 +2257,22 @@ much more problems).
return errors
pkgs.append(po)
- # download presto metadata
+ # download presto metadata and use drpms
presto = DeltaInfo(self, pkgs)
+ deltasize = rpmsize = 0
for po in pkgs:
- if presto.to_drpm(po) and verify_local(po):
- # there's .drpm already, use it
- presto.rebuild(po, adderror)
- continue
+ if isinstance(po, DeltaPackage):
+ if verify_local(po):
+ # there's .drpm already, use it
+ presto.rebuild(po, adderror)
+ continue
+ deltasize += po.size
+ rpmsize = po.rpm.size
remote_pkgs.append(po)
remote_size += po.size
- if presto.deltasize:
+ if deltasize:
self.verbose_logger.info(_('Delta RPMs reduced %s of updates to %s (%d%% saved)'),
- format_number(presto.rpmsize), format_number(presto.deltasize),
- 100 - presto.deltasize*100.0/presto.rpmsize)
+ format_number(rpmsize), format_number(deltasize), 100 - deltasize*100.0/rpmsize)
if downloadonly:
# close DBs, unlock
@@ -2298,7 +2301,7 @@ much more problems).
if hasattr(urlgrabber.progress, 'text_meter_total_size'):
urlgrabber.progress.text_meter_total_size(remote_size,
local_size[0])
- if po in presto.deltas:
+ if isinstance(po, DeltaPackage):
presto.rebuild(po, adderror)
return
if po.repoid not in done_repos:
@@ -2355,19 +2358,21 @@ much more problems).
fatal = False
for po in errors:
- if po not in presto.deltas:
- fatal = True; break
+ if not isinstance(po, DeltaPackage):
+ fatal = True
+ break
if not errors or fatal:
break
# there were drpm related errors *only*
- remote_pkgs = errors.keys()
+ remote_pkgs = []
remote_size = 0
- for po in remote_pkgs:
- presto.to_rpm(po) # needed, we don't rebuild() when DL fails
+ for po in errors:
+ po = po.rpm
+ remote_pkgs.append(po)
remote_size += po.size
+ errors.clear()
self.verbose_logger.warn(_('Some delta RPMs failed to download or rebuild. Retrying..'))
- presto.deltas.clear() # any error is now considered fatal
if not downloadonly:
# XXX: Run unlocked? Skip this for now..
diff --git a/yum/drpm.py b/yum/drpm.py
index 81afe80..8620e37 100644
--- a/yum/drpm.py
+++ b/yum/drpm.py
@@ -20,6 +20,8 @@
from yum.constants import TS_UPDATE
from yum.Errors import RepoError
from yum.i18n import exception2msg, _
+from yum.Errors import MiscError
+from misc import checksum
from urlgrabber import grabber
async = hasattr(grabber, 'parallel_wait')
from xml.etree.cElementTree import iterparse
@@ -27,25 +29,56 @@ import os, gzip
APPLYDELTA = '/usr/bin/applydeltarpm'
+class DeltaPackage:
+ def __init__(self, rpm, size, remote, csum):
+ # copy what needed
+ self.rpm = rpm
+ self.repo = rpm.repo
+ self.basepath = rpm.basepath
+ self.pkgtup = rpm.pkgtup
+
+ # set up drpm attributes
+ self.size = size
+ self.relativepath = remote
+ self.localpath = os.path.dirname(rpm.localpath) +'/'+ os.path.basename(remote)
+ self.csum = csum
+
+ def __str__(self):
+ return 'Delta RPM of %s' % self.rpm
+
+ def localPkg(self):
+ return self.localpath
+
+ def verifyLocalPkg(self):
+ # check file size first
+ try: fsize = os.path.getsize(self.localpath)
+ except OSError: return False
+ if fsize != self.size: return False
+
+ # checksum
+ ctype, csum = self.csum
+ try: fsum = checksum(ctype, self.localpath)
+ except MiscError: return False
+ if fsum != csum: return False
+
+ # hooray
+ return True
+
class DeltaInfo:
def __init__(self, ayum, pkgs):
self.verbose_logger = ayum.verbose_logger
- self.deltas = {}
- self._rpmsave = {}
- self.rpmsize = 0
- self.deltasize = 0
self.jobs = {}
self.limit = ayum.conf.deltarpm
# calculate update sizes
pinfo = {}
reposize = {}
- for po in pkgs:
+ for index, po in enumerate(pkgs):
if not po.repo.deltarpm:
continue
if po.state != TS_UPDATE and po.name not in ayum.conf.installonlypkgs:
continue
- pinfo.setdefault(po.repo, {})[po.pkgtup] = po
+ pinfo.setdefault(po.repo, {})[po.pkgtup] = index
reposize[po.repo] = reposize.get(po.repo, 0) + po.size
# don't use deltas when deltarpm not installed
@@ -90,8 +123,9 @@ class DeltaInfo:
for ev, el in iterparse(path):
if el.tag != 'newpackage': continue
new = el.get('name'), el.get('arch'), el.get('epoch'), el.get('version'), el.get('release')
- po = pinfo_repo.get(new)
- if po:
+ index = pinfo_repo.get(new)
+ if index is not None:
+ po = pkgs[index]
best = po.size * 0.75 # make this configurable?
have = ayum._up.installdict.get(new[:2], [])
for el in el.findall('delta'):
@@ -100,34 +134,12 @@ class DeltaInfo:
if size >= best or old not in have:
continue
best = size
+ remote = el.find('filename').text
csum = el.find('checksum')
csum = csum.get('type'), csum.text
- self.deltas[po] = size, el.find('filename').text, csum
+ pkgs[index] = DeltaPackage(po, size, remote, csum)
el.clear()
- def to_drpm(self, po):
- try: size, remote, csum = self.deltas[po]
- except KeyError: return False
- self._rpmsave[po] = po.packagesize, po.relativepath, po.localpath
-
- # update stats
- self.rpmsize += po.packagesize
- self.deltasize += size
-
- # update size/path/checksum to drpm values
- po.packagesize = size
- po.relativepath = remote
- po.localpath = os.path.dirname(po.localpath) +'/'+ os.path.basename(remote)
- po.returnIdSum = lambda: csum
- return True
-
- def to_rpm(self, po):
- if po not in self._rpmsave:
- return
- # revert back to RPM
- po.packagesize, po.relativepath, po.localpath = self._rpmsave.pop(po)
- del po.returnIdSum
-
def wait(self, limit = 1):
# wait for some jobs, run callbacks
while len(self.jobs) >= limit:
@@ -139,20 +151,16 @@ class DeltaInfo:
callback(code)
def rebuild(self, po, adderror):
- # restore rpm values
- deltapath = po.localpath
- po.packagesize, po.relativepath, po.localpath = self._rpmsave.pop(po)
- del po.returnIdSum
-
# this runs when worker finishes
def callback(code):
if code != 0:
return adderror(po, _('Delta RPM rebuild failed'))
- if not po.verifyLocalPkg():
+ if not po.rpm.verifyLocalPkg():
return adderror(po, _('Checksum of the delta-rebuilt RPM failed'))
- os.unlink(deltapath)
+ os.unlink(po.localpath)
+ po.localpath = po.rpm.localpath # for --downloadonly
# spawn a worker process
self.wait(self.limit)
- pid = os.spawnl(os.P_NOWAIT, APPLYDELTA, APPLYDELTA, deltapath, po.localpath)
+ pid = os.spawnl(os.P_NOWAIT, APPLYDELTA, APPLYDELTA, po.localpath, po.rpm.localpath)
self.jobs[pid] = callback
commit 462d2bb54c1e500d464b697ca3fd105dd42ba218
Author: Zdenek Pavlas <zpavlas at redhat.com>
Date: Tue Feb 26 14:12:49 2013 +0100
rename option "presto" to "deltarpm"
diff --git a/docs/yum.conf.5 b/docs/yum.conf.5
index 25f0c19..f9a924a 100644
--- a/docs/yum.conf.5
+++ b/docs/yum.conf.5
@@ -372,7 +372,7 @@ default of 5 connections. Note that there are also implicit per-mirror limits
and the downloader honors these too.
.IP
-\fBpresto\fR
+\fBdeltarpm\fR
When non-zero, delta-RPM files are used if available. The value specifies
the maximum number of "applydeltarpm" processes Yum will spawn. (4 by default).
diff --git a/yum/__init__.py b/yum/__init__.py
index fca6d2d..8ef7989 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -90,7 +90,7 @@ from packages import YumUrlPackage, YumNotFoundPackage
from constants import *
from yum.rpmtrans import RPMTransaction,SimpleCliCallBack
from yum.i18n import to_unicode, to_str, exception2msg
-from yum.presto import Presto
+from yum.drpm import DeltaInfo
import string
import StringIO
@@ -2258,7 +2258,7 @@ much more problems).
pkgs.append(po)
# download presto metadata
- presto = Presto(self, pkgs)
+ presto = DeltaInfo(self, pkgs)
for po in pkgs:
if presto.to_drpm(po) and verify_local(po):
# there's .drpm already, use it
diff --git a/yum/config.py b/yum/config.py
index 4aae831..d2ca937 100644
--- a/yum/config.py
+++ b/yum/config.py
@@ -791,7 +791,7 @@ class YumConf(StartupConf):
allowed = ('ipv4', 'ipv6', 'whatever'),
mapper = {'4': 'ipv4', '6': 'ipv6'})
max_connections = IntOption(0)
- presto = IntOption(4)
+ deltarpm = IntOption(4)
http_caching = SelectionOption('all', ('none', 'packages', 'all'))
metadata_expire = SecondsOption(60 * 60 * 6) # Time in seconds (6h).
@@ -950,7 +950,7 @@ class RepoConf(BaseConfig):
throttle = Inherit(YumConf.throttle)
timeout = Inherit(YumConf.timeout)
ip_resolve = Inherit(YumConf.ip_resolve)
- presto = Inherit(YumConf.presto)
+ deltarpm = Inherit(YumConf.deltarpm)
http_caching = Inherit(YumConf.http_caching)
metadata_expire = Inherit(YumConf.metadata_expire)
diff --git a/yum/drpm.py b/yum/drpm.py
new file mode 100644
index 0000000..81afe80
--- /dev/null
+++ b/yum/drpm.py
@@ -0,0 +1,158 @@
+# Integrated delta rpm support
+# Copyright 2013 Zdenek Pavlas
+
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc.,
+# 59 Temple Place, Suite 330,
+# Boston, MA 02111-1307 USA
+
+from yum.constants import TS_UPDATE
+from yum.Errors import RepoError
+from yum.i18n import exception2msg, _
+from urlgrabber import grabber
+async = hasattr(grabber, 'parallel_wait')
+from xml.etree.cElementTree import iterparse
+import os, gzip
+
+APPLYDELTA = '/usr/bin/applydeltarpm'
+
+class DeltaInfo:
+ def __init__(self, ayum, pkgs):
+ self.verbose_logger = ayum.verbose_logger
+ self.deltas = {}
+ self._rpmsave = {}
+ self.rpmsize = 0
+ self.deltasize = 0
+ self.jobs = {}
+ self.limit = ayum.conf.deltarpm
+
+ # calculate update sizes
+ pinfo = {}
+ reposize = {}
+ for po in pkgs:
+ if not po.repo.deltarpm:
+ continue
+ if po.state != TS_UPDATE and po.name not in ayum.conf.installonlypkgs:
+ continue
+ pinfo.setdefault(po.repo, {})[po.pkgtup] = po
+ reposize[po.repo] = reposize.get(po.repo, 0) + po.size
+
+ # don't use deltas when deltarpm not installed
+ if reposize and not os.access(APPLYDELTA, os.X_OK):
+ self.verbose_logger.info(_('Delta RPMs disabled because %s not installed.'), APPLYDELTA)
+ return
+
+ # download delta metadata
+ mdpath = {}
+ for repo in reposize:
+ self.limit = max(self.limit, repo.deltarpm)
+ for name in ('prestodelta', 'deltainfo'):
+ try: data = repo.repoXML.getData(name); break
+ except: pass
+ else:
+ self.verbose_logger.warn(_('No Presto metadata available for %s'), repo)
+ continue
+ path = repo.cachedir +'/'+ os.path.basename(data.location[1])
+ if not os.path.exists(path) and int(data.size) > reposize[repo]:
+ self.verbose_logger.info(_('Not downloading Presto metadata for %s'), repo)
+ continue
+
+ def failfunc(e, name=name, repo=repo):
+ mdpath.pop(repo, None)
+ if hasattr(e, 'exception'): e = e.exception
+ self.verbose_logger.warn(_('Failed to download %s for repository %s: %s'),
+ name, repo, exception2msg(e))
+ kwargs = {}
+ if async and repo._async:
+ kwargs['failfunc'] = failfunc
+ kwargs['async'] = True
+ try: mdpath[repo] = repo._retrieveMD(name, **kwargs)
+ except Errors.RepoError, e: failfunc(e)
+ if async:
+ grabber.parallel_wait()
+
+ # parse metadata, populate self.deltas
+ for repo, path in mdpath.items():
+ pinfo_repo = pinfo[repo]
+ if path.endswith('.gz'):
+ path = gzip.open(path)
+ for ev, el in iterparse(path):
+ if el.tag != 'newpackage': continue
+ new = el.get('name'), el.get('arch'), el.get('epoch'), el.get('version'), el.get('release')
+ po = pinfo_repo.get(new)
+ if po:
+ best = po.size * 0.75 # make this configurable?
+ have = ayum._up.installdict.get(new[:2], [])
+ for el in el.findall('delta'):
+ size = int(el.find('size').text)
+ old = el.get('oldepoch'), el.get('oldversion'), el.get('oldrelease')
+ if size >= best or old not in have:
+ continue
+ best = size
+ csum = el.find('checksum')
+ csum = csum.get('type'), csum.text
+ self.deltas[po] = size, el.find('filename').text, csum
+ el.clear()
+
+ def to_drpm(self, po):
+ try: size, remote, csum = self.deltas[po]
+ except KeyError: return False
+ self._rpmsave[po] = po.packagesize, po.relativepath, po.localpath
+
+ # update stats
+ self.rpmsize += po.packagesize
+ self.deltasize += size
+
+ # update size/path/checksum to drpm values
+ po.packagesize = size
+ po.relativepath = remote
+ po.localpath = os.path.dirname(po.localpath) +'/'+ os.path.basename(remote)
+ po.returnIdSum = lambda: csum
+ return True
+
+ def to_rpm(self, po):
+ if po not in self._rpmsave:
+ return
+ # revert back to RPM
+ po.packagesize, po.relativepath, po.localpath = self._rpmsave.pop(po)
+ del po.returnIdSum
+
+ def wait(self, limit = 1):
+ # wait for some jobs, run callbacks
+ while len(self.jobs) >= limit:
+ pid, code = os.wait()
+ # urlgrabber spawns child jobs, too. But they exit synchronously,
+ # so we should never see an unknown pid here.
+ assert pid in self.jobs
+ callback = self.jobs.pop(pid)
+ callback(code)
+
+ def rebuild(self, po, adderror):
+ # restore rpm values
+ deltapath = po.localpath
+ po.packagesize, po.relativepath, po.localpath = self._rpmsave.pop(po)
+ del po.returnIdSum
+
+ # this runs when worker finishes
+ def callback(code):
+ if code != 0:
+ return adderror(po, _('Delta RPM rebuild failed'))
+ if not po.verifyLocalPkg():
+ return adderror(po, _('Checksum of the delta-rebuilt RPM failed'))
+ os.unlink(deltapath)
+
+ # spawn a worker process
+ self.wait(self.limit)
+ pid = os.spawnl(os.P_NOWAIT, APPLYDELTA, APPLYDELTA, deltapath, po.localpath)
+ self.jobs[pid] = callback
diff --git a/yum/presto.py b/yum/presto.py
deleted file mode 100644
index 002a82a..0000000
--- a/yum/presto.py
+++ /dev/null
@@ -1,158 +0,0 @@
-# Integrated delta rpm support
-# Copyright 2013 Zdenek Pavlas
-
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the
-# Free Software Foundation, Inc.,
-# 59 Temple Place, Suite 330,
-# Boston, MA 02111-1307 USA
-
-from yum.constants import TS_UPDATE
-from yum.Errors import RepoError
-from yum.i18n import exception2msg, _
-from urlgrabber import grabber
-async = hasattr(grabber, 'parallel_wait')
-from xml.etree.cElementTree import iterparse
-import os, gzip
-
-APPLYDELTA = '/usr/bin/applydeltarpm'
-
-class Presto:
- def __init__(self, ayum, pkgs):
- self.verbose_logger = ayum.verbose_logger
- self.deltas = {}
- self._rpmsave = {}
- self.rpmsize = 0
- self.deltasize = 0
- self.jobs = {}
- self.limit = ayum.conf.presto
-
- # calculate update sizes
- pinfo = {}
- reposize = {}
- for po in pkgs:
- if not po.repo.presto:
- continue
- if po.state != TS_UPDATE and po.name not in ayum.conf.installonlypkgs:
- continue
- self.limit = max(self.limit, po.repo.presto)
- pinfo.setdefault(po.repo, {})[po.pkgtup] = po
- reposize[po.repo] = reposize.get(po.repo, 0) + po.size
-
- # don't use deltas when deltarpm not installed
- if reposize and not os.access(APPLYDELTA, os.X_OK):
- self.verbose_logger.info(_('Delta RPMs disabled because %s not installed.'), APPLYDELTA)
- return
-
- # download delta metadata
- mdpath = {}
- for repo in reposize:
- for name in ('prestodelta', 'deltainfo'):
- try: data = repo.repoXML.getData(name); break
- except: pass
- else:
- self.verbose_logger.warn(_('No Presto metadata available for %s'), repo)
- continue
- path = repo.cachedir +'/'+ os.path.basename(data.location[1])
- if not os.path.exists(path) and int(data.size) > reposize[repo]:
- self.verbose_logger.info(_('Not downloading Presto metadata for %s'), repo)
- continue
-
- def failfunc(e, name=name, repo=repo):
- mdpath.pop(repo, None)
- if hasattr(e, 'exception'): e = e.exception
- self.verbose_logger.warn(_('Failed to download %s for repository %s: %s'),
- name, repo, exception2msg(e))
- kwargs = {}
- if async and repo._async:
- kwargs['failfunc'] = failfunc
- kwargs['async'] = True
- try: mdpath[repo] = repo._retrieveMD(name, **kwargs)
- except Errors.RepoError, e: failfunc(e)
- if async:
- grabber.parallel_wait()
-
- # parse metadata, populate self.deltas
- for repo, path in mdpath.items():
- pinfo_repo = pinfo[repo]
- if path.endswith('.gz'):
- path = gzip.open(path)
- for ev, el in iterparse(path):
- if el.tag != 'newpackage': continue
- new = el.get('name'), el.get('arch'), el.get('epoch'), el.get('version'), el.get('release')
- po = pinfo_repo.get(new)
- if po:
- best = po.size * 0.75 # make this configurable?
- have = ayum._up.installdict.get(new[:2], [])
- for el in el.findall('delta'):
- size = int(el.find('size').text)
- old = el.get('oldepoch'), el.get('oldversion'), el.get('oldrelease')
- if size >= best or old not in have:
- continue
- best = size
- csum = el.find('checksum')
- csum = csum.get('type'), csum.text
- self.deltas[po] = size, el.find('filename').text, csum
- el.clear()
-
- def to_drpm(self, po):
- try: size, remote, csum = self.deltas[po]
- except KeyError: return False
- self._rpmsave[po] = po.packagesize, po.relativepath, po.localpath
-
- # update stats
- self.rpmsize += po.packagesize
- self.deltasize += size
-
- # update size/path/checksum to drpm values
- po.packagesize = size
- po.relativepath = remote
- po.localpath = os.path.dirname(po.localpath) +'/'+ os.path.basename(remote)
- po.returnIdSum = lambda: csum
- return True
-
- def to_rpm(self, po):
- if po not in self._rpmsave:
- return
- # revert back to RPM
- po.packagesize, po.relativepath, po.localpath = self._rpmsave.pop(po)
- del po.returnIdSum
-
- def wait(self, limit = 1):
- # wait for some jobs, run callbacks
- while len(self.jobs) >= limit:
- pid, code = os.wait()
- # urlgrabber spawns child jobs, too. But they exit synchronously,
- # so we should never see an unknown pid here.
- assert pid in self.jobs
- callback = self.jobs.pop(pid)
- callback(code)
-
- def rebuild(self, po, adderror):
- # restore rpm values
- deltapath = po.localpath
- po.packagesize, po.relativepath, po.localpath = self._rpmsave.pop(po)
- del po.returnIdSum
-
- # this runs when worker finishes
- def callback(code):
- if code != 0:
- return adderror(po, _('Delta RPM rebuild failed'))
- if not po.verifyLocalPkg():
- return adderror(po, _('Checksum of the delta-rebuilt RPM failed'))
- os.unlink(deltapath)
-
- # spawn a worker process
- self.wait(self.limit)
- pid = os.spawnl(os.P_NOWAIT, APPLYDELTA, APPLYDELTA, deltapath, po.localpath)
- self.jobs[pid] = callback
commit f7a296a6e105b3014ae76da675a8cc6da5ca9999
Author: Zdenek Pavlas <zpavlas at redhat.com>
Date: Tue Feb 26 09:53:17 2013 +0100
Deactivate presto if deltarpm is not installed. BZ 915426
diff --git a/yum/presto.py b/yum/presto.py
index cb33918..002a82a 100644
--- a/yum/presto.py
+++ b/yum/presto.py
@@ -25,6 +25,8 @@ async = hasattr(grabber, 'parallel_wait')
from xml.etree.cElementTree import iterparse
import os, gzip
+APPLYDELTA = '/usr/bin/applydeltarpm'
+
class Presto:
def __init__(self, ayum, pkgs):
self.verbose_logger = ayum.verbose_logger
@@ -47,6 +49,11 @@ class Presto:
pinfo.setdefault(po.repo, {})[po.pkgtup] = po
reposize[po.repo] = reposize.get(po.repo, 0) + po.size
+ # don't use deltas when deltarpm not installed
+ if reposize and not os.access(APPLYDELTA, os.X_OK):
+ self.verbose_logger.info(_('Delta RPMs disabled because %s not installed.'), APPLYDELTA)
+ return
+
# download delta metadata
mdpath = {}
for repo in reposize:
@@ -147,5 +154,5 @@ class Presto:
# spawn a worker process
self.wait(self.limit)
- pid = os.spawnl(os.P_NOWAIT, '/usr/bin/applydeltarpm', 'applydeltarpm', deltapath, po.localpath)
+ pid = os.spawnl(os.P_NOWAIT, APPLYDELTA, APPLYDELTA, deltapath, po.localpath)
self.jobs[pid] = callback
commit 659d3003b77aef6603b490bd2920638f4bbcab9e
Author: Zdenek Pavlas <zpavlas at redhat.com>
Date: Tue Feb 26 09:52:02 2013 +0100
Download drpms first, to overlap delta rebuild with rpm download.
diff --git a/yum/__init__.py b/yum/__init__.py
index 2128785..fca6d2d 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -2190,7 +2190,9 @@ much more problems).
a = apo.getDiscNum()
b = bpo.getDiscNum()
if a is None and b is None:
- return cmp(apo, bpo)
+ # deltas first to start rebuilding asap
+ return cmp(bpo in presto.deltas, apo in presto.deltas) \
+ or cmp(apo, bpo)
if a is None:
return -1
if b is None:
commit 95ffe1d098895849923795a438ded3a189c8d2f6
Author: Zdenek Pavlas <zpavlas at redhat.com>
Date: Thu Feb 21 17:16:28 2013 +0100
Use multiple applydeltarpm workers
diff --git a/docs/yum.conf.5 b/docs/yum.conf.5
index ba3f8e1..25f0c19 100644
--- a/docs/yum.conf.5
+++ b/docs/yum.conf.5
@@ -374,9 +374,8 @@ and the downloader honors these too.
.IP
\fBpresto\fR
-Either `0' or `1'. Set this to `1' to use delta-RPM files, if available.
-This reduces the download size of updates significantly, but local rebuild
-is CPU intensive. Default is `1' (on).
+When non-zero, delta-RPM files are used if available. The value specifies
+the maximum number of "applydeltarpm" processes Yum will spawn. (4 by default).
.IP
\fBsslcacert \fR
diff --git a/yum/__init__.py b/yum/__init__.py
index d378080..2128785 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -2328,6 +2328,7 @@ much more problems).
adderror(po, exception2msg(e))
if async:
urlgrabber.grabber.parallel_wait()
+ presto.wait()
if hasattr(urlgrabber.progress, 'text_meter_total_size'):
urlgrabber.progress.text_meter_total_size(0)
diff --git a/yum/config.py b/yum/config.py
index d279ab3..4aae831 100644
--- a/yum/config.py
+++ b/yum/config.py
@@ -791,7 +791,7 @@ class YumConf(StartupConf):
allowed = ('ipv4', 'ipv6', 'whatever'),
mapper = {'4': 'ipv4', '6': 'ipv6'})
max_connections = IntOption(0)
- presto = BoolOption(True)
+ presto = IntOption(4)
http_caching = SelectionOption('all', ('none', 'packages', 'all'))
metadata_expire = SecondsOption(60 * 60 * 6) # Time in seconds (6h).
diff --git a/yum/presto.py b/yum/presto.py
index 1805974..cb33918 100644
--- a/yum/presto.py
+++ b/yum/presto.py
@@ -23,7 +23,7 @@ from yum.i18n import exception2msg, _
from urlgrabber import grabber
async = hasattr(grabber, 'parallel_wait')
from xml.etree.cElementTree import iterparse
-import os, gzip, subprocess
+import os, gzip
class Presto:
def __init__(self, ayum, pkgs):
@@ -32,6 +32,8 @@ class Presto:
self._rpmsave = {}
self.rpmsize = 0
self.deltasize = 0
+ self.jobs = {}
+ self.limit = ayum.conf.presto
# calculate update sizes
pinfo = {}
@@ -41,6 +43,7 @@ class Presto:
continue
if po.state != TS_UPDATE and po.name not in ayum.conf.installonlypkgs:
continue
+ self.limit = max(self.limit, po.repo.presto)
pinfo.setdefault(po.repo, {})[po.pkgtup] = po
reposize[po.repo] = reposize.get(po.repo, 0) + po.size
@@ -118,17 +121,31 @@ class Presto:
po.packagesize, po.relativepath, po.localpath = self._rpmsave.pop(po)
del po.returnIdSum
+ def wait(self, limit = 1):
+ # wait for some jobs, run callbacks
+ while len(self.jobs) >= limit:
+ pid, code = os.wait()
+ # urlgrabber spawns child jobs, too. But they exit synchronously,
+ # so we should never see an unknown pid here.
+ assert pid in self.jobs
+ callback = self.jobs.pop(pid)
+ callback(code)
+
def rebuild(self, po, adderror):
# restore rpm values
deltapath = po.localpath
po.packagesize, po.relativepath, po.localpath = self._rpmsave.pop(po)
del po.returnIdSum
- # rebuild it from drpm
- if subprocess.call(['/usr/bin/applydeltarpm', deltapath, po.localpath]) != 0:
- return adderror(po, _('Delta RPM rebuild failed'))
- # source drpm was already checksummed.. is this necessary?
- if not po.verifyLocalPkg():
- return adderror(po, _('Checksum of the delta-rebuilt RPM failed'))
- # no need to keep this
- os.unlink(deltapath)
+ # this runs when worker finishes
+ def callback(code):
+ if code != 0:
+ return adderror(po, _('Delta RPM rebuild failed'))
+ if not po.verifyLocalPkg():
+ return adderror(po, _('Checksum of the delta-rebuilt RPM failed'))
+ os.unlink(deltapath)
+
+ # spawn a worker process
+ self.wait(self.limit)
+ pid = os.spawnl(os.P_NOWAIT, '/usr/bin/applydeltarpm', 'applydeltarpm', deltapath, po.localpath)
+ self.jobs[pid] = callback
commit 738b2bc2073546a24a289a97d80353edd27e3a5f
Author: Zdenek Pavlas <zpavlas at redhat.com>
Date: Thu Feb 21 15:40:34 2013 +0100
downloadonly: don't update localpath on errors, so we retry correctly.
diff --git a/yum/__init__.py b/yum/__init__.py
index f70cdc0..d378080 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -2348,7 +2348,7 @@ much more problems).
# verifyPkg() didn't complain, so (potentially)
# overwriting another copy should not be a problem
os.rename(po.localpath, rpmfile)
- po.localpath = rpmfile
+ po.localpath = rpmfile
fatal = False
for po in errors:
commit 4fa5ceefa1f63f774a56340a50735877e2e74eb4
Author: Zdenek Pavlas <zpavlas at redhat.com>
Date: Thu Feb 21 12:19:21 2013 +0100
remove the sequence check
The only case where this makes a difference is a broken old
package install. Delta rebuild fails, and we retry downloading
a full rpm file instead. Cost and probability of this is low.
OTOH, the check is quite slow, and avoiding it is a big win.
diff --git a/yum/presto.py b/yum/presto.py
index 3bcd391..1805974 100644
--- a/yum/presto.py
+++ b/yum/presto.py
@@ -89,12 +89,6 @@ class Presto:
old = el.get('oldepoch'), el.get('oldversion'), el.get('oldrelease')
if size >= best or old not in have:
continue
- # the old version is installed, seq check should never fail. kill this?
- seq = el.find('sequence').text
- if subprocess.call(['/usr/bin/applydeltarpm', '-C', '-s', seq]) != 0:
- self.verbose_logger.warn(_('Deltarpm sequence check failed for %s'), seq)
- continue
-
best = size
csum = el.find('checksum')
csum = csum.get('type'), csum.text
commit 76a5787a799de1e15ebc6bf89fc52346b75d1075
Author: Zdenek Pavlas <zpavlas at redhat.com>
Date: Wed Feb 20 18:17:25 2013 +0100
Add DRPM => RPM retries
It's ok to try downloading rpms when delta rebuild fails,
but when we can't DL a drpm, rpm probably fails, too.
To handle possibly obsolete presto MD, retry anyway..
diff --git a/yum/__init__.py b/yum/__init__.py
index bb4d4a7..f70cdc0 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -2275,7 +2275,7 @@ much more problems).
self.closeRpmDB()
self.doUnlock()
- if 1:
+ while True:
remote_pkgs.sort(mediasort)
# This is kind of a hack and does nothing in non-Fedora versions,
# we'll fix it one way or anther soon.
@@ -2350,6 +2350,22 @@ much more problems).
os.rename(po.localpath, rpmfile)
po.localpath = rpmfile
+ fatal = False
+ for po in errors:
+ if po not in presto.deltas:
+ fatal = True; break
+ if not errors or fatal:
+ break
+
+ # there were drpm related errors *only*
+ remote_pkgs = errors.keys()
+ remote_size = 0
+ for po in remote_pkgs:
+ presto.to_rpm(po) # needed, we don't rebuild() when DL fails
+ remote_size += po.size
+ self.verbose_logger.warn(_('Some delta RPMs failed to download or rebuild. Retrying..'))
+ presto.deltas.clear() # any error is now considered fatal
+
if not downloadonly:
# XXX: Run unlocked? Skip this for now..
self.plugins.run('postdownload', pkglist=pkglist, errors=errors)
diff --git a/yum/presto.py b/yum/presto.py
index 08462f5..3bcd391 100644
--- a/yum/presto.py
+++ b/yum/presto.py
@@ -99,8 +99,6 @@ class Presto:
csum = el.find('checksum')
csum = csum.get('type'), csum.text
self.deltas[po] = size, el.find('filename').text, csum
- if po not in self.deltas:
- self.verbose_logger.warn(_('No suitable drpm files for %s'), po)
el.clear()
def to_drpm(self, po):
@@ -119,10 +117,17 @@ class Presto:
po.returnIdSum = lambda: csum
return True
+ def to_rpm(self, po):
+ if po not in self._rpmsave:
+ return
+ # revert back to RPM
+ po.packagesize, po.relativepath, po.localpath = self._rpmsave.pop(po)
+ del po.returnIdSum
+
def rebuild(self, po, adderror):
# restore rpm values
deltapath = po.localpath
- po.packagesize, po.relativepath, po.localpath = self._rpmsave[po]
+ po.packagesize, po.relativepath, po.localpath = self._rpmsave.pop(po)
del po.returnIdSum
# rebuild it from drpm
commit 931828c457175376f8bbd8d94129c060b7252b1a
Author: Zdenek Pavlas <zpavlas at redhat.com>
Date: Wed Feb 20 18:09:07 2013 +0100
indent only, to make next patch easier to read
diff --git a/yum/__init__.py b/yum/__init__.py
index c202d36..bb4d4a7 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -2275,80 +2275,82 @@ much more problems).
self.closeRpmDB()
self.doUnlock()
- remote_pkgs.sort(mediasort)
- # This is kind of a hack and does nothing in non-Fedora versions,
- # we'll fix it one way or anther soon.
- if (hasattr(urlgrabber.progress, 'text_meter_total_size') and
- len(remote_pkgs) > 1):
- urlgrabber.progress.text_meter_total_size(remote_size)
- beg_download = time.time()
- i = 0
- local_size = [0]
- done_repos = set()
- async = hasattr(urlgrabber.grabber, 'parallel_wait')
- for po in remote_pkgs:
- i += 1
-
- def checkfunc(obj, po=po):
- self.verifyPkg(obj, po, 1)
- local_size[0] += po.size
- if hasattr(urlgrabber.progress, 'text_meter_total_size'):
- urlgrabber.progress.text_meter_total_size(remote_size,
- local_size[0])
- if po in presto.deltas:
- presto.rebuild(po, adderror)
- return
- if po.repoid not in done_repos:
- done_repos.add(po.repoid)
- # Check a single package per. repo. ... to give a hint to
- # the user on big downloads.
- result, errmsg = self.sigCheckPkg(po)
- if result != 0:
- self.verbose_logger.warn("%s", errmsg)
- po.localpath = obj.filename
- if po in errors:
- del errors[po]
-
- text = os.path.basename(po.relativepath)
- kwargs = {}
- if async and po.repo._async:
- kwargs['failfunc'] = lambda obj, po=po: adderror(po, exception2msg(obj.exception))
- kwargs['async'] = True
- elif not (i == 1 and not local_size[0] and remote_size == po.size):
- text = '(%s/%s): %s' % (i, len(remote_pkgs), text)
- try:
- po.repo.getPackage(po,
- checkfunc=checkfunc,
- text=text,
- cache=po.repo.http_caching != 'none',
- **kwargs
- )
- except Errors.RepoError, e:
- adderror(po, exception2msg(e))
- if async:
- urlgrabber.grabber.parallel_wait()
-
- if hasattr(urlgrabber.progress, 'text_meter_total_size'):
- urlgrabber.progress.text_meter_total_size(0)
- if callback_total is not None and not errors:
- callback_total(remote_pkgs, remote_size, beg_download)
-
- if downloadonly:
+ if 1:
+ remote_pkgs.sort(mediasort)
+ # This is kind of a hack and does nothing in non-Fedora versions,
+ # we'll fix it one way or anther soon.
+ if (hasattr(urlgrabber.progress, 'text_meter_total_size') and
+ len(remote_pkgs) > 1):
+ urlgrabber.progress.text_meter_total_size(remote_size)
+ beg_download = time.time()
+ i = 0
+ local_size = [0]
+ done_repos = set()
+ async = hasattr(urlgrabber.grabber, 'parallel_wait')
for po in remote_pkgs:
- rpmfile = po.localpath.rsplit('.', 2)[0]
- if po in errors:
- # we may throw away partial file here- but we don't lock,
- # so can't rename tempfile to rpmfile safely
- misc.unlink_f(po.localpath)
-
- # Note that for file:// repos. urlgrabber won't "download"
- # so we have to check that po.localpath exists.
- elif os.path.exists(po.localpath):
- # verifyPkg() didn't complain, so (potentially)
- # overwriting another copy should not be a problem
- os.rename(po.localpath, rpmfile)
- po.localpath = rpmfile
- else:
+ i += 1
+
+ def checkfunc(obj, po=po):
+ self.verifyPkg(obj, po, 1)
+ local_size[0] += po.size
+ if hasattr(urlgrabber.progress, 'text_meter_total_size'):
+ urlgrabber.progress.text_meter_total_size(remote_size,
+ local_size[0])
+ if po in presto.deltas:
+ presto.rebuild(po, adderror)
+ return
+ if po.repoid not in done_repos:
+ done_repos.add(po.repoid)
+ # Check a single package per. repo. ... to give a hint to
+ # the user on big downloads.
+ result, errmsg = self.sigCheckPkg(po)
+ if result != 0:
+ self.verbose_logger.warn("%s", errmsg)
+ po.localpath = obj.filename
+ if po in errors:
+ del errors[po]
+
+ text = os.path.basename(po.relativepath)
+ kwargs = {}
+ if async and po.repo._async:
+ kwargs['failfunc'] = lambda obj, po=po: adderror(po, exception2msg(obj.exception))
+ kwargs['async'] = True
+ elif not (i == 1 and not local_size[0] and remote_size == po.size):
+ text = '(%s/%s): %s' % (i, len(remote_pkgs), text)
+ try:
+ po.repo.getPackage(po,
+ checkfunc=checkfunc,
+ text=text,
+ cache=po.repo.http_caching != 'none',
+ **kwargs
+ )
+ except Errors.RepoError, e:
+ adderror(po, exception2msg(e))
+ if async:
+ urlgrabber.grabber.parallel_wait()
+
+ if hasattr(urlgrabber.progress, 'text_meter_total_size'):
+ urlgrabber.progress.text_meter_total_size(0)
+ if callback_total is not None and not errors:
+ callback_total(remote_pkgs, remote_size, beg_download)
+
+ if downloadonly:
+ for po in remote_pkgs:
+ rpmfile = po.localpath.rsplit('.', 2)[0]
+ if po in errors:
+ # we may throw away partial file here- but we don't lock,
+ # so can't rename tempfile to rpmfile safely
+ misc.unlink_f(po.localpath)
+
+ # Note that for file:// repos. urlgrabber won't "download"
+ # so we have to check that po.localpath exists.
+ elif os.path.exists(po.localpath):
+ # verifyPkg() didn't complain, so (potentially)
+ # overwriting another copy should not be a problem
+ os.rename(po.localpath, rpmfile)
+ po.localpath = rpmfile
+
+ if not downloadonly:
# XXX: Run unlocked? Skip this for now..
self.plugins.run('postdownload', pkglist=pkglist, errors=errors)
commit 68d94e39bfa0d4b33ab72321318e9e5435d609af
Author: Zdenek Pavlas <zpavlas at redhat.com>
Date: Wed Feb 20 12:43:53 2013 +0100
Integrate presto support to yum
Add the "presto" bool config option, document it.
Add hooks to downloadPkgs().
diff --git a/docs/yum.conf.5 b/docs/yum.conf.5
index 6f400cf..ba3f8e1 100644
--- a/docs/yum.conf.5
+++ b/docs/yum.conf.5
@@ -372,6 +372,13 @@ default of 5 connections. Note that there are also implicit per-mirror limits
and the downloader honors these too.
.IP
+\fBpresto\fR
+
+Either `0' or `1'. Set this to `1' to use delta-RPM files, if available.
+This reduces the download size of updates significantly, but local rebuild
+is CPU intensive. Default is `1' (on).
+
+.IP
\fBsslcacert \fR
Path to the directory containing the databases of the certificate authorities
yum should use to verify SSL certificates. Defaults to none - uses system
@@ -930,6 +937,11 @@ repository.
Overrides the \fBip_resolve\fR option from the [main] section for this
repository.
+.IP
+\fBpresto\fR
+
+Overrides the \fBpresto\fR option from the [main] section for this
+repository.
.IP
\fBsslcacert \fR
diff --git a/yum/__init__.py b/yum/__init__.py
index b445831..c202d36 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -90,6 +90,7 @@ from packages import YumUrlPackage, YumNotFoundPackage
from constants import *
from yum.rpmtrans import RPMTransaction,SimpleCliCallBack
from yum.i18n import to_unicode, to_str, exception2msg
+from yum.presto import Presto
import string
import StringIO
@@ -2244,6 +2245,7 @@ much more problems).
po.basepath # prefetch now; fails when repos are closed
return False
+ pkgs = []
for po in pkglist:
if hasattr(po, 'pkgtype') and po.pkgtype == 'local':
continue
@@ -2251,8 +2253,21 @@ much more problems).
continue
if errors:
return errors
+ pkgs.append(po)
+
+ # download presto metadata
+ presto = Presto(self, pkgs)
+ for po in pkgs:
+ if presto.to_drpm(po) and verify_local(po):
+ # there's .drpm already, use it
+ presto.rebuild(po, adderror)
+ continue
remote_pkgs.append(po)
remote_size += po.size
+ if presto.deltasize:
+ self.verbose_logger.info(_('Delta RPMs reduced %s of updates to %s (%d%% saved)'),
+ format_number(presto.rpmsize), format_number(presto.deltasize),
+ 100 - presto.deltasize*100.0/presto.rpmsize)
if downloadonly:
# close DBs, unlock
@@ -2280,6 +2295,9 @@ much more problems).
if hasattr(urlgrabber.progress, 'text_meter_total_size'):
urlgrabber.progress.text_meter_total_size(remote_size,
local_size[0])
+ if po in presto.deltas:
+ presto.rebuild(po, adderror)
+ return
if po.repoid not in done_repos:
done_repos.add(po.repoid)
# Check a single package per. repo. ... to give a hint to
diff --git a/yum/config.py b/yum/config.py
index 3b22e41..d279ab3 100644
--- a/yum/config.py
+++ b/yum/config.py
@@ -791,6 +791,7 @@ class YumConf(StartupConf):
allowed = ('ipv4', 'ipv6', 'whatever'),
mapper = {'4': 'ipv4', '6': 'ipv6'})
max_connections = IntOption(0)
+ presto = BoolOption(True)
http_caching = SelectionOption('all', ('none', 'packages', 'all'))
metadata_expire = SecondsOption(60 * 60 * 6) # Time in seconds (6h).
@@ -949,6 +950,7 @@ class RepoConf(BaseConfig):
throttle = Inherit(YumConf.throttle)
timeout = Inherit(YumConf.timeout)
ip_resolve = Inherit(YumConf.ip_resolve)
+ presto = Inherit(YumConf.presto)
http_caching = Inherit(YumConf.http_caching)
metadata_expire = Inherit(YumConf.metadata_expire)
diff --git a/yum/presto.py b/yum/presto.py
new file mode 100644
index 0000000..08462f5
--- /dev/null
+++ b/yum/presto.py
@@ -0,0 +1,135 @@
+# Integrated delta rpm support
+# Copyright 2013 Zdenek Pavlas
+
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc.,
+# 59 Temple Place, Suite 330,
+# Boston, MA 02111-1307 USA
+
+from yum.constants import TS_UPDATE
+from yum.Errors import RepoError
+from yum.i18n import exception2msg, _
+from urlgrabber import grabber
+async = hasattr(grabber, 'parallel_wait')
+from xml.etree.cElementTree import iterparse
+import os, gzip, subprocess
+
+class Presto:
+ def __init__(self, ayum, pkgs):
+ self.verbose_logger = ayum.verbose_logger
+ self.deltas = {}
+ self._rpmsave = {}
+ self.rpmsize = 0
+ self.deltasize = 0
+
+ # calculate update sizes
+ pinfo = {}
+ reposize = {}
+ for po in pkgs:
+ if not po.repo.presto:
+ continue
+ if po.state != TS_UPDATE and po.name not in ayum.conf.installonlypkgs:
+ continue
+ pinfo.setdefault(po.repo, {})[po.pkgtup] = po
+ reposize[po.repo] = reposize.get(po.repo, 0) + po.size
+
+ # download delta metadata
+ mdpath = {}
+ for repo in reposize:
+ for name in ('prestodelta', 'deltainfo'):
+ try: data = repo.repoXML.getData(name); break
+ except: pass
+ else:
+ self.verbose_logger.warn(_('No Presto metadata available for %s'), repo)
+ continue
+ path = repo.cachedir +'/'+ os.path.basename(data.location[1])
+ if not os.path.exists(path) and int(data.size) > reposize[repo]:
+ self.verbose_logger.info(_('Not downloading Presto metadata for %s'), repo)
+ continue
+
+ def failfunc(e, name=name, repo=repo):
+ mdpath.pop(repo, None)
+ if hasattr(e, 'exception'): e = e.exception
+ self.verbose_logger.warn(_('Failed to download %s for repository %s: %s'),
+ name, repo, exception2msg(e))
+ kwargs = {}
+ if async and repo._async:
+ kwargs['failfunc'] = failfunc
+ kwargs['async'] = True
+ try: mdpath[repo] = repo._retrieveMD(name, **kwargs)
+ except Errors.RepoError, e: failfunc(e)
+ if async:
+ grabber.parallel_wait()
+
+ # parse metadata, populate self.deltas
+ for repo, path in mdpath.items():
+ pinfo_repo = pinfo[repo]
+ if path.endswith('.gz'):
+ path = gzip.open(path)
+ for ev, el in iterparse(path):
+ if el.tag != 'newpackage': continue
+ new = el.get('name'), el.get('arch'), el.get('epoch'), el.get('version'), el.get('release')
+ po = pinfo_repo.get(new)
+ if po:
+ best = po.size * 0.75 # make this configurable?
+ have = ayum._up.installdict.get(new[:2], [])
+ for el in el.findall('delta'):
+ size = int(el.find('size').text)
+ old = el.get('oldepoch'), el.get('oldversion'), el.get('oldrelease')
+ if size >= best or old not in have:
+ continue
+ # the old version is installed, seq check should never fail. kill this?
+ seq = el.find('sequence').text
+ if subprocess.call(['/usr/bin/applydeltarpm', '-C', '-s', seq]) != 0:
+ self.verbose_logger.warn(_('Deltarpm sequence check failed for %s'), seq)
+ continue
+
+ best = size
+ csum = el.find('checksum')
+ csum = csum.get('type'), csum.text
+ self.deltas[po] = size, el.find('filename').text, csum
+ if po not in self.deltas:
+ self.verbose_logger.warn(_('No suitable drpm files for %s'), po)
+ el.clear()
+
+ def to_drpm(self, po):
+ try: size, remote, csum = self.deltas[po]
+ except KeyError: return False
+ self._rpmsave[po] = po.packagesize, po.relativepath, po.localpath
+
+ # update stats
+ self.rpmsize += po.packagesize
+ self.deltasize += size
+
+ # update size/path/checksum to drpm values
+ po.packagesize = size
+ po.relativepath = remote
+ po.localpath = os.path.dirname(po.localpath) +'/'+ os.path.basename(remote)
+ po.returnIdSum = lambda: csum
+ return True
+
+ def rebuild(self, po, adderror):
+ # restore rpm values
+ deltapath = po.localpath
+ po.packagesize, po.relativepath, po.localpath = self._rpmsave[po]
+ del po.returnIdSum
+
+ # rebuild it from drpm
+ if subprocess.call(['/usr/bin/applydeltarpm', deltapath, po.localpath]) != 0:
+ return adderror(po, _('Delta RPM rebuild failed'))
+ # source drpm was already checksummed.. is this necessary?
+ if not po.verifyLocalPkg():
+ return adderror(po, _('Checksum of the delta-rebuilt RPM failed'))
+ # no need to keep this
+ os.unlink(deltapath)
commit e3e092ea326d1038631414097e88c8d0e7f86b94
Author: Zdenek Pavlas <zpavlas at redhat.com>
Date: Fri Feb 8 15:20:30 2013 +0100
downloadPkgs() cleanup
add check_local() function. Kill the "Recheck if the file is there"
code, add a duplicated package check and warning instead.
diff --git a/yum/__init__.py b/yum/__init__.py
index d485d05..b445831 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -2200,9 +2200,6 @@ much more problems).
return 1
return 0
- """download list of package objects handed to you, output based on
- callback, raise yum.Errors.YumBaseError on problems"""
-
errors = {}
def adderror(po, msg):
errors.setdefault(po, []).append(msg)
@@ -2218,41 +2215,45 @@ much more problems).
self.history.close()
self.plugins.run('predownload', pkglist=pkglist)
+ beenthere = set() # only once, please. BZ 468401
downloadonly = getattr(self.conf, 'downloadonly', False)
- repo_cached = False
remote_pkgs = []
remote_size = 0
- for po in pkglist:
- if hasattr(po, 'pkgtype') and po.pkgtype == 'local':
- continue
-
+
+ def verify_local(po):
local = po.localPkg()
+ if local in beenthere:
+ # This is definitely a depsolver bug. Make it fatal?
+ self.verbose_logger.warn(_("ignoring a dupe of %s") % po)
+ return True
+ beenthere.add(local)
if os.path.exists(local):
- if not self.verifyPkg(local, po, False):
- if po.repo.cache:
- repo_cached = True
- adderror(po, _('package fails checksum but caching is '
- 'enabled for %s') % po.repo.id)
- else:
- self.verbose_logger.debug(_("using local copy of %s") %(po,))
- continue
-
+ if self.verifyPkg(local, po, False):
+ self.verbose_logger.debug(_("using local copy of %s") % po)
+ return True
+ if po.repo.cache:
+ adderror(po, _('package fails checksum but caching is '
+ 'enabled for %s') % po.repo.id)
+ return False
+ if os.path.getsize(local) >= po.size:
+ os.unlink(local)
if downloadonly:
- # download to temp file
- rpmfile = po.localpath
po.localpath += '.%d.tmp' % os.getpid()
- try: os.rename(rpmfile, po.localpath)
+ try: os.rename(local, po.localpath)
except OSError: pass
po.basepath # prefetch now; fails when repos are closed
+ return False
+ for po in pkglist:
+ if hasattr(po, 'pkgtype') and po.pkgtype == 'local':
+ continue
+ if verify_local(po):
+ continue
+ if errors:
+ return errors
remote_pkgs.append(po)
remote_size += po.size
-
- # caching is enabled and the package
- # just failed to check out there's no
- # way to save this, report the error and return
- if (self.conf.cache or repo_cached) and errors:
- return errors
+
if downloadonly:
# close DBs, unlock
self.repos.close()
@@ -2271,20 +2272,7 @@ much more problems).
done_repos = set()
async = hasattr(urlgrabber.grabber, 'parallel_wait')
for po in remote_pkgs:
- # Recheck if the file is there, works around a couple of weird
- # edge cases.
- local = po.localPkg()
i += 1
- if os.path.exists(local):
- if self.verifyPkg(local, po, False):
- self.verbose_logger.debug(_("using local copy of %s") %(po,))
- remote_size -= po.size
- if hasattr(urlgrabber.progress, 'text_meter_total_size'):
- urlgrabber.progress.text_meter_total_size(remote_size,
- local_size[0])
- continue
- if os.path.getsize(local) >= po.size:
- os.unlink(local)
def checkfunc(obj, po=po):
self.verifyPkg(obj, po, 1)
@@ -2337,7 +2325,7 @@ much more problems).
# Note that for file:// repos. urlgrabber won't "download"
# so we have to check that po.localpath exists.
- if po not in errors and os.path.exists(po.localpath):
+ elif os.path.exists(po.localpath):
# verifyPkg() didn't complain, so (potentially)
# overwriting another copy should not be a problem
os.rename(po.localpath, rpmfile)
More information about the Yum-commits
mailing list