[Yum-devel] [PATCH 3/3] Use parallel urlgrab in downloadPkgs().
Zdeněk Pavlas
zpavlas at redhat.com
Fri Oct 21 14:30:40 UTC 2011
Changes are local to the last 'po' loop. Error path is elliminated
with the 'errors' argument to getPackage(). OK path is merged to
checkfunc.
---
yum/__init__.py | 58 ++++++++++++++++++++++++------------------------------
1 files changed, 26 insertions(+), 32 deletions(-)
diff --git a/yum/__init__.py b/yum/__init__.py
index b07bbb2..e29b219 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -65,6 +65,7 @@ from parser import ConfigPreProcessor, varReplace
import transactioninfo
import urlgrabber
from urlgrabber.grabber import URLGrabber, URLGrabError
+from urlgrabber.grabber import parallel_wait
from urlgrabber.progress import format_number
from packageSack import packagesNewestByName, packagesNewestByNameArch, ListPackageSack
import depsolve
@@ -2138,14 +2139,26 @@ class YumBase(depsolve.Depsolve):
len(remote_pkgs) > 1):
urlgrabber.progress.text_meter_total_size(remote_size)
beg_download = time.time()
- i = 0
local_size = 0
+
+ # check function
done_repos = set()
+ def checkfunc(obj, po):
+ self.verifyPkg(obj, po, 1)
+ if po.repoid not in done_repos:
+ done_repos.add(po.repoid)
+ # Check a single package per. repo. ... to give a hint to
+ # the user on big downloads.
+ result, errmsg = self.sigCheckPkg(po)
+ if result != 0:
+ self.verbose_logger.warn("%s", errmsg)
+ if po in errors:
+ del errors[po]
+
for po in remote_pkgs:
# Recheck if the file is there, works around a couple of weird
# edge cases.
local = po.localPkg()
- i += 1
if os.path.exists(local):
if self.verifyPkg(local, po, False):
self.verbose_logger.debug(_("using local copy of %s") %(po,))
@@ -2157,36 +2170,17 @@ class YumBase(depsolve.Depsolve):
if os.path.getsize(local) >= po.size:
os.unlink(local)
- checkfunc = (self.verifyPkg, (po, 1), {})
- try:
- if i == 1 and not local_size and remote_size == po.size:
- text = os.path.basename(po.relativepath)
- else:
- text = '(%s/%s): %s' % (i, len(remote_pkgs),
- os.path.basename(po.relativepath))
- mylocal = po.repo.getPackage(po,
- checkfunc=checkfunc,
- text=text,
- cache=po.repo.http_caching != 'none',
- )
- local_size += po.size
- if hasattr(urlgrabber.progress, 'text_meter_total_size'):
- urlgrabber.progress.text_meter_total_size(remote_size,
- local_size)
- if po.repoid not in done_repos:
- # Check a single package per. repo. ... to give a hint to
- # the user on big downloads.
- result, errmsg = self.sigCheckPkg(po)
- if result != 0:
- self.verbose_logger.warn("%s", errmsg)
- done_repos.add(po.repoid)
-
- except Errors.RepoError, e:
- adderror(po, exception2msg(e))
- else:
- po.localpath = mylocal
- if po in errors:
- del errors[po]
+ po.localpath = po.repo.getPackage(
+ po,
+ checkfunc=(checkfunc, (po,), {}),
+ text=os.path.basename(po.relativepath),
+ cache=po.repo.http_caching != 'none',
+ errors=lambda msg, po=po: adderror(po, msg),
+ )
+ local_size += po.size
+ if hasattr(urlgrabber.progress, 'text_meter_total_size'):
+ urlgrabber.progress.text_meter_total_size(remote_size, local_size)
+ parallel_wait()
if hasattr(urlgrabber.progress, 'text_meter_total_size'):
urlgrabber.progress.text_meter_total_size(0)
--
1.7.4.4
More information about the Yum-devel
mailing list