[Yum-devel] [PATCH 2/2] Use parallel urlgrab in downloadPkgs().

Zdeněk Pavlas zpavlas at redhat.com
Mon Nov 7 11:02:15 UTC 2011


Changes are local to the last 'po' loop.  Error path is elliminated
with the 'errors' argument to getPackage().  OK path is merged to
checkfunc.
---
 yum/__init__.py |   58 ++++++++++++++++++++++++------------------------------
 1 files changed, 26 insertions(+), 32 deletions(-)

diff --git a/yum/__init__.py b/yum/__init__.py
index b07bbb2..e29b219 100644
--- a/yum/__init__.py
+++ b/yum/__init__.py
@@ -65,6 +65,7 @@ from parser import ConfigPreProcessor, varReplace
 import transactioninfo
 import urlgrabber
 from urlgrabber.grabber import URLGrabber, URLGrabError
+from urlgrabber.grabber import parallel_wait
 from urlgrabber.progress import format_number
 from packageSack import packagesNewestByName, packagesNewestByNameArch, ListPackageSack
 import depsolve
@@ -2138,14 +2139,26 @@ class YumBase(depsolve.Depsolve):
             len(remote_pkgs) > 1):
             urlgrabber.progress.text_meter_total_size(remote_size)
         beg_download = time.time()
-        i = 0
         local_size = 0
+
+        # check function
         done_repos = set()
+        def checkfunc(obj, po):
+            self.verifyPkg(obj, po, 1)
+            if po.repoid not in done_repos:
+                done_repos.add(po.repoid)
+                #  Check a single package per. repo. ... to give a hint to
+                # the user on big downloads.
+                result, errmsg = self.sigCheckPkg(po)
+                if result != 0:
+                    self.verbose_logger.warn("%s", errmsg)
+            if po in errors:
+                del errors[po]
+
         for po in remote_pkgs:
             #  Recheck if the file is there, works around a couple of weird
             # edge cases.
             local = po.localPkg()
-            i += 1
             if os.path.exists(local):
                 if self.verifyPkg(local, po, False):
                     self.verbose_logger.debug(_("using local copy of %s") %(po,))
@@ -2157,36 +2170,17 @@ class YumBase(depsolve.Depsolve):
                 if os.path.getsize(local) >= po.size:
                     os.unlink(local)
 
-            checkfunc = (self.verifyPkg, (po, 1), {})
-            try:
-                if i == 1 and not local_size and remote_size == po.size:
-                    text = os.path.basename(po.relativepath)
-                else:
-                    text = '(%s/%s): %s' % (i, len(remote_pkgs),
-                                            os.path.basename(po.relativepath))
-                mylocal = po.repo.getPackage(po,
-                                   checkfunc=checkfunc,
-                                   text=text,
-                                   cache=po.repo.http_caching != 'none',
-                                   )
-                local_size += po.size
-                if hasattr(urlgrabber.progress, 'text_meter_total_size'):
-                    urlgrabber.progress.text_meter_total_size(remote_size,
-                                                              local_size)
-                if po.repoid not in done_repos:
-                    #  Check a single package per. repo. ... to give a hint to
-                    # the user on big downloads.
-                    result, errmsg = self.sigCheckPkg(po)
-                    if result != 0:
-                        self.verbose_logger.warn("%s", errmsg)
-                done_repos.add(po.repoid)
-
-            except Errors.RepoError, e:
-                adderror(po, exception2msg(e))
-            else:
-                po.localpath = mylocal
-                if po in errors:
-                    del errors[po]
+            po.localpath = po.repo.getPackage(
+                po,
+                checkfunc=(checkfunc, (po,), {}),
+                text=os.path.basename(po.relativepath),
+                cache=po.repo.http_caching != 'none',
+                errors=lambda msg, po=po: adderror(po, msg),
+            )
+            local_size += po.size
+            if hasattr(urlgrabber.progress, 'text_meter_total_size'):
+                urlgrabber.progress.text_meter_total_size(remote_size, local_size)
+        parallel_wait()
 
         if hasattr(urlgrabber.progress, 'text_meter_total_size'):
             urlgrabber.progress.text_meter_total_size(0)
-- 
1.7.4.4



More information about the Yum-devel mailing list