[Yum-devel] [PATCH 2/4] use pipe instead of tempfiles.. ~5% speedup

[Zdeněk Pavlas]

---
 createrepo/__init__.py |   45 ++++++++++++---------------------------------
 1 files changed, 12 insertions(+), 33 deletions(-)

diff --git a/createrepo/__init__.py b/createrepo/__init__.py
index 167d384..622e442 100644
--- a/createrepo/__init__.py
+++ b/createrepo/__init__.py
@@ -26,7 +26,6 @@ import tempfile
 import stat
 import fcntl
 import subprocess
-from select import select
 
 from yum import misc, Errors
 from yum.repoMDObject import RepoMD, RepoData
@@ -636,38 +635,30 @@ class MetaDataGenerator:
                 f.write('\n'.join(worker_chunks[worker_num]))
                 f.close()
                 
-                # make the worker directory
                 workercmdline = []
                 workercmdline.extend(base_worker_cmdline)
-                thisdir = self._worker_tmp_path + '/' + str(worker_num)
-                if checkAndMakeDir(thisdir):
-                    workercmdline.append('--tmpmdpath=%s' % thisdir)
-                else:
-                    raise MDError, "Unable to create worker path: %s" % thisdir
                 workercmdline.append('--pkglist=%s/pkglist-%s' % (self._worker_tmp_path, worker_num))
                 worker_cmd_dict[worker_num] = workercmdline
-            
-                
 
-            fds = {}
             for (num, cmdline) in worker_cmd_dict.items():
                 if not self.conf.quiet:
                     self.callback.log("Spawning worker %s with %s pkgs" % (num, 
                                                       len(worker_chunks[num])))
-                job = subprocess.Popen(cmdline, stdout=subprocess.PIPE,
-                                        stderr=subprocess.PIPE)
+                job = subprocess.Popen(cmdline, stdout=subprocess.PIPE)
                 worker_jobs[num] = job
-                fds[job.stdout.fileno()] = num, job.stdout, self.callback.log
-                fds[job.stderr.fileno()] = num, job.stderr, self.callback.errorlog
             
-            while fds:
-                for fd in select(fds, [], [])[0]:
-                    num, stream, logger = fds[fd]
+            files = self.primaryfile, self.flfile, self.otherfile
+            for i, pkg in enumerate(pkgfiles):
+                num = i % self.conf.workers
+                stream = worker_jobs[num].stdout
+                while True:
                     line = stream.readline()
-                    if line == '':
-                        del fds[fd]
-                        continue
-                    logger('Worker %s: %s' % (num, line.rstrip()))
+                    if line.startswith('*** '): break
+                    self.callback.log('Worker %s: %s' % (num, line.rstrip()))
+
+                # save the output to local files
+                for out, size in zip(files, line[4:].split()):
+                    out.write(stream.read(int(size)))
 
             for (num, job) in worker_jobs.items():
                 if job.wait() != 0:
@@ -677,18 +668,6 @@ class MetaDataGenerator:
                     
             if not self.conf.quiet:
                 self.callback.log("Workers Finished")
-            # finished with workers
-            # go to their dirs and add the contents
-            if not self.conf.quiet:
-                self.callback.log("Gathering worker results")
-            for num in range(self.conf.workers):
-                for (fn, fo) in (('primary.xml', self.primaryfile), 
-                           ('filelists.xml', self.flfile),
-                           ('other.xml', self.otherfile)):
-                    fnpath = self._worker_tmp_path + '/' + str(num) + '/' + fn
-                    if os.path.exists(fnpath):
-                        fo.write(open(fnpath, 'r').read())
-
                     
             for pkgfile in pkgfiles:
                 if self.conf.deltas:
-- 
1.7.4.4