[Yum-devel] [UG] progress
Michael Stenner
mstenner at linux.duke.edu
Thu Jul 22 03:08:53 UTC 2004
The subject is a bit of a pun. I am pleased to see some progress on
urlgrabber, but my major commit just now was the new progress meter
code. It doesn't include any test code at the moment, but I'm
attaching some rough test code.
Also, much of the code in there now will only really be used by the
threaded stuff. There is a version that basically reproduces what the
old one did. Ryan, I'd really love if you could spend some time
figuring out what I did and seeing if you would do things
differently. I ran into a few serious hurdles that I haven't entirely
solved yet, mostly related to threading, failures, regets, etc. I
need to remind myself about them, too, but I think I'm ready (probably
past ready) for another pair of eyes.
-Michael
--
Michael D. Stenner mstenner at ece.arizona.edu
ECE Department, the University of Arizona 520-626-1619
1230 E. Speedway Blvd., Tucson, AZ 85721-0104 ECE 524G
-------------- next part --------------
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330,
# Boston, MA 02111-1307 USA
# This file is part of urlgrabber, a high-level cross-protocol url-grabber
# Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko
"""Tests of the new progress meter. Not making this real live unit
tests yet because it's a real PITA. This is just for getting things working
"""
from urlgrabber.progress import *
def timing_tester(size, rates, step, functions, realtime=0, now=0.0):
startfunc, midfunc, endfunc = functions[0:3]
functions = list(functions[3:])
if functions: failurefunc = functions.pop(0)
remaining = size
if startfunc: startfunc(0, now)
i = 0
while remaining > 0:
rate, cutoff = rates[i]
if rate is None:
failurefunc('failure message\nfoo', now)
endfunc = None
break
if midfunc: midfunc(size - remaining, now)
now += step
if realtime: time.sleep(step)
remaining -= int(step * rate)
if cutoff and cutoff < now: i += 1
if endfunc: endfunc(size, now)
return now
def test_text_meter():
meter = text_progress_meter(fo=sys.stdout)
size = 1 << 20
def start(read, now):
meter.start('filename', 'http:/url.com/filename', 'basename',
size, now)
def mid(read, now):
meter.update(read, now)
def end(read, now):
meter.end(read, now)
functions = (start, mid, end)
rates = [(0.1*size, 4), (0.05*size, None)]
timing_tester(size, rates, 0.1, functions, realtime=0)
def test_text_mf_meter():
meter = TextMultiFileMeter(fo=sys.stdout)
size = 1 << 20
numchildren = 4
numfiles = 20
iter = 0
children = [meter.newMeter() for i in range(4)]
def start(read, now):
i = 0
for m in children:
id = i + numchildren * iter
m.start('filename', 'http:/url.com/filename', 'basename %i' % id,
size, now)
i += 1
def mid(read, now):
for m in children:
m.update(read, now)
def end(read, now):
for m in children:
m.end(read, now)
functions = (start, mid, end)
rates = [(0.1*size, None)]
now = 0.0
meter.start(numfiles, numfiles * size, now)
while iter < numfiles / numchildren:
now = timing_tester(size, rates, 0.1, functions, realtime=1, now=now)
iter += 1
meter.end(now)
def test_re():
re = RateEstimator()
megabyte = 1 << 20
size = megabyte
rate = 0.1 * megabyte # per second
updatestep = 0.5
remaining = size
starttime = 0.0
fakenow = starttime
re.start(size, starttime)
changed = 0
while remaining > 0:
re.update(size - remaining, fakenow)
et, ar, rt = (fakenow-starttime, re.average_rate(),
re.remaining_time())
if not ar: ar = 0
if not rt: rt = 0
rrt = remaining / rate # real remaining time
diff = rt - rrt
print '%10.2f %10f %10f %10f %10f' % (et, ar/megabyte, rt, rrt, diff)
#time.sleep(updatestep)
fakenow += updatestep
remaining -= int(updatestep * rate)
if not changed and remaining < size/2:
changed = 1
rate = rate /2
import random
import threading
class ThreadedFetcher(threading.Thread):
def __init__(self, meter, filelist):
self.meter = meter
self.filelist = filelist
threading.Thread.__init__(self)
def _start(self, read, now):
self.meter.start('filename', 'http:/url.com/filename', self.fn,
self.size, now)
def _mid(self, read, now):
self.meter.update(read, now)
def _end(self, read, now):
self.meter.end(read, now)
def _failure(self, message, now):
self.meter.failure(message, now)
def run(self):
now = 0
functions = (self._start, self._mid, self._end, self._failure)
for self.fn, self.size, rate in self.filelist:
rates = [(rate, None)]
now = timing_tester(self.size, rates, 0.1, functions, realtime=1,now=now)
def threading_tester():
numthreads = 5
filenum = 0
file_lists = []
total_size = 0
for i in range(numthreads):
filelist = []
files_in_this_thread = random.randrange(3,6)
for j in range(files_in_this_thread):
fn = 'file_%02i (thread %i: %i/%i)' % \
(filenum+1, i, j+1, files_in_this_thread)
#fn = 'file_%02i' % (filenum+1,)
filenum += 1
sizeexp = random.randrange(15, 21)
size = 1 << sizeexp
total_size += size
rate = random.randrange(1<<12, 1<<18)
if random.random() < 0.1: rate = None # failure
filelist.append( (fn, size, rate) )
file_lists.append(filelist)
master = TextMultiFileMeter()
master.start(filenum, total_size, now=0.0)
threads = []
for i in range(numthreads):
meter = master.newMeter()
t = ThreadedFetcher(meter, file_lists[i])
t.start()
threads.append(t)
for t in threads: t.join()
master.end()
#while 1:
# time.sleep(0.1)
# print master.lock.locked()
def grabber_tests():
from urlgrabber import grabber
pm = text_progress_meter()
g = grabber.URLGrabber(progress_obj=pm)
g.urlread('http://www.linux.duke.edu/projects/urlgrabber/test/reference')
if __name__ == '__main__':
#test_text_mf_meter()
#test_text_meter()
#test_re()
#threading_tester()
grabber_tests()
More information about the Yum-devel
mailing list