[yum-commits] 8 commits - urlgrabber/grabber.py urlgrabber/progress.py
James Antill
james at osuosl.org
Fri Oct 21 15:57:20 UTC 2011
urlgrabber/grabber.py | 110 ++++++++++++++++++++++++++-----------------------
urlgrabber/progress.py | 7 +--
2 files changed, 64 insertions(+), 53 deletions(-)
New commits:
commit 5443754de676ed4b173502522143460ae8a50013
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date: Fri Oct 21 16:28:02 2011 +0200
move closing of target file to _do_close_fo().
diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index b53760c..5614bcd 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -1594,29 +1594,7 @@ class PyCurlFileObject(object):
raise e
if _was_filename:
- # close it up
- self.fo.flush()
- self.fo.close()
-
- # Set the URL where we got it from:
- if xattr is not None:
- # See: http://www.freedesktop.org/wiki/CommonExtendedAttributes
- try:
- xattr.set(self.filename, 'user.xdg.origin.url', self.url)
- except:
- pass # URL too long. = IOError ... ignore everything.
-
- # set the time
- mod_time = self.curl_obj.getinfo(pycurl.INFO_FILETIME)
- if mod_time != -1:
- try:
- os.utime(self.filename, (mod_time, mod_time))
- except OSError, e:
- err = URLGrabError(16, _(\
- 'error setting timestamp on file %s from %s, OSError: %s')
- % (self.filename, self.url, e))
- err.url = self.url
- raise err
+ self._do_close_fo()
# re open it
try:
self.fo = open(self.filename, 'r')
@@ -1648,6 +1626,31 @@ class PyCurlFileObject(object):
err.url = self.url
raise err
+ def _do_close_fo(self):
+ # close it up
+ self.fo.flush()
+ self.fo.close()
+
+ # Set the URL where we got it from:
+ if xattr is not None:
+ # See: http://www.freedesktop.org/wiki/CommonExtendedAttributes
+ try:
+ xattr.set(self.filename, 'user.xdg.origin.url', self.url)
+ except:
+ pass # URL too long. = IOError ... ignore everything.
+
+ # set the time
+ mod_time = self.curl_obj.getinfo(pycurl.INFO_FILETIME)
+ if mod_time != -1:
+ try:
+ os.utime(self.filename, (mod_time, mod_time))
+ except OSError, e:
+ err = URLGrabError(16, _(\
+ 'error setting timestamp on file %s from %s, OSError: %s')
+ % (self.filename, self.url, e))
+ err.url = self.url
+ raise err
+
def _fill_buffer(self, amt=None):
"""fill the buffer to contain at least 'amt' bytes by reading
from the underlying file object. If amt is None, then it will
commit eb365596a44bd8ca15fe290d50290f0f84a268dd
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date: Fri Oct 21 16:28:01 2011 +0200
move opening of target file to _do_open_fo().
diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index 10b7330..b53760c 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -1574,22 +1574,7 @@ class PyCurlFileObject(object):
_was_filename = False
if type(self.filename) in types.StringTypes and self.filename:
_was_filename = True
- self._prog_reportname = str(self.filename)
- self._prog_basename = os.path.basename(self.filename)
-
- if self.append: mode = 'ab'
- else: mode = 'wb'
-
- if DEBUG: DEBUG.info('opening local file "%s" with mode %s' % \
- (self.filename, mode))
- try:
- self.fo = open(self.filename, mode)
- except IOError, e:
- err = URLGrabError(16, _(\
- 'error opening local file from %s, IOError: %s') % (self.url, e))
- err.url = self.url
- raise err
-
+ self._do_open_fo()
else:
self._prog_reportname = 'MEMORY'
self._prog_basename = 'MEMORY'
@@ -1647,6 +1632,22 @@ class PyCurlFileObject(object):
self._complete = True
+ def _do_open_fo(self):
+ self._prog_reportname = str(self.filename)
+ self._prog_basename = os.path.basename(self.filename)
+ if self.append: mode = 'ab'
+ else: mode = 'wb'
+
+ if DEBUG: DEBUG.info('opening local file "%s" with mode %s' % \
+ (self.filename, mode))
+ try:
+ self.fo = open(self.filename, mode)
+ except IOError, e:
+ err = URLGrabError(16, _(\
+ 'error opening local file from %s, IOError: %s') % (self.url, e))
+ err.url = self.url
+ raise err
+
def _fill_buffer(self, amt=None):
"""fill the buffer to contain at least 'amt' bytes by reading
from the underlying file object. If amt is None, then it will
commit 54ee979ada490cdae12fff038a838602fa404075
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date: Fri Oct 21 16:28:00 2011 +0200
move pycurl.error handling to _do_perform_exc()
diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index 3330b49..10b7330 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -1313,8 +1313,14 @@ class PyCurlFileObject(object):
return
try:
+ e = None
self.curl_obj.perform()
- except pycurl.error, e:
+ except pycurl.error, e: pass
+ self._do_perform_exc(e)
+
+ def _do_perform_exc(self, e):
+ # handle pycurl exception 'e'
+ if e:
# XXX - break some of these out a bit more clearly
# to other URLGrabErrors from
# http://curl.haxx.se/libcurl/c/libcurl-errors.html
commit cd0d68ce83b320527e4bace3598ac103a27e4788
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date: Fri Oct 21 16:27:58 2011 +0200
Use re.total instead of total_size.
Total percentages are calculated in re.fraction_read(), and are relative
to re.last_amount_read and re.total. Use it also for done/total sizes.
Keep total_size for compatibility.
diff --git a/urlgrabber/progress.py b/urlgrabber/progress.py
index 1de0abd..ecef8ca 100644
--- a/urlgrabber/progress.py
+++ b/urlgrabber/progress.py
@@ -481,7 +481,7 @@ class TextMultiFileMeter(MultiFileMeter):
tf = self.numfiles or 1
pf = 100 * float(df)/tf + 0.49
dd = self.re.last_amount_read
- td = self.total_size
+ td = self.re.total
pd = 100 * (self.re.fraction_read() or 0) + 0.49
dt = self.re.elapsed_time()
rt = self.re.remaining_time()
commit 2147035b206d61986daf6fed55a9b206d79154e0
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date: Fri Oct 21 16:27:59 2011 +0200
Documentation: fix urlgrab options
Remove 'self.' prefix from the option name.
diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index f6f57bd..3330b49 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -217,44 +217,44 @@ GENERAL ARGUMENTS (kwargs)
No-op when using the curl backend (default)
- self.ssl_verify_peer = True
+ ssl_verify_peer = True
Check the server's certificate to make sure it is valid with what our CA validates
- self.ssl_verify_host = True
+ ssl_verify_host = True
Check the server's hostname to make sure it matches the certificate DN
- self.ssl_key = None
+ ssl_key = None
Path to the key the client should use to connect/authenticate with
- self.ssl_key_type = 'PEM'
+ ssl_key_type = 'PEM'
PEM or DER - format of key
- self.ssl_cert = None
+ ssl_cert = None
Path to the ssl certificate the client should use to to authenticate with
- self.ssl_cert_type = 'PEM'
+ ssl_cert_type = 'PEM'
PEM or DER - format of certificate
- self.ssl_key_pass = None
+ ssl_key_pass = None
password to access the ssl_key
- self.size = None
+ size = None
size (in bytes) or Maximum size of the thing being downloaded.
This is mostly to keep us from exploding with an endless datastream
- self.max_header_size = 2097152
+ max_header_size = 2097152
Maximum size (in bytes) of the headers.
- self.ip_resolve = 'whatever'
+ ip_resolve = 'whatever'
What type of name to IP resolving to use, default is to do both IPV4 and
IPV6.
commit ea29f98e6f29301591f0a03d34c0e907f4f49590
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date: Fri Oct 21 16:27:57 2011 +0200
TextMultiFileMeter: use 'text' instead of 'basename'.
See also: TextMeter._do_update()
diff --git a/urlgrabber/progress.py b/urlgrabber/progress.py
index 2a6894f..1de0abd 100644
--- a/urlgrabber/progress.py
+++ b/urlgrabber/progress.py
@@ -503,7 +503,7 @@ class TextMultiFileMeter(MultiFileMeter):
self._lock.acquire()
try:
format = "%-30.30s %6.6s %8.8s %9.9s"
- fn = meter.basename
+ fn = meter.text or meter.basename
size = meter.last_amount_read
fsize = format_number(size) + 'B'
et = meter.re.elapsed_time()
commit 7da6eec053c351b3d3b0d4fb753785501d0a8fd7
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date: Fri Oct 21 16:27:55 2011 +0200
Prevent float division by zero
There may be exactly one update, then last_update_time == start_time.
diff --git a/urlgrabber/progress.py b/urlgrabber/progress.py
index 0beef3d..2a6894f 100644
--- a/urlgrabber/progress.py
+++ b/urlgrabber/progress.py
@@ -508,7 +508,7 @@ class TextMultiFileMeter(MultiFileMeter):
fsize = format_number(size) + 'B'
et = meter.re.elapsed_time()
fet = format_time(et, 1)
- frate = format_number(size / et) + 'B/s'
+ frate = format_number(et and size / et) + 'B/s'
out = '%-79.79s' % (format % (fn, fsize, fet, frate))
self.fo.write('\r' + out + '\n')
commit 09c351fcaae5293470310d964e2b11fbe003430f
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date: Fri Oct 21 16:27:56 2011 +0200
MultiFileMeter: show correct finished size
Rate estimator is updated only partially in update_meter().
Make sure the downloaded size reported at the end includes
data read in last 0.3 seconds.
diff --git a/urlgrabber/progress.py b/urlgrabber/progress.py
index 45eb248..0beef3d 100644
--- a/urlgrabber/progress.py
+++ b/urlgrabber/progress.py
@@ -369,6 +369,7 @@ class MultiFileMeter:
def end(self, now=None):
if now is None: now = time.time()
+ self.re.update(self._amount_read(), now)
self._do_end(now)
def _do_end(self, now):
More information about the Yum-commits
mailing list