[yum-commits] scripts/urlgrabber scripts/urlgrabber-ext-down urlgrabber/byterange.py urlgrabber/grabber.py urlgrabber/__init__.py urlgrabber/mirror.py urlgrabber/progress.py
zpavlas at osuosl.org
zpavlas at osuosl.org
Wed Aug 8 15:53:47 UTC 2012
scripts/urlgrabber | 46 +++++-----
scripts/urlgrabber-ext-down | 4
urlgrabber/__init__.py | 2
urlgrabber/byterange.py | 99 ++++++++++------------
urlgrabber/grabber.py | 192 +++++++++++++++++++-------------------------
urlgrabber/mirror.py | 18 +---
urlgrabber/progress.py | 15 +--
7 files changed, 168 insertions(+), 208 deletions(-)
New commits:
commit 7355f28f04fcf0a444d6ebc84e8568b89db0dbdb
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date: Wed Aug 8 17:35:26 2012 +0200
Sorry, didn't want to push this to master yet.
Revert "python3-compat: handle renamed modules and types"
Revert "python3-compat: print"
Revert "python3-compat: except"
Revert "python3-compat: raise"
diff --git a/scripts/urlgrabber b/scripts/urlgrabber
index c8d915a..09cd896 100644
--- a/scripts/urlgrabber
+++ b/scripts/urlgrabber
@@ -139,7 +139,6 @@ import re
import urlgrabber.grabber
from urlgrabber.grabber import URLGrabber, URLGrabberOptions, URLGrabError
-from __future__ import print_function
class client_options:
def __init__(self):
@@ -162,6 +161,7 @@ class client_options:
for k in ugo.__dict__.keys():
if (k not in ug_options) and (k not in options_exclude):
ug_options.append(k)
+ #print k
ug_defaults = {}
for k in list(ug_options):
try:
@@ -178,8 +178,8 @@ class client_options:
try:
optlist, args = getopt.getopt(sys.argv[1:], short_options,
long_options + ug_long)
- except getopt.GetoptError as e:
- print("Error:", e, file=sys.stderr)
+ except getopt.GetoptError, e:
+ print >>sys.stderr, "Error:", e
self.help([], ret=1)
self.verbose = 0
@@ -209,7 +209,7 @@ class client_options:
self.repeat = int(v)
if self.repeat < 1: raise ValueError()
except ValueError:
- print('ERROR: repeat value must be an int >= 1')
+ print 'ERROR: repeat value must be an int >= 1'
sys.exit(1)
if o == '-D':
self.verbose = 3
@@ -218,20 +218,20 @@ class client_options:
if o in ug_dash:
try:
val = eval(v)
- except Exception as e:
- print("error processing option value: %s" % v)
- print(e)
+ except Exception, e:
+ print "error processing option value: %s" % v
+ print e
sys.exit(1)
else:
self.ugops[o[2:]] = val
if len(self.args) > 1 and self.outputfile is not None:
- print("ERROR: cannot use -o when grabbing multiple files")
+ print "ERROR: cannot use -o when grabbing multiple files"
sys.exit(1)
def help(self, args, ret=0):
if not args:
- print(MAINHELP)
+ print MAINHELP
else:
for a in args:
m = getattr(self, 'help_'+a, None)
@@ -240,20 +240,20 @@ class client_options:
elif a in self.ug_options:
self.help_ug_option(a)
else:
- print('ERROR: no help on command "%s"' % a)
+ print 'ERROR: no help on command "%s"' % a
sys.exit(ret)
def help_doc(self):
- print(__doc__)
+ print __doc__
def help_options(self):
width = max(map(len, self.ug_options))
format = ' %-' + str(width) + 's = %s'
hformat = ' %-' + str(width) + 's %s'
- print(hformat % ('OPTION', 'DEFAULT'))
- print('-'*(width + 20))
+ print hformat % ('OPTION', 'DEFAULT')
+ print '-'*(width + 20)
for k in self.ug_options:
- print(format % (k, self.ug_defaults[k]))
+ print format % (k, self.ug_defaults[k])
def help_all(self):
for k in self.ug_options:
@@ -264,21 +264,21 @@ class client_options:
m = re.search(r'^( '+option+'.*?)\s*^ {,2}\S',
urlgrabber.grabber.__doc__, re.M|re.S)
if m:
- print(m.group(1))
+ print m.group(1)
else:
- print(' %s: no help found for this option' % option)
- print('')
+ print ' %s: no help found for this option' % option
+ print ''
class ugclient:
def __init__(self):
op = client_options()
self.op = op
if op.verbose >= 2 and op.ugops:
- print("Module Options:")
+ print "Module Options:"
width = max(map(len, op.ugops.keys()))
format = " %-" + str(width) + "s = %s"
for k, v in op.ugops.items():
- print(format % (k, repr(v)))
+ print format % (k, repr(v))
if op.debug:
self.set_debug_logger(op.debug)
@@ -294,13 +294,13 @@ class ugclient:
def run(self):
for url in self.op.args:
- if self.op.verbose: print('grabbing: %s' % url)
+ if self.op.verbose: print 'grabbing: %s' % url
try:
for i in range(0, self.op.repeat):
f = self.g.urlgrab(url, self.op.outputfile)
- if self.op.localfile: print(f)
- except URLGrabError as e:
- print(e)
+ if self.op.localfile: print f
+ except URLGrabError, e:
+ print e
def set_debug_logger(self, dbspec):
try:
diff --git a/scripts/urlgrabber-ext-down b/scripts/urlgrabber-ext-down
index 3fcc8e2..3da55a4 100755
--- a/scripts/urlgrabber-ext-down
+++ b/scripts/urlgrabber-ext-down
@@ -25,7 +25,7 @@ from urlgrabber.grabber import \
def write(fmt, *arg):
try: os.write(1, fmt % arg)
- except OSError as e:
+ except OSError, e:
if e.args[0] != errno.EPIPE: raise
sys.exit(1)
@@ -63,7 +63,7 @@ def main():
size = fo._amount_read
dlsz = size - fo._reget_length
ug_err = 'OK'
- except URLGrabError as e:
+ except URLGrabError, e:
size = dlsz = 0
ug_err = '%d %s' % e.args
write('%d %d %d %.3f %s\n', opts._id, size, dlsz, time.time() - tm, ug_err)
diff --git a/urlgrabber/__init__.py b/urlgrabber/__init__.py
index ed0e526..ddd5204 100644
--- a/urlgrabber/__init__.py
+++ b/urlgrabber/__init__.py
@@ -51,4 +51,4 @@ __author__ = 'Michael D. Stenner <mstenner at linux.duke.edu>, ' \
'Seth Vidal <skvidal at fedoraproject.org>'
__url__ = 'http://linux.duke.edu/projects/urlgrabber/'
-from urlgrabber.grabber import urlgrab, urlopen, urlread
+from grabber import urlgrab, urlopen, urlread
diff --git a/urlgrabber/byterange.py b/urlgrabber/byterange.py
index 5461da5..8eeaeda 100644
--- a/urlgrabber/byterange.py
+++ b/urlgrabber/byterange.py
@@ -20,36 +20,22 @@
import os
import stat
-import sys
+import urllib
+import urllib2
+import rfc822
DEBUG = None
-# python2+3 compatible imports
-if sys.version_info.major < 3:
- import urllib2 as urllib_request
- import urlparse as urllib_parse
- import urllib2 as urllib_error
- from urllib2 import unquote
- from mimetools import Message
- from rfc822 import formatdate
- try:
- from cStringIO import StringIO
- except ImportError:
- from StringIO import StringIO
-else:
- import urllib.request as urllib_request
- import urllib.parse as urllib_parse
- import urllib.error as urllib_error
- from urllib.parse import unquote
- from email.message import Message
- from email.utils import formatdate
- from io import StringIO
+try:
+ from cStringIO import StringIO
+except ImportError, msg:
+ from StringIO import StringIO
class RangeError(IOError):
"""Error raised when an unsatisfiable range is requested."""
pass
-class HTTPRangeHandler(urllib_request.BaseHandler):
+class HTTPRangeHandler(urllib2.BaseHandler):
"""Handler that enables HTTP Range headers.
This was extremely simple. The Range header is a HTTP feature to
@@ -75,7 +61,7 @@ class HTTPRangeHandler(urllib_request.BaseHandler):
def http_error_206(self, req, fp, code, msg, hdrs):
# 206 Partial Content Response
- r = urllib_request.addinfourl(fp, hdrs, req.get_full_url())
+ r = urllib.addinfourl(fp, hdrs, req.get_full_url())
r.code = code
r.msg = msg
return r
@@ -134,7 +120,7 @@ class RangeableFileObject:
in self.fo. This includes methods."""
if hasattr(self.fo, name):
return getattr(self.fo, name)
- raise AttributeError(name)
+ raise AttributeError, name
def tell(self):
"""Return the position within the range.
@@ -225,23 +211,25 @@ class RangeableFileObject:
raise RangeError(9, 'Requested Range Not Satisfiable')
pos+= bufsize
-class FileRangeHandler(urllib_request.FileHandler):
+class FileRangeHandler(urllib2.FileHandler):
"""FileHandler subclass that adds Range support.
This class handles Range headers exactly like an HTTP
server would.
"""
def open_local_file(self, req):
+ import mimetypes
+ import mimetools
host = req.get_host()
file = req.get_selector()
- localfile = urllib_request.url2pathname(file)
+ localfile = urllib.url2pathname(file)
stats = os.stat(localfile)
size = stats[stat.ST_SIZE]
- modified = formatdate(stats[stat.ST_MTIME])
+ modified = rfc822.formatdate(stats[stat.ST_MTIME])
mtype = mimetypes.guess_type(file)[0]
if host:
- host, port = urllib_parse.splitport(host)
+ host, port = urllib.splitport(host)
if port or socket.gethostbyname(host) not in self.get_names():
- raise urllib_error.URLError('file not on local host')
+ raise urllib2.URLError('file not on local host')
fo = open(localfile,'rb')
brange = req.headers.get('Range',None)
brange = range_header_to_tuple(brange)
@@ -253,10 +241,10 @@ class FileRangeHandler(urllib_request.FileHandler):
raise RangeError(9, 'Requested Range Not Satisfiable')
size = (lb - fb)
fo = RangeableFileObject(fo, (fb,lb))
- headers = Message(StringIO(
+ headers = mimetools.Message(StringIO(
'Content-Type: %s\nContent-Length: %d\nLast-modified: %s\n' %
(mtype or 'text/plain', size, modified)))
- return urllib_request.addinfourl(fo, headers, 'file:'+file)
+ return urllib.addinfourl(fo, headers, 'file:'+file)
# FTP Range Support
@@ -266,25 +254,29 @@ class FileRangeHandler(urllib_request.FileHandler):
# follows:
# -- range support modifications start/end here
+from urllib import splitport, splituser, splitpasswd, splitattr, \
+ unquote, addclosehook, addinfourl
import ftplib
import socket
+import sys
import mimetypes
+import mimetools
-class FTPRangeHandler(urllib_request.FTPHandler):
+class FTPRangeHandler(urllib2.FTPHandler):
def ftp_open(self, req):
host = req.get_host()
if not host:
- raise IOError('ftp error', 'no host given')
- host, port = urllib_parse.splitport(host)
+ raise IOError, ('ftp error', 'no host given')
+ host, port = splitport(host)
if port is None:
port = ftplib.FTP_PORT
else:
port = int(port)
# username/password handling
- user, host = urllib_parse.splituser(host)
+ user, host = splituser(host)
if user:
- user, passwd = urllib_parse.splitpasswd(user)
+ user, passwd = splitpasswd(user)
else:
passwd = None
host = unquote(host)
@@ -293,18 +285,19 @@ class FTPRangeHandler(urllib_request.FTPHandler):
try:
host = socket.gethostbyname(host)
- except socket.error as msg:
- raise urllib_error.URLError(msg)
- path, attrs = urllib_parse.splitattr(req.get_selector())
- dirs = [unquote(i) for i in path.split('/')]
- file = dirs.pop()
+ except socket.error, msg:
+ raise urllib2.URLError(msg)
+ path, attrs = splitattr(req.get_selector())
+ dirs = path.split('/')
+ dirs = map(unquote, dirs)
+ dirs, file = dirs[:-1], dirs[-1]
if dirs and not dirs[0]:
- del dirs[0]
+ dirs = dirs[1:]
try:
fw = self.connect_ftp(user, passwd, host, port, dirs)
type = file and 'I' or 'D'
for attr in attrs:
- attr, value = urllib_parse.splitattr(attr)
+ attr, value = splitattr(attr)
if attr.lower() == 'type' and \
value in ('a', 'A', 'i', 'I', 'd', 'D'):
type = value.upper()
@@ -343,16 +336,16 @@ class FTPRangeHandler(urllib_request.FTPHandler):
if retrlen is not None and retrlen >= 0:
headers += "Content-Length: %d\n" % retrlen
sf = StringIO(headers)
- headers = Message(sf)
- return urllib_request.addinfourl(fp, headers, req.get_full_url())
- except ftplib.all_errors as msg:
- raise IOError('ftp error', msg)
+ headers = mimetools.Message(sf)
+ return addinfourl(fp, headers, req.get_full_url())
+ except ftplib.all_errors, msg:
+ raise IOError, ('ftp error', msg), sys.exc_info()[2]
def connect_ftp(self, user, passwd, host, port, dirs):
fw = ftpwrapper(user, passwd, host, port, dirs)
return fw
-class ftpwrapper(urllib_request.ftpwrapper):
+class ftpwrapper(urllib.ftpwrapper):
# range support note:
# this ftpwrapper code is copied directly from
# urllib. The only enhancement is to add the rest
@@ -371,22 +364,22 @@ class ftpwrapper(urllib_request.ftpwrapper):
# Use nlst to see if the file exists at all
try:
self.ftp.nlst(file)
- except ftplib.error_perm as reason:
- raise IOError('ftp error', reason)
+ except ftplib.error_perm, reason:
+ raise IOError, ('ftp error', reason), sys.exc_info()[2]
# Restore the transfer mode!
self.ftp.voidcmd(cmd)
# Try to retrieve as a file
try:
cmd = 'RETR ' + file
conn = self.ftp.ntransfercmd(cmd, rest)
- except ftplib.error_perm as reason:
+ except ftplib.error_perm, reason:
if str(reason)[:3] == '501':
# workaround for REST not supported error
fp, retrlen = self.retrfile(file, type)
fp = RangeableFileObject(fp, (rest,''))
return (fp, retrlen)
elif str(reason)[:3] != '550':
- raise IOError('ftp error', reason)
+ raise IOError, ('ftp error', reason), sys.exc_info()[2]
if not conn:
# Set transfer mode to ASCII!
self.ftp.voidcmd('TYPE A')
@@ -396,7 +389,7 @@ class ftpwrapper(urllib_request.ftpwrapper):
conn = self.ftp.ntransfercmd(cmd)
self.busy = 1
# Pass back both a suitably decorated object and a retrieval length
- return (urllib_request.addclosehook(conn[0].makefile('rb'),
+ return (addclosehook(conn[0].makefile('rb'),
self.endtransfer), conn[1])
diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index bf4964b..bdcdfe3 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -463,46 +463,25 @@ BANDWIDTH THROTTLING
"""
-from __future__ import print_function
import os
import sys
+import urlparse
import time
import string
+import urllib
+import urllib2
+from httplib import responses
+import mimetools
+import thread
import types
import stat
import pycurl
from ftplib import parse150
+from StringIO import StringIO
+from httplib import HTTPException
import socket, select, fcntl
-from urlgrabber.byterange import range_tuple_normalize, range_tuple_to_header, RangeError
-
-# python2+3 compatible imports
-if sys.version_info.major < 3:
- import urllib2 as urllib_request
- import urlparse as urllib_parse
- import urllib2 as urllib_error
- from urllib2 import quote, unquote
- from httplib import responses, HTTPException
- from mimetools import Message
- try:
- from cStringIO import StringIO
- except ImportError:
- from StringIO import StringIO
- from types import StringTypes
-else:
- import urllib.request as urllib_request
- import urllib.parse as urllib_parse
- import urllib.error as urllib_error
- from urllib.parse import quote, unquote
- from http.client import responses, HTTPException
- from email.message import Message
- from io import StringIO
- import collections
- def callable(cb):
- return isinstance(cb, collections.Callable)
- StringTypes = bytes, str
- unicode = str
- long = int
+from byterange import range_tuple_normalize, range_tuple_to_header, RangeError
try:
import xattr
@@ -523,7 +502,7 @@ except:
try:
# this part isn't going to do much - need to talk to gettext
from i18n import _
-except ImportError:
+except ImportError, msg:
def _(st): return st
########################################################################
@@ -771,14 +750,14 @@ class URLParser:
if opts.prefix:
url = self.add_prefix(url, opts.prefix)
- parts = urllib_parse.urlparse(url)
+ parts = urlparse.urlparse(url)
(scheme, host, path, parm, query, frag) = parts
if not scheme or (len(scheme) == 1 and scheme in string.letters):
# if a scheme isn't specified, we guess that it's "file:"
if url[0] not in '/\\': url = os.path.abspath(url)
- url = 'file:' + urllib_request.pathname2url(url)
- parts = urllib_parse.urlparse(url)
+ url = 'file:' + urllib.pathname2url(url)
+ parts = urlparse.urlparse(url)
quote = 0 # pathname2url quotes, so we won't do it again
if scheme in ['http', 'https']:
@@ -789,7 +768,7 @@ class URLParser:
if quote:
parts = self.quote(parts)
- url = urllib_parse.urlunparse(parts)
+ url = urlparse.urlunparse(parts)
return url, parts
def add_prefix(self, url, prefix):
@@ -813,7 +792,7 @@ class URLParser:
passing into urlgrabber.
"""
(scheme, host, path, parm, query, frag) = parts
- path = quote(path)
+ path = urllib.quote(path)
return (scheme, host, path, parm, query, frag)
hexvals = '0123456789ABCDEF'
@@ -859,7 +838,7 @@ class URLGrabberOptions:
def __getattr__(self, name):
if self.delegate and hasattr(self.delegate, name):
return getattr(self.delegate, name)
- raise AttributeError(name)
+ raise AttributeError, name
def raw_throttle(self):
"""Calculate raw throttle value from throttle and bandwidth
@@ -917,7 +896,7 @@ class URLGrabberOptions:
def _set_attributes(self, **kwargs):
"""Update object attributes with those provided in kwargs."""
self.__dict__.update(kwargs)
- if 'range' in kwargs:
+ if kwargs.has_key('range'):
# normalize the supplied range value
self.range = range_tuple_normalize(self.range)
if not self.reget in [None, 'simple', 'check_timestamp']:
@@ -983,7 +962,7 @@ class URLGrabberOptions:
return self.format()
def format(self, indent=' '):
- keys = list(self.__dict__.keys())
+ keys = self.__dict__.keys()
if self.delegate is not None:
keys.remove('delegate')
keys.sort()
@@ -1038,13 +1017,13 @@ class URLGrabber(object):
if DEBUG: DEBUG.info('attempt %i/%s: %s',
tries, opts.retry, args[0])
try:
- r = func(*(opts,) + args)
+ r = apply(func, (opts,) + args, {})
if DEBUG: DEBUG.info('success')
return r
- except URLGrabError as e:
+ except URLGrabError, e:
exception = e
callback = opts.failure_callback
- except KeyboardInterrupt as e:
+ except KeyboardInterrupt, e:
exception = e
callback = opts.interrupt_callback
if not callback:
@@ -1096,14 +1075,14 @@ class URLGrabber(object):
(scheme, host, path, parm, query, frag) = parts
opts.find_proxy(url, scheme)
if filename is None:
- filename = os.path.basename(unquote(path))
+ filename = os.path.basename( urllib.unquote(path) )
if not filename:
# This is better than nothing.
filename = 'index.html'
if scheme == 'file' and not opts.copy_local:
# just return the name of the local file - don't make a
# copy currently
- path = urllib_request.url2pathname(path)
+ path = urllib.url2pathname(path)
if host:
path = os.path.normpath('//' + host + path)
if not os.path.exists(path):
@@ -1145,7 +1124,7 @@ class URLGrabber(object):
try:
return self._retry(opts, retryfunc, url, filename)
- except URLGrabError as e:
+ except URLGrabError, e:
_TH.update(url, 0, 0, e)
opts.exception = e
return _run_callback(opts.failfunc, opts)
@@ -1210,13 +1189,13 @@ class PyCurlFileObject(object):
self._hdr_dump = ''
self._parsed_hdr = None
self.url = url
- self.scheme = urllib_parse.urlsplit(self.url)[0]
+ self.scheme = urlparse.urlsplit(self.url)[0]
self.filename = filename
self.append = False
self.reget_time = None
self.opts = opts
if self.opts.reget == 'check_timestamp':
- raise NotImplementedError("check_timestamp regets are not implemented in this ver of urlgrabber. Please report this.")
+ raise NotImplementedError, "check_timestamp regets are not implemented in this ver of urlgrabber. Please report this."
self._complete = False
self._rbuf = ''
self._rbufsize = 1024*8
@@ -1238,7 +1217,7 @@ class PyCurlFileObject(object):
if hasattr(self.fo, name):
return getattr(self.fo, name)
- raise AttributeError(name)
+ raise AttributeError, name
def _retrieve(self, buf):
try:
@@ -1246,7 +1225,7 @@ class PyCurlFileObject(object):
if self.opts.progress_obj:
size = self.size + self._reget_length
self.opts.progress_obj.start(self._prog_reportname,
- unquote(self.url),
+ urllib.unquote(self.url),
self._prog_basename,
size=size,
text=self.opts.text)
@@ -1288,7 +1267,7 @@ class PyCurlFileObject(object):
if buf.lower().find('location') != -1:
location = ':'.join(buf.split(':')[1:])
location = location.strip()
- self.scheme = urllib_parse.urlsplit(location)[0]
+ self.scheme = urlparse.urlsplit(location)[0]
self.url = location
if len(self._hdr_dump) != 0 and buf == '\r\n':
@@ -1307,7 +1286,7 @@ class PyCurlFileObject(object):
hdrfp = StringIO()
hdrfp.write(self._hdr_dump[statusend:])
hdrfp.seek(0)
- self._parsed_hdr = Message(hdrfp)
+ self._parsed_hdr = mimetools.Message(hdrfp)
return self._parsed_hdr
hdr = property(_return_hdr_obj)
@@ -1428,7 +1407,7 @@ class PyCurlFileObject(object):
try:
self.curl_obj.perform()
- except pycurl.error as e:
+ except pycurl.error, e:
# XXX - break some of these out a bit more clearly
# to other URLGrabErrors from
# http://curl.haxx.se/libcurl/c/libcurl-errors.html
@@ -1436,7 +1415,7 @@ class PyCurlFileObject(object):
code = self.http_code
errcode = e.args[0]
- errurl = unquote(self.url)
+ errurl = urllib.unquote(self.url)
if self._error[0]:
errcode = self._error[0]
@@ -1568,7 +1547,7 @@ class PyCurlFileObject(object):
if self._error[1]:
msg = self._error[1]
err = URLGrabError(14, msg)
- err.url = unquote(self.url)
+ err.url = urllib.unquote(self.url)
raise err
def _do_open(self):
@@ -1585,7 +1564,7 @@ class PyCurlFileObject(object):
def _build_range(self):
reget_length = 0
rt = None
- if self.opts.reget and type(self.filename) in StringTypes:
+ if self.opts.reget and type(self.filename) in types.StringTypes:
# we have reget turned on and we're dumping to a file
try:
s = os.stat(self.filename)
@@ -1635,22 +1614,22 @@ class PyCurlFileObject(object):
else:
fo = opener.open(req)
hdr = fo.info()
- except ValueError as e:
+ except ValueError, e:
err = URLGrabError(1, _('Bad URL: %s : %s') % (self.url, e, ))
err.url = self.url
raise err
- except RangeError as e:
+ except RangeError, e:
err = URLGrabError(9, _('%s on %s') % (e, self.url))
err.url = self.url
raise err
- except urllib_error.HTTPError as e:
+ except urllib2.HTTPError, e:
new_e = URLGrabError(14, _('%s on %s') % (e, self.url))
new_e.code = e.code
new_e.exception = e
new_e.url = self.url
raise new_e
- except IOError as e:
+ except IOError, e:
if hasattr(e, 'reason') and isinstance(e.reason, socket.timeout):
err = URLGrabError(12, _('Timeout on %s: %s') % (self.url, e))
err.url = self.url
@@ -1660,12 +1639,12 @@ class PyCurlFileObject(object):
err.url = self.url
raise err
- except OSError as e:
+ except OSError, e:
err = URLGrabError(5, _('%s on %s') % (e, self.url))
err.url = self.url
raise err
- except HTTPException as e:
+ except HTTPException, e:
err = URLGrabError(7, _('HTTP Exception (%s) on %s: %s') % \
(e.__class__.__name__, self.url, e))
err.url = self.url
@@ -1680,7 +1659,7 @@ class PyCurlFileObject(object):
if self._complete:
return
_was_filename = False
- if type(self.filename) in StringTypes and self.filename:
+ if type(self.filename) in types.StringTypes and self.filename:
_was_filename = True
self._prog_reportname = str(self.filename)
self._prog_basename = os.path.basename(self.filename)
@@ -1692,7 +1671,7 @@ class PyCurlFileObject(object):
(self.filename, mode))
try:
self.fo = open(self.filename, mode)
- except IOError as e:
+ except IOError, e:
err = URLGrabError(16, _(\
'error opening local file from %s, IOError: %s') % (self.url, e))
err.url = self.url
@@ -1711,7 +1690,7 @@ class PyCurlFileObject(object):
try:
self._do_perform()
- except URLGrabError as e:
+ except URLGrabError, e:
self.fo.flush()
self.fo.close()
raise e
@@ -1734,7 +1713,7 @@ class PyCurlFileObject(object):
if mod_time != -1:
try:
os.utime(self.filename, (mod_time, mod_time))
- except OSError as e:
+ except OSError, e:
err = URLGrabError(16, _(\
'error setting timestamp on file %s from %s, OSError: %s')
% (self.filename, self.url, e))
@@ -1743,7 +1722,7 @@ class PyCurlFileObject(object):
# re open it
try:
self.fo = open(self.filename, 'r')
- except IOError as e:
+ except IOError, e:
err = URLGrabError(16, _(\
'error opening file from %s, IOError: %s') % (self.url, e))
err.url = self.url
@@ -1789,17 +1768,17 @@ class PyCurlFileObject(object):
else: readamount = min(amt, self._rbufsize)
try:
new = self.fo.read(readamount)
- except socket.error as e:
+ except socket.error, e:
err = URLGrabError(4, _('Socket Error on %s: %s') % (self.url, e))
err.url = self.url
raise err
- except socket.timeout as e:
+ except socket.timeout, e:
raise URLGrabError(12, _('Timeout on %s: %s') % (self.url, e))
err.url = self.url
raise err
- except IOError as e:
+ except IOError, e:
raise URLGrabError(4, _('IOError on %s: %s') %(self.url, e))
err.url = self.url
raise err
@@ -1956,7 +1935,7 @@ def _dumps(v):
return "(%s)" % ','.join(map(_dumps, v))
if type(v) == list:
return "[%s]" % ','.join(map(_dumps, v))
- raise TypeError('Can\'t serialize %s' % v)
+ raise TypeError, 'Can\'t serialize %s' % v
def _loads(s):
def decode(v):
@@ -2067,7 +2046,7 @@ class _ExternalDownloader:
for line in lines:
# parse downloader output
line = line.split(' ', 5)
- _id, size = int(line[0]), int(line[1])
+ _id, size = map(int, line[:2])
if len(line) == 2:
self.running[_id].progress_obj.update(size)
continue
@@ -2095,7 +2074,7 @@ class _ExternalDownloaderPool:
self.cache = {}
def start(self, opts):
- host = urllib_parse.urlsplit(opts.url).netloc
+ host = urlparse.urlsplit(opts.url).netloc
dl = self.cache.pop(host, None)
if not dl:
dl = _ExternalDownloader()
@@ -2118,7 +2097,7 @@ class _ExternalDownloaderPool:
ret.extend(done)
# dl finished, move it to the cache
- host = urllib_parse.urlsplit(done[0][0].url).netloc
+ host = urlparse.urlsplit(done[0][0].url).netloc
if host in self.cache: self.cache[host].abort()
self.epoll.unregister(fd)
self.cache[host] = self.running.pop(fd)
@@ -2149,7 +2128,7 @@ def parallel_wait(meter = 'text'):
count += 1
total += opts.size
if meter == 'text':
- from urlgrabber.progress import TextMultiFileMeter
+ from progress import TextMultiFileMeter
meter = TextMultiFileMeter()
meter.start(count, total)
@@ -2185,8 +2164,7 @@ def parallel_wait(meter = 'text'):
if ug_err is None:
if opts.checkfunc:
try: _run_callback(opts.checkfunc, opts)
- except URLGrabError as e:
- ug_err = e
+ except URLGrabError, ug_err: pass
if ug_err is None:
continue
@@ -2194,7 +2172,7 @@ def parallel_wait(meter = 'text'):
if opts.failure_callback:
opts.exception = ug_err
try: _run_callback(opts.failure_callback, opts)
- except URLGrabError:
+ except URLGrabError, ug_err:
retry = 0 # no retries
if opts.tries < retry and ug_err.errno in opts.retrycodes:
start(opts, opts.tries + 1) # simple retry
@@ -2279,7 +2257,7 @@ def parallel_wait(meter = 'text'):
while host_con.get(key, 0) >= limit:
perform()
start(opts, 1)
- except IOError as e:
+ except IOError, e:
if e.errno != 4: raise
raise KeyboardInterrupt
@@ -2329,7 +2307,7 @@ class _TH:
# Use hostname from URL. If it's a file:// URL, use baseurl.
# If no baseurl, do not update timedhosts.
- host = urllib_parse.urlsplit(url).netloc.split('@')[-1] or baseurl
+ host = urlparse.urlsplit(url).netloc.split('@')[-1] or baseurl
if not host: return
speed, fail, ts = _TH.hosts.get(host) or (0, 0, 0)
@@ -2357,7 +2335,7 @@ class _TH:
_TH.load()
# Use just the hostname, unless it's a file:// baseurl.
- host = urllib_parse.urlsplit(baseurl).netloc.split('@')[-1] or baseurl
+ host = urlparse.urlsplit(baseurl).netloc.split('@')[-1] or baseurl
default_speed = default_grabber.opts.default_speed
try: speed, fail, ts = _TH.hosts[host]
@@ -2370,12 +2348,11 @@ class _TH:
#####################################################################
# TESTING
-
def _main_test():
try: url, filename = sys.argv[1:3]
except ValueError:
- print('usage:', sys.argv[0], \
- '<url> <filename> [copy_local=0|1] [close_connection=0|1]')
+ print 'usage:', sys.argv[0], \
+ '<url> <filename> [copy_local=0|1] [close_connection=0|1]'
sys.exit()
kwargs = {}
@@ -2385,23 +2362,23 @@ def _main_test():
set_throttle(1.0)
set_bandwidth(32 * 1024)
- print("throttle: %s, throttle bandwidth: %s B/s" % (default_grabber.throttle,
- default_grabber.bandwidth))
+ print "throttle: %s, throttle bandwidth: %s B/s" % (default_grabber.throttle,
+ default_grabber.bandwidth)
- try: from urlgrabber.progress import text_progress_meter
- except ImportError: pass
+ try: from progress import text_progress_meter
+ except ImportError, e: pass
else: kwargs['progress_obj'] = text_progress_meter()
- try: name = urlgrab(url, filename, **kwargs)
- except URLGrabError as e: print(e)
- else: print('LOCAL FILE:', name)
+ try: name = apply(urlgrab, (url, filename), kwargs)
+ except URLGrabError, e: print e
+ else: print 'LOCAL FILE:', name
def _retry_test():
try: url, filename = sys.argv[1:3]
except ValueError:
- print('usage:', sys.argv[0], \
- '<url> <filename> [copy_local=0|1] [close_connection=0|1]')
+ print 'usage:', sys.argv[0], \
+ '<url> <filename> [copy_local=0|1] [close_connection=0|1]'
sys.exit()
kwargs = {}
@@ -2409,32 +2386,33 @@ def _retry_test():
k, v = string.split(a, '=', 1)
kwargs[k] = int(v)
- try: from urlgrabber.progress import text_progress_meter
- except ImportError: pass
+ try: from progress import text_progress_meter
+ except ImportError, e: pass
else: kwargs['progress_obj'] = text_progress_meter()
def cfunc(filename, hello, there='foo'):
- print(hello, there)
+ print hello, there
import random
rnum = random.random()
if rnum < .5:
- print('forcing retry')
+ print 'forcing retry'
raise URLGrabError(-1, 'forcing retry')
if rnum < .75:
- print('forcing failure')
+ print 'forcing failure'
raise URLGrabError(-2, 'forcing immediate failure')
- print('success')
+ print 'success'
return
kwargs['checkfunc'] = (cfunc, ('hello',), {'there':'there'})
- try: name = retrygrab(url, filename, **kwargs)
- except URLGrabError as e: print(e)
- else: print('LOCAL FILE:', name)
+ try: name = apply(retrygrab, (url, filename), kwargs)
+ except URLGrabError, e: print e
+ else: print 'LOCAL FILE:', name
def _file_object_test(filename=None):
+ import cStringIO
if filename is None:
filename = __file__
- print('using file "%s" for comparisons' % filename)
+ print 'using file "%s" for comparisons' % filename
fo = open(filename)
s_input = fo.read()
fo.close()
@@ -2443,14 +2421,14 @@ def _file_object_test(filename=None):
_test_file_object_readall,
_test_file_object_readline,
_test_file_object_readlines]:
- fo_input = StringIO(s_input)
- fo_output = StringIO()
+ fo_input = cStringIO.StringIO(s_input)
+ fo_output = cStringIO.StringIO()
wrapper = PyCurlFileObject(fo_input, None, 0)
- print('testing %-30s ' % testfunc.__name__, end='')
+ print 'testing %-30s ' % testfunc.__name__,
testfunc(wrapper, fo_output)
s_output = fo_output.getvalue()
- if s_output == s_input: print('passed')
- else: print('FAILED')
+ if s_output == s_input: print 'passed'
+ else: print 'FAILED'
def _test_file_object_smallread(wrapper, fo_output):
while 1:
diff --git a/urlgrabber/mirror.py b/urlgrabber/mirror.py
index bd76603..ac78b34 100644
--- a/urlgrabber/mirror.py
+++ b/urlgrabber/mirror.py
@@ -91,16 +91,10 @@ CUSTOMIZATION
import random
-import sys
-if sys.version_info.major < 3:
- import thread # needed for locking to make this threadsafe
- from types import StringTypes
-else:
- import _thread as thread
- StringTypes = bytes, str
+import thread # needed for locking to make this threadsafe
-from urlgrabber.grabber import URLGrabError, CallbackObject, DEBUG, _to_utf8
-from urlgrabber.grabber import _run_callback, _do_raise
+from grabber import URLGrabError, CallbackObject, DEBUG, _to_utf8
+from grabber import _run_callback, _do_raise
def _(st):
return st
@@ -274,7 +268,7 @@ class MirrorGroup:
def _parse_mirrors(self, mirrors):
parsed_mirrors = []
for m in mirrors:
- if type(m) in StringTypes:
+ if isinstance(m, basestring):
m = {'mirror': _to_utf8(m)}
parsed_mirrors.append(m)
return parsed_mirrors
@@ -406,7 +400,7 @@ class MirrorGroup:
if DEBUG: DEBUG.info('MIRROR: trying %s -> %s', url, fullurl)
try:
return func_ref( *(fullurl,), **kwargs )
- except URLGrabError as e:
+ except URLGrabError, e:
if DEBUG: DEBUG.info('MIRROR: failed')
obj = CallbackObject()
obj.exception = e
@@ -428,7 +422,7 @@ class MirrorGroup:
func = 'urlgrab'
try:
return self._mirror_try(func, url, kw)
- except URLGrabError as e:
+ except URLGrabError, e:
obj = CallbackObject(url=url, filename=filename, exception=e, **kwargs)
return _run_callback(kwargs.get('failfunc', _do_raise), obj)
diff --git a/urlgrabber/progress.py b/urlgrabber/progress.py
index f59afb0..5a02707 100644
--- a/urlgrabber/progress.py
+++ b/urlgrabber/progress.py
@@ -21,18 +21,11 @@
import sys
import time
import math
+import thread
import fcntl
import struct
import termios
-if sys.version_info.major < 3:
- import thread
- from types import StringTypes
-else:
- import _thread as thread
- StringTypes = bytes, str
- long = int
-
# Code from http://mail.python.org/pipermail/python-list/2000-May/033365.html
def terminal_width(fd=1):
""" Get the real terminal width """
@@ -596,7 +589,7 @@ class TextMultiFileMeter(MultiFileMeter):
try:
format = "%-30.30s %6.6s %s"
fn = meter.basename
- if type(message) in StringTypes:
+ if type(message) in (type(''), type(u'')):
message = message.splitlines()
if not message: message = ['']
out = '%-79s' % (format % (fn, 'FAILED', message[0] or ''))
@@ -637,6 +630,7 @@ class RateEstimator:
self.ave_rate = None
return
+ #print 'times', now, self.last_update_time
time_diff = now - self.last_update_time
read_diff = amount_read - self.last_amount_read
# First update, on reget is the file size
@@ -645,6 +639,7 @@ class RateEstimator:
self.ave_rate = self._temporal_rolling_ave(\
time_diff, read_diff, self.ave_rate, self.timescale)
self.last_amount_read = amount_read
+ #print 'results', time_diff, read_diff, self.ave_rate
#####################################################################
# result methods
@@ -762,7 +757,7 @@ def format_number(number, SI=0, space=' '):
depth = depth + 1
number = number / step
- if type(number) in (int, long):
+ if type(number) == type(1) or type(number) == type(1L):
# it's an int or a long, which means it didn't get divided,
# which means it's already short enough
format = '%i%s%s'
More information about the Yum-commits
mailing list