[yum-commits] 5 commits - scripts/urlgrabber scripts/urlgrabber-ext-down urlgrabber/byterange.py urlgrabber/grabber.py urlgrabber/__init__.py urlgrabber/mirror.py urlgrabber/progress.py
zpavlas at osuosl.org
zpavlas at osuosl.org
Tue Aug 7 11:46:02 UTC 2012
scripts/urlgrabber | 46 +++++-----
scripts/urlgrabber-ext-down | 4
urlgrabber/__init__.py | 2
urlgrabber/byterange.py | 99 +++++++++++-----------
urlgrabber/grabber.py | 195 ++++++++++++++++++++++++--------------------
urlgrabber/mirror.py | 18 ++--
urlgrabber/progress.py | 15 ++-
7 files changed, 211 insertions(+), 168 deletions(-)
New commits:
commit 309efb6f976a70ddd10a86ae9a94e3c9d1d9b9ed
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date: Tue Aug 7 13:38:53 2012 +0200
Handle EPOLLHUP, abort instead of traceback. BZ 846266.
diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index 72d654d..32e6410 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -2108,6 +2108,9 @@ class _ExternalDownloaderPool:
def perform(self):
ret = []
for fd, event in self.epoll.poll():
+ if event & select.EPOLLHUP:
+ if DEBUG: DEBUG.info('downloader died')
+ raise KeyboardInterrupt
assert event & select.EPOLLIN
done = self.running[fd].perform()
if not done: continue
commit 6ae8bc9a6e95bac41d50a4c555b935cd8ed9b865
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date: Thu Jul 26 12:25:11 2012 +0200
python3-compat: handle renamed modules and types
diff --git a/urlgrabber/__init__.py b/urlgrabber/__init__.py
index ddd5204..ed0e526 100644
--- a/urlgrabber/__init__.py
+++ b/urlgrabber/__init__.py
@@ -51,4 +51,4 @@ __author__ = 'Michael D. Stenner <mstenner at linux.duke.edu>, ' \
'Seth Vidal <skvidal at fedoraproject.org>'
__url__ = 'http://linux.duke.edu/projects/urlgrabber/'
-from grabber import urlgrab, urlopen, urlread
+from urlgrabber.grabber import urlgrab, urlopen, urlread
diff --git a/urlgrabber/byterange.py b/urlgrabber/byterange.py
index f7caec4..5461da5 100644
--- a/urlgrabber/byterange.py
+++ b/urlgrabber/byterange.py
@@ -20,22 +20,36 @@
import os
import stat
-import urllib
-import urllib2
-import rfc822
+import sys
DEBUG = None
-try:
- from cStringIO import StringIO
-except ImportError:
- from StringIO import StringIO
+# python2+3 compatible imports
+if sys.version_info.major < 3:
+ import urllib2 as urllib_request
+ import urlparse as urllib_parse
+ import urllib2 as urllib_error
+ from urllib2 import unquote
+ from mimetools import Message
+ from rfc822 import formatdate
+ try:
+ from cStringIO import StringIO
+ except ImportError:
+ from StringIO import StringIO
+else:
+ import urllib.request as urllib_request
+ import urllib.parse as urllib_parse
+ import urllib.error as urllib_error
+ from urllib.parse import unquote
+ from email.message import Message
+ from email.utils import formatdate
+ from io import StringIO
class RangeError(IOError):
"""Error raised when an unsatisfiable range is requested."""
pass
-class HTTPRangeHandler(urllib2.BaseHandler):
+class HTTPRangeHandler(urllib_request.BaseHandler):
"""Handler that enables HTTP Range headers.
This was extremely simple. The Range header is a HTTP feature to
@@ -61,7 +75,7 @@ class HTTPRangeHandler(urllib2.BaseHandler):
def http_error_206(self, req, fp, code, msg, hdrs):
# 206 Partial Content Response
- r = urllib.addinfourl(fp, hdrs, req.get_full_url())
+ r = urllib_request.addinfourl(fp, hdrs, req.get_full_url())
r.code = code
r.msg = msg
return r
@@ -211,25 +225,23 @@ class RangeableFileObject:
raise RangeError(9, 'Requested Range Not Satisfiable')
pos+= bufsize
-class FileRangeHandler(urllib2.FileHandler):
+class FileRangeHandler(urllib_request.FileHandler):
"""FileHandler subclass that adds Range support.
This class handles Range headers exactly like an HTTP
server would.
"""
def open_local_file(self, req):
- import mimetypes
- import mimetools
host = req.get_host()
file = req.get_selector()
- localfile = urllib.url2pathname(file)
+ localfile = urllib_request.url2pathname(file)
stats = os.stat(localfile)
size = stats[stat.ST_SIZE]
- modified = rfc822.formatdate(stats[stat.ST_MTIME])
+ modified = formatdate(stats[stat.ST_MTIME])
mtype = mimetypes.guess_type(file)[0]
if host:
- host, port = urllib.splitport(host)
+ host, port = urllib_parse.splitport(host)
if port or socket.gethostbyname(host) not in self.get_names():
- raise urllib2.URLError('file not on local host')
+ raise urllib_error.URLError('file not on local host')
fo = open(localfile,'rb')
brange = req.headers.get('Range',None)
brange = range_header_to_tuple(brange)
@@ -241,10 +253,10 @@ class FileRangeHandler(urllib2.FileHandler):
raise RangeError(9, 'Requested Range Not Satisfiable')
size = (lb - fb)
fo = RangeableFileObject(fo, (fb,lb))
- headers = mimetools.Message(StringIO(
+ headers = Message(StringIO(
'Content-Type: %s\nContent-Length: %d\nLast-modified: %s\n' %
(mtype or 'text/plain', size, modified)))
- return urllib.addinfourl(fo, headers, 'file:'+file)
+ return urllib_request.addinfourl(fo, headers, 'file:'+file)
# FTP Range Support
@@ -254,29 +266,25 @@ class FileRangeHandler(urllib2.FileHandler):
# follows:
# -- range support modifications start/end here
-from urllib import splitport, splituser, splitpasswd, splitattr, \
- unquote, addclosehook, addinfourl
import ftplib
import socket
-import sys
import mimetypes
-import mimetools
-class FTPRangeHandler(urllib2.FTPHandler):
+class FTPRangeHandler(urllib_request.FTPHandler):
def ftp_open(self, req):
host = req.get_host()
if not host:
raise IOError('ftp error', 'no host given')
- host, port = splitport(host)
+ host, port = urllib_parse.splitport(host)
if port is None:
port = ftplib.FTP_PORT
else:
port = int(port)
# username/password handling
- user, host = splituser(host)
+ user, host = urllib_parse.splituser(host)
if user:
- user, passwd = splitpasswd(user)
+ user, passwd = urllib_parse.splitpasswd(user)
else:
passwd = None
host = unquote(host)
@@ -286,18 +294,17 @@ class FTPRangeHandler(urllib2.FTPHandler):
try:
host = socket.gethostbyname(host)
except socket.error as msg:
- raise urllib2.URLError(msg)
- path, attrs = splitattr(req.get_selector())
- dirs = path.split('/')
- dirs = map(unquote, dirs)
- dirs, file = dirs[:-1], dirs[-1]
+ raise urllib_error.URLError(msg)
+ path, attrs = urllib_parse.splitattr(req.get_selector())
+ dirs = [unquote(i) for i in path.split('/')]
+ file = dirs.pop()
if dirs and not dirs[0]:
- dirs = dirs[1:]
+ del dirs[0]
try:
fw = self.connect_ftp(user, passwd, host, port, dirs)
type = file and 'I' or 'D'
for attr in attrs:
- attr, value = splitattr(attr)
+ attr, value = urllib_parse.splitattr(attr)
if attr.lower() == 'type' and \
value in ('a', 'A', 'i', 'I', 'd', 'D'):
type = value.upper()
@@ -336,8 +343,8 @@ class FTPRangeHandler(urllib2.FTPHandler):
if retrlen is not None and retrlen >= 0:
headers += "Content-Length: %d\n" % retrlen
sf = StringIO(headers)
- headers = mimetools.Message(sf)
- return addinfourl(fp, headers, req.get_full_url())
+ headers = Message(sf)
+ return urllib_request.addinfourl(fp, headers, req.get_full_url())
except ftplib.all_errors as msg:
raise IOError('ftp error', msg)
@@ -345,7 +352,7 @@ class FTPRangeHandler(urllib2.FTPHandler):
fw = ftpwrapper(user, passwd, host, port, dirs)
return fw
-class ftpwrapper(urllib.ftpwrapper):
+class ftpwrapper(urllib_request.ftpwrapper):
# range support note:
# this ftpwrapper code is copied directly from
# urllib. The only enhancement is to add the rest
@@ -389,7 +396,7 @@ class ftpwrapper(urllib.ftpwrapper):
conn = self.ftp.ntransfercmd(cmd)
self.busy = 1
# Pass back both a suitably decorated object and a retrieval length
- return (addclosehook(conn[0].makefile('rb'),
+ return (urllib_request.addclosehook(conn[0].makefile('rb'),
self.endtransfer), conn[1])
diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index 465acd2..72d654d 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -467,22 +467,42 @@ from __future__ import print_function
import os
import sys
-import urlparse
import time
import string
-import urllib
-import urllib2
-from httplib import responses
-import mimetools
-import thread
import types
import stat
import pycurl
from ftplib import parse150
-from StringIO import StringIO
-from httplib import HTTPException
import socket, select, fcntl
-from byterange import range_tuple_normalize, range_tuple_to_header, RangeError
+from urlgrabber.byterange import range_tuple_normalize, range_tuple_to_header, RangeError
+
+# python2+3 compatible imports
+if sys.version_info.major < 3:
+ import urllib2 as urllib_request
+ import urlparse as urllib_parse
+ import urllib2 as urllib_error
+ from urllib2 import quote, unquote
+ from httplib import responses, HTTPException
+ from mimetools import Message
+ try:
+ from cStringIO import StringIO
+ except ImportError:
+ from StringIO import StringIO
+ from types import StringTypes
+else:
+ import urllib.request as urllib_request
+ import urllib.parse as urllib_parse
+ import urllib.error as urllib_error
+ from urllib.parse import quote, unquote
+ from http.client import responses, HTTPException
+ from email.message import Message
+ from io import StringIO
+ import collections
+ def callable(cb):
+ return isinstance(cb, collections.Callable)
+ StringTypes = bytes, str
+ unicode = str
+ long = int
try:
import xattr
@@ -751,14 +771,14 @@ class URLParser:
if opts.prefix:
url = self.add_prefix(url, opts.prefix)
- parts = urlparse.urlparse(url)
+ parts = urllib_parse.urlparse(url)
(scheme, host, path, parm, query, frag) = parts
if not scheme or (len(scheme) == 1 and scheme in string.letters):
# if a scheme isn't specified, we guess that it's "file:"
if url[0] not in '/\\': url = os.path.abspath(url)
- url = 'file:' + urllib.pathname2url(url)
- parts = urlparse.urlparse(url)
+ url = 'file:' + urllib_request.pathname2url(url)
+ parts = urllib_parse.urlparse(url)
quote = 0 # pathname2url quotes, so we won't do it again
if scheme in ['http', 'https']:
@@ -769,7 +789,7 @@ class URLParser:
if quote:
parts = self.quote(parts)
- url = urlparse.urlunparse(parts)
+ url = urllib_parse.urlunparse(parts)
return url, parts
def add_prefix(self, url, prefix):
@@ -793,7 +813,7 @@ class URLParser:
passing into urlgrabber.
"""
(scheme, host, path, parm, query, frag) = parts
- path = urllib.quote(path)
+ path = quote(path)
return (scheme, host, path, parm, query, frag)
hexvals = '0123456789ABCDEF'
@@ -897,7 +917,7 @@ class URLGrabberOptions:
def _set_attributes(self, **kwargs):
"""Update object attributes with those provided in kwargs."""
self.__dict__.update(kwargs)
- if kwargs.has_key('range'):
+ if 'range' in kwargs:
# normalize the supplied range value
self.range = range_tuple_normalize(self.range)
if not self.reget in [None, 'simple', 'check_timestamp']:
@@ -963,7 +983,7 @@ class URLGrabberOptions:
return self.format()
def format(self, indent=' '):
- keys = self.__dict__.keys()
+ keys = list(self.__dict__.keys())
if self.delegate is not None:
keys.remove('delegate')
keys.sort()
@@ -1018,7 +1038,7 @@ class URLGrabber(object):
if DEBUG: DEBUG.info('attempt %i/%s: %s',
tries, opts.retry, args[0])
try:
- r = apply(func, (opts,) + args, {})
+ r = func(*(opts,) + args)
if DEBUG: DEBUG.info('success')
return r
except URLGrabError as e:
@@ -1076,14 +1096,14 @@ class URLGrabber(object):
(scheme, host, path, parm, query, frag) = parts
opts.find_proxy(url, scheme)
if filename is None:
- filename = os.path.basename( urllib.unquote(path) )
+ filename = os.path.basename(unquote(path))
if not filename:
# This is better than nothing.
filename = 'index.html'
if scheme == 'file' and not opts.copy_local:
# just return the name of the local file - don't make a
# copy currently
- path = urllib.url2pathname(path)
+ path = urllib_request.url2pathname(path)
if host:
path = os.path.normpath('//' + host + path)
if not os.path.exists(path):
@@ -1190,7 +1210,7 @@ class PyCurlFileObject(object):
self._hdr_dump = ''
self._parsed_hdr = None
self.url = url
- self.scheme = urlparse.urlsplit(self.url)[0]
+ self.scheme = urllib_parse.urlsplit(self.url)[0]
self.filename = filename
self.append = False
self.reget_time = None
@@ -1226,7 +1246,7 @@ class PyCurlFileObject(object):
if self.opts.progress_obj:
size = self.size + self._reget_length
self.opts.progress_obj.start(self._prog_reportname,
- urllib.unquote(self.url),
+ unquote(self.url),
self._prog_basename,
size=size,
text=self.opts.text)
@@ -1268,7 +1288,7 @@ class PyCurlFileObject(object):
if buf.lower().find('location') != -1:
location = ':'.join(buf.split(':')[1:])
location = location.strip()
- self.scheme = urlparse.urlsplit(location)[0]
+ self.scheme = urllib_parse.urlsplit(location)[0]
self.url = location
if len(self._hdr_dump) != 0 and buf == '\r\n':
@@ -1287,7 +1307,7 @@ class PyCurlFileObject(object):
hdrfp = StringIO()
hdrfp.write(self._hdr_dump[statusend:])
hdrfp.seek(0)
- self._parsed_hdr = mimetools.Message(hdrfp)
+ self._parsed_hdr = Message(hdrfp)
return self._parsed_hdr
hdr = property(_return_hdr_obj)
@@ -1416,7 +1436,7 @@ class PyCurlFileObject(object):
code = self.http_code
errcode = e.args[0]
- errurl = urllib.unquote(self.url)
+ errurl = unquote(self.url)
if self._error[0]:
errcode = self._error[0]
@@ -1548,7 +1568,7 @@ class PyCurlFileObject(object):
if self._error[1]:
msg = self._error[1]
err = URLGrabError(14, msg)
- err.url = urllib.unquote(self.url)
+ err.url = unquote(self.url)
raise err
def _do_open(self):
@@ -1565,7 +1585,7 @@ class PyCurlFileObject(object):
def _build_range(self):
reget_length = 0
rt = None
- if self.opts.reget and type(self.filename) in types.StringTypes:
+ if self.opts.reget and type(self.filename) in StringTypes:
# we have reget turned on and we're dumping to a file
try:
s = os.stat(self.filename)
@@ -1624,7 +1644,7 @@ class PyCurlFileObject(object):
err = URLGrabError(9, _('%s on %s') % (e, self.url))
err.url = self.url
raise err
- except urllib2.HTTPError as e:
+ except urllib_error.HTTPError as e:
new_e = URLGrabError(14, _('%s on %s') % (e, self.url))
new_e.code = e.code
new_e.exception = e
@@ -1660,7 +1680,7 @@ class PyCurlFileObject(object):
if self._complete:
return
_was_filename = False
- if type(self.filename) in types.StringTypes and self.filename:
+ if type(self.filename) in StringTypes and self.filename:
_was_filename = True
self._prog_reportname = str(self.filename)
self._prog_basename = os.path.basename(self.filename)
@@ -2047,7 +2067,7 @@ class _ExternalDownloader:
for line in lines:
# parse downloader output
line = line.split(' ', 5)
- _id, size = map(int, line[:2])
+ _id, size = int(line[0]), int(line[1])
if len(line) == 2:
self.running[_id].progress_obj.update(size)
continue
@@ -2075,7 +2095,7 @@ class _ExternalDownloaderPool:
self.cache = {}
def start(self, opts):
- host = urlparse.urlsplit(opts.url).netloc
+ host = urllib_parse.urlsplit(opts.url).netloc
dl = self.cache.pop(host, None)
if not dl:
dl = _ExternalDownloader()
@@ -2095,7 +2115,7 @@ class _ExternalDownloaderPool:
ret.extend(done)
# dl finished, move it to the cache
- host = urlparse.urlsplit(done[0][0].url).netloc
+ host = urllib_parse.urlsplit(done[0][0].url).netloc
if host in self.cache: self.cache[host].abort()
self.epoll.unregister(fd)
self.cache[host] = self.running.pop(fd)
@@ -2126,7 +2146,7 @@ def parallel_wait(meter = 'text'):
count += 1
total += opts.size
if meter == 'text':
- from progress import TextMultiFileMeter
+ from urlgrabber.progress import TextMultiFileMeter
meter = TextMultiFileMeter()
meter.start(count, total)
@@ -2306,7 +2326,7 @@ class _TH:
# Use hostname from URL. If it's a file:// URL, use baseurl.
# If no baseurl, do not update timedhosts.
- host = urlparse.urlsplit(url).netloc.split('@')[-1] or baseurl
+ host = urllib_parse.urlsplit(url).netloc.split('@')[-1] or baseurl
if not host: return
speed, fail, ts = _TH.hosts.get(host) or (0, 0, 0)
@@ -2334,7 +2354,7 @@ class _TH:
_TH.load()
# Use just the hostname, unless it's a file:// baseurl.
- host = urlparse.urlsplit(baseurl).netloc.split('@')[-1] or baseurl
+ host = urllib_parse.urlsplit(baseurl).netloc.split('@')[-1] or baseurl
default_speed = default_grabber.opts.default_speed
try: speed, fail, ts = _TH.hosts[host]
@@ -2365,11 +2385,11 @@ def _main_test():
print("throttle: %s, throttle bandwidth: %s B/s" % (default_grabber.throttle,
default_grabber.bandwidth))
- try: from progress import text_progress_meter
+ try: from urlgrabber.progress import text_progress_meter
except ImportError: pass
else: kwargs['progress_obj'] = text_progress_meter()
- try: name = apply(urlgrab, (url, filename), kwargs)
+ try: name = urlgrab(url, filename, **kwargs)
except URLGrabError as e: print(e)
else: print('LOCAL FILE:', name)
@@ -2386,7 +2406,7 @@ def _retry_test():
k, v = string.split(a, '=', 1)
kwargs[k] = int(v)
- try: from progress import text_progress_meter
+ try: from urlgrabber.progress import text_progress_meter
except ImportError: pass
else: kwargs['progress_obj'] = text_progress_meter()
@@ -2404,12 +2424,11 @@ def _retry_test():
return
kwargs['checkfunc'] = (cfunc, ('hello',), {'there':'there'})
- try: name = apply(retrygrab, (url, filename), kwargs)
+ try: name = retrygrab(url, filename, **kwargs)
except URLGrabError as e: print(e)
else: print('LOCAL FILE:', name)
def _file_object_test(filename=None):
- import cStringIO
if filename is None:
filename = __file__
print('using file "%s" for comparisons' % filename)
@@ -2421,8 +2440,8 @@ def _file_object_test(filename=None):
_test_file_object_readall,
_test_file_object_readline,
_test_file_object_readlines]:
- fo_input = cStringIO.StringIO(s_input)
- fo_output = cStringIO.StringIO()
+ fo_input = StringIO(s_input)
+ fo_output = StringIO()
wrapper = PyCurlFileObject(fo_input, None, 0)
print('testing %-30s ' % testfunc.__name__, end='')
testfunc(wrapper, fo_output)
diff --git a/urlgrabber/mirror.py b/urlgrabber/mirror.py
index 75d9cae..bd76603 100644
--- a/urlgrabber/mirror.py
+++ b/urlgrabber/mirror.py
@@ -91,10 +91,16 @@ CUSTOMIZATION
import random
-import thread # needed for locking to make this threadsafe
+import sys
+if sys.version_info.major < 3:
+ import thread # needed for locking to make this threadsafe
+ from types import StringTypes
+else:
+ import _thread as thread
+ StringTypes = bytes, str
-from grabber import URLGrabError, CallbackObject, DEBUG, _to_utf8
-from grabber import _run_callback, _do_raise
+from urlgrabber.grabber import URLGrabError, CallbackObject, DEBUG, _to_utf8
+from urlgrabber.grabber import _run_callback, _do_raise
def _(st):
return st
@@ -268,7 +274,7 @@ class MirrorGroup:
def _parse_mirrors(self, mirrors):
parsed_mirrors = []
for m in mirrors:
- if isinstance(m, basestring):
+ if type(m) in StringTypes:
m = {'mirror': _to_utf8(m)}
parsed_mirrors.append(m)
return parsed_mirrors
diff --git a/urlgrabber/progress.py b/urlgrabber/progress.py
index 0d4d7bb..9888bef 100644
--- a/urlgrabber/progress.py
+++ b/urlgrabber/progress.py
@@ -21,11 +21,18 @@
import sys
import time
import math
-import thread
import fcntl
import struct
import termios
+if sys.version_info.major < 3:
+ import thread
+ from types import StringTypes
+else:
+ import _thread as thread
+ StringTypes = bytes, str
+ long = int
+
# Code from http://mail.python.org/pipermail/python-list/2000-May/033365.html
def terminal_width(fd=1):
""" Get the real terminal width """
@@ -582,7 +589,7 @@ class TextMultiFileMeter(MultiFileMeter):
try:
format = "%-30.30s %6.6s %s"
fn = meter.basename
- if type(message) in (type(''), type(u'')):
+ if type(message) in StringTypes:
message = message.splitlines()
if not message: message = ['']
out = '%-79s' % (format % (fn, 'FAILED', message[0] or ''))
@@ -748,7 +755,7 @@ def format_number(number, SI=0, space=' '):
depth = depth + 1
number = number / step
- if type(number) == type(1) or type(number) == type(1L):
+ if type(number) in (int, long):
# it's an int or a long, which means it didn't get divided,
# which means it's already short enough
format = '%i%s%s'
commit 3c33b88d8f74779729375511e281690b9544bae6
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date: Wed Jul 25 16:04:20 2012 +0200
python3-compat: print
diff --git a/scripts/urlgrabber b/scripts/urlgrabber
index a23a11c..c8d915a 100644
--- a/scripts/urlgrabber
+++ b/scripts/urlgrabber
@@ -139,6 +139,7 @@ import re
import urlgrabber.grabber
from urlgrabber.grabber import URLGrabber, URLGrabberOptions, URLGrabError
+from __future__ import print_function
class client_options:
def __init__(self):
@@ -161,7 +162,6 @@ class client_options:
for k in ugo.__dict__.keys():
if (k not in ug_options) and (k not in options_exclude):
ug_options.append(k)
- #print k
ug_defaults = {}
for k in list(ug_options):
try:
@@ -179,7 +179,7 @@ class client_options:
optlist, args = getopt.getopt(sys.argv[1:], short_options,
long_options + ug_long)
except getopt.GetoptError as e:
- print >>sys.stderr, "Error:", e
+ print("Error:", e, file=sys.stderr)
self.help([], ret=1)
self.verbose = 0
@@ -209,7 +209,7 @@ class client_options:
self.repeat = int(v)
if self.repeat < 1: raise ValueError()
except ValueError:
- print 'ERROR: repeat value must be an int >= 1'
+ print('ERROR: repeat value must be an int >= 1')
sys.exit(1)
if o == '-D':
self.verbose = 3
@@ -219,19 +219,19 @@ class client_options:
try:
val = eval(v)
except Exception as e:
- print "error processing option value: %s" % v
- print e
+ print("error processing option value: %s" % v)
+ print(e)
sys.exit(1)
else:
self.ugops[o[2:]] = val
if len(self.args) > 1 and self.outputfile is not None:
- print "ERROR: cannot use -o when grabbing multiple files"
+ print("ERROR: cannot use -o when grabbing multiple files")
sys.exit(1)
def help(self, args, ret=0):
if not args:
- print MAINHELP
+ print(MAINHELP)
else:
for a in args:
m = getattr(self, 'help_'+a, None)
@@ -240,20 +240,20 @@ class client_options:
elif a in self.ug_options:
self.help_ug_option(a)
else:
- print 'ERROR: no help on command "%s"' % a
+ print('ERROR: no help on command "%s"' % a)
sys.exit(ret)
def help_doc(self):
- print __doc__
+ print(__doc__)
def help_options(self):
width = max(map(len, self.ug_options))
format = ' %-' + str(width) + 's = %s'
hformat = ' %-' + str(width) + 's %s'
- print hformat % ('OPTION', 'DEFAULT')
- print '-'*(width + 20)
+ print(hformat % ('OPTION', 'DEFAULT'))
+ print('-'*(width + 20))
for k in self.ug_options:
- print format % (k, self.ug_defaults[k])
+ print(format % (k, self.ug_defaults[k]))
def help_all(self):
for k in self.ug_options:
@@ -264,21 +264,21 @@ class client_options:
m = re.search(r'^( '+option+'.*?)\s*^ {,2}\S',
urlgrabber.grabber.__doc__, re.M|re.S)
if m:
- print m.group(1)
+ print(m.group(1))
else:
- print ' %s: no help found for this option' % option
- print ''
+ print(' %s: no help found for this option' % option)
+ print('')
class ugclient:
def __init__(self):
op = client_options()
self.op = op
if op.verbose >= 2 and op.ugops:
- print "Module Options:"
+ print("Module Options:")
width = max(map(len, op.ugops.keys()))
format = " %-" + str(width) + "s = %s"
for k, v in op.ugops.items():
- print format % (k, repr(v))
+ print(format % (k, repr(v)))
if op.debug:
self.set_debug_logger(op.debug)
@@ -294,13 +294,13 @@ class ugclient:
def run(self):
for url in self.op.args:
- if self.op.verbose: print 'grabbing: %s' % url
+ if self.op.verbose: print('grabbing: %s' % url)
try:
for i in range(0, self.op.repeat):
f = self.g.urlgrab(url, self.op.outputfile)
- if self.op.localfile: print f
+ if self.op.localfile: print(f)
except URLGrabError as e:
- print e
+ print(e)
def set_debug_logger(self, dbspec):
try:
diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index 9e1c6ee..465acd2 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -463,6 +463,7 @@ BANDWIDTH THROTTLING
"""
+from __future__ import print_function
import os
import sys
@@ -2346,11 +2347,12 @@ class _TH:
#####################################################################
# TESTING
+
def _main_test():
try: url, filename = sys.argv[1:3]
except ValueError:
- print 'usage:', sys.argv[0], \
- '<url> <filename> [copy_local=0|1] [close_connection=0|1]'
+ print('usage:', sys.argv[0], \
+ '<url> <filename> [copy_local=0|1] [close_connection=0|1]')
sys.exit()
kwargs = {}
@@ -2360,23 +2362,23 @@ def _main_test():
set_throttle(1.0)
set_bandwidth(32 * 1024)
- print "throttle: %s, throttle bandwidth: %s B/s" % (default_grabber.throttle,
- default_grabber.bandwidth)
+ print("throttle: %s, throttle bandwidth: %s B/s" % (default_grabber.throttle,
+ default_grabber.bandwidth))
try: from progress import text_progress_meter
except ImportError: pass
else: kwargs['progress_obj'] = text_progress_meter()
try: name = apply(urlgrab, (url, filename), kwargs)
- except URLGrabError as e: print e
- else: print 'LOCAL FILE:', name
+ except URLGrabError as e: print(e)
+ else: print('LOCAL FILE:', name)
def _retry_test():
try: url, filename = sys.argv[1:3]
except ValueError:
- print 'usage:', sys.argv[0], \
- '<url> <filename> [copy_local=0|1] [close_connection=0|1]'
+ print('usage:', sys.argv[0], \
+ '<url> <filename> [copy_local=0|1] [close_connection=0|1]')
sys.exit()
kwargs = {}
@@ -2389,28 +2391,28 @@ def _retry_test():
else: kwargs['progress_obj'] = text_progress_meter()
def cfunc(filename, hello, there='foo'):
- print hello, there
+ print(hello, there)
import random
rnum = random.random()
if rnum < .5:
- print 'forcing retry'
+ print('forcing retry')
raise URLGrabError(-1, 'forcing retry')
if rnum < .75:
- print 'forcing failure'
+ print('forcing failure')
raise URLGrabError(-2, 'forcing immediate failure')
- print 'success'
+ print('success')
return
kwargs['checkfunc'] = (cfunc, ('hello',), {'there':'there'})
try: name = apply(retrygrab, (url, filename), kwargs)
- except URLGrabError as e: print e
- else: print 'LOCAL FILE:', name
+ except URLGrabError as e: print(e)
+ else: print('LOCAL FILE:', name)
def _file_object_test(filename=None):
import cStringIO
if filename is None:
filename = __file__
- print 'using file "%s" for comparisons' % filename
+ print('using file "%s" for comparisons' % filename)
fo = open(filename)
s_input = fo.read()
fo.close()
@@ -2422,11 +2424,11 @@ def _file_object_test(filename=None):
fo_input = cStringIO.StringIO(s_input)
fo_output = cStringIO.StringIO()
wrapper = PyCurlFileObject(fo_input, None, 0)
- print 'testing %-30s ' % testfunc.__name__,
+ print('testing %-30s ' % testfunc.__name__, end='')
testfunc(wrapper, fo_output)
s_output = fo_output.getvalue()
- if s_output == s_input: print 'passed'
- else: print 'FAILED'
+ if s_output == s_input: print('passed')
+ else: print('FAILED')
def _test_file_object_smallread(wrapper, fo_output):
while 1:
diff --git a/urlgrabber/progress.py b/urlgrabber/progress.py
index 4c126c5..0d4d7bb 100644
--- a/urlgrabber/progress.py
+++ b/urlgrabber/progress.py
@@ -623,7 +623,6 @@ class RateEstimator:
self.ave_rate = None
return
- #print 'times', now, self.last_update_time
time_diff = now - self.last_update_time
read_diff = amount_read - self.last_amount_read
# First update, on reget is the file size
@@ -632,7 +631,6 @@ class RateEstimator:
self.ave_rate = self._temporal_rolling_ave(\
time_diff, read_diff, self.ave_rate, self.timescale)
self.last_amount_read = amount_read
- #print 'results', time_diff, read_diff, self.ave_rate
#####################################################################
# result methods
commit 9242ed695c641d9089eea542717889d85faabcbc
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date: Wed Jul 25 14:19:03 2012 +0200
python3-compat: except
diff --git a/scripts/urlgrabber b/scripts/urlgrabber
index 09cd896..a23a11c 100644
--- a/scripts/urlgrabber
+++ b/scripts/urlgrabber
@@ -178,7 +178,7 @@ class client_options:
try:
optlist, args = getopt.getopt(sys.argv[1:], short_options,
long_options + ug_long)
- except getopt.GetoptError, e:
+ except getopt.GetoptError as e:
print >>sys.stderr, "Error:", e
self.help([], ret=1)
@@ -218,7 +218,7 @@ class client_options:
if o in ug_dash:
try:
val = eval(v)
- except Exception, e:
+ except Exception as e:
print "error processing option value: %s" % v
print e
sys.exit(1)
@@ -299,7 +299,7 @@ class ugclient:
for i in range(0, self.op.repeat):
f = self.g.urlgrab(url, self.op.outputfile)
if self.op.localfile: print f
- except URLGrabError, e:
+ except URLGrabError as e:
print e
def set_debug_logger(self, dbspec):
diff --git a/scripts/urlgrabber-ext-down b/scripts/urlgrabber-ext-down
index 3da55a4..3fcc8e2 100755
--- a/scripts/urlgrabber-ext-down
+++ b/scripts/urlgrabber-ext-down
@@ -25,7 +25,7 @@ from urlgrabber.grabber import \
def write(fmt, *arg):
try: os.write(1, fmt % arg)
- except OSError, e:
+ except OSError as e:
if e.args[0] != errno.EPIPE: raise
sys.exit(1)
@@ -63,7 +63,7 @@ def main():
size = fo._amount_read
dlsz = size - fo._reget_length
ug_err = 'OK'
- except URLGrabError, e:
+ except URLGrabError as e:
size = dlsz = 0
ug_err = '%d %s' % e.args
write('%d %d %d %.3f %s\n', opts._id, size, dlsz, time.time() - tm, ug_err)
diff --git a/urlgrabber/byterange.py b/urlgrabber/byterange.py
index e30cd48..f7caec4 100644
--- a/urlgrabber/byterange.py
+++ b/urlgrabber/byterange.py
@@ -28,7 +28,7 @@ DEBUG = None
try:
from cStringIO import StringIO
-except ImportError, msg:
+except ImportError:
from StringIO import StringIO
class RangeError(IOError):
@@ -285,7 +285,7 @@ class FTPRangeHandler(urllib2.FTPHandler):
try:
host = socket.gethostbyname(host)
- except socket.error, msg:
+ except socket.error as msg:
raise urllib2.URLError(msg)
path, attrs = splitattr(req.get_selector())
dirs = path.split('/')
@@ -338,7 +338,7 @@ class FTPRangeHandler(urllib2.FTPHandler):
sf = StringIO(headers)
headers = mimetools.Message(sf)
return addinfourl(fp, headers, req.get_full_url())
- except ftplib.all_errors, msg:
+ except ftplib.all_errors as msg:
raise IOError('ftp error', msg)
def connect_ftp(self, user, passwd, host, port, dirs):
@@ -364,7 +364,7 @@ class ftpwrapper(urllib.ftpwrapper):
# Use nlst to see if the file exists at all
try:
self.ftp.nlst(file)
- except ftplib.error_perm, reason:
+ except ftplib.error_perm as reason:
raise IOError('ftp error', reason)
# Restore the transfer mode!
self.ftp.voidcmd(cmd)
@@ -372,7 +372,7 @@ class ftpwrapper(urllib.ftpwrapper):
try:
cmd = 'RETR ' + file
conn = self.ftp.ntransfercmd(cmd, rest)
- except ftplib.error_perm, reason:
+ except ftplib.error_perm as reason:
if str(reason)[:3] == '501':
# workaround for REST not supported error
fp, retrlen = self.retrfile(file, type)
diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index 02898cb..9e1c6ee 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -502,7 +502,7 @@ except:
try:
# this part isn't going to do much - need to talk to gettext
from i18n import _
-except ImportError, msg:
+except ImportError:
def _(st): return st
########################################################################
@@ -1020,10 +1020,10 @@ class URLGrabber(object):
r = apply(func, (opts,) + args, {})
if DEBUG: DEBUG.info('success')
return r
- except URLGrabError, e:
+ except URLGrabError as e:
exception = e
callback = opts.failure_callback
- except KeyboardInterrupt, e:
+ except KeyboardInterrupt as e:
exception = e
callback = opts.interrupt_callback
if not callback:
@@ -1124,7 +1124,7 @@ class URLGrabber(object):
try:
return self._retry(opts, retryfunc, url, filename)
- except URLGrabError, e:
+ except URLGrabError as e:
_TH.update(url, 0, 0, e)
opts.exception = e
return _run_callback(opts.failfunc, opts)
@@ -1407,7 +1407,7 @@ class PyCurlFileObject(object):
try:
self.curl_obj.perform()
- except pycurl.error, e:
+ except pycurl.error as e:
# XXX - break some of these out a bit more clearly
# to other URLGrabErrors from
# http://curl.haxx.se/libcurl/c/libcurl-errors.html
@@ -1614,22 +1614,22 @@ class PyCurlFileObject(object):
else:
fo = opener.open(req)
hdr = fo.info()
- except ValueError, e:
+ except ValueError as e:
err = URLGrabError(1, _('Bad URL: %s : %s') % (self.url, e, ))
err.url = self.url
raise err
- except RangeError, e:
+ except RangeError as e:
err = URLGrabError(9, _('%s on %s') % (e, self.url))
err.url = self.url
raise err
- except urllib2.HTTPError, e:
+ except urllib2.HTTPError as e:
new_e = URLGrabError(14, _('%s on %s') % (e, self.url))
new_e.code = e.code
new_e.exception = e
new_e.url = self.url
raise new_e
- except IOError, e:
+ except IOError as e:
if hasattr(e, 'reason') and isinstance(e.reason, socket.timeout):
err = URLGrabError(12, _('Timeout on %s: %s') % (self.url, e))
err.url = self.url
@@ -1639,12 +1639,12 @@ class PyCurlFileObject(object):
err.url = self.url
raise err
- except OSError, e:
+ except OSError as e:
err = URLGrabError(5, _('%s on %s') % (e, self.url))
err.url = self.url
raise err
- except HTTPException, e:
+ except HTTPException as e:
err = URLGrabError(7, _('HTTP Exception (%s) on %s: %s') % \
(e.__class__.__name__, self.url, e))
err.url = self.url
@@ -1671,7 +1671,7 @@ class PyCurlFileObject(object):
(self.filename, mode))
try:
self.fo = open(self.filename, mode)
- except IOError, e:
+ except IOError as e:
err = URLGrabError(16, _(\
'error opening local file from %s, IOError: %s') % (self.url, e))
err.url = self.url
@@ -1690,7 +1690,7 @@ class PyCurlFileObject(object):
try:
self._do_perform()
- except URLGrabError, e:
+ except URLGrabError as e:
self.fo.flush()
self.fo.close()
raise e
@@ -1713,7 +1713,7 @@ class PyCurlFileObject(object):
if mod_time != -1:
try:
os.utime(self.filename, (mod_time, mod_time))
- except OSError, e:
+ except OSError as e:
err = URLGrabError(16, _(\
'error setting timestamp on file %s from %s, OSError: %s')
% (self.filename, self.url, e))
@@ -1722,7 +1722,7 @@ class PyCurlFileObject(object):
# re open it
try:
self.fo = open(self.filename, 'r')
- except IOError, e:
+ except IOError as e:
err = URLGrabError(16, _(\
'error opening file from %s, IOError: %s') % (self.url, e))
err.url = self.url
@@ -1768,17 +1768,17 @@ class PyCurlFileObject(object):
else: readamount = min(amt, self._rbufsize)
try:
new = self.fo.read(readamount)
- except socket.error, e:
+ except socket.error as e:
err = URLGrabError(4, _('Socket Error on %s: %s') % (self.url, e))
err.url = self.url
raise err
- except socket.timeout, e:
+ except socket.timeout as e:
raise URLGrabError(12, _('Timeout on %s: %s') % (self.url, e))
err.url = self.url
raise err
- except IOError, e:
+ except IOError as e:
raise URLGrabError(4, _('IOError on %s: %s') %(self.url, e))
err.url = self.url
raise err
@@ -2161,7 +2161,8 @@ def parallel_wait(meter = 'text'):
if ug_err is None:
if opts.checkfunc:
try: _run_callback(opts.checkfunc, opts)
- except URLGrabError, ug_err: pass
+ except URLGrabError as e:
+ ug_err = e
if ug_err is None:
continue
@@ -2169,7 +2170,7 @@ def parallel_wait(meter = 'text'):
if opts.failure_callback:
opts.exception = ug_err
try: _run_callback(opts.failure_callback, opts)
- except URLGrabError, ug_err:
+ except URLGrabError:
retry = 0 # no retries
if opts.tries < retry and ug_err.errno in opts.retrycodes:
start(opts, opts.tries + 1) # simple retry
@@ -2254,7 +2255,7 @@ def parallel_wait(meter = 'text'):
while host_con.get(key, 0) >= limit:
perform()
start(opts, 1)
- except IOError, e:
+ except IOError as e:
if e.errno != 4: raise
raise KeyboardInterrupt
@@ -2363,11 +2364,11 @@ def _main_test():
default_grabber.bandwidth)
try: from progress import text_progress_meter
- except ImportError, e: pass
+ except ImportError: pass
else: kwargs['progress_obj'] = text_progress_meter()
try: name = apply(urlgrab, (url, filename), kwargs)
- except URLGrabError, e: print e
+ except URLGrabError as e: print e
else: print 'LOCAL FILE:', name
@@ -2384,7 +2385,7 @@ def _retry_test():
kwargs[k] = int(v)
try: from progress import text_progress_meter
- except ImportError, e: pass
+ except ImportError: pass
else: kwargs['progress_obj'] = text_progress_meter()
def cfunc(filename, hello, there='foo'):
@@ -2402,7 +2403,7 @@ def _retry_test():
kwargs['checkfunc'] = (cfunc, ('hello',), {'there':'there'})
try: name = apply(retrygrab, (url, filename), kwargs)
- except URLGrabError, e: print e
+ except URLGrabError as e: print e
else: print 'LOCAL FILE:', name
def _file_object_test(filename=None):
diff --git a/urlgrabber/mirror.py b/urlgrabber/mirror.py
index ac78b34..75d9cae 100644
--- a/urlgrabber/mirror.py
+++ b/urlgrabber/mirror.py
@@ -400,7 +400,7 @@ class MirrorGroup:
if DEBUG: DEBUG.info('MIRROR: trying %s -> %s', url, fullurl)
try:
return func_ref( *(fullurl,), **kwargs )
- except URLGrabError, e:
+ except URLGrabError as e:
if DEBUG: DEBUG.info('MIRROR: failed')
obj = CallbackObject()
obj.exception = e
@@ -422,7 +422,7 @@ class MirrorGroup:
func = 'urlgrab'
try:
return self._mirror_try(func, url, kw)
- except URLGrabError, e:
+ except URLGrabError as e:
obj = CallbackObject(url=url, filename=filename, exception=e, **kwargs)
return _run_callback(kwargs.get('failfunc', _do_raise), obj)
commit 60f79bb7c4a9728851f078dd2ce8c52eb2c4b804
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date: Wed Jul 25 10:43:11 2012 +0200
python3-compat: raise
diff --git a/urlgrabber/byterange.py b/urlgrabber/byterange.py
index 8eeaeda..e30cd48 100644
--- a/urlgrabber/byterange.py
+++ b/urlgrabber/byterange.py
@@ -120,7 +120,7 @@ class RangeableFileObject:
in self.fo. This includes methods."""
if hasattr(self.fo, name):
return getattr(self.fo, name)
- raise AttributeError, name
+ raise AttributeError(name)
def tell(self):
"""Return the position within the range.
@@ -266,7 +266,7 @@ class FTPRangeHandler(urllib2.FTPHandler):
def ftp_open(self, req):
host = req.get_host()
if not host:
- raise IOError, ('ftp error', 'no host given')
+ raise IOError('ftp error', 'no host given')
host, port = splitport(host)
if port is None:
port = ftplib.FTP_PORT
@@ -339,7 +339,7 @@ class FTPRangeHandler(urllib2.FTPHandler):
headers = mimetools.Message(sf)
return addinfourl(fp, headers, req.get_full_url())
except ftplib.all_errors, msg:
- raise IOError, ('ftp error', msg), sys.exc_info()[2]
+ raise IOError('ftp error', msg)
def connect_ftp(self, user, passwd, host, port, dirs):
fw = ftpwrapper(user, passwd, host, port, dirs)
@@ -365,7 +365,7 @@ class ftpwrapper(urllib.ftpwrapper):
try:
self.ftp.nlst(file)
except ftplib.error_perm, reason:
- raise IOError, ('ftp error', reason), sys.exc_info()[2]
+ raise IOError('ftp error', reason)
# Restore the transfer mode!
self.ftp.voidcmd(cmd)
# Try to retrieve as a file
@@ -379,7 +379,7 @@ class ftpwrapper(urllib.ftpwrapper):
fp = RangeableFileObject(fp, (rest,''))
return (fp, retrlen)
elif str(reason)[:3] != '550':
- raise IOError, ('ftp error', reason), sys.exc_info()[2]
+ raise IOError('ftp error', reason)
if not conn:
# Set transfer mode to ASCII!
self.ftp.voidcmd('TYPE A')
diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index 83823ea..02898cb 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -838,7 +838,7 @@ class URLGrabberOptions:
def __getattr__(self, name):
if self.delegate and hasattr(self.delegate, name):
return getattr(self.delegate, name)
- raise AttributeError, name
+ raise AttributeError(name)
def raw_throttle(self):
"""Calculate raw throttle value from throttle and bandwidth
@@ -1195,7 +1195,7 @@ class PyCurlFileObject(object):
self.reget_time = None
self.opts = opts
if self.opts.reget == 'check_timestamp':
- raise NotImplementedError, "check_timestamp regets are not implemented in this ver of urlgrabber. Please report this."
+ raise NotImplementedError("check_timestamp regets are not implemented in this ver of urlgrabber. Please report this.")
self._complete = False
self._rbuf = ''
self._rbufsize = 1024*8
@@ -1217,7 +1217,7 @@ class PyCurlFileObject(object):
if hasattr(self.fo, name):
return getattr(self.fo, name)
- raise AttributeError, name
+ raise AttributeError(name)
def _retrieve(self, buf):
try:
@@ -1935,7 +1935,7 @@ def _dumps(v):
return "(%s)" % ','.join(map(_dumps, v))
if type(v) == list:
return "[%s]" % ','.join(map(_dumps, v))
- raise TypeError, 'Can\'t serialize %s' % v
+ raise TypeError('Can\'t serialize %s' % v)
def _loads(s):
def decode(v):
More information about the Yum-commits
mailing list