[yum-commits] 5 commits - scripts/urlgrabber urlgrabber/grabber.py
James Antill
james at osuosl.org
Wed Jul 14 15:40:20 UTC 2010
scripts/urlgrabber | 19 +++++++++++++------
urlgrabber/grabber.py | 18 ++++++++++++++++++
2 files changed, 31 insertions(+), 6 deletions(-)
New commits:
commit 1bfef62a275d0b5f6231579c4add92384509303b
Author: James Antill <james at and.org>
Date: Wed Jul 14 11:33:04 2010 -0400
Don't traceback when we get an unknown option.
diff --git a/scripts/urlgrabber b/scripts/urlgrabber
index d0fa181..09cd896 100644
--- a/scripts/urlgrabber
+++ b/scripts/urlgrabber
@@ -175,8 +175,13 @@ class client_options:
long_options = ['profile', 'repeat=', 'verbose=',
'debug=', 'help', 'progress', 'output=']
ug_long = [ o + '=' for o in self.ug_options ]
- optlist, args = getopt.getopt(sys.argv[1:], short_options,
- long_options + ug_long)
+ try:
+ optlist, args = getopt.getopt(sys.argv[1:], short_options,
+ long_options + ug_long)
+ except getopt.GetoptError, e:
+ print >>sys.stderr, "Error:", e
+ self.help([], ret=1)
+
self.verbose = 0
self.debug = None
self.outputfile = None
@@ -224,7 +229,7 @@ class client_options:
print "ERROR: cannot use -o when grabbing multiple files"
sys.exit(1)
- def help(self, args):
+ def help(self, args, ret=0):
if not args:
print MAINHELP
else:
@@ -236,7 +241,7 @@ class client_options:
self.help_ug_option(a)
else:
print 'ERROR: no help on command "%s"' % a
- sys.exit(0)
+ sys.exit(ret)
def help_doc(self):
print __doc__
commit 7b4c3097a118904f6c6dba12fafc1de8c32fa7d7
Author: James Antill <james at and.org>
Date: Wed Jul 14 11:27:37 2010 -0400
If there is no basename, use index.html
diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index af07d43..27d8eb3 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -943,6 +943,9 @@ class URLGrabber:
(scheme, host, path, parm, query, frag) = parts
if filename is None:
filename = os.path.basename( urllib.unquote(path) )
+ if not filename:
+ # This is better than nothing.
+ filename = 'index.html'
if scheme == 'file' and not opts.copy_local:
# just return the name of the local file - don't make a
# copy currently
commit 6858688f3050e7bb35d7fb5295140cf2143a9182
Author: James Antill <james at and.org>
Date: Wed Jul 14 11:25:15 2010 -0400
Add --output option.
diff --git a/scripts/urlgrabber b/scripts/urlgrabber
index 73a343f..d0fa181 100644
--- a/scripts/urlgrabber
+++ b/scripts/urlgrabber
@@ -115,6 +115,7 @@ options:
including quotes in the case of strings.
e.g. --user_agent='"foobar/2.0"'
+ --output FILE
-o FILE write output to FILE, otherwise the basename of the
url will be used
-O print the names of saved files to STDOUT
@@ -172,7 +173,7 @@ class client_options:
def process_command_line(self):
short_options = 'vd:ho:OpD'
long_options = ['profile', 'repeat=', 'verbose=',
- 'debug=', 'help', 'progress']
+ 'debug=', 'help', 'progress', 'output=']
ug_long = [ o + '=' for o in self.ug_options ]
optlist, args = getopt.getopt(sys.argv[1:], short_options,
long_options + ug_long)
@@ -193,6 +194,7 @@ class client_options:
if o == '--verbose': self.verbose = v
if o == '-v': self.verbose += 1
if o == '-o': self.outputfile = v
+ if o == '--output': self.outputfile = v
if o == '-p' or o == '--progress': self.progress = 1
if o == '-d' or o == '--debug': self.debug = v
if o == '--profile': self.profile = 1
commit a693f67d4933305c9c87adeefdd8d060765cf143
Author: James Antill <james at and.org>
Date: Wed Jul 14 11:24:07 2010 -0400
Fix the -o option
diff --git a/scripts/urlgrabber b/scripts/urlgrabber
index 518e512..73a343f 100644
--- a/scripts/urlgrabber
+++ b/scripts/urlgrabber
@@ -170,7 +170,7 @@ class client_options:
return ug_options, ug_defaults
def process_command_line(self):
- short_options = 'vd:hoOpD'
+ short_options = 'vd:ho:OpD'
long_options = ['profile', 'repeat=', 'verbose=',
'debug=', 'help', 'progress']
ug_long = [ o + '=' for o in self.ug_options ]
commit 76243b4da055f480f4979c4305df49c468c5cb0a
Author: James Antill <james at and.org>
Date: Tue Apr 20 16:23:27 2010 -0400
Add ip_resolve option, because ipv6 is ugly and smells bad
diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index 4797436..af07d43 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -248,6 +248,11 @@ GENERAL ARGUMENTS (kwargs)
Maximum size (in bytes) of the headers.
+ self.ip_resolve = 'whatever'
+
+ What type of name to IP resolving to use, default is to do both IPV4 and
+ IPV6.
+
RETRY RELATED ARGUMENTS
@@ -806,6 +811,7 @@ class URLGrabberOptions:
self.close_connection = 0
self.range = None
self.user_agent = 'urlgrabber/%s' % __version__
+ self.ip_resolve = None
self.keepalive = 1
self.proxies = None
self.reget = None
@@ -1171,6 +1177,15 @@ class PyCurlFileObject():
self.curl_obj.setopt(pycurl.VERBOSE, True)
if opts.user_agent:
self.curl_obj.setopt(pycurl.USERAGENT, opts.user_agent)
+ if opts.ip_resolve:
+ # Default is: IPRESOLVE_WHATEVER
+ ipr = opts.ip_resolve.lower()
+ if ipr == 'whatever': # Do we need this?
+ self.curl_obj.setopt(pycurl.IPRESOLVE,pycurl.IPRESOLVE_WHATEVER)
+ if ipr == 'ipv4':
+ self.curl_obj.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4)
+ if ipr == 'ipv6':
+ self.curl_obj.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V6)
# maybe to be options later
self.curl_obj.setopt(pycurl.FOLLOWLOCATION, True)
More information about the Yum-commits
mailing list