[yum-commits] test/test_mirror.py urlgrabber/grabber.py

[zpavlas at osuosl.org]

 test/test_mirror.py   |   13 +++++++++++++
 urlgrabber/grabber.py |   14 ++++++++++++--
 2 files changed, 25 insertions(+), 2 deletions(-)

New commits:
commit d6f9b4c1bc7db1a9f663e5c3e453d69257871232
Author: Zdenek Pavlas <zpavlas at redhat.com>
Date:   Wed Dec 18 10:01:50 2013 +0100

    Process HTTP ranges when the server didn't comply. BZ 435076.
    
    If user wants a range and proxy eats the "Range" header, server
    responds with "200 OK".  Clamp the stream at the client side.

diff --git a/test/test_mirror.py b/test/test_mirror.py
index 4ce790c..7f493d0 100644
--- a/test/test_mirror.py
+++ b/test/test_mirror.py
@@ -323,6 +323,19 @@ class HttpReplyCode(TestCase):
         self.assertEquals([e.exception.errno for e in err], [256])
         self.assertEquals(self.code, 503); del self.code
 
+    def test_range(self):
+        'test client-side processing of HTTP ranges'
+        # server does not process ranges
+        self.reply = 200, "OK"
+        self.content = 'ABCDEF'
+
+        # no range specified
+        data = self.mg.urlread('foo')
+        self.assertEquals(data, 'ABCDEF')
+
+        data = self.mg.urlread('foo', range = (3, 5))
+        self.assertEquals(data, 'DE')
+
 def suite():
     tl = TestLoader()
     return tl.loadTestsFromModule(sys.modules[__name__])
diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index 13f7919..03fa1ba 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -1249,6 +1249,7 @@ class PyCurlFileObject(object):
         self._tsize = 0
         self._amount_read = 0
         self._reget_length = 0
+        self._range = None
         self._prog_running = False
         self._error = (None, None)
         self.size = 0
@@ -1288,7 +1289,15 @@ class PyCurlFileObject(object):
 
             self._amount_read += len(buf)
             try:
-                self.fo.write(buf)
+                if self._range:
+                    # client-side ranges
+                    pos = self._amount_read - len(buf)
+                    start = self._range[0] - pos
+                    stop = self._range[1] - pos
+                    if start < len(buf) and stop > 0:
+                        self.fo.write(buf[max(start, 0):stop])
+                else:
+                    self.fo.write(buf)
             except IOError, e:
                 self._cb_error = URLGrabError(16, exception2msg(e))
                 return -1
@@ -1313,13 +1322,14 @@ class PyCurlFileObject(object):
                 if buf.lower().find('content-length:') != -1:
                     length = buf.split(':')[1]
                     self.size = int(length)
-                elif self.append and self._hdr_dump == '' and ' 200 ' in buf:
+                elif (self.append or self.opts.range) and self._hdr_dump == '' and ' 200 ' in buf:
                     # reget was attempted but server sends it all
                     # undo what we did in _build_range()
                     self.append = False
                     self.reget_time = None
                     self._amount_read = 0
                     self._reget_length = 0
+                    self._range = self.opts.range
                     self.fo.truncate(0)
             elif self.scheme in ['ftp']:
                 s = None