[yum-commits] 2 commits - scripts/urlgrabber-ext-down test/test_mirror.py urlgrabber/grabber.py

[zpavlas at osuosl.org]

 scripts/urlgrabber-ext-down |    2 +-
 test/test_mirror.py         |   32 ++++++++++++++++++++++++++++++++
 urlgrabber/grabber.py       |    6 ++++--
 3 files changed, 37 insertions(+), 3 deletions(-)

New commits:
commit 701a4b09c405c274ea7cb2edb1958241162b2117
Author: Zdenek Pavlas <zpavlas at redhat.com>
Date:   Tue Apr 16 14:17:37 2013 +0200

    add URLGrabError.code to the external downloader API

diff --git a/scripts/urlgrabber-ext-down b/scripts/urlgrabber-ext-down
index 3dafb12..9ea0e70 100755
--- a/scripts/urlgrabber-ext-down
+++ b/scripts/urlgrabber-ext-down
@@ -68,7 +68,7 @@ def main():
                 ug_err = 'OK'
             except URLGrabError, e:
                 size = 0
-                ug_err = '%d %s' % e.args
+                ug_err = '%d %d %s' % (e.errno, getattr(e, 'code', 0), e.strerror)
             write('%d %d %d %.3f %s\n', opts._id, size, dlsz, dltm, ug_err)
 
 if __name__ == '__main__':
diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index 9526dc1..da15319 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -2052,7 +2052,7 @@ class _ExternalDownloader:
             raise KeyboardInterrupt
         for line in lines:
             # parse downloader output
-            line = line.split(' ', 5)
+            line = line.split(' ', 6)
             _id, size = map(int, line[:2])
             if len(line) == 2:
                 self.running[_id]._progress.update(size)
@@ -2063,7 +2063,9 @@ class _ExternalDownloader:
                 ug_err = None
                 if DEBUG: DEBUG.info('success')
             else:
-                ug_err = URLGrabError(int(line[4]), line[5])
+                ug_err = URLGrabError(int(line[4]), line[6])
+                if line[5] != '0':
+                    ug_err.code = int(line[5])
                 if DEBUG: DEBUG.info('failure: %s', ug_err)
             _TH.update(opts.url, int(line[2]), float(line[3]), ug_err, opts.async[0])
             ret.append((opts, size, ug_err))
commit fe639831896aa1b561231a0f6e7bb3ed753eda3e
Author: Zdenek Pavlas <zpavlas at redhat.com>
Date:   Tue Apr 16 14:03:53 2013 +0200

    test added: URLGrabError should include the HTTP error code.

diff --git a/test/test_mirror.py b/test/test_mirror.py
index cb63a41..6fdb668 100644
--- a/test/test_mirror.py
+++ b/test/test_mirror.py
@@ -269,6 +269,38 @@ class ActionTests(TestCase):
         self.assertEquals(urlgrabber.mirror.DEBUG.logs, expected_logs)
                 
 
+class HttpReplyCode(TestCase):
+    def setUp(self):
+        def server():
+            import socket
+            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+            s.bind(('localhost', 2000)); s.listen(1)
+            while 1:
+                c, a = s.accept()
+                while not c.recv(4096).endswith('\r\n\r\n'): pass
+                c.sendall('HTTP/1.1 %d %s\r\n' % self.reply)
+                c.close()
+        import thread
+        self.reply = 503, "Busy"
+        thread.start_new_thread(server, ())
+
+        def failure(obj):
+            self.code = getattr(obj.exception, 'code', None)
+            return {}
+        self.g  = URLGrabber()
+        self.mg = MirrorGroup(self.g, ['http://localhost:2000/'], failure_callback = failure)
+
+    def test_grab(self):
+        self.assertRaises(URLGrabError, self.mg.urlgrab, 'foo')
+        self.assertEquals(self.code, 503); del self.code
+
+        err = []
+        self.mg.urlgrab('foo', async = True, failfunc = err.append)
+        urlgrabber.grabber.parallel_wait()
+        self.assertEquals([e.exception.errno for e in err], [256])
+        self.assertEquals(self.code, 503); del self.code
+
 def suite():
     tl = TestLoader()
     return tl.loadTestsFromModule(sys.modules[__name__])