[yum-commits] urlgrabber/grabber.py

skvidal at osuosl.org skvidal at osuosl.org
Wed Jul 29 14:59:51 UTC 2009


 urlgrabber/grabber.py |   17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

New commits:
commit a45fcc76b9fb7897d1044c42d50e91c84b54ad2c
Author: Seth Vidal <skvidal at fedoraproject.org>
Date:   Wed Jul 29 10:57:23 2009 -0400

    get the content-length/size for ftp pkgs too
    - steals parse150 from ftplib. Should work for A LOT of ftp servers, but not all of them
    - add self.scheme for which protocol we're using here.

diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
index a8aae0e..15803fb 100644
--- a/urlgrabber/grabber.py
+++ b/urlgrabber/grabber.py
@@ -403,6 +403,7 @@ import mimetools
 import thread
 from stat import *  # S_* and ST_*
 import pycurl
+from ftplib import parse150
 from StringIO import StringIO
 
 ########################################################################
@@ -1448,6 +1449,7 @@ class PyCurlFileObject():
         self._hdr_dump = ''
         self._parsed_hdr = None
         self.url = url
+        self.scheme = urlparse.urlsplit(self.url)[0]
         self.filename = filename
         self.append = False
         self.opts = opts
@@ -1489,10 +1491,21 @@ class PyCurlFileObject():
     
     def _hdr_retrieve(self, buf):
         self._hdr_dump += buf
-        if buf.lower().find('content-length') != -1:
+        # we have to get the size before we do the progress obj start
+        # but we can't do that w/o making it do 2 connects, which sucks
+        # so we cheat and stuff it in here in the hdr_retrieve
+        if self.scheme in ['http','https'] and buf.lower().find('content-length') != -1:
             length = buf.split(':')[1]
             self.size = int(length)
-            
+        elif self.scheme in ['ftp']:
+            s = None
+            if buf.startswith('213 '):
+                s = buf[3:].strip()
+            elif buf.startswith('150 '):
+                s = parse150(buf)
+            if s:
+                self.size = s
+        
         return len(buf)
 
     def _return_hdr_obj(self):


More information about the Yum-commits mailing list