[yum-cvs] yum/yum __init__.py, 1.336, 1.337 config.py, 1.122, 1.123 yumRepo.py, 1.50, 1.51

Seth Vidal skvidal at linux.duke.edu
Mon Jul 2 19:40:27 UTC 2007


Update of /home/groups/yum/cvs/yum/yum
In directory login1.linux.duke.edu:/tmp/cvs-serv2021/yum

Modified Files:
	__init__.py config.py yumRepo.py 
Log Message:

download and save mirrorlists locally when using them
expire mirrorlist data based on mirrorlist_expire config variable in [main]
and each repo config
deprecate metadataCurrent() and getMirrorList from yumRepo.py


Index: __init__.py
===================================================================
RCS file: /home/groups/yum/cvs/yum/yum/__init__.py,v
retrieving revision 1.336
retrieving revision 1.337
diff -u -r1.336 -r1.337
--- __init__.py	29 Jun 2007 10:19:40 -0000	1.336
+++ __init__.py	2 Jul 2007 19:40:25 -0000	1.337
@@ -978,7 +978,7 @@
         return self._cleanFiles(exts, 'cachedir', 'sqlite')
 
     def cleanMetadata(self):
-        exts = ['xml.gz', 'xml', 'cachecookie']
+        exts = ['xml.gz', 'xml', 'cachecookie', 'mirrorlist.txt']
         return self._cleanFiles(exts, 'cachedir', 'metadata') 
 
     def _cleanFiles(self, exts, pathattr, filetype):

Index: config.py
===================================================================
RCS file: /home/groups/yum/cvs/yum/yum/config.py,v
retrieving revision 1.122
retrieving revision 1.123
diff -u -r1.122 -r1.123
--- config.py	25 Jun 2007 18:46:52 -0000	1.122
+++ config.py	2 Jul 2007 19:40:25 -0000	1.123
@@ -520,7 +520,8 @@
 
     http_caching = SelectionOption('all', ('none', 'packages', 'all'))
     metadata_expire = IntOption(1800)   # time in seconds
-
+    mirrorlist_expire = IntOption(86400) # time in seconds (1 day)
+    
     _reposlist = []
 
 class RepoConf(BaseConfig):
@@ -551,7 +552,8 @@
     timeout = Inherit(YumConf.timeout)
     http_caching = Inherit(YumConf.http_caching)
     metadata_expire = Inherit(YumConf.metadata_expire)
-
+    mirrorlist_expire = Inherit(YumConf.mirrorlist_expire)
+    
 def readStartupConfig(configfile, root):
     '''
     Parse Yum's main configuration file and return a StartupConf instance.

Index: yumRepo.py
===================================================================
RCS file: /home/groups/yum/cvs/yum/yum/yumRepo.py,v
retrieving revision 1.50
retrieving revision 1.51
diff -u -r1.50 -r1.51
--- yumRepo.py	27 Jun 2007 19:39:48 -0000	1.50
+++ yumRepo.py	2 Jul 2007 19:40:25 -0000	1.51
@@ -448,25 +448,29 @@
     def _baseurlSetup(self):
         """go through the baseurls and mirrorlists and populate self.urls
            with valid ones, run  self.check() at the end to make sure it worked"""
-        goodurls = []
+
+        mirrorurls = []
         if self.mirrorlist and not self.mirrorlistparsed:
-            mirrorurls = getMirrorList(self.mirrorlist, self.proxy_dict)
-            self.mirrorlistparsed = 1
-            for url in mirrorurls:
-                url = parser.varReplace(url, self.yumvar)
-                self.baseurl.append(url)
+            mirrorurls.extend(self._getMirrorList())
+            self.mirrorlistparsed = True
 
-        for url in self.baseurl:
+        self.baseurl = self._replace_and_check_url(self.baseurl)
+        self.mirrorurls = self._replace_and_check_url(mirrorurls)
+        self._urls = self.baseurl + self.mirrorurls
+        self.check()
+        
+    def _replace_and_check_url(self, url_list):
+        goodurls = []
+        for url in url_list:
             url = parser.varReplace(url, self.yumvar)
             (s,b,p,q,f,o) = urlparse.urlparse(url)
             if s not in ['http', 'ftp', 'file', 'https']:
-                print 'not using ftp, http[s], or file for repos, skipping - %s' % (url)
+                print 'YumRepo Warning: not using ftp, http[s], or file for repos, skipping - %s' % (url)
                 continue
             else:
                 goodurls.append(url)
-    
-        self._urls = goodurls
-        self.check()
+
+        return goodurls
 
     def _geturls(self):
         if not self._urls:
@@ -613,17 +617,29 @@
         """Check if there is a metadata_cookie and check its age. If the
         age of the cookie is less than metadata_expire time then return true
         else return False"""
+        warnings.warn('metadataCurrent() will go away in a future version of Yum.\n \
+                       please use withinCacheAge() instead.',
+                Errors.YumFutureDeprecationWarning, stacklevel=2)
+
+        return self.withinCacheAge(self.metadata_cookie, self.metadata_expire)
+
+    def withinCacheAge(self, myfile, expiration_time):
+        """check if any file is older than a certain amount of time. Used for 
+           the cachecookie and the mirrorlist
+           return True if w/i the expiration time limit
+           false if the time limit has expired
+           """
 
         val = False
-        if os.path.exists(self.metadata_cookie):
-            cookie_info = os.stat(self.metadata_cookie)
-            if cookie_info[8] + self.metadata_expire > time.time():
+        if os.path.exists(myfile):
+            cookie_info = os.stat(myfile)
+            if cookie_info[8] + expiration_time > time.time():
                 val = True
             # WE ARE FROM THE FUTURE!!!!
             elif cookie_info[8] > time.time():
                 val = False
         return val
-
+           
     def setMetadataCookie(self):
         """if possible, set touch the metadata_cookie file"""
 
@@ -654,7 +670,7 @@
         if self._repoXML is not None:
             return
     
-        if self.cache or self.metadataCurrent():
+        if self.cache or self.withinCacheAge(self.metadata_cookie, self.metadata_expire):
             if not os.path.exists(local):
                 raise Errors.RepoError, 'Cannot find repomd.xml file for %s' % (self)
             else:
@@ -827,8 +843,55 @@
     def setInterruptCallback(self, callback):
         self.interrupt_callback = callback
         self._callbacks_changed = True
+    def _getMirrorList(self):
+        """retrieve an up2date-style mirrorlist file from our mirrorlist url,
+           also save the file to the local repo dir and use that if cache expiry
+           not expired
+
+           we also s/$ARCH/$BASEARCH/ and move along
+           return the baseurls from the mirrorlist file
+           """
+        returnlist = []
+        
+        self.mirrorlist_file = self.cachedir + '/' + 'mirrorlist.txt'
+        fo = None
+        
+        cacheok = False
+        if self.withinCacheAge(self.mirrorlist_file, self.mirrorlist_expire):
+            cacheok = True
+            fo = open(self.mirrorlist_file, 'r')
+        else:
+            url = self.mirrorlist
+            scheme = urlparse.urlparse(url)[0]
+            if scheme == '':
+                url = 'file://' + url
+            try:
+                fo = urlgrabber.grabber.urlopen(url, proxies=self.proxy_dict)
+            except urlgrabber.grabber.URLGrabError, e:
+                print "Could not retrieve mirrorlist %s error was\n%s" % (url, e)
+                fo = None
+        
+        if fo is not None:
+            content = fo.readlines()
+            for line in content:
+                if re.match('^\s*\#.*', line) or re.match('^\s*$', line):
+                    continue
+                mirror = re.sub('\n$', '', line) # no more trailing \n's
+                (mirror, count) = re.subn('\$ARCH', '$BASEARCH', mirror)
+                returnlist.append(mirror)
+
+            if not self.cache and not cacheok:
+                output = open(self.mirrorlist_file, 'w')
+                for line in content:
+                    output.write(line)
+                output.close()
+
+        return returnlist
+
 
 def getMirrorList(mirrorlist, pdict = None):
+    warnings.warn('getMirrorList() will go away in a future version of Yum.\n',
+            Errors.YumFutureDeprecationWarning, stacklevel=2)    
     """retrieve an up2date-style mirrorlist file from a url,
        we also s/$ARCH/$BASEARCH/ and move along
        returns a list of the urls from that file"""




More information about the Yum-cvs-commits mailing list