[yum-commits] reposync.py yumdownloader.py
zpavlas at osuosl.org
zpavlas at osuosl.org
Wed Aug 8 16:09:57 UTC 2012
reposync.py | 91 ++++++++++++++++++-------------------------------------
yumdownloader.py | 66 +++++++++++++--------------------------
2 files changed, 53 insertions(+), 104 deletions(-)
New commits:
commit 989a02bb7fc0d4c25fe549c93cf58c3f936ca6ac
Author: Zdeněk Pavlas <zpavlas at redhat.com>
Date: Wed Aug 8 12:15:14 2012 +0200
yumdownloader, reposync: use YumBase.downloadPkgs() to download pkgs
This saves some code, and runs downloads in parallel, if possible.
diff --git a/reposync.py b/reposync.py
index fb2c592..ad4e441 100755
--- a/reposync.py
+++ b/reposync.py
@@ -276,75 +276,46 @@ def main():
urlgrabber.progress.text_meter_total_size(remote_size)
download_list.sort(sortPkgObj)
- n = 0
- exit_code = 0
+ if opts.urls:
+ for pkg in download_list:
+ print urljoin(pkg.repo.urls[0], pkg.relativepath)
+ return 0
+
+ # create dest dir
+ if not os.path.exists(local_repo_path):
+ os.makedirs(local_repo_path)
+
+ # set localpaths
for pkg in download_list:
- n = n + 1
- repo = my.repos.getRepo(pkg.repoid)
- remote = pkg.returnSimple('relativepath')
- local = local_repo_path + '/' + remote
- localdir = os.path.dirname(local)
- if not os.path.exists(localdir):
- os.makedirs(localdir)
-
- sz = int(pkg.returnSimple('packagesize'))
- if os.path.exists(local) and os.path.getsize(local) == sz:
-
- if not opts.quiet:
- my.logger.error("[%s: %-5d of %-5d ] Skipping existing %s" % (repo.id, n, len(download_list), remote))
- continue
-
- if opts.urls:
- baseurl = None
- if repo.urls[0][-1] != '/':
- baseurl = repo.urls[0] + '/'
- else:
- baseurl = repo.urls[0]
- url = urljoin(baseurl,remote)
- print '%s' % url
- continue
-
- # make sure the repo subdir is here before we go on.
- if not os.path.exists(local_repo_path):
- try:
- os.makedirs(local_repo_path)
- except IOError, e:
- my.logger.error("Could not make repo subdir: %s" % e)
- my.closeRpmDB()
- sys.exit(1)
-
- # Disable cache otherwise things won't download
- repo.cache = 0
- if not opts.quiet:
- my.logger.info( '[%s: %-5d of %-5d ] Downloading %s' % (repo.id, n, len(download_list), remote))
- pkg.localpath = local # Hack: to set the localpath we want.
- try:
- path = repo.getPackage(pkg)
- except yum.Errors.RepoError, e:
- my.logger.error("Could not retrieve package %s. Error was %s" % (pkg, str(e)))
- local_size += sz
- exit_code = 1
- continue
-
- local_size += sz
- if hasattr(urlgrabber.progress, 'text_meter_total_size'):
- urlgrabber.progress.text_meter_total_size(remote_size, local_size)
- if opts.gpgcheck:
+ rpmfn = os.path.basename(pkg.remote_path)
+ pkg.localpath = os.path.join(local_repo_path, rpmfn)
+ pkg.repo.copy_local = True
+ pkg.repo.cache = 0
+
+ # use downloader from YumBase
+ exit_code = 0
+ probs = my.downloadPkgs(download_list)
+ if probs:
+ exit_code = 1
+ for key in probs:
+ for error in probs[key]:
+ self.logger.error('%s: %s', key, error)
+
+ if opts.gpgcheck:
+ for pkg in download_list:
result, error = my.sigCheckPkg(pkg)
if result != 0:
+ rpmfn = os.path.basename(pkg.remote_path)
if result == 1:
- my.logger.warning('Removing %s, due to missing GPG key.' % os.path.basename(remote))
+ my.logger.warning('Removing %s, due to missing GPG key.' % rpmfn)
elif result == 2:
- my.logger.warning('Removing %s due to failed signature check.' % os.path.basename(remote))
+ my.logger.warning('Removing %s due to failed signature check.' % rpmfn)
else:
- my.logger.warning('Removing %s due to failed signature check: %s' % (os.path.basename(remote), error))
- os.unlink(path)
+ my.logger.warning('Removing %s due to failed signature check: %s' % rpmfn)
+ os.unlink(pkg.localpath)
exit_code = 1
continue
- if not os.path.exists(local) or not os.path.samefile(path, local):
- shutil.copy2(path, local)
-
my.closeRpmDB()
sys.exit(exit_code)
diff --git a/yumdownloader.py b/yumdownloader.py
index 4cdd595..ba17edd 100755
--- a/yumdownloader.py
+++ b/yumdownloader.py
@@ -192,52 +192,30 @@ class YumDownloader(YumUtilBase):
if len(toDownload) == 0:
self.logger.error('Nothing to download')
sys.exit(1)
+ if opts.urls:
+ for pkg in toDownload:
+ print urljoin(pkg.repo.urls[0], pkg.relativepath)
+ return 0
- exit_code = 0
+ # create dest dir
+ if not os.path.exists(opts.destdir):
+ os.makedirs(opts.destdir)
+
+ # set localpaths
for pkg in toDownload:
- n,a,e,v,r = pkg.pkgtup
- packages = self.pkgSack.searchNevra(n,e,v,r,a)
- packages.sort()
- last = None
- for download in packages:
- if download.pkgtup == last :
- continue
- last = download.pkgtup
- repo = self.repos.getRepo(download.repoid)
- remote = download.returnSimple('relativepath')
- if opts.urls:
- url = urljoin(repo.urls[0]+'/',remote)
- self.logger.info('%s' % url)
- continue
- local = os.path.basename(remote)
- if not os.path.exists(opts.destdir):
- os.makedirs(opts.destdir)
- local = os.path.join(opts.destdir, local)
- if (os.path.exists(local) and
- os.path.getsize(local) == int(download.returnSimple('packagesize'))):
- self.logger.error("%s already exists and appears to be complete" % local)
- continue
- # Disable cache otherwise things won't download
- repo.cache = 0
- download.localpath = local # Hack: to set the localpath we want.
- try:
- checkfunc = (self.verifyPkg, (download, 1), {})
- path = repo.getPackage(download, checkfunc=checkfunc)
- except IOError, e:
- self.logger.error("Cannot write to file %s. Error was: %s" % (local, e))
- exit_code = 2
- continue
- except RepoError, e:
- self.logger.error("Could not download/verify pkg %s: %s" % (download, e))
- exit_code = 2
- continue
-
- if not os.path.exists(local) or not os.path.samefile(path, local):
- progress = TextMeter()
- progress.start(basename=os.path.basename(local),
- size=os.stat(path).st_size)
- shutil.copy2(path, local)
- progress.end(progress.size)
+ rpmfn = os.path.basename(pkg.remote_path)
+ pkg.localpath = os.path.join(opts.destdir, rpmfn)
+ pkg.repo.copy_local = True
+ pkg.repo.cache = 0
+
+ # use downloader from YumBase
+ exit_code = 0
+ probs = self.downloadPkgs(toDownload)
+ if probs:
+ exit_code = 2
+ for key in probs:
+ for error in probs[key]:
+ self.logger.error('%s: %s', key, error)
return exit_code
def _groupPackages(self,pkglist):
More information about the Yum-commits
mailing list