summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorZac Medico <zmedico@gentoo.org>2008-04-03 19:23:16 +0000
committerZac Medico <zmedico@gentoo.org>2008-04-03 19:23:16 +0000
commite2983690e7a22d69cb1507b55050438a6ec3d924 (patch)
treed899d272b1bf7ba6110aab278ab2d5e928327063
parentFor old-style virtuals, we need to repeat the package.provided (diff)
downloadportage-idfetch-e2983690e7a22d69cb1507b55050438a6ec3d924.tar.gz
portage-idfetch-e2983690e7a22d69cb1507b55050438a6ec3d924.tar.bz2
portage-idfetch-e2983690e7a22d69cb1507b55050438a6ec3d924.zip
Implement a PORTAGE_FETCH_RESUME_MIN_SIZE variable that is useful for
helping to ensure that small garbage files such as html 404 pages are properly discarded. svn path=/main/trunk/; revision=9687
-rw-r--r--cnf/make.globals3
-rw-r--r--man/make.conf.58
-rw-r--r--pym/portage/__init__.py60
3 files changed, 68 insertions, 3 deletions
diff --git a/cnf/make.globals b/cnf/make.globals
index 36d8591a..5e815013 100644
--- a/cnf/make.globals
+++ b/cnf/make.globals
@@ -59,6 +59,9 @@ AUTOCLEAN="yes"
# Number of mirrors to try when a downloaded file has an incorrect checksum.
PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS="5"
+# Minimum size of existing file for RESUMECOMMAND to be called.
+PORTAGE_FETCH_RESUME_MIN_SIZE="350K"
+
# Number of times 'emerge --sync' will run before giving up.
PORTAGE_RSYNC_RETRIES="3"
diff --git a/man/make.conf.5 b/man/make.conf.5
index 18d750a2..d35acf16 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -412,6 +412,14 @@ Please see /etc/make.conf.example for elog documentation.
\fBPORTAGE_FETCH_CHECKSUM_TRY_MIRRORS\fR = \fI5\fR
Number of mirrors to try when a downloaded file has an incorrect checksum.
.TP
+\PORTAGE_FETCH_RESUME_MIN_SIZE\fR = \fI350K\fR
+Minimum size of existing file for RESUMECOMMAND to be called. Files smaller
+than this size will be removed and FETCHCOMMAND will be called to download
+the file from the beginning. This is useful for helping to ensure that small
+garbage files such as html 404 pages are properly discarded. The variable
+should contain an integer number of bytes and may have a suffix such as
+K, M, or G.
+.TP
\fBPORTAGE_NICENESS\fR = \fI[number]\fR
The value of this variable will be added to the current nice level that
emerge is running at. In other words, this will not set the nice level,
diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py
index a64e90c5..4233c2ee 100644
--- a/pym/portage/__init__.py
+++ b/pym/portage/__init__.py
@@ -938,7 +938,8 @@ class config(object):
"PORTAGE_ECLASS_WARNING_ENABLE", "PORTAGE_ELOG_CLASSES",
"PORTAGE_ELOG_MAILFROM", "PORTAGE_ELOG_MAILSUBJECT",
"PORTAGE_ELOG_MAILURI", "PORTAGE_ELOG_SYSTEM",
- "PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS", "PORTAGE_GPG_DIR",
+ "PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS", "PORTAGE_FETCH_RESUME_MIN_SIZE",
+ "PORTAGE_GPG_DIR",
"PORTAGE_GPG_KEY", "PORTAGE_PACKAGE_EMPTY_ABORT",
"PORTAGE_RSYNC_EXTRA_OPTS", "PORTAGE_RSYNC_OPTS",
"PORTAGE_RSYNC_RETRIES", "PORTAGE_USE", "PORT_LOGDIR",
@@ -3010,6 +3011,20 @@ def _checksum_failure_temp_file(distdir, basename):
os.rename(filename, temp_filename)
return temp_filename
+_fetch_resume_size_re = re.compile('(^[\d]+)([KMGTPEZY]?$)')
+
+_size_suffix_map = {
+ '' : 0,
+ 'K' : 10,
+ 'M' : 20,
+ 'G' : 30,
+ 'T' : 40,
+ 'P' : 50,
+ 'E' : 60,
+ 'Z' : 70,
+ 'Y' : 80,
+}
+
def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",use_locks=1, try_mirrors=1):
"fetch files. Will use digest file if available."
@@ -3050,6 +3065,26 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
checksum_failure_max_tries = v
del v
+ fetch_resume_size_default = "350K"
+ fetch_resume_size = mysettings.get("PORTAGE_FETCH_RESUME_MIN_SIZE")
+ if fetch_resume_size is not None:
+ fetch_resume_size = "".join(fetch_resume_size.split())
+ match = _fetch_resume_size_re.match(fetch_resume_size)
+ if match is None or \
+ (match.group(2).upper() not in _size_suffix_map):
+ writemsg("!!! Variable PORTAGE_FETCH_RESUME_MIN_SIZE" + \
+ " contains an unrecognized format: '%s'\n" % \
+ mysettings["PORTAGE_FETCH_RESUME_MIN_SIZE"], noiselevel=-1)
+ writemsg("!!! Using PORTAGE_FETCH_RESUME_MIN_SIZE " + \
+ "default value: %s\n" % fetch_resume_size_default,
+ noiselevel=-1)
+ fetch_resume_size = None
+ if fetch_resume_size is None:
+ fetch_resume_size = fetch_resume_size_default
+ match = _fetch_resume_size_re.match(fetch_resume_size)
+ fetch_resume_size = int(match.group(1)) * \
+ 2 ** _size_suffix_map[match.group(2)]
+
# Behave like the package has RESTRICT="primaryuri" after a
# couple of checksum failures, to increase the probablility
# of success before checksum_failure_max_tries is reached.
@@ -3405,8 +3440,27 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
if fetched != 2 and has_space:
#we either need to resume or start the download
- #you can't use "continue" when you're inside a "try" block
- if fetched==1:
+ if fetched == 1:
+ try:
+ mystat = os.stat(myfile_path)
+ except OSError, e:
+ if e.errno != errno.ENOENT:
+ raise
+ del e
+ fetched = 0
+ else:
+ if mystat.st_size < fetch_resume_size:
+ writemsg((">>> Deleting distfile with size " + \
+ "%d (smaller than " "PORTAGE_FETCH_RESU" + \
+ "ME_MIN_SIZE)\n") % mystat.st_size)
+ try:
+ os.unlink(myfile_path)
+ except OSError, e:
+ if e.errno != errno.ENOENT:
+ raise
+ del e
+ fetched = 0
+ if fetched == 1:
#resume mode:
writemsg(">>> Resuming download...\n")
locfetch=resumecommand