updated portage to 2.2.8-r1HEADmaster

1 files changed, 170 insertions, 64 deletions

diff --git a/portage_with_autodep/pym/_emerge/Scheduler.py b/portage_with_autodep/pym/_emerge/Scheduler.py
index 30a7e10..d663e97 100644
--- a/portage_with_autodep/pym/_emerge/Scheduler.py
+++ b/portage_with_autodep/pym/_emerge/Scheduler.py
@@ -1,7 +1,7 @@
-# Copyright 1999-2012 Gentoo Foundation
+# Copyright 1999-2013 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
-from __future__ import print_function
+from __future__ import print_function, unicode_literals
 
 from collections import deque
 import gc
@@ -18,7 +18,7 @@ import zlib
 import portage
 from portage import os
 from portage import _encodings
-from portage import _unicode_decode, _unicode_encode
+from portage import _unicode_encode
 from portage.cache.mappings import slot_dict_class
 from portage.elog.messages import eerror
 from portage.localization import _
@@ -28,6 +28,8 @@ from portage._sets import SETPREFIX
 from portage._sets.base import InternalPackageSet
 from portage.util import ensure_dirs, writemsg, writemsg_level
 from portage.util.SlotObject import SlotObject
+from portage.util._async.SchedulerInterface import SchedulerInterface
+from portage.util._eventloop.EventLoop import EventLoop
 from portage.package.ebuild.digestcheck import digestcheck
 from portage.package.ebuild.digestgen import digestgen
 from portage.package.ebuild.doebuild import (_check_temp_dir,
@@ -50,6 +52,7 @@ from _emerge.EbuildFetcher import EbuildFetcher
 from _emerge.EbuildPhase import EbuildPhase
 from _emerge.emergelog import emergelog
 from _emerge.FakeVartree import FakeVartree
+from _emerge.getloadavg import getloadavg
 from _emerge._find_deep_system_runtime_deps import _find_deep_system_runtime_deps
 from _emerge._flush_elog_mod_echo import _flush_elog_mod_echo
 from _emerge.JobStatusDisplay import JobStatusDisplay
@@ -64,6 +67,9 @@ if sys.hexversion >= 0x3000000:
 
 class Scheduler(PollScheduler):
 
+	# max time between loadavg checks (milliseconds)
+	_loadavg_latency = 30000
+
 	# max time between display status updates (milliseconds)
 	_max_display_latency = 3000
 
@@ -79,7 +85,7 @@ class Scheduler(PollScheduler):
 	_opts_no_self_update = frozenset(["--buildpkgonly",
 		"--fetchonly", "--fetch-all-uri", "--pretend"])
 
-	class _iface_class(PollScheduler._sched_iface_class):
+	class _iface_class(SchedulerInterface):
 		__slots__ = ("fetch",
 			"scheduleSetup", "scheduleUnpack")
 
@@ -135,8 +141,7 @@ class Scheduler(PollScheduler):
 			portage.exception.PortageException.__init__(self, value)
 
 	def __init__(self, settings, trees, mtimedb, myopts,
-		spinner, mergelist=None, favorites=None, graph_config=None,
-		uninstall_only=False):
+		spinner, mergelist=None, favorites=None, graph_config=None):
 		PollScheduler.__init__(self, main=True)
 
 		if mergelist is not None:
@@ -152,7 +157,6 @@ class Scheduler(PollScheduler):
 		self._spinner = spinner
 		self._mtimedb = mtimedb
 		self._favorites = favorites
-		self._uninstall_only = uninstall_only
 		self._args_set = InternalPackageSet(favorites, allow_repo=True)
 		self._build_opts = self._build_opts_class()
 
@@ -217,14 +221,15 @@ class Scheduler(PollScheduler):
 		fetch_iface = self._fetch_iface_class(log_file=self._fetch_log,
 			schedule=self._schedule_fetch)
 		self._sched_iface = self._iface_class(
+			self._event_loop,
+			is_background=self._is_background,
 			fetch=fetch_iface,
 			scheduleSetup=self._schedule_setup,
-			scheduleUnpack=self._schedule_unpack,
-			**dict((k, getattr(self.sched_iface, k))
-			for k in self.sched_iface.__slots__))
+			scheduleUnpack=self._schedule_unpack)
 
 		self._prefetchers = weakref.WeakValueDictionary()
 		self._pkg_queue = []
+		self._jobs = 0
 		self._running_tasks = {}
 		self._completed_tasks = set()
 
@@ -243,10 +248,15 @@ class Scheduler(PollScheduler):
 		# The load average takes some time to respond when new
 		# jobs are added, so we need to limit the rate of adding
 		# new jobs.
-		self._job_delay_max = 10
-		self._job_delay_factor = 1.0
-		self._job_delay_exp = 1.5
+		self._job_delay_max = 5
 		self._previous_job_start_time = None
+		self._job_delay_timeout_id = None
+
+		# The load average takes some time to respond when after
+		# a SIGSTOP/SIGCONT cycle, so delay scheduling for some
+		# time after SIGCONT is received.
+		self._sigcont_delay = 5
+		self._sigcont_time = None
 
 		# This is used to memoize the _choose_pkg() result when
 		# no packages can be chosen until one of the existing
@@ -300,15 +310,10 @@ class Scheduler(PollScheduler):
 			if not portage.dep.match_from_list(
 				portage.const.PORTAGE_PACKAGE_ATOM, [x]):
 				continue
-			if self._running_portage is None or \
-				self._running_portage.cpv != x.cpv or \
-				'9999' in x.cpv or \
-				'git' in x.inherited or \
-				'git-2' in x.inherited:
-				rval = _check_temp_dir(self.settings)
-				if rval != os.EX_OK:
-					return rval
-				_prepare_self_update(self.settings)
+			rval = _check_temp_dir(self.settings)
+			if rval != os.EX_OK:
+				return rval
+			_prepare_self_update(self.settings)
 			break
 
 		return os.EX_OK
@@ -328,12 +333,13 @@ class Scheduler(PollScheduler):
 		self._set_graph_config(graph_config)
 		self._blocker_db = {}
 		dynamic_deps = self.myopts.get("--dynamic-deps", "y") != "n"
+		ignore_built_slot_operator_deps = self.myopts.get(
+			"--ignore-built-slot-operator-deps", "n") == "y"
 		for root in self.trees:
-			if self._uninstall_only:
-				continue
 			if graph_config is None:
 				fake_vartree = FakeVartree(self.trees[root]["root_config"],
-					pkg_cache=self._pkg_cache, dynamic_deps=dynamic_deps)
+					pkg_cache=self._pkg_cache, dynamic_deps=dynamic_deps,
+					ignore_built_slot_operator_deps=ignore_built_slot_operator_deps)
 				fake_vartree.sync()
 			else:
 				fake_vartree = graph_config.trees[root]['vartree']
@@ -410,7 +416,7 @@ class Scheduler(PollScheduler):
 			if not (isinstance(task, Package) and \
 				task.operation == "merge"):
 				continue
-			if 'interactive' in task.metadata.properties:
+			if 'interactive' in task.properties:
 				interactive_tasks.append(task)
 		return interactive_tasks
 
@@ -655,10 +661,11 @@ class Scheduler(PollScheduler):
 				if value and value.strip():
 					continue
 				msg = _("%(var)s is not set... "
-					"Are you missing the '%(configroot)setc/make.profile' symlink? "
+					"Are you missing the '%(configroot)s%(profile_path)s' symlink? "
 					"Is the symlink correct? "
 					"Is your portage tree complete?") % \
-					{"var": var, "configroot": settings["PORTAGE_CONFIGROOT"]}
+					{"var": var, "configroot": settings["PORTAGE_CONFIGROOT"],
+					"profile_path": portage.const.PROFILE_PATH}
 
 				out = portage.output.EOutput()
 				for line in textwrap.wrap(msg, 70):
@@ -718,7 +725,6 @@ class Scheduler(PollScheduler):
 			return
 
 		if self._parallel_fetch:
-			self._status_msg("Starting parallel fetch")
 
 			prefetchers = self._prefetchers
 
@@ -771,10 +777,10 @@ class Scheduler(PollScheduler):
 
 		failures = 0
 
-		# Use a local PollScheduler instance here, since we don't
+		# Use a local EventLoop instance here, since we don't
 		# want tasks here to trigger the usual Scheduler callbacks
 		# that handle job scheduling and status display.
-		sched_iface = PollScheduler().sched_iface
+		sched_iface = SchedulerInterface(EventLoop(main=False))
 
 		for x in self._mergelist:
 			if not isinstance(x, Package):
@@ -783,10 +789,10 @@ class Scheduler(PollScheduler):
 			if x.operation == "uninstall":
 				continue
 
-			if x.metadata["EAPI"] in ("0", "1", "2", "3"):
+			if x.eapi in ("0", "1", "2", "3"):
 				continue
 
-			if "pretend" not in x.metadata.defined_phases:
+			if "pretend" not in x.defined_phases:
 				continue
 
 			out_str =">>> Running pre-merge checks for " + colorize("INFORM", x.cpv) + "\n"
@@ -805,7 +811,7 @@ class Scheduler(PollScheduler):
 			build_dir_path = os.path.join(
 				os.path.realpath(settings["PORTAGE_TMPDIR"]),
 				"portage", x.category, x.pf)
-			existing_buildir = os.path.isdir(build_dir_path)
+			existing_builddir = os.path.isdir(build_dir_path)
 			settings["PORTAGE_BUILDDIR"] = build_dir_path
 			build_dir = EbuildBuildDir(scheduler=sched_iface,
 				settings=settings)
@@ -816,7 +822,7 @@ class Scheduler(PollScheduler):
 
 				# Clean up the existing build dir, in case pkg_pretend
 				# checks for available space (bug #390711).
-				if existing_buildir:
+				if existing_builddir:
 					if x.built:
 						tree = "bintree"
 						infloc = os.path.join(build_dir_path, "build-info")
@@ -905,13 +911,18 @@ class Scheduler(PollScheduler):
 					failures += 1
 				portage.elog.elog_process(x.cpv, settings)
 			finally:
-				if current_task is not None and current_task.isAlive():
-					current_task.cancel()
-					current_task.wait()
-				clean_phase = EbuildPhase(background=False,
-					phase='clean', scheduler=sched_iface, settings=settings)
-				clean_phase.start()
-				clean_phase.wait()
+
+				if current_task is not None:
+					if current_task.isAlive():
+						current_task.cancel()
+						current_task.wait()
+					if current_task.returncode == os.EX_OK:
+						clean_phase = EbuildPhase(background=False,
+							phase='clean', scheduler=sched_iface,
+							settings=settings)
+						clean_phase.start()
+						clean_phase.wait()
+
 				build_dir.unlock()
 
 		if failures:
@@ -1001,6 +1012,8 @@ class Scheduler(PollScheduler):
 
 			earlier_sigint_handler = signal.signal(signal.SIGINT, sighandler)
 			earlier_sigterm_handler = signal.signal(signal.SIGTERM, sighandler)
+			earlier_sigcont_handler = \
+				signal.signal(signal.SIGCONT, self._sigcont_handler)
 
 			try:
 				rval = self._merge()
@@ -1014,6 +1027,10 @@ class Scheduler(PollScheduler):
 					signal.signal(signal.SIGTERM, earlier_sigterm_handler)
 				else:
 					signal.signal(signal.SIGTERM, signal.SIG_DFL)
+				if earlier_sigcont_handler is not None:
+					signal.signal(signal.SIGCONT, earlier_sigcont_handler)
+				else:
+					signal.signal(signal.SIGCONT, signal.SIG_DFL)
 
 			if received_signal:
 				sys.exit(received_signal[0])
@@ -1060,7 +1077,8 @@ class Scheduler(PollScheduler):
 		printer = portage.output.EOutput()
 		background = self._background
 		failure_log_shown = False
-		if background and len(self._failed_pkgs_all) == 1:
+		if background and len(self._failed_pkgs_all) == 1 and \
+			self.myopts.get('--quiet-fail', 'n') != 'y':
 			# If only one package failed then just show it's
 			# whole log for easy viewing.
 			failed_pkg = self._failed_pkgs_all[-1]
@@ -1139,9 +1157,9 @@ class Scheduler(PollScheduler):
 				printer.eerror(line)
 			printer.eerror("")
 			for failed_pkg in self._failed_pkgs_all:
-				# Use _unicode_decode() to force unicode format string so
+				# Use unicode_literals to force unicode format string so
 				# that Package.__unicode__() is called in python2.
-				msg = _unicode_decode(" %s") % (failed_pkg.pkg,)
+				msg = " %s" % (failed_pkg.pkg,)
 				log_path = self._locate_failure_log(failed_pkg)
 				if log_path is not None:
 					msg += ", Log file:"
@@ -1338,6 +1356,38 @@ class Scheduler(PollScheduler):
 		blocker_db = self._blocker_db[pkg.root]
 		blocker_db.discardBlocker(pkg)
 
+	def _main_loop(self):
+		term_check_id = self._event_loop.idle_add(self._termination_check)
+		loadavg_check_id = None
+		if self._max_load is not None and \
+			self._loadavg_latency is not None and \
+			(self._max_jobs is True or self._max_jobs > 1):
+			# We have to schedule periodically, in case the load
+			# average has changed since the last call.
+			loadavg_check_id = self._event_loop.timeout_add(
+				self._loadavg_latency, self._schedule)
+
+		try:
+			# Populate initial event sources. Unless we're scheduling
+			# based on load average, we only need to do this once
+			# here, since it can be called during the loop from within
+			# event handlers.
+			self._schedule()
+
+			# Loop while there are jobs to be scheduled.
+			while self._keep_scheduling():
+				self._event_loop.iteration()
+
+			# Clean shutdown of previously scheduled jobs. In the
+			# case of termination, this allows for basic cleanup
+			# such as flushing of buffered output to logs.
+			while self._is_work_scheduled():
+				self._event_loop.iteration()
+		finally:
+			self._event_loop.source_remove(term_check_id)
+			if loadavg_check_id is not None:
+				self._event_loop.source_remove(loadavg_check_id)
+
 	def _merge(self):
 
 		if self._opts_no_background.intersection(self.myopts):
@@ -1348,8 +1398,10 @@ class Scheduler(PollScheduler):
 		failed_pkgs = self._failed_pkgs
 		portage.locks._quiet = self._background
 		portage.elog.add_listener(self._elog_listener)
-		display_timeout_id = self.sched_iface.timeout_add(
-			self._max_display_latency, self._status_display.display)
+		display_timeout_id = None
+		if self._status_display._isatty and not self._status_display.quiet:
+			display_timeout_id = self._event_loop.timeout_add(
+				self._max_display_latency, self._status_display.display)
 		rval = os.EX_OK
 
 		try:
@@ -1358,7 +1410,8 @@ class Scheduler(PollScheduler):
 			self._main_loop_cleanup()
 			portage.locks._quiet = False
 			portage.elog.remove_listener(self._elog_listener)
-			self.sched_iface.source_remove(display_timeout_id)
+			if display_timeout_id is not None:
+				self._event_loop.source_remove(display_timeout_id)
 			if failed_pkgs:
 				rval = failed_pkgs[-1].returncode
 
@@ -1490,12 +1543,15 @@ class Scheduler(PollScheduler):
 		self._config_pool[settings['EROOT']].append(settings)
 
 	def _keep_scheduling(self):
-		return bool(not self._terminated_tasks and self._pkg_queue and \
+		return bool(not self._terminated.is_set() and self._pkg_queue and \
 			not (self._failed_pkgs and not self._build_opts.fetchonly))
 
 	def _is_work_scheduled(self):
 		return bool(self._running_tasks)
 
+	def _running_job_count(self):
+		return self._jobs
+
 	def _schedule_tasks(self):
 
 		while True:
@@ -1536,6 +1592,9 @@ class Scheduler(PollScheduler):
 				not self._task_queues.merge)):
 				break
 
+	def _sigcont_handler(self, signum, frame):
+		self._sigcont_time = time.time()
+
 	def _job_delay(self):
 		"""
 		@rtype: bool
@@ -1546,14 +1605,53 @@ class Scheduler(PollScheduler):
 
 			current_time = time.time()
 
-			delay = self._job_delay_factor * self._jobs ** self._job_delay_exp
+			if self._sigcont_time is not None:
+
+				elapsed_seconds = current_time - self._sigcont_time
+				# elapsed_seconds < 0 means the system clock has been adjusted
+				if elapsed_seconds > 0 and \
+					elapsed_seconds < self._sigcont_delay:
+
+					if self._job_delay_timeout_id is not None:
+						self._event_loop.source_remove(
+							self._job_delay_timeout_id)
+
+					self._job_delay_timeout_id = self._event_loop.timeout_add(
+						1000 * (self._sigcont_delay - elapsed_seconds),
+						self._schedule_once)
+					return True
+
+				# Only set this to None after the delay has expired,
+				# since this method may be called again before the
+				# delay has expired.
+				self._sigcont_time = None
+
+			try:
+				avg1, avg5, avg15 = getloadavg()
+			except OSError:
+				return False
+
+			delay = self._job_delay_max * avg1 / self._max_load
 			if delay > self._job_delay_max:
 				delay = self._job_delay_max
-			if (current_time - self._previous_job_start_time) < delay:
+			elapsed_seconds = current_time - self._previous_job_start_time
+			# elapsed_seconds < 0 means the system clock has been adjusted
+			if elapsed_seconds > 0 and elapsed_seconds < delay:
+
+				if self._job_delay_timeout_id is not None:
+					self._event_loop.source_remove(
+						self._job_delay_timeout_id)
+
+				self._job_delay_timeout_id = self._event_loop.timeout_add(
+					1000 * (delay - elapsed_seconds), self._schedule_once)
 				return True
 
 		return False
 
+	def _schedule_once(self):
+		self._schedule()
+		return False
+
 	def _schedule_tasks_imp(self):
 		"""
 		@rtype: bool
@@ -1735,7 +1833,7 @@ class Scheduler(PollScheduler):
 			#              scope
 			e = exc
 			mydepgraph = e.depgraph
-			dropped_tasks = set()
+			dropped_tasks = {}
 
 		if e is not None:
 			def unsatisfied_resume_dep_msg():
@@ -1772,11 +1870,7 @@ class Scheduler(PollScheduler):
 			return False
 
 		if success and self._show_list():
-			mylist = mydepgraph.altlist()
-			if mylist:
-				if "--tree" in self.myopts:
-					mylist.reverse()
-				mydepgraph.display(mylist, favorites=self._favorites)
+			mydepgraph.display(mydepgraph.altlist(), favorites=self._favorites)
 
 		if not success:
 			self._post_mod_echo_msgs.append(mydepgraph.display_problems)
@@ -1785,7 +1879,7 @@ class Scheduler(PollScheduler):
 		self._init_graph(mydepgraph.schedulerGraph())
 
 		msg_width = 75
-		for task in dropped_tasks:
+		for task, atoms in dropped_tasks.items():
 			if not (isinstance(task, Package) and task.operation == "merge"):
 				continue
 			pkg = task
@@ -1793,7 +1887,10 @@ class Scheduler(PollScheduler):
 				" %s" % (pkg.cpv,)
 			if pkg.root_config.settings["ROOT"] != "/":
 				msg += " for %s" % (pkg.root,)
-			msg += " dropped due to unsatisfied dependency."
+			if not atoms:
+				msg += " dropped because it is masked or unavailable"
+			else:
+				msg += " dropped because it requires %s" % ", ".join(atoms)
 			for line in textwrap.wrap(msg, msg_width):
 				eerror(line, phase="other", key=pkg.cpv)
 			settings = self.pkgsettings[pkg.root]
@@ -1838,11 +1935,21 @@ class Scheduler(PollScheduler):
 		root_config = pkg.root_config
 		world_set = root_config.sets["selected"]
 		world_locked = False
-		if hasattr(world_set, "lock"):
-			world_set.lock()
-			world_locked = True
+		atom = None
+
+		if pkg.operation != "uninstall":
+			# Do this before acquiring the lock, since it queries the
+			# portdbapi which can call the global event loop, triggering
+			# a concurrent call to this method or something else that
+			# needs an exclusive (non-reentrant) lock on the world file.
+			atom = create_world_atom(pkg, args_set, root_config)
 
 		try:
+
+			if hasattr(world_set, "lock"):
+				world_set.lock()
+				world_locked = True
+
 			if hasattr(world_set, "load"):
 				world_set.load() # maybe it's changed on disk
 
@@ -1854,8 +1961,7 @@ class Scheduler(PollScheduler):
 					for s in pkg.root_config.setconfig.active:
 						world_set.remove(SETPREFIX+s)
 			else:
-				atom = create_world_atom(pkg, args_set, root_config)
-				if atom:
+				if atom is not None:
 					if hasattr(world_set, "add"):
 						self._status_msg(('Recording %s in "world" ' + \
 							'favorites file...') % atom)