diff options
| author |   Alexander Bersenev <bay@hackerdom.ru> | 2014-02-17 17:57:05 +0600 |
|---|---|---|
| committer | Alexander Bersenev <bay@hackerdom.ru> | 2014-02-17 17:57:05 +0600 |
| commit | 6563293d18daed502ccdb663f3c72b4bae5fe23a (patch) | |
| tree | d0a7d53a7c137feb4073c963408829f88ea75c92 /portage_with_autodep/pym/_emerge | |
| parent | updated portage to 2.2.8-r1 (diff) | |
| download | autodep-master.tar.gz autodep-master.tar.bz2 autodep-master.zip | |
Diffstat (limited to 'portage_with_autodep/pym/_emerge')
157 files changed, 5331 insertions, 3684 deletions
diff --git a/portage_with_autodep/pym/_emerge/AbstractDepPriority.py b/portage_with_autodep/pym/_emerge/AbstractDepPriority.py index 94f26ef..1fcd043 100644 --- a/portage_with_autodep/pym/_emerge/AbstractDepPriority.py +++ b/portage_with_autodep/pym/_emerge/AbstractDepPriority.py @@ -1,11 +1,12 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import copy from portage.util.SlotObject import SlotObject class AbstractDepPriority(SlotObject): - __slots__ = ("buildtime", "runtime", "runtime_post") + __slots__ = ("buildtime", "buildtime_slot_op", + "runtime", "runtime_post", "runtime_slot_op") def __lt__(self, other): return self.__int__() < other diff --git a/portage_with_autodep/pym/_emerge/AbstractDepPriority.pyo b/portage_with_autodep/pym/_emerge/AbstractDepPriority.pyo Binary files differindex b6a9871..0108460 100644 --- a/portage_with_autodep/pym/_emerge/AbstractDepPriority.pyo +++ b/portage_with_autodep/pym/_emerge/AbstractDepPriority.pyo diff --git a/portage_with_autodep/pym/_emerge/AbstractEbuildProcess.py b/portage_with_autodep/pym/_emerge/AbstractEbuildProcess.py index c7b8f83..31127f4 100644 --- a/portage_with_autodep/pym/_emerge/AbstractEbuildProcess.py +++ b/portage_with_autodep/pym/_emerge/AbstractEbuildProcess.py @@ -1,8 +1,10 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import io +import platform import stat +import subprocess import textwrap from _emerge.SpawnProcess import SpawnProcess from _emerge.EbuildBuildDir import EbuildBuildDir @@ -20,8 +22,10 @@ class AbstractEbuildProcess(SpawnProcess): __slots__ = ('phase', 'settings',) + \ ('_build_dir', '_ipc_daemon', '_exit_command', '_exit_timeout_id') + _phases_without_builddir = ('clean', 'cleanrm', 'depend', 'help',) _phases_interactive_whitelist = ('config',) + _phases_without_cgroup = ('preinst', 'postinst', 'prerm', 'postrm', 'config') # Number of milliseconds to allow natural exit of the ebuild # process after it has called the exit command via IPC. It @@ -52,13 +56,48 @@ class AbstractEbuildProcess(SpawnProcess): if need_builddir and \ not os.path.isdir(self.settings['PORTAGE_BUILDDIR']): msg = _("The ebuild phase '%s' has been aborted " - "since PORTAGE_BUILDIR does not exist: '%s'") % \ + "since PORTAGE_BUILDDIR does not exist: '%s'") % \ (self.phase, self.settings['PORTAGE_BUILDDIR']) self._eerror(textwrap.wrap(msg, 72)) self._set_returncode((self.pid, 1 << 8)) - self.wait() + self._async_wait() return + # Check if the cgroup hierarchy is in place. If it's not, mount it. + if (os.geteuid() == 0 and platform.system() == 'Linux' + and 'cgroup' in self.settings.features + and self.phase not in self._phases_without_cgroup): + cgroup_root = '/sys/fs/cgroup' + cgroup_portage = os.path.join(cgroup_root, 'portage') + cgroup_path = os.path.join(cgroup_portage, + '%s:%s' % (self.settings["CATEGORY"], + self.settings["PF"])) + try: + # cgroup tmpfs + if not os.path.ismount(cgroup_root): + # we expect /sys/fs to be there already + if not os.path.isdir(cgroup_root): + os.mkdir(cgroup_root, 0o755) + subprocess.check_call(['mount', '-t', 'tmpfs', + '-o', 'rw,nosuid,nodev,noexec,mode=0755', + 'tmpfs', cgroup_root]) + + # portage subsystem + if not os.path.ismount(cgroup_portage): + if not os.path.isdir(cgroup_portage): + os.mkdir(cgroup_portage, 0o755) + subprocess.check_call(['mount', '-t', 'cgroup', + '-o', 'rw,nosuid,nodev,noexec,none,name=portage', + 'tmpfs', cgroup_portage]) + + # the ebuild cgroup + if not os.path.isdir(cgroup_path): + os.mkdir(cgroup_path) + except (subprocess.CalledProcessError, OSError): + pass + else: + self.cgroup = cgroup_path + if self.background: # Automatically prevent color codes from showing up in logs, # since we're not displaying to a terminal anyway. @@ -67,7 +106,7 @@ class AbstractEbuildProcess(SpawnProcess): if self._enable_ipc_daemon: self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None) if self.phase not in self._phases_without_builddir: - if 'PORTAGE_BUILDIR_LOCKED' not in self.settings: + if 'PORTAGE_BUILDDIR_LOCKED' not in self.settings: self._build_dir = EbuildBuildDir( scheduler=self.scheduler, settings=self.settings) self._build_dir.lock() @@ -143,9 +182,14 @@ class AbstractEbuildProcess(SpawnProcess): self._exit_command.reply_hook = self._exit_command_callback query_command = QueryCommand(self.settings, self.phase) commands = { - 'best_version' : query_command, - 'exit' : self._exit_command, - 'has_version' : query_command, + 'available_eclasses' : query_command, + 'best_version' : query_command, + 'eclass_path' : query_command, + 'exit' : self._exit_command, + 'has_version' : query_command, + 'license_path' : query_command, + 'master_repositories' : query_command, + 'repository_path' : query_command, } input_fifo, output_fifo = self._init_ipc_fifos() self._ipc_daemon = EbuildIpcDaemon(commands=commands, diff --git a/portage_with_autodep/pym/_emerge/AbstractEbuildProcess.pyo b/portage_with_autodep/pym/_emerge/AbstractEbuildProcess.pyo Binary files differindex b55f9c2..35828fe 100644 --- a/portage_with_autodep/pym/_emerge/AbstractEbuildProcess.pyo +++ b/portage_with_autodep/pym/_emerge/AbstractEbuildProcess.pyo diff --git a/portage_with_autodep/pym/_emerge/AbstractPollTask.py b/portage_with_autodep/pym/_emerge/AbstractPollTask.py index 2c84709..3f6dd6c 100644 --- a/portage_with_autodep/pym/_emerge/AbstractPollTask.py +++ b/portage_with_autodep/pym/_emerge/AbstractPollTask.py @@ -151,4 +151,4 @@ class AbstractPollTask(AsynchronousTask): while self._registered and not timeout_cb.timed_out: self.scheduler.iteration() finally: - self.scheduler.unregister(timeout_cb.timeout_id) + self.scheduler.source_remove(timeout_cb.timeout_id) diff --git a/portage_with_autodep/pym/_emerge/AbstractPollTask.pyo b/portage_with_autodep/pym/_emerge/AbstractPollTask.pyo Binary files differindex 06ef6b9..c6208bd 100644 --- a/portage_with_autodep/pym/_emerge/AbstractPollTask.pyo +++ b/portage_with_autodep/pym/_emerge/AbstractPollTask.pyo diff --git a/portage_with_autodep/pym/_emerge/AsynchronousLock.py b/portage_with_autodep/pym/_emerge/AsynchronousLock.py index 587aa46..c0b9b26 100644 --- a/portage_with_autodep/pym/_emerge/AsynchronousLock.py +++ b/portage_with_autodep/pym/_emerge/AsynchronousLock.py @@ -1,4 +1,4 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import dummy_threading @@ -49,7 +49,7 @@ class AsynchronousLock(AsynchronousTask): pass else: self.returncode = os.EX_OK - self.wait() + self._async_wait() return if self._force_process or \ @@ -105,44 +105,27 @@ class _LockThread(AbstractPollTask): """ __slots__ = ('path',) + \ - ('_files', '_force_dummy', '_lock_obj', - '_thread', '_reg_id',) + ('_force_dummy', '_lock_obj', '_thread',) def _start(self): - pr, pw = os.pipe() - self._files = {} - self._files['pipe_read'] = pr - self._files['pipe_write'] = pw - for f in self._files.values(): - fcntl.fcntl(f, fcntl.F_SETFL, - fcntl.fcntl(f, fcntl.F_GETFL) | os.O_NONBLOCK) - self._reg_id = self.scheduler.register(self._files['pipe_read'], - self.scheduler.IO_IN, self._output_handler) self._registered = True threading_mod = threading if self._force_dummy: threading_mod = dummy_threading self._thread = threading_mod.Thread(target=self._run_lock) + self._thread.daemon = True self._thread.start() def _run_lock(self): self._lock_obj = lockfile(self.path, wantnewlockfile=True) - os.write(self._files['pipe_write'], b'\0') - - def _output_handler(self, f, event): - buf = None - if event & self.scheduler.IO_IN: - try: - buf = os.read(self._files['pipe_read'], self._bufsize) - except OSError as e: - if e.errno not in (errno.EAGAIN,): - raise - if buf: - self._unregister() - self.returncode = os.EX_OK - self.wait() + # Thread-safe callback to EventLoop + self.scheduler.idle_add(self._run_lock_cb) - return True + def _run_lock_cb(self): + self._unregister() + self.returncode = os.EX_OK + self.wait() + return False def _cancel(self): # There's currently no way to force thread termination. @@ -163,15 +146,6 @@ class _LockThread(AbstractPollTask): self._thread.join() self._thread = None - if self._reg_id is not None: - self.scheduler.unregister(self._reg_id) - self._reg_id = None - - if self._files is not None: - for f in self._files.values(): - os.close(f) - self._files = None - class _LockProcess(AbstractPollTask): """ This uses the portage.locks module to acquire a lock asynchronously, @@ -190,16 +164,28 @@ class _LockProcess(AbstractPollTask): self._files = {} self._files['pipe_in'] = in_pr self._files['pipe_out'] = out_pw + fcntl.fcntl(in_pr, fcntl.F_SETFL, fcntl.fcntl(in_pr, fcntl.F_GETFL) | os.O_NONBLOCK) - self._reg_id = self.scheduler.register(in_pr, + + # FD_CLOEXEC is enabled by default in Python >=3.4. + if sys.hexversion < 0x3040000: + try: + fcntl.FD_CLOEXEC + except AttributeError: + pass + else: + fcntl.fcntl(in_pr, fcntl.F_SETFD, + fcntl.fcntl(in_pr, fcntl.F_GETFD) | fcntl.FD_CLOEXEC) + + self._reg_id = self.scheduler.io_add_watch(in_pr, self.scheduler.IO_IN, self._output_handler) self._registered = True self._proc = SpawnProcess( args=[portage._python_interpreter, os.path.join(portage._bin_path, 'lock-helper.py'), self.path], env=dict(os.environ, PORTAGE_PYM_PATH=portage._pym_path), - fd_pipes={0:out_pr, 1:in_pw, 2:sys.stderr.fileno()}, + fd_pipes={0:out_pr, 1:in_pw, 2:sys.__stderr__.fileno()}, scheduler=self.scheduler) self._proc.addExitListener(self._proc_exit) self._proc.start() @@ -273,7 +259,7 @@ class _LockProcess(AbstractPollTask): self._registered = False if self._reg_id is not None: - self.scheduler.unregister(self._reg_id) + self.scheduler.source_remove(self._reg_id) self._reg_id = None if self._files is not None: diff --git a/portage_with_autodep/pym/_emerge/AsynchronousLock.pyo b/portage_with_autodep/pym/_emerge/AsynchronousLock.pyo Binary files differindex 5f3cfbb..5b9031c 100644 --- a/portage_with_autodep/pym/_emerge/AsynchronousLock.pyo +++ b/portage_with_autodep/pym/_emerge/AsynchronousLock.pyo diff --git a/portage_with_autodep/pym/_emerge/AsynchronousTask.py b/portage_with_autodep/pym/_emerge/AsynchronousTask.py index 7a193ce..da58261 100644 --- a/portage_with_autodep/pym/_emerge/AsynchronousTask.py +++ b/portage_with_autodep/pym/_emerge/AsynchronousTask.py @@ -60,6 +60,20 @@ class AsynchronousTask(SlotObject): def _wait(self): return self.returncode + def _async_wait(self): + """ + For cases where _start exits synchronously, this method is a + convenient way to trigger an asynchronous call to self.wait() + (in order to notify exit listeners), avoiding excessive event + loop recursion (or stack overflow) that synchronous calling of + exit listeners can cause. This method is thread-safe. + """ + self.scheduler.idle_add(self._async_wait_cb) + + def _async_wait_cb(self): + self.wait() + return False + def cancel(self): """ Cancel the task, but do not wait for exit status. If asynchronous exit diff --git a/portage_with_autodep/pym/_emerge/AsynchronousTask.pyo b/portage_with_autodep/pym/_emerge/AsynchronousTask.pyo Binary files differindex b8d67ea..f679452 100644 --- a/portage_with_autodep/pym/_emerge/AsynchronousTask.pyo +++ b/portage_with_autodep/pym/_emerge/AsynchronousTask.pyo diff --git a/portage_with_autodep/pym/_emerge/AtomArg.py b/portage_with_autodep/pym/_emerge/AtomArg.py index a929b43..343d7aa 100644 --- a/portage_with_autodep/pym/_emerge/AtomArg.py +++ b/portage_with_autodep/pym/_emerge/AtomArg.py @@ -1,10 +1,13 @@ -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from portage._sets.base import InternalPackageSet from _emerge.DependencyArg import DependencyArg class AtomArg(DependencyArg): + + __slots__ = ('atom', 'pset') + def __init__(self, atom=None, **kwargs): DependencyArg.__init__(self, **kwargs) self.atom = atom diff --git a/portage_with_autodep/pym/_emerge/AtomArg.pyo b/portage_with_autodep/pym/_emerge/AtomArg.pyo Binary files differindex b8f59cf..8d9fe2a 100644 --- a/portage_with_autodep/pym/_emerge/AtomArg.pyo +++ b/portage_with_autodep/pym/_emerge/AtomArg.pyo diff --git a/portage_with_autodep/pym/_emerge/Binpkg.py b/portage_with_autodep/pym/_emerge/Binpkg.py index ea8a1ad..a740efd 100644 --- a/portage_with_autodep/pym/_emerge/Binpkg.py +++ b/portage_with_autodep/pym/_emerge/Binpkg.py @@ -1,4 +1,4 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.EbuildPhase import EbuildPhase @@ -298,6 +298,7 @@ class Binpkg(CompositeTask): extractor = BinpkgExtractorAsync(background=self.background, env=self.settings.environ(), + features=self.settings.features, image_dir=self._image_dir, pkg=self.pkg, pkg_path=self._pkg_path, logfile=self.settings.get("PORTAGE_LOG_FILE"), @@ -328,11 +329,13 @@ class Binpkg(CompositeTask): self.wait() return + env = self.settings.environ() + env["PYTHONPATH"] = self.settings["PORTAGE_PYTHONPATH"] chpathtool = SpawnProcess( args=[portage._python_interpreter, os.path.join(self.settings["PORTAGE_BIN_PATH"], "chpathtool.py"), self.settings["D"], self._build_prefix, self.settings["EPREFIX"]], - background=self.background, env=self.settings.environ(), + background=self.background, env=env, scheduler=self.scheduler, logfile=self.settings.get('PORTAGE_LOG_FILE')) self._writemsg_level(">>> Adjusting Prefix to %s\n" % self.settings["EPREFIX"]) diff --git a/portage_with_autodep/pym/_emerge/Binpkg.pyo b/portage_with_autodep/pym/_emerge/Binpkg.pyo Binary files differindex 4499b9d..7b6472d 100644 --- a/portage_with_autodep/pym/_emerge/Binpkg.pyo +++ b/portage_with_autodep/pym/_emerge/Binpkg.pyo diff --git a/portage_with_autodep/pym/_emerge/BinpkgEnvExtractor.pyo b/portage_with_autodep/pym/_emerge/BinpkgEnvExtractor.pyo Binary files differindex 21c2e13..a86a949 100644 --- a/portage_with_autodep/pym/_emerge/BinpkgEnvExtractor.pyo +++ b/portage_with_autodep/pym/_emerge/BinpkgEnvExtractor.pyo diff --git a/portage_with_autodep/pym/_emerge/BinpkgExtractorAsync.py b/portage_with_autodep/pym/_emerge/BinpkgExtractorAsync.py index f25cbf9..be74c2f 100644 --- a/portage_with_autodep/pym/_emerge/BinpkgExtractorAsync.py +++ b/portage_with_autodep/pym/_emerge/BinpkgExtractorAsync.py @@ -1,23 +1,31 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.SpawnProcess import SpawnProcess import portage import signal +import subprocess class BinpkgExtractorAsync(SpawnProcess): - __slots__ = ("image_dir", "pkg", "pkg_path") + __slots__ = ("features", "image_dir", "pkg", "pkg_path") _shell_binary = portage.const.BASH_BINARY def _start(self): + tar_options = "" + if "xattr" in self.features: + process = subprocess.Popen(["tar", "--help"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output = process.communicate()[0] + if b"--xattrs" in output: + tar_options = "--xattrs" + # Add -q to bzip2 opts, in order to avoid "trailing garbage after # EOF ignored" warning messages due to xpak trailer. # SIGPIPE handling (128 + SIGPIPE) should be compatible with # assert_sigpipe_ok() that's used by the ebuild unpack() helper. self.args = [self._shell_binary, "-c", - ("${PORTAGE_BUNZIP2_COMMAND:-${PORTAGE_BZIP2_COMMAND} -d} -cq -- %s | tar -xp -C %s -f - ; " + \ + ("${PORTAGE_BUNZIP2_COMMAND:-${PORTAGE_BZIP2_COMMAND} -d} -cq -- %s | tar -xp %s -C %s -f - ; " + \ "p=(${PIPESTATUS[@]}) ; " + \ "if [[ ${p[0]} != 0 && ${p[0]} != %d ]] ; then " % (128 + signal.SIGPIPE) + \ "echo bzip2 failed with status ${p[0]} ; exit ${p[0]} ; fi ; " + \ @@ -25,6 +33,7 @@ class BinpkgExtractorAsync(SpawnProcess): "echo tar failed with status ${p[1]} ; exit ${p[1]} ; fi ; " + \ "exit 0 ;") % \ (portage._shell_quote(self.pkg_path), + tar_options, portage._shell_quote(self.image_dir))] SpawnProcess._start(self) diff --git a/portage_with_autodep/pym/_emerge/BinpkgExtractorAsync.pyo b/portage_with_autodep/pym/_emerge/BinpkgExtractorAsync.pyo Binary files differindex f8498f7..83d4d9c 100644 --- a/portage_with_autodep/pym/_emerge/BinpkgExtractorAsync.pyo +++ b/portage_with_autodep/pym/_emerge/BinpkgExtractorAsync.pyo diff --git a/portage_with_autodep/pym/_emerge/BinpkgFetcher.py b/portage_with_autodep/pym/_emerge/BinpkgFetcher.py index f415e2e..543881e 100644 --- a/portage_with_autodep/pym/_emerge/BinpkgFetcher.py +++ b/portage_with_autodep/pym/_emerge/BinpkgFetcher.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.AsynchronousLock import AsynchronousLock @@ -63,7 +63,7 @@ class BinpkgFetcher(SpawnProcess): if pretend: portage.writemsg_stdout("\n%s\n" % uri, noiselevel=-1) self._set_returncode((self.pid, os.EX_OK << 8)) - self.wait() + self._async_wait() return protocol = urllib_parse_urlparse(uri)[0] @@ -80,6 +80,12 @@ class BinpkgFetcher(SpawnProcess): "FILE" : os.path.basename(pkg_path) } + for k in ("PORTAGE_SSH_OPTS",): + try: + fcmd_vars[k] = settings[k] + except KeyError: + pass + fetch_env = dict(settings.items()) fetch_args = [portage.util.varexpand(x, mydict=fcmd_vars) \ for x in portage.util.shlex_split(fcmd)] @@ -91,9 +97,9 @@ class BinpkgFetcher(SpawnProcess): # Redirect all output to stdout since some fetchers like # wget pollute stderr (if portage detects a problem then it # can send it's own message to stderr). - fd_pipes.setdefault(0, sys.stdin.fileno()) - fd_pipes.setdefault(1, sys.stdout.fileno()) - fd_pipes.setdefault(2, sys.stdout.fileno()) + fd_pipes.setdefault(0, portage._get_stdin().fileno()) + fd_pipes.setdefault(1, sys.__stdout__.fileno()) + fd_pipes.setdefault(2, sys.__stdout__.fileno()) self.args = fetch_args self.env = fetch_env @@ -104,7 +110,7 @@ class BinpkgFetcher(SpawnProcess): def _pipe(self, fd_pipes): """When appropriate, use a pty so that fetcher progress bars, like wget has, will work properly.""" - if self.background or not sys.stdout.isatty(): + if self.background or not sys.__stdout__.isatty(): # When the output only goes to a log file, # there's no point in creating a pty. return os.pipe() diff --git a/portage_with_autodep/pym/_emerge/BinpkgFetcher.pyo b/portage_with_autodep/pym/_emerge/BinpkgFetcher.pyo Binary files differindex 482e55e..1514fb9 100644 --- a/portage_with_autodep/pym/_emerge/BinpkgFetcher.pyo +++ b/portage_with_autodep/pym/_emerge/BinpkgFetcher.pyo diff --git a/portage_with_autodep/pym/_emerge/BinpkgPrefetcher.pyo b/portage_with_autodep/pym/_emerge/BinpkgPrefetcher.pyo Binary files differindex c890cac..cfe9e45 100644 --- a/portage_with_autodep/pym/_emerge/BinpkgPrefetcher.pyo +++ b/portage_with_autodep/pym/_emerge/BinpkgPrefetcher.pyo diff --git a/portage_with_autodep/pym/_emerge/BinpkgVerifier.py b/portage_with_autodep/pym/_emerge/BinpkgVerifier.py index 0052967..2c69792 100644 --- a/portage_with_autodep/pym/_emerge/BinpkgVerifier.py +++ b/portage_with_autodep/pym/_emerge/BinpkgVerifier.py @@ -1,75 +1,120 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from _emerge.AsynchronousTask import AsynchronousTask -from portage.util import writemsg +import errno import io import sys + +from _emerge.CompositeTask import CompositeTask import portage from portage import os +from portage.checksum import (_apply_hash_filter, + _filter_unaccelarated_hashes, _hash_filter) +from portage.output import EOutput +from portage.util._async.FileDigester import FileDigester from portage.package.ebuild.fetch import _checksum_failure_temp_file -class BinpkgVerifier(AsynchronousTask): - __slots__ = ("logfile", "pkg", "scheduler") +class BinpkgVerifier(CompositeTask): + __slots__ = ("logfile", "pkg", "_digests", "_pkg_path") def _start(self): - """ - Note: Unlike a normal AsynchronousTask.start() method, - this one does all work is synchronously. The returncode - attribute will be set before it returns. - """ - - pkg = self.pkg - root_config = pkg.root_config - bintree = root_config.trees["bintree"] - rval = os.EX_OK + + bintree = self.pkg.root_config.trees["bintree"] + digests = bintree._get_digests(self.pkg) + if "size" not in digests: + self.returncode = os.EX_OK + self._async_wait() + return + + digests = _filter_unaccelarated_hashes(digests) + hash_filter = _hash_filter( + bintree.settings.get("PORTAGE_CHECKSUM_FILTER", "")) + if not hash_filter.transparent: + digests = _apply_hash_filter(digests, hash_filter) + + self._digests = digests + self._pkg_path = bintree.getname(self.pkg.cpv) + + try: + size = os.stat(self._pkg_path).st_size + except OSError as e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + raise + self.scheduler.output(("!!! Fetching Binary failed " + "for '%s'\n") % self.pkg.cpv, log_path=self.logfile, + background=self.background) + self.returncode = 1 + self._async_wait() + return + else: + if size != digests["size"]: + self._digest_exception("size", size, digests["size"]) + self.returncode = 1 + self._async_wait() + return + + self._start_task(FileDigester(file_path=self._pkg_path, + hash_names=(k for k in digests if k != "size"), + background=self.background, logfile=self.logfile, + scheduler=self.scheduler), + self._digester_exit) + + def _digester_exit(self, digester): + + if self._default_exit(digester) != os.EX_OK: + self.wait() + return + + for hash_name in digester.hash_names: + if digester.digests[hash_name] != self._digests[hash_name]: + self._digest_exception(hash_name, + digester.digests[hash_name], self._digests[hash_name]) + self.returncode = 1 + self.wait() + return + + if self.pkg.root_config.settings.get("PORTAGE_QUIET") != "1": + self._display_success() + + self.returncode = os.EX_OK + self.wait() + + def _display_success(self): stdout_orig = sys.stdout stderr_orig = sys.stderr global_havecolor = portage.output.havecolor out = io.StringIO() - file_exists = True try: sys.stdout = out sys.stderr = out if portage.output.havecolor: portage.output.havecolor = not self.background - try: - bintree.digestCheck(pkg) - except portage.exception.FileNotFound: - writemsg("!!! Fetching Binary failed " + \ - "for '%s'\n" % pkg.cpv, noiselevel=-1) - rval = 1 - file_exists = False - except portage.exception.DigestException as e: - writemsg("\n!!! Digest verification failed:\n", - noiselevel=-1) - writemsg("!!! %s\n" % e.value[0], - noiselevel=-1) - writemsg("!!! Reason: %s\n" % e.value[1], - noiselevel=-1) - writemsg("!!! Got: %s\n" % e.value[2], - noiselevel=-1) - writemsg("!!! Expected: %s\n" % e.value[3], - noiselevel=-1) - rval = 1 - if rval == os.EX_OK: - pass - elif file_exists: - pkg_path = bintree.getname(pkg.cpv) - head, tail = os.path.split(pkg_path) - temp_filename = _checksum_failure_temp_file(head, tail) - writemsg("File renamed to '%s'\n" % (temp_filename,), - noiselevel=-1) + + eout = EOutput() + eout.ebegin("%s %s ;-)" % (os.path.basename(self._pkg_path), + " ".join(sorted(self._digests)))) + eout.eend(0) + finally: sys.stdout = stdout_orig sys.stderr = stderr_orig portage.output.havecolor = global_havecolor - msg = out.getvalue() - if msg: - self.scheduler.output(msg, log_path=self.logfile, - background=self.background) + self.scheduler.output(out.getvalue(), log_path=self.logfile, + background=self.background) - self.returncode = rval - self.wait() + def _digest_exception(self, name, value, expected): + + head, tail = os.path.split(self._pkg_path) + temp_filename = _checksum_failure_temp_file(head, tail) + self.scheduler.output(( + "\n!!! Digest verification failed:\n" + "!!! %s\n" + "!!! Reason: Failed on %s verification\n" + "!!! Got: %s\n" + "!!! Expected: %s\n" + "File renamed to '%s'\n") % + (self._pkg_path, name, value, expected, temp_filename), + log_path=self.logfile, + background=self.background) diff --git a/portage_with_autodep/pym/_emerge/BinpkgVerifier.pyo b/portage_with_autodep/pym/_emerge/BinpkgVerifier.pyo Binary files differindex 21f770e..1c84b08 100644 --- a/portage_with_autodep/pym/_emerge/BinpkgVerifier.pyo +++ b/portage_with_autodep/pym/_emerge/BinpkgVerifier.pyo diff --git a/portage_with_autodep/pym/_emerge/Blocker.pyo b/portage_with_autodep/pym/_emerge/Blocker.pyo Binary files differindex b9e56bc..8905faf 100644 --- a/portage_with_autodep/pym/_emerge/Blocker.pyo +++ b/portage_with_autodep/pym/_emerge/Blocker.pyo diff --git a/portage_with_autodep/pym/_emerge/BlockerCache.py b/portage_with_autodep/pym/_emerge/BlockerCache.py index fce81f8..53342d6 100644 --- a/portage_with_autodep/pym/_emerge/BlockerCache.py +++ b/portage_with_autodep/pym/_emerge/BlockerCache.py @@ -1,4 +1,4 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import errno @@ -62,7 +62,9 @@ class BlockerCache(portage.cache.mappings.MutableMapping): self._cache_data = mypickle.load() f.close() del f - except (AttributeError, EOFError, EnvironmentError, ValueError, pickle.UnpicklingError) as e: + except (SystemExit, KeyboardInterrupt): + raise + except Exception as e: if isinstance(e, EnvironmentError) and \ getattr(e, 'errno', None) in (errno.ENOENT, errno.EACCES): pass @@ -126,9 +128,9 @@ class BlockerCache(portage.cache.mappings.MutableMapping): self._modified.clear() def flush(self): - """If the current user has permission and the internal blocker cache + """If the current user has permission and the internal blocker cache has been updated, save it to disk and mark it unmodified. This is called - by emerge after it has proccessed blockers for all installed packages. + by emerge after it has processed blockers for all installed packages. Currently, the cache is only written if the user has superuser privileges (since that's required to obtain a lock), but all users have read access and benefit from faster blocker lookups (as long as diff --git a/portage_with_autodep/pym/_emerge/BlockerCache.pyo b/portage_with_autodep/pym/_emerge/BlockerCache.pyo Binary files differindex 41554e1..632df92 100644 --- a/portage_with_autodep/pym/_emerge/BlockerCache.pyo +++ b/portage_with_autodep/pym/_emerge/BlockerCache.pyo diff --git a/portage_with_autodep/pym/_emerge/BlockerDB.py b/portage_with_autodep/pym/_emerge/BlockerDB.py index 459affd..8bb8f5f 100644 --- a/portage_with_autodep/pym/_emerge/BlockerDB.py +++ b/portage_with_autodep/pym/_emerge/BlockerDB.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import sys @@ -9,6 +9,7 @@ from portage import digraph from portage._sets.base import InternalPackageSet from _emerge.BlockerCache import BlockerCache +from _emerge.Package import Package from _emerge.show_invalid_depstring_notice import show_invalid_depstring_notice if sys.hexversion >= 0x3000000: @@ -38,7 +39,7 @@ class BlockerDB(object): """ blocker_cache = BlockerCache(None, self._vartree.dbapi) - dep_keys = ["RDEPEND", "PDEPEND"] + dep_keys = Package._runtime_keys settings = self._vartree.settings stale_cache = set(blocker_cache) fake_vartree = self._fake_vartree @@ -50,7 +51,7 @@ class BlockerDB(object): stale_cache.discard(inst_pkg.cpv) cached_blockers = blocker_cache.get(inst_pkg.cpv) if cached_blockers is not None and \ - cached_blockers.counter != long(inst_pkg.metadata["COUNTER"]): + cached_blockers.counter != inst_pkg.counter: cached_blockers = None if cached_blockers is not None: blocker_atoms = cached_blockers.atoms @@ -71,9 +72,8 @@ class BlockerDB(object): blocker_atoms = [atom for atom in atoms \ if atom.startswith("!")] blocker_atoms.sort() - counter = long(inst_pkg.metadata["COUNTER"]) blocker_cache[inst_pkg.cpv] = \ - blocker_cache.BlockerData(counter, blocker_atoms) + blocker_cache.BlockerData(inst_pkg.counter, blocker_atoms) for cpv in stale_cache: del blocker_cache[cpv] blocker_cache.flush() @@ -92,7 +92,7 @@ class BlockerDB(object): blocking_pkgs.update(blocker_parents.parent_nodes(atom)) # Check for blockers in the other direction. - depstr = " ".join(new_pkg.metadata[k] for k in dep_keys) + depstr = " ".join(new_pkg._metadata[k] for k in dep_keys) success, atoms = portage.dep_check(depstr, vardb, settings, myuse=new_pkg.use.enabled, trees=dep_check_trees, myroot=new_pkg.root) diff --git a/portage_with_autodep/pym/_emerge/BlockerDB.pyo b/portage_with_autodep/pym/_emerge/BlockerDB.pyo Binary files differindex dfab0aa..15ab6cd 100644 --- a/portage_with_autodep/pym/_emerge/BlockerDB.pyo +++ b/portage_with_autodep/pym/_emerge/BlockerDB.pyo diff --git a/portage_with_autodep/pym/_emerge/BlockerDepPriority.pyo b/portage_with_autodep/pym/_emerge/BlockerDepPriority.pyo Binary files differindex c3b554c..c998728 100644 --- a/portage_with_autodep/pym/_emerge/BlockerDepPriority.pyo +++ b/portage_with_autodep/pym/_emerge/BlockerDepPriority.pyo diff --git a/portage_with_autodep/pym/_emerge/CompositeTask.py b/portage_with_autodep/pym/_emerge/CompositeTask.py index 3e43478..40cf859 100644 --- a/portage_with_autodep/pym/_emerge/CompositeTask.py +++ b/portage_with_autodep/pym/_emerge/CompositeTask.py @@ -142,6 +142,10 @@ class CompositeTask(AsynchronousTask): a task. """ + try: + task.scheduler = self.scheduler + except AttributeError: + pass task.addExitListener(exit_handler) self._current_task = task task.start() diff --git a/portage_with_autodep/pym/_emerge/CompositeTask.pyo b/portage_with_autodep/pym/_emerge/CompositeTask.pyo Binary files differindex adc8cae..f41c565 100644 --- a/portage_with_autodep/pym/_emerge/CompositeTask.pyo +++ b/portage_with_autodep/pym/_emerge/CompositeTask.pyo diff --git a/portage_with_autodep/pym/_emerge/DepPriority.py b/portage_with_autodep/pym/_emerge/DepPriority.py index 3c2256a..34fdb48 100644 --- a/portage_with_autodep/pym/_emerge/DepPriority.py +++ b/portage_with_autodep/pym/_emerge/DepPriority.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.AbstractDepPriority import AbstractDepPriority @@ -16,31 +16,38 @@ class DepPriority(AbstractDepPriority): Attributes Hardness - buildtime 0 - runtime -1 - runtime_post -2 - optional -3 - (none of the above) -4 + buildtime_slot_op 0 + buildtime -1 + runtime -2 + runtime_post -3 + optional -4 + (none of the above) -5 """ if self.optional: - return -3 - if self.buildtime: + return -4 + if self.buildtime_slot_op: return 0 - if self.runtime: + if self.buildtime: return -1 - if self.runtime_post: + if self.runtime: return -2 - return -4 + if self.runtime_post: + return -3 + return -5 def __str__(self): if self.ignored: return "ignored" if self.optional: return "optional" + if self.buildtime_slot_op: + return "buildtime_slot_op" if self.buildtime: return "buildtime" + if self.runtime_slot_op: + return "runtime_slot_op" if self.runtime: return "runtime" if self.runtime_post: diff --git a/portage_with_autodep/pym/_emerge/DepPriority.pyo b/portage_with_autodep/pym/_emerge/DepPriority.pyo Binary files differindex 4028a36..ade8cdd 100644 --- a/portage_with_autodep/pym/_emerge/DepPriority.pyo +++ b/portage_with_autodep/pym/_emerge/DepPriority.pyo diff --git a/portage_with_autodep/pym/_emerge/DepPriorityNormalRange.pyo b/portage_with_autodep/pym/_emerge/DepPriorityNormalRange.pyo Binary files differindex 5e0f710..a255f1d 100644 --- a/portage_with_autodep/pym/_emerge/DepPriorityNormalRange.pyo +++ b/portage_with_autodep/pym/_emerge/DepPriorityNormalRange.pyo diff --git a/portage_with_autodep/pym/_emerge/DepPrioritySatisfiedRange.py b/portage_with_autodep/pym/_emerge/DepPrioritySatisfiedRange.py index edb29df..391f540 100644 --- a/portage_with_autodep/pym/_emerge/DepPrioritySatisfiedRange.py +++ b/portage_with_autodep/pym/_emerge/DepPrioritySatisfiedRange.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.DepPriority import DepPriority @@ -7,17 +7,18 @@ class DepPrioritySatisfiedRange(object): DepPriority Index Category not satisfied and buildtime HARD - not satisfied and runtime 6 MEDIUM - not satisfied and runtime_post 5 MEDIUM_SOFT + not satisfied and runtime 7 MEDIUM + not satisfied and runtime_post 6 MEDIUM_SOFT + satisfied and buildtime_slot_op 5 SOFT satisfied and buildtime 4 SOFT satisfied and runtime 3 SOFT satisfied and runtime_post 2 SOFT optional 1 SOFT (none of the above) 0 NONE """ - MEDIUM = 6 - MEDIUM_SOFT = 5 - SOFT = 4 + MEDIUM = 7 + MEDIUM_SOFT = 6 + SOFT = 5 NONE = 0 @classmethod @@ -50,6 +51,16 @@ class DepPrioritySatisfiedRange(object): def _ignore_satisfied_buildtime(cls, priority): if priority.__class__ is not DepPriority: return False + if priority.optional: + return True + if priority.buildtime_slot_op: + return False + return bool(priority.satisfied) + + @classmethod + def _ignore_satisfied_buildtime_slot_op(cls, priority): + if priority.__class__ is not DepPriority: + return False return bool(priority.optional or \ priority.satisfied) @@ -80,6 +91,7 @@ DepPrioritySatisfiedRange.ignore_priority = ( DepPrioritySatisfiedRange._ignore_satisfied_runtime_post, DepPrioritySatisfiedRange._ignore_satisfied_runtime, DepPrioritySatisfiedRange._ignore_satisfied_buildtime, + DepPrioritySatisfiedRange._ignore_satisfied_buildtime_slot_op, DepPrioritySatisfiedRange._ignore_runtime_post, DepPrioritySatisfiedRange._ignore_runtime ) diff --git a/portage_with_autodep/pym/_emerge/DepPrioritySatisfiedRange.pyo b/portage_with_autodep/pym/_emerge/DepPrioritySatisfiedRange.pyo Binary files differindex 5309bcd..7f95d84 100644 --- a/portage_with_autodep/pym/_emerge/DepPrioritySatisfiedRange.pyo +++ b/portage_with_autodep/pym/_emerge/DepPrioritySatisfiedRange.pyo diff --git a/portage_with_autodep/pym/_emerge/Dependency.py b/portage_with_autodep/pym/_emerge/Dependency.py index c2d36b2..2ec860f 100644 --- a/portage_with_autodep/pym/_emerge/Dependency.py +++ b/portage_with_autodep/pym/_emerge/Dependency.py @@ -6,7 +6,7 @@ from _emerge.DepPriority import DepPriority class Dependency(SlotObject): __slots__ = ("atom", "blocker", "child", "depth", - "parent", "onlydeps", "priority", "root", + "parent", "onlydeps", "priority", "root", "want_update", "collapsed_parent", "collapsed_priority") def __init__(self, **kwargs): SlotObject.__init__(self, **kwargs) diff --git a/portage_with_autodep/pym/_emerge/Dependency.pyo b/portage_with_autodep/pym/_emerge/Dependency.pyo Binary files differindex f53e0ed..b1428d5 100644 --- a/portage_with_autodep/pym/_emerge/Dependency.pyo +++ b/portage_with_autodep/pym/_emerge/Dependency.pyo diff --git a/portage_with_autodep/pym/_emerge/DependencyArg.py b/portage_with_autodep/pym/_emerge/DependencyArg.py index 861d837..29a0072 100644 --- a/portage_with_autodep/pym/_emerge/DependencyArg.py +++ b/portage_with_autodep/pym/_emerge/DependencyArg.py @@ -1,13 +1,26 @@ -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + import sys -from portage import _encodings, _unicode_encode, _unicode_decode +from portage import _encodings, _unicode_encode class DependencyArg(object): - def __init__(self, arg=None, root_config=None): + + __slots__ = ('arg', 'force_reinstall', 'internal', 'reset_depth', 'root_config') + + def __init__(self, arg=None, force_reinstall=False, internal=False, + reset_depth=True, root_config=None): + """ + Use reset_depth=False for special arguments that should not interact + with depth calculations (see the emerge --deep=DEPTH option). + """ self.arg = arg + self.force_reinstall = force_reinstall + self.internal = internal + self.reset_depth = reset_depth self.root_config = root_config def __eq__(self, other): @@ -20,10 +33,10 @@ class DependencyArg(object): return hash((self.arg, self.root_config.root)) def __str__(self): - # Force unicode format string for python-2.x safety, + # Use unicode_literals format string for python-2.x safety, # ensuring that self.arg.__unicode__() is used # when necessary. - return _unicode_decode("%s") % (self.arg,) + return "%s" % (self.arg,) if sys.hexversion < 0x3000000: diff --git a/portage_with_autodep/pym/_emerge/DependencyArg.pyo b/portage_with_autodep/pym/_emerge/DependencyArg.pyo Binary files differindex 916a762..536670c 100644 --- a/portage_with_autodep/pym/_emerge/DependencyArg.pyo +++ b/portage_with_autodep/pym/_emerge/DependencyArg.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildBinpkg.pyo b/portage_with_autodep/pym/_emerge/EbuildBinpkg.pyo Binary files differindex 2acfc87..3190c6a 100644 --- a/portage_with_autodep/pym/_emerge/EbuildBinpkg.pyo +++ b/portage_with_autodep/pym/_emerge/EbuildBinpkg.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildBuild.py b/portage_with_autodep/pym/_emerge/EbuildBuild.py index 5a48f8e..f680434 100644 --- a/portage_with_autodep/pym/_emerge/EbuildBuild.py +++ b/portage_with_autodep/pym/_emerge/EbuildBuild.py @@ -1,4 +1,4 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.EbuildExecuter import EbuildExecuter @@ -12,11 +12,14 @@ from _emerge.EbuildBuildDir import EbuildBuildDir from _emerge.EventsAnalyser import EventsAnalyser, FilterProcGenerator from _emerge.EventsLogger import EventsLogger from _emerge.MiscFunctionsProcess import MiscFunctionsProcess +from _emerge.TaskSequence import TaskSequence + from portage.util import writemsg import portage from portage import os from portage.output import colorize from portage.package.ebuild.digestcheck import digestcheck +from portage.package.ebuild.digestgen import digestgen from portage.package.ebuild.doebuild import _check_temp_dir from portage.package.ebuild._spawn_nofetch import spawn_nofetch @@ -37,7 +40,7 @@ class EbuildBuild(CompositeTask): if rval != os.EX_OK: self.returncode = rval self._current_task = None - self.wait() + self._async_wait() return root_config = pkg.root_config @@ -62,7 +65,7 @@ class EbuildBuild(CompositeTask): if not self._check_manifest(): self.returncode = 1 self._current_task = None - self.wait() + self._async_wait() return prefetcher = self.prefetcher @@ -93,7 +96,8 @@ class EbuildBuild(CompositeTask): success = True settings = self.settings - if 'strict' in settings.features: + if 'strict' in settings.features and \ + 'digest' not in settings.features: settings['O'] = os.path.dirname(self._ebuild_path) quiet_setting = settings.get('PORTAGE_QUIET') settings['PORTAGE_QUIET'] = '1' @@ -162,6 +166,10 @@ class EbuildBuild(CompositeTask): if self.returncode != os.EX_OK: portdb = self.pkg.root_config.trees[self._tree].dbapi spawn_nofetch(portdb, self._ebuild_path, settings=self.settings) + elif 'digest' in self.settings.features: + if not digestgen(mysettings=self.settings, + myportdb=self.pkg.root_config.trees[self._tree].dbapi): + self.returncode = 1 self.wait() def _pre_clean_exit(self, pre_clean_phase): @@ -264,7 +272,7 @@ class EbuildBuild(CompositeTask): "depcheckstrict" in self.settings["FEATURES"]: # Lets start a log listening server temp_path=self.settings.get("T",self.settings["PORTAGE_TMPDIR"]) - + if "depcheckstrict" not in self.settings["FEATURES"]: # use default filter_proc self.logserver=EventsLogger(socket_dir=temp_path) @@ -273,11 +281,11 @@ class EbuildBuild(CompositeTask): "This may take some time\n") filter_gen=FilterProcGenerator(self.pkg.cpv, self.settings) filter_proc=filter_gen.get_filter_proc() - self.logserver=EventsLogger(socket_dir=temp_path, + self.logserver=EventsLogger(socket_dir=temp_path, filter_proc=filter_proc) - + self.logserver.start() - + # Copy socket path to LOG_SOCKET environment variable env=self.settings.configdict["pkg"] env['LOG_SOCKET'] = self.logserver.socket_name @@ -291,15 +299,13 @@ class EbuildBuild(CompositeTask): env=self.settings.configdict["pkg"] if 'LOG_SOCKET' in env: del env['LOG_SOCKET'] - + events=self.logserver.stop() self.logserver=None analyser=EventsAnalyser(self.pkg.cpv, events, self.settings) analyser.display() # show the analyse #import pdb; pdb.set_trace() - - def _fetch_failed(self): # We only call the pkg_nofetch phase if either RESTRICT=fetch @@ -308,8 +314,8 @@ class EbuildBuild(CompositeTask): # to be displayed for problematic packages even though they do # not set RESTRICT=fetch (bug #336499). - if 'fetch' not in self.pkg.metadata.restrict and \ - 'nofetch' not in self.pkg.metadata.defined_phases: + if 'fetch' not in self.pkg.restrict and \ + 'nofetch' not in self.pkg.defined_phases: self._unlock_builddir() self.wait() return @@ -348,10 +354,20 @@ class EbuildBuild(CompositeTask): self.scheduler.output(msg, log_path=self.settings.get("PORTAGE_LOG_FILE")) - packager = EbuildBinpkg(background=self.background, pkg=self.pkg, - scheduler=self.scheduler, settings=self.settings) + binpkg_tasks = TaskSequence() + requested_binpkg_formats = self.settings.get("PORTAGE_BINPKG_FORMAT", "tar").split() + for pkg_fmt in portage.const.SUPPORTED_BINPKG_FORMATS: + if pkg_fmt in requested_binpkg_formats: + if pkg_fmt == "rpm": + binpkg_tasks.add(EbuildPhase(background=self.background, + phase="rpm", scheduler=self.scheduler, + settings=self.settings)) + else: + binpkg_tasks.add(EbuildBinpkg(background=self.background, + pkg=self.pkg, scheduler=self.scheduler, + settings=self.settings)) - self._start_task(packager, self._buildpkg_exit) + self._start_task(binpkg_tasks, self._buildpkg_exit) def _buildpkg_exit(self, packager): """ diff --git a/portage_with_autodep/pym/_emerge/EbuildBuild.pyo b/portage_with_autodep/pym/_emerge/EbuildBuild.pyo Binary files differindex 19d913c..78bf68d 100644 --- a/portage_with_autodep/pym/_emerge/EbuildBuild.pyo +++ b/portage_with_autodep/pym/_emerge/EbuildBuild.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildBuildDir.py b/portage_with_autodep/pym/_emerge/EbuildBuildDir.py index 9773bd7..58905c2 100644 --- a/portage_with_autodep/pym/_emerge/EbuildBuildDir.py +++ b/portage_with_autodep/pym/_emerge/EbuildBuildDir.py @@ -7,7 +7,6 @@ import portage from portage import os from portage.exception import PortageException from portage.util.SlotObject import SlotObject -import errno class EbuildBuildDir(SlotObject): @@ -60,7 +59,7 @@ class EbuildBuildDir(SlotObject): builddir_lock.wait() self._assert_lock(builddir_lock) self._lock_obj = builddir_lock - self.settings['PORTAGE_BUILDIR_LOCKED'] = '1' + self.settings['PORTAGE_BUILDDIR_LOCKED'] = '1' finally: self.locked = self._lock_obj is not None catdir_lock.unlock() @@ -92,16 +91,14 @@ class EbuildBuildDir(SlotObject): self._lock_obj.unlock() self._lock_obj = None self.locked = False - self.settings.pop('PORTAGE_BUILDIR_LOCKED', None) + self.settings.pop('PORTAGE_BUILDDIR_LOCKED', None) catdir_lock = AsynchronousLock(path=self._catdir, scheduler=self.scheduler) catdir_lock.start() if catdir_lock.wait() == os.EX_OK: try: os.rmdir(self._catdir) - except OSError as e: - if e.errno not in (errno.ENOENT, - errno.ENOTEMPTY, errno.EEXIST, errno.EPERM): - raise + except OSError: + pass finally: catdir_lock.unlock() diff --git a/portage_with_autodep/pym/_emerge/EbuildBuildDir.pyo b/portage_with_autodep/pym/_emerge/EbuildBuildDir.pyo Binary files differindex 2846579..290f8d9 100644 --- a/portage_with_autodep/pym/_emerge/EbuildBuildDir.pyo +++ b/portage_with_autodep/pym/_emerge/EbuildBuildDir.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildExecuter.py b/portage_with_autodep/pym/_emerge/EbuildExecuter.py index fd663a4..5587d4e 100644 --- a/portage_with_autodep/pym/_emerge/EbuildExecuter.py +++ b/portage_with_autodep/pym/_emerge/EbuildExecuter.py @@ -16,16 +16,7 @@ class EbuildExecuter(CompositeTask): _phases = ("prepare", "configure", "compile", "test", "install") - _live_eclasses = frozenset([ - "bzr", - "cvs", - "darcs", - "git", - "git-2", - "mercurial", - "subversion", - "tla", - ]) + _live_eclasses = portage.const.LIVE_ECLASSES def _start(self): pkg = self.pkg @@ -83,7 +74,7 @@ class EbuildExecuter(CompositeTask): pkg = self.pkg phases = self._phases - eapi = pkg.metadata["EAPI"] + eapi = pkg.eapi if not eapi_has_src_prepare_and_src_configure(eapi): # skip src_prepare and src_configure phases = phases[2:] diff --git a/portage_with_autodep/pym/_emerge/EbuildExecuter.pyo b/portage_with_autodep/pym/_emerge/EbuildExecuter.pyo Binary files differindex 592a0c9..21fc3d3 100644 --- a/portage_with_autodep/pym/_emerge/EbuildExecuter.pyo +++ b/portage_with_autodep/pym/_emerge/EbuildExecuter.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildFetcher.py b/portage_with_autodep/pym/_emerge/EbuildFetcher.py index c0a7fdd..d98d007 100644 --- a/portage_with_autodep/pym/_emerge/EbuildFetcher.py +++ b/portage_with_autodep/pym/_emerge/EbuildFetcher.py @@ -1,23 +1,22 @@ # Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -import traceback - -from _emerge.SpawnProcess import SpawnProcess import copy import io -import signal import sys + import portage from portage import os from portage import _encodings from portage import _unicode_encode from portage import _unicode_decode +from portage.checksum import _hash_filter from portage.elog.messages import eerror from portage.package.ebuild.fetch import _check_distfile, fetch +from portage.util._async.ForkProcess import ForkProcess from portage.util._pty import _create_pty_or_pipe -class EbuildFetcher(SpawnProcess): +class EbuildFetcher(ForkProcess): __slots__ = ("config_pool", "ebuild_path", "fetchonly", "fetchall", "pkg", "prefetch") + \ @@ -57,6 +56,9 @@ class EbuildFetcher(SpawnProcess): if st.st_size != expected_size: return False + hash_filter = _hash_filter(settings.get("PORTAGE_CHECKSUM_FILTER", "")) + if hash_filter.transparent: + hash_filter = None stdout_orig = sys.stdout stderr_orig = sys.stderr global_havecolor = portage.output.havecolor @@ -78,7 +80,7 @@ class EbuildFetcher(SpawnProcess): break continue ok, st = _check_distfile(os.path.join(distdir, filename), - mydigests, eout, show_errors=False) + mydigests, eout, show_errors=False, hash_filter=hash_filter) if not ok: success = False break @@ -115,13 +117,13 @@ class EbuildFetcher(SpawnProcess): msg_lines.append(msg) self._eerror(msg_lines) self._set_returncode((self.pid, 1 << 8)) - self.wait() + self._async_wait() return if not uri_map: # Nothing to fetch. self._set_returncode((self.pid, os.EX_OK << 8)) - self.wait() + self._async_wait() return settings = self.config_pool.allocate() @@ -133,7 +135,7 @@ class EbuildFetcher(SpawnProcess): self._prefetch_size_ok(uri_map, settings, ebuild_path): self.config_pool.deallocate(settings) self._set_returncode((self.pid, os.EX_OK << 8)) - self.wait() + self._async_wait() return nocolor = settings.get("NOCOLOR") @@ -148,7 +150,7 @@ class EbuildFetcher(SpawnProcess): settings["NOCOLOR"] = nocolor self._settings = settings - SpawnProcess._start(self) + ForkProcess._start(self) # Free settings now since it's no longer needed in # this process (the subprocess has a private copy). @@ -156,48 +158,20 @@ class EbuildFetcher(SpawnProcess): settings = None self._settings = None - def _spawn(self, args, fd_pipes=None, **kwargs): - """ - Fork a subprocess, apply local settings, and call fetch(). - """ - - pid = os.fork() - if pid != 0: - if not isinstance(pid, int): - raise AssertionError( - "fork returned non-integer: %s" % (repr(pid),)) - portage.process.spawned_pids.append(pid) - return [pid] - - portage.locks._close_fds() - # Disable close_fds since we don't exec (see _setup_pipes docstring). - portage.process._setup_pipes(fd_pipes, close_fds=False) - - # Use default signal handlers in order to avoid problems - # killing subprocesses as reported in bug #353239. - signal.signal(signal.SIGINT, signal.SIG_DFL) - signal.signal(signal.SIGTERM, signal.SIG_DFL) - + def _run(self): # Force consistent color output, in case we are capturing fetch # output through a normal pipe due to unavailability of ptys. portage.output.havecolor = self._settings.get('NOCOLOR') \ not in ('yes', 'true') rval = 1 - allow_missing = self._get_manifest().allow_missing - try: - if fetch(self._uri_map, self._settings, fetchonly=self.fetchonly, - digests=copy.deepcopy(self._get_digests()), - allow_missing_digests=allow_missing): - rval = os.EX_OK - except SystemExit: - raise - except: - traceback.print_exc() - finally: - # Call os._exit() from finally block, in order to suppress any - # finally blocks from earlier in the call stack. See bug #345289. - os._exit(rval) + allow_missing = self._get_manifest().allow_missing or \ + 'digest' in self._settings.features + if fetch(self._uri_map, self._settings, fetchonly=self.fetchonly, + digests=copy.deepcopy(self._get_digests()), + allow_missing_digests=allow_missing): + rval = os.EX_OK + return rval def _get_ebuild_path(self): if self.ebuild_path is not None: @@ -297,7 +271,7 @@ class EbuildFetcher(SpawnProcess): self.scheduler.output(msg, log_path=self.logfile) def _set_returncode(self, wait_retval): - SpawnProcess._set_returncode(self, wait_retval) + ForkProcess._set_returncode(self, wait_retval) # Collect elog messages that might have been # created by the pkg_nofetch phase. # Skip elog messages for prefetch, in order to avoid duplicates. diff --git a/portage_with_autodep/pym/_emerge/EbuildFetcher.pyo b/portage_with_autodep/pym/_emerge/EbuildFetcher.pyo Binary files differindex ddc92d1..e87abd9 100644 --- a/portage_with_autodep/pym/_emerge/EbuildFetcher.pyo +++ b/portage_with_autodep/pym/_emerge/EbuildFetcher.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildFetchonly.pyo b/portage_with_autodep/pym/_emerge/EbuildFetchonly.pyo Binary files differindex c54a1db..947ab78 100644 --- a/portage_with_autodep/pym/_emerge/EbuildFetchonly.pyo +++ b/portage_with_autodep/pym/_emerge/EbuildFetchonly.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildIpcDaemon.pyo b/portage_with_autodep/pym/_emerge/EbuildIpcDaemon.pyo Binary files differindex 7a9588f..fbc4edc 100644 --- a/portage_with_autodep/pym/_emerge/EbuildIpcDaemon.pyo +++ b/portage_with_autodep/pym/_emerge/EbuildIpcDaemon.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildMerge.pyo b/portage_with_autodep/pym/_emerge/EbuildMerge.pyo Binary files differindex 662c681..b281450 100644 --- a/portage_with_autodep/pym/_emerge/EbuildMerge.pyo +++ b/portage_with_autodep/pym/_emerge/EbuildMerge.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildMetadataPhase.py b/portage_with_autodep/pym/_emerge/EbuildMetadataPhase.py index c2d3747..bbb1ca9 100644 --- a/portage_with_autodep/pym/_emerge/EbuildMetadataPhase.py +++ b/portage_with_autodep/pym/_emerge/EbuildMetadataPhase.py @@ -1,4 +1,4 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.SubProcess import SubProcess @@ -6,12 +6,14 @@ import sys from portage.cache.mappings import slot_dict_class import portage portage.proxy.lazyimport.lazyimport(globals(), - 'portage.package.ebuild._eapi_invalid:eapi_invalid', + 'portage.package.ebuild._metadata_invalid:eapi_invalid', ) from portage import os from portage import _encodings from portage import _unicode_decode from portage import _unicode_encode +from portage.dep import extract_unpack_dependencies +from portage.eapi import eapi_has_automatic_unpack_dependencies import errno import fcntl @@ -25,12 +27,11 @@ class EbuildMetadataPhase(SubProcess): """ __slots__ = ("cpv", "eapi_supported", "ebuild_hash", "fd_pipes", - "metadata", "portdb", "repo_path", "settings") + \ + "metadata", "portdb", "repo_path", "settings", "write_auxdb") + \ ("_eapi", "_eapi_lineno", "_raw_metadata",) _file_names = ("ebuild",) _files_dict = slot_dict_class(_file_names, prefix="") - _metadata_fd = 9 def _start(self): ebuild_path = self.ebuild_hash.location @@ -49,14 +50,14 @@ class EbuildMetadataPhase(SubProcess): # An empty EAPI setting is invalid. self._eapi_invalid(None) self._set_returncode((self.pid, 1 << 8)) - self.wait() + self._async_wait() return self.eapi_supported = portage.eapi_is_supported(parsed_eapi) if not self.eapi_supported: self.metadata = {"EAPI": parsed_eapi} self._set_returncode((self.pid, os.EX_OK << 8)) - self.wait() + self._async_wait() return settings = self.settings @@ -74,28 +75,41 @@ class EbuildMetadataPhase(SubProcess): null_input = open('/dev/null', 'rb') fd_pipes.setdefault(0, null_input.fileno()) - fd_pipes.setdefault(1, sys.stdout.fileno()) - fd_pipes.setdefault(2, sys.stderr.fileno()) + fd_pipes.setdefault(1, sys.__stdout__.fileno()) + fd_pipes.setdefault(2, sys.__stderr__.fileno()) # flush any pending output + stdout_filenos = (sys.__stdout__.fileno(), sys.__stderr__.fileno()) for fd in fd_pipes.values(): - if fd == sys.stdout.fileno(): - sys.stdout.flush() - if fd == sys.stderr.fileno(): - sys.stderr.flush() + if fd in stdout_filenos: + sys.__stdout__.flush() + sys.__stderr__.flush() + break self._files = self._files_dict() files = self._files master_fd, slave_fd = os.pipe() + fcntl.fcntl(master_fd, fcntl.F_SETFL, fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK) - fd_pipes[self._metadata_fd] = slave_fd + # FD_CLOEXEC is enabled by default in Python >=3.4. + if sys.hexversion < 0x3040000: + try: + fcntl.FD_CLOEXEC + except AttributeError: + pass + else: + fcntl.fcntl(master_fd, fcntl.F_SETFD, + fcntl.fcntl(master_fd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC) + + fd_pipes[slave_fd] = slave_fd + settings["PORTAGE_PIPE_FD"] = str(slave_fd) self._raw_metadata = [] files.ebuild = master_fd - self._reg_id = self.scheduler.register(files.ebuild, + self._reg_id = self.scheduler.io_add_watch(files.ebuild, self._registered_events, self._output_handler) self._registered = True @@ -103,6 +117,7 @@ class EbuildMetadataPhase(SubProcess): settings=settings, debug=debug, mydbapi=self.portdb, tree="porttree", fd_pipes=fd_pipes, returnpid=True) + settings.pop("PORTAGE_PIPE_FD", None) os.close(slave_fd) null_input.close() @@ -111,11 +126,10 @@ class EbuildMetadataPhase(SubProcess): # doebuild failed before spawning self._unregister() self._set_returncode((self.pid, retval << 8)) - self.wait() + self._async_wait() return self.pid = retval[0] - portage.process.spawned_pids.remove(self.pid) def _output_handler(self, fd, event): @@ -141,8 +155,7 @@ class EbuildMetadataPhase(SubProcess): def _set_returncode(self, wait_retval): SubProcess._set_returncode(self, wait_retval) # self._raw_metadata is None when _start returns - # early due to an unsupported EAPI detected with - # FEATURES=parse-eapi-ebuild-head + # early due to an unsupported EAPI if self.returncode == os.EX_OK and \ self._raw_metadata is not None: metadata_lines = _unicode_decode(b''.join(self._raw_metadata), @@ -163,8 +176,7 @@ class EbuildMetadataPhase(SubProcess): if (not metadata["EAPI"] or self.eapi_supported) and \ metadata["EAPI"] != parsed_eapi: self._eapi_invalid(metadata) - if 'parse-eapi-ebuild-head' in self.settings.features: - metadata_valid = False + metadata_valid = False if metadata_valid: # Since we're supposed to be able to efficiently obtain the @@ -181,8 +193,18 @@ class EbuildMetadataPhase(SubProcess): metadata["_eclasses_"] = {} metadata.pop("INHERITED", None) - self.portdb._write_cache(self.cpv, - self.repo_path, metadata, self.ebuild_hash) + if eapi_has_automatic_unpack_dependencies(metadata["EAPI"]): + repo = self.portdb.repositories.get_name_for_location(self.repo_path) + unpackers = self.settings.unpack_dependencies.get(repo, {}).get(metadata["EAPI"], {}) + unpack_dependencies = extract_unpack_dependencies(metadata["SRC_URI"], unpackers) + if unpack_dependencies: + metadata["DEPEND"] += (" " if metadata["DEPEND"] else "") + unpack_dependencies + + # If called by egencache, this cache write is + # undesirable when metadata-transfer is disabled. + if self.write_auxdb is not False: + self.portdb._write_cache(self.cpv, + self.repo_path, metadata, self.ebuild_hash) else: metadata = {"EAPI": metadata["EAPI"]} self.metadata = metadata diff --git a/portage_with_autodep/pym/_emerge/EbuildMetadataPhase.pyo b/portage_with_autodep/pym/_emerge/EbuildMetadataPhase.pyo Binary files differindex fcc0874..20c9574 100644 --- a/portage_with_autodep/pym/_emerge/EbuildMetadataPhase.pyo +++ b/portage_with_autodep/pym/_emerge/EbuildMetadataPhase.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildPhase.py b/portage_with_autodep/pym/_emerge/EbuildPhase.py index 36ca8b0..0916d73 100644 --- a/portage_with_autodep/pym/_emerge/EbuildPhase.py +++ b/portage_with_autodep/pym/_emerge/EbuildPhase.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import gzip @@ -11,16 +11,26 @@ from _emerge.BinpkgEnvExtractor import BinpkgEnvExtractor from _emerge.MiscFunctionsProcess import MiscFunctionsProcess from _emerge.EbuildProcess import EbuildProcess from _emerge.CompositeTask import CompositeTask +from portage.package.ebuild.prepare_build_dirs import _prepare_workdir from portage.util import writemsg -from portage.xml.metadata import MetaDataXML + +try: + from portage.xml.metadata import MetaDataXML +except (SystemExit, KeyboardInterrupt): + raise +except (ImportError, SystemError, RuntimeError, Exception): + # broken or missing xml support + # http://bugs.python.org/issue14988 + MetaDataXML = None + import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.elog:messages@elog_messages', 'portage.package.ebuild.doebuild:_check_build_log,' + \ '_post_phase_cmds,_post_phase_userpriv_perms,' + \ - '_post_src_install_chost_fix,' + \ '_post_src_install_soname_symlinks,' + \ '_post_src_install_uid_fix,_postinst_bsdflags,' + \ + '_post_src_install_write_metadata,' + \ '_preinst_bsdflags' ) from portage import os @@ -29,12 +39,12 @@ from portage import _unicode_encode class EbuildPhase(CompositeTask): - __slots__ = ("actionmap", "phase", "settings") + \ + __slots__ = ("actionmap", "fd_pipes", "phase", "settings") + \ ("_ebuild_lock",) # FEATURES displayed prior to setup phase _features_display = ( - "ccache", "compressdebug", "depcheck", "depcheckstrict", + "ccache", "compressdebug", "depcheck", "depcheckstrict", "distcc", "distcc-pump", "fakeroot", "installsources", "keeptemp", "keepwork", "nostrip", "preserve-libs", "sandbox", "selinux", "sesandbox", @@ -72,7 +82,7 @@ class EbuildPhase(CompositeTask): maint_str = "" upstr_str = "" metadata_xml_path = os.path.join(os.path.dirname(self.settings['EBUILD']), "metadata.xml") - if os.path.isfile(metadata_xml_path): + if MetaDataXML is not None and os.path.isfile(metadata_xml_path): herds_path = os.path.join(self.settings['PORTDIR'], 'metadata/herds.xml') try: @@ -148,8 +158,7 @@ class EbuildPhase(CompositeTask): return self._start_ebuild() - def _start_ebuild(self): - + def _get_log_path(self): # Don't open the log file during the clean phase since the # open file can result in an nfs lock on $T/build.log which # prevents the clean phase from removing $T. @@ -157,17 +166,21 @@ class EbuildPhase(CompositeTask): if self.phase not in ("clean", "cleanrm") and \ self.settings.get("PORTAGE_BACKGROUND") != "subprocess": logfile = self.settings.get("PORTAGE_LOG_FILE") + return logfile - fd_pipes = None - if not self.background and self.phase == 'nofetch': - # All the pkg_nofetch output goes to stderr since - # it's considered to be an error message. - fd_pipes = {1 : sys.stderr.fileno()} + def _start_ebuild(self): + + fd_pipes = self.fd_pipes + if fd_pipes is None: + if not self.background and self.phase == 'nofetch': + # All the pkg_nofetch output goes to stderr since + # it's considered to be an error message. + fd_pipes = {1 : sys.__stderr__.fileno()} ebuild_process = EbuildProcess(actionmap=self.actionmap, - background=self.background, fd_pipes=fd_pipes, logfile=logfile, - phase=self.phase, scheduler=self.scheduler, - settings=self.settings) + background=self.background, fd_pipes=fd_pipes, + logfile=self._get_log_path(), phase=self.phase, + scheduler=self.scheduler, settings=self.settings) self._start_task(ebuild_process, self._ebuild_exit) @@ -181,16 +194,21 @@ class EbuildPhase(CompositeTask): if self._default_exit(ebuild_process) != os.EX_OK: if self.phase == "test" and \ "test-fail-continue" in self.settings.features: - pass + # mark test phase as complete (bug #452030) + try: + open(_unicode_encode(os.path.join( + self.settings["PORTAGE_BUILDDIR"], ".tested"), + encoding=_encodings['fs'], errors='strict'), + 'wb').close() + except OSError: + pass else: fail = True if not fail: self.returncode = None - logfile = None - if self.settings.get("PORTAGE_BACKGROUND") != "subprocess": - logfile = self.settings.get("PORTAGE_LOG_FILE") + logfile = self._get_log_path() if self.phase == "install": out = io.StringIO() @@ -205,9 +223,16 @@ class EbuildPhase(CompositeTask): settings = self.settings _post_phase_userpriv_perms(settings) - if self.phase == "install": + if self.phase == "unpack": + # Bump WORKDIR timestamp, in case tar gave it a timestamp + # that will interfere with distfiles / WORKDIR timestamp + # comparisons as reported in bug #332217. Also, fix + # ownership since tar can change that too. + os.utime(settings["WORKDIR"], None) + _prepare_workdir(settings) + elif self.phase == "install": out = io.StringIO() - _post_src_install_chost_fix(settings) + _post_src_install_write_metadata(settings) _post_src_install_uid_fix(settings, out) msg = out.getvalue() if msg: @@ -227,8 +252,9 @@ class EbuildPhase(CompositeTask): fd, logfile = tempfile.mkstemp() os.close(fd) post_phase = MiscFunctionsProcess(background=self.background, - commands=post_phase_cmds, logfile=logfile, phase=self.phase, - scheduler=self.scheduler, settings=settings) + commands=post_phase_cmds, fd_pipes=self.fd_pipes, + logfile=logfile, phase=self.phase, scheduler=self.scheduler, + settings=settings) self._start_task(post_phase, self._post_phase_exit) return @@ -303,8 +329,9 @@ class EbuildPhase(CompositeTask): self.returncode = None phase = 'die_hooks' die_hooks = MiscFunctionsProcess(background=self.background, - commands=[phase], phase=phase, - scheduler=self.scheduler, settings=self.settings) + commands=[phase], phase=phase, logfile=self._get_log_path(), + fd_pipes=self.fd_pipes, scheduler=self.scheduler, + settings=self.settings) self._start_task(die_hooks, self._die_hooks_exit) def _die_hooks_exit(self, die_hooks): @@ -323,7 +350,8 @@ class EbuildPhase(CompositeTask): portage.elog.elog_process(self.settings.mycpv, self.settings) phase = "clean" clean_phase = EbuildPhase(background=self.background, - phase=phase, scheduler=self.scheduler, settings=self.settings) + fd_pipes=self.fd_pipes, phase=phase, scheduler=self.scheduler, + settings=self.settings) self._start_task(clean_phase, self._fail_clean_exit) return diff --git a/portage_with_autodep/pym/_emerge/EbuildPhase.py.rej b/portage_with_autodep/pym/_emerge/EbuildPhase.py.rej deleted file mode 100644 index 0f061da..0000000 --- a/portage_with_autodep/pym/_emerge/EbuildPhase.py.rej +++ /dev/null @@ -1,12 +0,0 @@ ---- pym/_emerge/EbuildPhase.py -+++ pym/_emerge/EbuildPhase.py -@@ -33,7 +33,8 @@ - ("_ebuild_lock",) - - # FEATURES displayed prior to setup phase -- _features_display = ("ccache", "distcc", "distcc-pump", "fakeroot", -+ _features_display = ("ccache", "depcheck", "depcheckstrict" "distcc", -+ "distcc-pump", "fakeroot", - "installsources", "keeptemp", "keepwork", "nostrip", - "preserve-libs", "sandbox", "selinux", "sesandbox", - "splitdebug", "suidctl", "test", "userpriv", diff --git a/portage_with_autodep/pym/_emerge/EbuildPhase.pyo b/portage_with_autodep/pym/_emerge/EbuildPhase.pyo Binary files differindex 4c73313..8c719a5 100644 --- a/portage_with_autodep/pym/_emerge/EbuildPhase.pyo +++ b/portage_with_autodep/pym/_emerge/EbuildPhase.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildProcess.py b/portage_with_autodep/pym/_emerge/EbuildProcess.py index ce97aff..333ad7b 100644 --- a/portage_with_autodep/pym/_emerge/EbuildProcess.py +++ b/portage_with_autodep/pym/_emerge/EbuildProcess.py @@ -1,4 +1,4 @@ -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.AbstractEbuildProcess import AbstractEbuildProcess @@ -17,5 +17,11 @@ class EbuildProcess(AbstractEbuildProcess): if actionmap is None: actionmap = _spawn_actionmap(self.settings) - return _doebuild_spawn(self.phase, self.settings, - actionmap=actionmap, **kwargs) + if self._dummy_pipe_fd is not None: + self.settings["PORTAGE_PIPE_FD"] = str(self._dummy_pipe_fd) + + try: + return _doebuild_spawn(self.phase, self.settings, + actionmap=actionmap, **kwargs) + finally: + self.settings.pop("PORTAGE_PIPE_FD", None) diff --git a/portage_with_autodep/pym/_emerge/EbuildProcess.pyo b/portage_with_autodep/pym/_emerge/EbuildProcess.pyo Binary files differindex 52f6cdf..7332563 100644 --- a/portage_with_autodep/pym/_emerge/EbuildProcess.pyo +++ b/portage_with_autodep/pym/_emerge/EbuildProcess.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildSpawnProcess.py b/portage_with_autodep/pym/_emerge/EbuildSpawnProcess.py index e1f682a..26d26fc 100644 --- a/portage_with_autodep/pym/_emerge/EbuildSpawnProcess.py +++ b/portage_with_autodep/pym/_emerge/EbuildSpawnProcess.py @@ -1,4 +1,4 @@ -# Copyright 2010 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.AbstractEbuildProcess import AbstractEbuildProcess @@ -13,4 +13,10 @@ class EbuildSpawnProcess(AbstractEbuildProcess): __slots__ = ('fakeroot_state', 'spawn_func') def _spawn(self, args, **kwargs): - return self.spawn_func(args, env=self.settings.environ(), **kwargs) + + env = self.settings.environ() + + if self._dummy_pipe_fd is not None: + env["PORTAGE_PIPE_FD"] = str(self._dummy_pipe_fd) + + return self.spawn_func(args, env=env, **kwargs) diff --git a/portage_with_autodep/pym/_emerge/EbuildSpawnProcess.pyo b/portage_with_autodep/pym/_emerge/EbuildSpawnProcess.pyo Binary files differindex 1f3e925..4cfe833 100644 --- a/portage_with_autodep/pym/_emerge/EbuildSpawnProcess.pyo +++ b/portage_with_autodep/pym/_emerge/EbuildSpawnProcess.pyo diff --git a/portage_with_autodep/pym/_emerge/EventsAnalyser.py b/portage_with_autodep/pym/_emerge/EventsAnalyser.py index 65ece7b..f740cf6 100644 --- a/portage_with_autodep/pym/_emerge/EventsAnalyser.py +++ b/portage_with_autodep/pym/_emerge/EventsAnalyser.py @@ -34,36 +34,36 @@ class PortageUtils: def get_dep(self,pkg,dep_type=["RDEPEND","DEPEND"]): """ Gets current dependencies of a package. Looks in portage db - + :param pkg: name of package - :param dep_type: type of dependencies to recurse. Can be ["DEPEND"] or + :param dep_type: type of dependencies to recurse. Can be ["DEPEND"] or ["RDEPEND", "DEPEND"] :returns: **set** of packages names """ ret=set() - - pkg = self.get_best_visible_pkg(pkg) + + pkg = self.get_best_visible_pkg(pkg) if not pkg: return ret - + # we found the best visible match in common tree - metadata = dict(zip(self.metadata_keys, + metadata = dict(zip(self.metadata_keys, self.portdbapi.aux_get(pkg, self.metadata_keys))) dep_str = " ".join(metadata[k] for k in dep_type) # the IUSE default are very important for us iuse_defaults=[ u[1:] for u in metadata.get("IUSE",'').split() if u.startswith("+")] - + use=self.use.split() - + for u in iuse_defaults: if u not in use: use.append(u) - success, atoms = portage.dep_check(dep_str, None, self.settings, + success, atoms = portage.dep_check(dep_str, None, self.settings, myuse=use, myroot=self.settings["ROOT"], trees={self.settings["ROOT"]:{"vartree":self.vartree, "porttree": self.vartree}}) if not success: @@ -74,7 +74,7 @@ class PortageUtils: if not atomname: continue - + for unvirt_pkg in expand_new_virt(self.vardbapi,'='+atomname): for pkg in self.vartree.dep_match(unvirt_pkg): ret.add(pkg) @@ -83,12 +83,12 @@ class PortageUtils: # recursive dependency getter def get_deps(self,pkg,dep_type=["RDEPEND","DEPEND"]): - """ - Gets current dependencies of a package on any depth + """ + Gets current dependencies of a package on any depth All dependencies **must** be installed - + :param pkg: name of package - :param dep_type: type of dependencies to recurse. Can be ["DEPEND"] or + :param dep_type: type of dependencies to recurse. Can be ["DEPEND"] or ["RDEPEND", "DEPEND"] :returns: **set** of packages names """ @@ -97,14 +97,14 @@ class PortageUtils: # get porttree dependencies on the first package - pkg = self.portdbapi.xmatch("bestmatch-visible", pkg) + pkg = self.portdbapi.xmatch("bestmatch-visible", pkg) if not pkg: return ret known_packages=set() unknown_packages=self.get_dep(pkg,dep_type) ret=ret.union(unknown_packages) - + while unknown_packages: p=unknown_packages.pop() if p in known_packages: @@ -114,18 +114,18 @@ class PortageUtils: metadata = dict(zip(self.metadata_keys, self.vardbapi.aux_get(p, self.metadata_keys))) dep_str = " ".join(metadata[k] for k in dep_type) - + # the IUSE default are very important for us iuse_defaults=[ u[1:] for u in metadata.get("IUSE",'').split() if u.startswith("+")] - + use=self.use.split() - + for u in iuse_defaults: if u not in use: use.append(u) - - success, atoms = portage.dep_check(dep_str, None, self.settings, + + success, atoms = portage.dep_check(dep_str, None, self.settings, myuse=use, myroot=self.settings["ROOT"], trees={self.settings["ROOT"]:{"vartree":self.vartree,"porttree": self.vartree}}) @@ -136,7 +136,7 @@ class PortageUtils: atomname = self.vartree.dep_bestmatch(atom) if not atomname: continue - + for unvirt_pkg in expand_new_virt(self.vardbapi,'='+atomname): for pkg in self.vartree.dep_match(unvirt_pkg): ret.add(pkg) @@ -144,8 +144,8 @@ class PortageUtils: return ret def get_deps_for_package_building(self, pkg): - """ - returns buildtime dependencies of current package and + """ + returns buildtime dependencies of current package and all runtime dependencies of that buildtime dependencies """ buildtime_deps=self.get_dep(pkg, ["DEPEND"]) @@ -157,9 +157,9 @@ class PortageUtils: return ret def get_system_packages_list(self): - """ + """ returns all packages from system set. They are always implicit dependencies - + :returns: **list** of package names """ ret=[] @@ -172,11 +172,12 @@ class PortageUtils: class GentoolkitUtils: - """ - Interface with qfile and qlist utils. They are much faster than + """ + Interface with qfile and qlist utils. They are much faster than internals. """ + @staticmethod def getpackagesbyfiles(files): """ :param files: list of filenames @@ -190,14 +191,14 @@ class GentoolkitUtils: ret[f]="directory" else: listtocheck.append(f) - + try: proc=subprocess.Popen(['qfile']+['--nocolor','--exact','','--from','-'], - stdin=subprocess.PIPE, stdout=subprocess.PIPE,stderr=subprocess.PIPE, + stdin=subprocess.PIPE, stdout=subprocess.PIPE,stderr=subprocess.PIPE, bufsize=4096) - + out,err=proc.communicate("\n".join(listtocheck).encode("utf8")) - + lines=out.decode("utf8").split("\n") #print lines line_re=re.compile(r"^([^ ]+)\s+\(([^)]+)\)$") @@ -212,24 +213,25 @@ class GentoolkitUtils: except OSError as e: portage.util.writemsg("Error while launching qfile: %s\n" % e) - - + + return ret - + + @staticmethod def getfilesbypackages(packagenames): """ - + :param packagename: name of package :returns: **list** of files in package with name *packagename* """ ret=[] try: proc=subprocess.Popen(['qlist']+['--nocolor',"--obj"]+packagenames, - stdout=subprocess.PIPE,stderr=subprocess.PIPE, + stdout=subprocess.PIPE,stderr=subprocess.PIPE, bufsize=4096) - + out,err=proc.communicate() - + ret=out.decode("utf8").split("\n") if ret==['']: ret=[] @@ -237,34 +239,35 @@ class GentoolkitUtils: portage.util.writemsg("Error while launching qfile: %s\n" % e) return ret - - def get_all_packages_files(): + + @staticmethod + def get_all_packages_files(): """ Memory-hungry operation - + :returns: **set** of all files that belongs to package """ ret=[] try: proc=subprocess.Popen(['qlist']+['--all',"--obj"], - stdout=subprocess.PIPE,stderr=subprocess.PIPE, + stdout=subprocess.PIPE,stderr=subprocess.PIPE, bufsize=4096) - + out,err=proc.communicate() - + ret=out.decode("utf8").split("\n") except OSError as e: portage.util.writemsg("Error while launching qfile: %s\n" % e) return set(ret) - + class FilterProcGenerator: def __init__(self, pkgname, settings): portageutils=PortageUtils(settings=settings) deps_all=portageutils.get_deps_for_package_building(pkgname) deps_portage=portageutils.get_dep('portage',["RDEPEND"]) - + system_packages=portageutils.get_system_packages_list() allfiles=GentoolkitUtils.get_all_packages_files() @@ -272,8 +275,8 @@ class FilterProcGenerator: "a list of allowed files\n") - allowedpkgs=system_packages+list(deps_portage)+list(deps_all) - + allowedpkgs=system_packages+list(deps_portage)+list(deps_all) + allowedfiles=GentoolkitUtils.getfilesbypackages(allowedpkgs) #for pkg in allowedpkgs: # allowedfiles+=GentoolkitUtils.getfilesbypackage(pkg) @@ -283,14 +286,14 @@ class FilterProcGenerator: # manually add all python interpreters to this list allowedfiles+=GentoolkitUtils.getfilesbypackages(['python']) allowedfiles=set(allowedfiles) - + deniedfiles=allfiles-allowedfiles def filter_proc(eventname,filename,stage): if filename in deniedfiles: return False return True - + self.filter_proc=filter_proc def get_filter_proc(self): return self.filter_proc @@ -305,10 +308,10 @@ class EventsAnalyser: self.deps_all=self.portageutils.get_deps_for_package_building(pkgname) self.deps_direct=self.portageutils.get_dep(pkgname,["DEPEND"]) self.deps_portage=self.portageutils.get_dep('portage',["RDEPEND"]) - + self.system_packages=self.portageutils.get_system_packages_list() # All analyse work is here - + # get unique filenames filenames=set() for stage in events: @@ -319,7 +322,7 @@ class EventsAnalyser: filenames=list(filenames) file_to_package=GentoolkitUtils.getpackagesbyfiles(filenames) - # This part is completly unreadable. + # This part is completly unreadable. # It converting one complex struct(returned by getfsevents) to another complex # struct which good for generating output. # @@ -330,24 +333,24 @@ class EventsAnalyser: for stage in sorted(events): succ_events=events[stage][0] fail_events=events[stage][1] - + for filename in succ_events: if filename in file_to_package: package=file_to_package[filename] else: package="unknown" - + if not package in packagesinfo: packagesinfo[package]={} stageinfo=packagesinfo[package] if not stage in stageinfo: stageinfo[stage]={} - + filesinfo=stageinfo[stage] if not filename in filesinfo: filesinfo[filename]={"found":[],"notfound":[]} filesinfo[filename]["found"]=succ_events[filename] - + for filename in fail_events: if filename in file_to_package: package=file_to_package[filename] @@ -358,13 +361,13 @@ class EventsAnalyser: stageinfo=packagesinfo[package] if not stage in stageinfo: stageinfo[stage]={} - + filesinfo=stageinfo[stage] if not filename in filesinfo: filesinfo[filename]={"found":[],"notfound":[]} filesinfo[filename]["notfound"]=fail_events[filename] self.packagesinfo=packagesinfo - + def display(self): portage.util.writemsg( portage.output.colorize( @@ -373,12 +376,12 @@ class EventsAnalyser: stagesorder={"clean":1,"setup":2,"unpack":3,"prepare":4,"configure":5,"compile":6,"test":7, "install":8,"preinst":9,"postinst":10,"prerm":11,"postrm":12,"unknown":13} packagesinfo=self.packagesinfo - # print information grouped by package + # print information grouped by package for package in sorted(packagesinfo): # not showing special directory package if package=="directory": continue - + if package=="unknown": continue @@ -395,7 +398,7 @@ class EventsAnalyser: if len(stages)==0: continue - + filenames={} for stage in stages: for filename in packagesinfo[package][stage]: @@ -406,7 +409,7 @@ class EventsAnalyser: else: status, old_was_readed, old_was_writed=filenames[filename] filenames[filename]=[ - 'ok',old_was_readed | was_readed, old_was_writed | was_writed + 'ok',old_was_readed | was_readed, old_was_writed | was_writed ] if len(packagesinfo[package][stage][filename]["notfound"])!=0: was_notfound,was_blocked=packagesinfo[package][stage][filename]["notfound"] @@ -415,9 +418,9 @@ class EventsAnalyser: else: status, old_was_notfound, old_was_blocked=filenames[filename] filenames[filename]=[ - 'err',old_was_notfound | was_notfound, old_was_blocked | was_blocked + 'err',old_was_notfound | was_notfound, old_was_blocked | was_blocked ] - + if is_pkg_in_dep: portage.util.writemsg("[OK]") @@ -446,9 +449,9 @@ class EventsAnalyser: ('err',False,True):"blocked", ('err',True,True):"not found and blocked" } - + filescounter=0 - + for filename in filenames: event_info=tuple(filenames[filename]) portage.util.writemsg(" %-56s %-21s\n" % (filename,action[event_info])) @@ -456,7 +459,7 @@ class EventsAnalyser: if filescounter>10: portage.util.writemsg(" ... and %d more ...\n" % (len(filenames)-10)) break - # ... and one more check. Making sure that direct build time + # ... and one more check. Making sure that direct build time # dependencies were accessed #import pdb; pdb.set_trace() not_accessed_deps=set(self.deps_direct)-set(self.packagesinfo.keys()) @@ -465,7 +468,7 @@ class EventsAnalyser: portage.util.writemsg("Warning! Some build time dependencies " + \ "of packages were not accessed: " + \ " ".join(not_accessed_deps) + "\n") - + def is_package_useful(self,pkg,stages,files): """ some basic heuristics here to cut part of packages """ @@ -499,13 +502,13 @@ class EventsAnalyser: for f in files: if is_file_excluded(f): continue - - # test 1: package is not useful if all files are *.desktop or *.xml or *.m4 + + # test 1: package is not useful if all files are *.desktop or *.xml or *.m4 if not (f.endswith(".desktop") or f.endswith(".xml") or f.endswith(".m4") or f.endswith(".pc")): break else: return False # we get here if cycle ends not with break - + return True - -
\ No newline at end of file + + diff --git a/portage_with_autodep/pym/_emerge/EventsLogger.py b/portage_with_autodep/pym/_emerge/EventsLogger.py index 68b3c67..a08c533 100644 --- a/portage_with_autodep/pym/_emerge/EventsLogger.py +++ b/portage_with_autodep/pym/_emerge/EventsLogger.py @@ -14,17 +14,17 @@ from portage import os class EventsLogger(threading.Thread): def default_filter(eventname, filename, stage): return True - + def __init__(self, socket_dir="/tmp/", filter_proc=default_filter): threading.Thread.__init__(self) # init the Thread - + self.alive=False - + self.main_thread=threading.currentThread() - + self.socket_dir=socket_dir self.filter_proc=filter_proc - + self.socket_name=None self.socket_logger=None @@ -33,16 +33,16 @@ class EventsLogger(threading.Thread): try: socket_dir_name = tempfile.mkdtemp(dir=self.socket_dir, prefix="log_socket_") - + socket_name = os.path.join(socket_dir_name, 'socket') except OSError as e: return - + self.socket_name=socket_name - + #print(self.socket_name) - + try: socket_logger=socket.socket(socket.AF_UNIX, socket.SOCK_SEQPACKET) socket_logger.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) @@ -65,21 +65,21 @@ class EventsLogger(threading.Thread): stat.S_IROTH|stat.S_IWOTH|stat.S_IXOTH) except OSError as e: return - + def run(self): """ Starts the log server """ self.alive=True self.listen_thread=threading.currentThread() clients={} - + epoll=select.epoll() epoll.register(self.socket_logger.fileno(), select.EPOLLIN) while self.alive: try: sock_events = epoll.poll(3) - + for fileno, sock_event in sock_events: if fileno == self.socket_logger.fileno(): ret = self.socket_logger.accept() @@ -92,13 +92,13 @@ class EventsLogger(threading.Thread): elif sock_event & select.EPOLLIN: s=clients[fileno] record=s.recv(8192) - + if not record: # if connection was closed epoll.unregister(fileno) clients[fileno].close() del clients[fileno] continue - + #import pdb; pdb.set_trace() try: message=record.decode("utf8").split("\0") @@ -109,7 +109,7 @@ class EventsLogger(threading.Thread): # continue #print(message) - + try: if message[4]=="ASKING": if self.filter_proc(message[1],message[2],message[3]): @@ -123,7 +123,7 @@ class EventsLogger(threading.Thread): if not stage in self.events: self.events[stage]=[{},{}] - + hashofsucesses=self.events[stage][0] hashoffailures=self.events[stage][1] @@ -133,19 +133,19 @@ class EventsLogger(threading.Thread): if result=="OK": if not filename in hashofsucesses: hashofsucesses[filename]=[False,False] - + readed_or_writed=hashofsucesses[filename] - + if eventname=="read": readed_or_writed[0]=True elif eventname=="write": readed_or_writed[1]=True - + elif result[0:3]=="ERR" or result=="DENIED": if not filename in hashoffailures: hashoffailures[filename]=[False,False] notfound_or_blocked=hashoffailures[filename] - + if result=="ERR/2": notfound_or_blocked[0]=True elif result=="DENIED": @@ -153,28 +153,28 @@ class EventsLogger(threading.Thread): else: print("Error in logger module<->analyser protocol") - + except IndexError: print("IndexError while parsing %s" % record) except IOError as e: if e.errno!=4: # handling "Interrupted system call" errors raise - - # if main thread doesnt exists then exit + + # if main thread doesnt exists then exit if not self.main_thread.is_alive(): break epoll.unregister(self.socket_logger.fileno()) epoll.close() self.socket_logger.close() - + def stop(self): """ Stops the log server. Returns all events """ self.alive=False - + # Block the main thread until listener exists self.listen_thread.join() - + # We assume portage clears tmp folder, so no deleting a socket file # We assume that no new socket data will arrive after this moment return self.events diff --git a/portage_with_autodep/pym/_emerge/FakeVartree.py b/portage_with_autodep/pym/_emerge/FakeVartree.py index d4dbe97..14be50c 100644 --- a/portage_with_autodep/pym/_emerge/FakeVartree.py +++ b/portage_with_autodep/pym/_emerge/FakeVartree.py @@ -1,6 +1,8 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + import sys import warnings @@ -10,11 +12,17 @@ from _emerge.Package import Package from _emerge.PackageVirtualDbapi import PackageVirtualDbapi from portage.const import VDB_PATH from portage.dbapi.vartree import vartree -from portage.repository.config import _gen_valid_repo +from portage.dep._slot_operator import find_built_slot_operator_atoms +from portage.eapi import _get_eapi_attrs +from portage.exception import InvalidData, InvalidDependString from portage.update import grab_updates, parse_updates, update_dbentries +from portage.versions import _pkg_str if sys.hexversion >= 0x3000000: long = int + _unicode = str +else: + _unicode = unicode class FakeVardbapi(PackageVirtualDbapi): """ @@ -27,6 +35,9 @@ class FakeVardbapi(PackageVirtualDbapi): path =os.path.join(path, filename) return path +class _DynamicDepsNotApplicable(Exception): + pass + class FakeVartree(vartree): """This is implements an in-memory copy of a vartree instance that provides all the interfaces required for use by the depgraph. The vardb is locked @@ -39,9 +50,10 @@ class FakeVartree(vartree): is not a matching ebuild in the tree). Instances of this class are not populated until the sync() method is called.""" def __init__(self, root_config, pkg_cache=None, pkg_root_config=None, - dynamic_deps=True): + dynamic_deps=True, ignore_built_slot_operator_deps=False): self._root_config = root_config self._dynamic_deps = dynamic_deps + self._ignore_built_slot_operator_deps = ignore_built_slot_operator_deps if pkg_root_config is None: pkg_root_config = self._root_config self._pkg_root_config = pkg_root_config @@ -68,7 +80,7 @@ class FakeVartree(vartree): self.dbapi.aux_get = self._aux_get_wrapper self.dbapi.match = self._match_wrapper self._aux_get_history = set() - self._portdb_keys = ["EAPI", "DEPEND", "RDEPEND", "PDEPEND"] + self._portdb_keys = Package._dep_keys + ("EAPI", "KEYWORDS") self._portdb = portdb self._global_updates = None @@ -95,18 +107,30 @@ class FakeVartree(vartree): self._aux_get_wrapper(cpv, []) return matches - def _aux_get_wrapper(self, pkg, wants, myrepo=None): - if pkg in self._aux_get_history: - return self._aux_get(pkg, wants) - self._aux_get_history.add(pkg) - # We need to check the EAPI, and this also raises - # a KeyError to the caller if appropriate. - installed_eapi, repo = self._aux_get(pkg, ["EAPI", "repository"]) + def _aux_get_wrapper(self, cpv, wants, myrepo=None): + if cpv in self._aux_get_history: + return self._aux_get(cpv, wants) + self._aux_get_history.add(cpv) + + # This raises a KeyError to the caller if appropriate. + pkg = self.dbapi._cpv_map[cpv] + try: - # Use the live ebuild metadata if possible. - repo = _gen_valid_repo(repo) live_metadata = dict(zip(self._portdb_keys, - self._portdb.aux_get(pkg, self._portdb_keys, myrepo=repo))) + self._portdb.aux_get(cpv, self._portdb_keys, + myrepo=pkg.repo))) + except (KeyError, portage.exception.PortageException): + live_metadata = None + + self._apply_dynamic_deps(pkg, live_metadata) + + return self._aux_get(cpv, wants) + + def _apply_dynamic_deps(self, pkg, live_metadata): + + try: + if live_metadata is None: + raise _DynamicDepsNotApplicable() # Use the metadata from the installed instance if the EAPI # of either instance is unsupported, since if the installed # instance has an unsupported or corrupt EAPI then we don't @@ -116,16 +140,46 @@ class FakeVartree(vartree): # order to respect dep updates without revision bump or EAPI # bump, as in bug #368725. if not (portage.eapi_is_supported(live_metadata["EAPI"]) and \ - portage.eapi_is_supported(installed_eapi)): - raise KeyError(pkg) - self.dbapi.aux_update(pkg, live_metadata) - except (KeyError, portage.exception.PortageException): + portage.eapi_is_supported(pkg.eapi)): + raise _DynamicDepsNotApplicable() + + # preserve built slot/sub-slot := operator deps + built_slot_operator_atoms = None + if not self._ignore_built_slot_operator_deps and \ + _get_eapi_attrs(pkg.eapi).slot_operator: + try: + built_slot_operator_atoms = \ + find_built_slot_operator_atoms(pkg) + except InvalidDependString: + pass + + if built_slot_operator_atoms: + live_eapi_attrs = _get_eapi_attrs(live_metadata["EAPI"]) + if not live_eapi_attrs.slot_operator: + raise _DynamicDepsNotApplicable() + for k, v in built_slot_operator_atoms.items(): + live_metadata[k] += (" " + + " ".join(_unicode(atom) for atom in v)) + + self.dbapi.aux_update(pkg.cpv, live_metadata) + except _DynamicDepsNotApplicable: if self._global_updates is None: self._global_updates = \ grab_global_updates(self._portdb) + + # Bypass _aux_get_wrapper, since calling that + # here would trigger infinite recursion. + aux_keys = Package._dep_keys + self.dbapi._pkg_str_aux_keys + aux_dict = dict(zip(aux_keys, self._aux_get(pkg.cpv, aux_keys))) perform_global_updates( - pkg, self.dbapi, self._global_updates) - return self._aux_get(pkg, wants) + pkg.cpv, aux_dict, self.dbapi, self._global_updates) + + def dynamic_deps_preload(self, pkg, metadata): + if metadata is not None: + metadata = dict((k, metadata.get(k, '')) + for k in self._portdb_keys) + self._apply_dynamic_deps(pkg, metadata) + self._aux_get_history.add(pkg.cpv) def cpv_discard(self, pkg): """ @@ -223,12 +277,6 @@ class FakeVartree(vartree): root_config=self._pkg_root_config, type_name="installed") - try: - mycounter = long(pkg.metadata["COUNTER"]) - except ValueError: - mycounter = 0 - pkg.metadata["COUNTER"] = str(mycounter) - self._pkg_cache[pkg] = pkg return pkg @@ -257,12 +305,14 @@ def grab_global_updates(portdb): return retupdates -def perform_global_updates(mycpv, mydb, myupdates): - aux_keys = ["DEPEND", "RDEPEND", "PDEPEND", 'repository'] - aux_dict = dict(zip(aux_keys, mydb.aux_get(mycpv, aux_keys))) - repository = aux_dict.pop('repository') +def perform_global_updates(mycpv, aux_dict, mydb, myupdates): + try: + pkg = _pkg_str(mycpv, metadata=aux_dict, settings=mydb.settings) + except InvalidData: + return + aux_dict = dict((k, aux_dict[k]) for k in Package._dep_keys) try: - mycommands = myupdates[repository] + mycommands = myupdates[pkg.repo] except KeyError: try: mycommands = myupdates['DEFAULT'] @@ -272,6 +322,6 @@ def perform_global_updates(mycpv, mydb, myupdates): if not mycommands: return - updates = update_dbentries(mycommands, aux_dict) + updates = update_dbentries(mycommands, aux_dict, parent=pkg) if updates: mydb.aux_update(mycpv, updates) diff --git a/portage_with_autodep/pym/_emerge/FakeVartree.pyo b/portage_with_autodep/pym/_emerge/FakeVartree.pyo Binary files differindex 8707391..740d33d 100644 --- a/portage_with_autodep/pym/_emerge/FakeVartree.pyo +++ b/portage_with_autodep/pym/_emerge/FakeVartree.pyo diff --git a/portage_with_autodep/pym/_emerge/FifoIpcDaemon.py b/portage_with_autodep/pym/_emerge/FifoIpcDaemon.py index fcc4ab4..7468de5 100644 --- a/portage_with_autodep/pym/_emerge/FifoIpcDaemon.py +++ b/portage_with_autodep/pym/_emerge/FifoIpcDaemon.py @@ -1,6 +1,14 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +import sys + +try: + import fcntl +except ImportError: + # http://bugs.jython.org/issue1074 + fcntl = None + from portage import os from _emerge.AbstractPollTask import AbstractPollTask from portage.cache.mappings import slot_dict_class @@ -21,7 +29,18 @@ class FifoIpcDaemon(AbstractPollTask): self._files.pipe_in = \ os.open(self.input_fifo, os.O_RDONLY|os.O_NONBLOCK) - self._reg_id = self.scheduler.register( + # FD_CLOEXEC is enabled by default in Python >=3.4. + if sys.hexversion < 0x3040000 and fcntl is not None: + try: + fcntl.FD_CLOEXEC + except AttributeError: + pass + else: + fcntl.fcntl(self._files.pipe_in, fcntl.F_SETFD, + fcntl.fcntl(self._files.pipe_in, + fcntl.F_GETFD) | fcntl.FD_CLOEXEC) + + self._reg_id = self.scheduler.io_add_watch( self._files.pipe_in, self._registered_events, self._input_handler) @@ -32,11 +51,23 @@ class FifoIpcDaemon(AbstractPollTask): Re-open the input stream, in order to suppress POLLHUP events (bug #339976). """ - self.scheduler.unregister(self._reg_id) + self.scheduler.source_remove(self._reg_id) os.close(self._files.pipe_in) self._files.pipe_in = \ os.open(self.input_fifo, os.O_RDONLY|os.O_NONBLOCK) - self._reg_id = self.scheduler.register( + + # FD_CLOEXEC is enabled by default in Python >=3.4. + if sys.hexversion < 0x3040000 and fcntl is not None: + try: + fcntl.FD_CLOEXEC + except AttributeError: + pass + else: + fcntl.fcntl(self._files.pipe_in, fcntl.F_SETFD, + fcntl.fcntl(self._files.pipe_in, + fcntl.F_GETFD) | fcntl.FD_CLOEXEC) + + self._reg_id = self.scheduler.io_add_watch( self._files.pipe_in, self._registered_events, self._input_handler) @@ -47,6 +78,8 @@ class FifoIpcDaemon(AbstractPollTask): if self.returncode is None: self.returncode = 1 self._unregister() + # notify exit listeners + self.wait() def _wait(self): if self.returncode is not None: @@ -67,7 +100,7 @@ class FifoIpcDaemon(AbstractPollTask): self._registered = False if self._reg_id is not None: - self.scheduler.unregister(self._reg_id) + self.scheduler.source_remove(self._reg_id) self._reg_id = None if self._files is not None: diff --git a/portage_with_autodep/pym/_emerge/FifoIpcDaemon.pyo b/portage_with_autodep/pym/_emerge/FifoIpcDaemon.pyo Binary files differindex 6d7c4f9..15f04ac 100644 --- a/portage_with_autodep/pym/_emerge/FifoIpcDaemon.pyo +++ b/portage_with_autodep/pym/_emerge/FifoIpcDaemon.pyo diff --git a/portage_with_autodep/pym/_emerge/JobStatusDisplay.py b/portage_with_autodep/pym/_emerge/JobStatusDisplay.py index 5b9b221..9f6f09b 100644 --- a/portage_with_autodep/pym/_emerge/JobStatusDisplay.py +++ b/portage_with_autodep/pym/_emerge/JobStatusDisplay.py @@ -1,6 +1,8 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + import formatter import io import sys @@ -9,7 +11,6 @@ import time import portage from portage import os from portage import _encodings -from portage import _unicode_decode from portage import _unicode_encode from portage.output import xtermTitle @@ -121,7 +122,8 @@ class JobStatusDisplay(object): term_codes = {} for k, capname in self._termcap_name_map.items(): - code = tigetstr(capname) + # Use _native_string for PyPy compat (bug #470258). + code = tigetstr(portage._native_string(capname)) if code is None: code = self._default_term_codes[capname] term_codes[k] = code @@ -233,10 +235,10 @@ class JobStatusDisplay(object): def _display_status(self): # Don't use len(self._completed_tasks) here since that also # can include uninstall tasks. - curval_str = str(self.curval) - maxval_str = str(self.maxval) - running_str = str(self.running) - failed_str = str(self.failed) + curval_str = "%s" % (self.curval,) + maxval_str = "%s" % (self.maxval,) + running_str = "%s" % (self.running,) + failed_str = "%s" % (self.failed,) load_avg_str = self._load_avg_str() color_output = io.StringIO() @@ -248,36 +250,36 @@ class JobStatusDisplay(object): f = formatter.AbstractFormatter(style_writer) number_style = "INFORM" - f.add_literal_data(_unicode_decode("Jobs: ")) + f.add_literal_data("Jobs: ") f.push_style(number_style) - f.add_literal_data(_unicode_decode(curval_str)) + f.add_literal_data(curval_str) f.pop_style() - f.add_literal_data(_unicode_decode(" of ")) + f.add_literal_data(" of ") f.push_style(number_style) - f.add_literal_data(_unicode_decode(maxval_str)) + f.add_literal_data(maxval_str) f.pop_style() - f.add_literal_data(_unicode_decode(" complete")) + f.add_literal_data(" complete") if self.running: - f.add_literal_data(_unicode_decode(", ")) + f.add_literal_data(", ") f.push_style(number_style) - f.add_literal_data(_unicode_decode(running_str)) + f.add_literal_data(running_str) f.pop_style() - f.add_literal_data(_unicode_decode(" running")) + f.add_literal_data(" running") if self.failed: - f.add_literal_data(_unicode_decode(", ")) + f.add_literal_data(", ") f.push_style(number_style) - f.add_literal_data(_unicode_decode(failed_str)) + f.add_literal_data(failed_str) f.pop_style() - f.add_literal_data(_unicode_decode(" failed")) + f.add_literal_data(" failed") padding = self._jobs_column_width - len(plain_output.getvalue()) if padding > 0: - f.add_literal_data(padding * _unicode_decode(" ")) + f.add_literal_data(padding * " ") - f.add_literal_data(_unicode_decode("Load avg: ")) - f.add_literal_data(_unicode_decode(load_avg_str)) + f.add_literal_data("Load avg: ") + f.add_literal_data(load_avg_str) # Truncate to fit width, to avoid making the terminal scroll if the # line overflows (happens when the load average is large). diff --git a/portage_with_autodep/pym/_emerge/JobStatusDisplay.pyo b/portage_with_autodep/pym/_emerge/JobStatusDisplay.pyo Binary files differindex f79b2c2..551188d 100644 --- a/portage_with_autodep/pym/_emerge/JobStatusDisplay.pyo +++ b/portage_with_autodep/pym/_emerge/JobStatusDisplay.pyo diff --git a/portage_with_autodep/pym/_emerge/MergeListItem.py b/portage_with_autodep/pym/_emerge/MergeListItem.py index 8086c68..172dfcc 100644 --- a/portage_with_autodep/pym/_emerge/MergeListItem.py +++ b/portage_with_autodep/pym/_emerge/MergeListItem.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from portage import os @@ -32,7 +32,7 @@ class MergeListItem(CompositeTask): if pkg.installed: # uninstall, executed by self.merge() self.returncode = os.EX_OK - self.wait() + self._async_wait() return args_set = self.args_set @@ -47,7 +47,9 @@ class MergeListItem(CompositeTask): action_desc = "Emerging" preposition = "for" + pkg_color = "PKG_MERGE" if pkg.type_name == "binary": + pkg_color = "PKG_BINARY_MERGE" action_desc += " binary" if build_opts.fetchonly: @@ -57,7 +59,7 @@ class MergeListItem(CompositeTask): (action_desc, colorize("MERGE_LIST_PROGRESS", str(pkg_count.curval)), colorize("MERGE_LIST_PROGRESS", str(pkg_count.maxval)), - colorize("GOOD", pkg.cpv)) + colorize(pkg_color, pkg.cpv)) portdb = pkg.root_config.trees["porttree"].dbapi portdir_repo_name = portdb.getRepositoryName(portdb.porttree_root) diff --git a/portage_with_autodep/pym/_emerge/MergeListItem.pyo b/portage_with_autodep/pym/_emerge/MergeListItem.pyo Binary files differindex 168a227..b5295f0 100644 --- a/portage_with_autodep/pym/_emerge/MergeListItem.pyo +++ b/portage_with_autodep/pym/_emerge/MergeListItem.pyo diff --git a/portage_with_autodep/pym/_emerge/MetadataRegen.py b/portage_with_autodep/pym/_emerge/MetadataRegen.py index e82015f..d92b6a0 100644 --- a/portage_with_autodep/pym/_emerge/MetadataRegen.py +++ b/portage_with_autodep/pym/_emerge/MetadataRegen.py @@ -1,18 +1,20 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import portage from portage import os from portage.dep import _repo_separator from _emerge.EbuildMetadataPhase import EbuildMetadataPhase -from _emerge.PollScheduler import PollScheduler +from portage.cache.cache_errors import CacheError +from portage.util._async.AsyncScheduler import AsyncScheduler -class MetadataRegen(PollScheduler): +class MetadataRegen(AsyncScheduler): def __init__(self, portdb, cp_iter=None, consumer=None, - max_jobs=None, max_load=None): - PollScheduler.__init__(self, main=True) + write_auxdb=True, **kwargs): + AsyncScheduler.__init__(self, **kwargs) self._portdb = portdb + self._write_auxdb = write_auxdb self._global_cleanse = False if cp_iter is None: cp_iter = self._iter_every_cp() @@ -22,34 +24,21 @@ class MetadataRegen(PollScheduler): self._cp_iter = cp_iter self._consumer = consumer - if max_jobs is None: - max_jobs = 1 - - self._max_jobs = max_jobs - self._max_load = max_load - self._valid_pkgs = set() self._cp_set = set() self._process_iter = self._iter_metadata_processes() - self.returncode = os.EX_OK - self._error_count = 0 self._running_tasks = set() - self._remaining_tasks = True - def _terminate_tasks(self): - for task in list(self._running_tasks): - task.cancel() + def _next_task(self): + return next(self._process_iter) def _iter_every_cp(self): - portage.writemsg_stdout("Listing available packages...\n") - every_cp = self._portdb.cp_all() - portage.writemsg_stdout("Regenerating cache entries...\n") - every_cp.sort(reverse=True) - try: - while not self._terminated_tasks: - yield every_cp.pop() - except IndexError: - pass + # List categories individually, in order to start yielding quicker, + # and in order to reduce latency in case of a signal interrupt. + cp_all = self._portdb.cp_all + for category in sorted(self._portdb.categories): + for cp in cp_all(categories=(category,)): + yield cp def _iter_metadata_processes(self): portdb = self._portdb @@ -57,8 +46,9 @@ class MetadataRegen(PollScheduler): cp_set = self._cp_set consumer = self._consumer + portage.writemsg_stdout("Regenerating cache entries...\n") for cp in self._cp_iter: - if self._terminated_tasks: + if self._terminated.is_set(): break cp_set.add(cp) portage.writemsg_stdout("Processing %s\n" % cp) @@ -68,7 +58,7 @@ class MetadataRegen(PollScheduler): repo = portdb.repositories.get_repo_for_location(mytree) cpv_list = portdb.cp_list(cp, mytree=[repo.location]) for cpv in cpv_list: - if self._terminated_tasks: + if self._terminated.is_set(): break valid_pkgs.add(cpv) ebuild_path, repo_path = portdb.findname2(cpv, myrepo=repo.name) @@ -84,22 +74,21 @@ class MetadataRegen(PollScheduler): yield EbuildMetadataPhase(cpv=cpv, ebuild_hash=ebuild_hash, portdb=portdb, repo_path=repo_path, - settings=portdb.doebuild_settings) + settings=portdb.doebuild_settings, + write_auxdb=self._write_auxdb) - def _keep_scheduling(self): - return self._remaining_tasks and not self._terminated_tasks + def _wait(self): - def run(self): + AsyncScheduler._wait(self) portdb = self._portdb - from portage.cache.cache_errors import CacheError dead_nodes = {} - self._main_loop() - + self._termination_check() if self._terminated_tasks: - self.returncode = 1 - return + portdb.flush_cache() + self.returncode = self._cancelled_returncode + return self.returncode if self._global_cleanse: for mytree in portdb.porttrees: @@ -142,29 +131,12 @@ class MetadataRegen(PollScheduler): except (KeyError, CacheError): pass - def _schedule_tasks(self): - if self._terminated_tasks: - return - - while self._can_add_job(): - try: - metadata_process = next(self._process_iter) - except StopIteration: - self._remaining_tasks = False - return - - self._jobs += 1 - self._running_tasks.add(metadata_process) - metadata_process.scheduler = self.sched_iface - metadata_process.addExitListener(self._metadata_exit) - metadata_process.start() - - def _metadata_exit(self, metadata_process): - self._jobs -= 1 - self._running_tasks.discard(metadata_process) + portdb.flush_cache() + return self.returncode + + def _task_exit(self, metadata_process): + if metadata_process.returncode != os.EX_OK: - self.returncode = 1 - self._error_count += 1 self._valid_pkgs.discard(metadata_process.cpv) if not self._terminated_tasks: portage.writemsg("Error processing %s, continuing...\n" % \ @@ -179,5 +151,4 @@ class MetadataRegen(PollScheduler): metadata_process.ebuild_hash, metadata_process.eapi_supported) - self._schedule() - + AsyncScheduler._task_exit(self, metadata_process) diff --git a/portage_with_autodep/pym/_emerge/MetadataRegen.pyo b/portage_with_autodep/pym/_emerge/MetadataRegen.pyo Binary files differindex 6c8788f..bcb940f 100644 --- a/portage_with_autodep/pym/_emerge/MetadataRegen.pyo +++ b/portage_with_autodep/pym/_emerge/MetadataRegen.pyo diff --git a/portage_with_autodep/pym/_emerge/MiscFunctionsProcess.py b/portage_with_autodep/pym/_emerge/MiscFunctionsProcess.py index afa44fb..bada79d 100644 --- a/portage_with_autodep/pym/_emerge/MiscFunctionsProcess.py +++ b/portage_with_autodep/pym/_emerge/MiscFunctionsProcess.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.AbstractEbuildProcess import AbstractEbuildProcess @@ -29,6 +29,10 @@ class MiscFunctionsProcess(AbstractEbuildProcess): AbstractEbuildProcess._start(self) def _spawn(self, args, **kwargs): + + if self._dummy_pipe_fd is not None: + self.settings["PORTAGE_PIPE_FD"] = str(self._dummy_pipe_fd) + # Temporarily unset EBUILD_PHASE so that bashrc code doesn't # think this is a real phase. phase_backup = self.settings.pop("EBUILD_PHASE", None) @@ -37,3 +41,4 @@ class MiscFunctionsProcess(AbstractEbuildProcess): finally: if phase_backup is not None: self.settings["EBUILD_PHASE"] = phase_backup + self.settings.pop("PORTAGE_PIPE_FD", None) diff --git a/portage_with_autodep/pym/_emerge/MiscFunctionsProcess.pyo b/portage_with_autodep/pym/_emerge/MiscFunctionsProcess.pyo Binary files differindex e3f5344..773256a 100644 --- a/portage_with_autodep/pym/_emerge/MiscFunctionsProcess.pyo +++ b/portage_with_autodep/pym/_emerge/MiscFunctionsProcess.pyo diff --git a/portage_with_autodep/pym/_emerge/Package.py b/portage_with_autodep/pym/_emerge/Package.py index c04fa1f..c795568 100644 --- a/portage_with_autodep/pym/_emerge/Package.py +++ b/portage_with_autodep/pym/_emerge/Package.py @@ -1,81 +1,98 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + import sys from itertools import chain +import warnings + import portage from portage import _encodings, _unicode_decode, _unicode_encode from portage.cache.mappings import slot_dict_class from portage.const import EBUILD_PHASES from portage.dep import Atom, check_required_use, use_reduce, \ - paren_enclose, _slot_re, _slot_separator, _repo_separator + paren_enclose, _slot_separator, _repo_separator from portage.versions import _pkg_str, _unknown_repo -from portage.eapi import eapi_has_iuse_defaults, eapi_has_required_use +from portage.eapi import _get_eapi_attrs, eapi_has_use_aliases from portage.exception import InvalidDependString +from portage.localization import _ from _emerge.Task import Task if sys.hexversion >= 0x3000000: basestring = str long = int + _unicode = str +else: + _unicode = unicode class Package(Task): __hash__ = Task.__hash__ __slots__ = ("built", "cpv", "depth", - "installed", "metadata", "onlydeps", "operation", + "installed", "onlydeps", "operation", "root_config", "type_name", "category", "counter", "cp", "cpv_split", - "inherited", "invalid", "iuse", "masks", "mtime", - "pf", "root", "slot", "slot_atom", "version", "visible",) + \ - ("_raw_metadata", "_use",) + "inherited", "iuse", "mtime", + "pf", "root", "slot", "sub_slot", "slot_atom", "version") + \ + ("_invalid", "_masks", "_metadata", "_raw_metadata", "_use", + "_validated_atoms", "_visible") metadata_keys = [ "BUILD_TIME", "CHOST", "COUNTER", "DEPEND", "EAPI", - "INHERITED", "IUSE", "KEYWORDS", + "HDEPEND", "INHERITED", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROVIDE", "RDEPEND", "repository", "PROPERTIES", "RESTRICT", "SLOT", "USE", "_mtime_", "DEFINED_PHASES", "REQUIRED_USE"] - _dep_keys = ('DEPEND', 'PDEPEND', 'RDEPEND',) + _dep_keys = ('DEPEND', 'HDEPEND', 'PDEPEND', 'RDEPEND') + _buildtime_keys = ('DEPEND', 'HDEPEND') + _runtime_keys = ('PDEPEND', 'RDEPEND') _use_conditional_misc_keys = ('LICENSE', 'PROPERTIES', 'RESTRICT') UNKNOWN_REPO = _unknown_repo def __init__(self, **kwargs): + metadata = _PackageMetadataWrapperBase(kwargs.pop('metadata')) Task.__init__(self, **kwargs) # the SlotObject constructor assigns self.root_config from keyword args # and is an instance of a '_emerge.RootConfig.RootConfig class self.root = self.root_config.root - self._raw_metadata = _PackageMetadataWrapperBase(self.metadata) - self.metadata = _PackageMetadataWrapper(self, self._raw_metadata) + self._raw_metadata = metadata + self._metadata = _PackageMetadataWrapper(self, metadata) if not self.built: - self.metadata['CHOST'] = self.root_config.settings.get('CHOST', '') - slot = self.slot - if _slot_re.match(slot) is None: + self._metadata['CHOST'] = self.root_config.settings.get('CHOST', '') + eapi_attrs = _get_eapi_attrs(self.eapi) + self.cpv = _pkg_str(self.cpv, metadata=self._metadata, + settings=self.root_config.settings) + if hasattr(self.cpv, 'slot_invalid'): self._invalid_metadata('SLOT.invalid', - "SLOT: invalid value: '%s'" % slot) - # Avoid an InvalidAtom exception when creating slot_atom. - # This package instance will be masked due to empty SLOT. - slot = '0' - self.cpv = _pkg_str(self.cpv, slot=slot, - repo=self.metadata.get('repository', '')) + "SLOT: invalid value: '%s'" % self._metadata["SLOT"]) + self.cpv_split = self.cpv.cpv_split + self.category, self.pf = portage.catsplit(self.cpv) self.cp = self.cpv.cp + self.version = self.cpv.version + self.slot = self.cpv.slot + self.sub_slot = self.cpv.sub_slot + self.slot_atom = Atom("%s%s%s" % (self.cp, _slot_separator, self.slot)) # sync metadata with validated repo (may be UNKNOWN_REPO) - self.metadata['repository'] = self.cpv.repo + self._metadata['repository'] = self.cpv.repo + + if eapi_attrs.iuse_effective: + implicit_match = self.root_config.settings._iuse_effective_match + else: + implicit_match = self.root_config.settings._iuse_implicit_match + usealiases = self.root_config.settings._use_manager.getUseAliases(self) + self.iuse = self._iuse(self, self._metadata["IUSE"].split(), implicit_match, + usealiases, self.eapi) + if (self.iuse.enabled or self.iuse.disabled) and \ - not eapi_has_iuse_defaults(self.metadata["EAPI"]): + not eapi_attrs.iuse_defaults: if not self.installed: self._invalid_metadata('EAPI.incompatible', "IUSE contains defaults, but EAPI doesn't allow them") - self.slot_atom = portage.dep.Atom("%s%s%s" % (self.cp, _slot_separator, slot)) - self.category, self.pf = portage.catsplit(self.cpv) - self.cpv_split = self.cpv.cpv_split - self.version = self.cpv.version if self.inherited is None: self.inherited = frozenset() - self._validate_deps() - self.masks = self._masks() - self.visible = self._visible(self.masks) if self.operation is None: if self.onlydeps or self.installed: self.operation = "nomerge" @@ -89,6 +106,74 @@ class Package(Task): type_name=self.type_name) self._hash_value = hash(self._hash_key) + @property + def eapi(self): + return self._metadata["EAPI"] + + @property + def build_time(self): + if not self.built: + raise AttributeError('build_time') + try: + return long(self._metadata['BUILD_TIME']) + except (KeyError, ValueError): + return 0 + + @property + def defined_phases(self): + return self._metadata.defined_phases + + @property + def properties(self): + return self._metadata.properties + + @property + def restrict(self): + return self._metadata.restrict + + @property + def metadata(self): + warnings.warn("_emerge.Package.Package.metadata is deprecated", + DeprecationWarning, stacklevel=3) + return self._metadata + + # These are calculated on-demand, so that they are calculated + # after FakeVartree applies its metadata tweaks. + @property + def invalid(self): + if self._invalid is None: + self._validate_deps() + if self._invalid is None: + self._invalid = False + return self._invalid + + @property + def masks(self): + if self._masks is None: + self._masks = self._eval_masks() + return self._masks + + @property + def visible(self): + if self._visible is None: + self._visible = self._eval_visiblity(self.masks) + return self._visible + + @property + def validated_atoms(self): + """ + Returns *all* validated atoms from the deps, regardless + of USE conditionals, with USE conditionals inside + atoms left unevaluated. + """ + if self._validated_atoms is None: + self._validate_deps() + return self._validated_atoms + + @property + def stable(self): + return self.cpv.stable + @classmethod def _gen_hash_key(cls, cpv=None, installed=None, onlydeps=None, operation=None, repo_name=None, root_config=None, @@ -123,15 +208,15 @@ class Package(Task): # So overwrite the repo_key with type_name. repo_key = type_name - return (type_name, root, cpv, operation, repo_key) + return (type_name, root, _unicode(cpv), operation, repo_key) def _validate_deps(self): """ Validate deps. This does not trigger USE calculation since that is expensive for ebuilds and therefore we want to avoid doing - in unnecessarily (like for masked packages). + it unnecessarily (like for masked packages). """ - eapi = self.metadata['EAPI'] + eapi = self.eapi dep_eapi = eapi dep_valid_flag = self.iuse.is_valid_flag if self.installed: @@ -142,28 +227,44 @@ class Package(Task): dep_eapi = None dep_valid_flag = None + validated_atoms = [] for k in self._dep_keys: - v = self.metadata.get(k) + v = self._metadata.get(k) if not v: continue try: - use_reduce(v, eapi=dep_eapi, matchall=True, - is_valid_flag=dep_valid_flag, token_class=Atom) + atoms = use_reduce(v, eapi=dep_eapi, + matchall=True, is_valid_flag=dep_valid_flag, + token_class=Atom, flat=True) except InvalidDependString as e: self._metadata_exception(k, e) + else: + validated_atoms.extend(atoms) + if not self.built: + for atom in atoms: + if not isinstance(atom, Atom): + continue + if atom.slot_operator_built: + e = InvalidDependString( + _("Improper context for slot-operator " + "\"built\" atom syntax: %s") % + (atom.unevaluated_atom,)) + self._metadata_exception(k, e) + + self._validated_atoms = tuple(set(atom for atom in + validated_atoms if isinstance(atom, Atom))) k = 'PROVIDE' - v = self.metadata.get(k) + v = self._metadata.get(k) if v: try: use_reduce(v, eapi=dep_eapi, matchall=True, is_valid_flag=dep_valid_flag, token_class=Atom) except InvalidDependString as e: - self._invalid_metadata("PROVIDE.syntax", - _unicode_decode("%s: %s") % (k, e)) + self._invalid_metadata("PROVIDE.syntax", "%s: %s" % (k, e)) for k in self._use_conditional_misc_keys: - v = self.metadata.get(k) + v = self._metadata.get(k) if not v: continue try: @@ -173,24 +274,20 @@ class Package(Task): self._metadata_exception(k, e) k = 'REQUIRED_USE' - v = self.metadata.get(k) - if v: - if not eapi_has_required_use(eapi): + v = self._metadata.get(k) + if v and not self.built: + if not _get_eapi_attrs(eapi).required_use: self._invalid_metadata('EAPI.incompatible', "REQUIRED_USE set, but EAPI='%s' doesn't allow it" % eapi) else: try: check_required_use(v, (), - self.iuse.is_valid_flag) + self.iuse.is_valid_flag, eapi=eapi) except InvalidDependString as e: - # Force unicode format string for python-2.x safety, - # ensuring that PortageException.__unicode__() is used - # when necessary. - self._invalid_metadata(k + ".syntax", - _unicode_decode("%s: %s") % (k, e)) + self._invalid_metadata(k + ".syntax", "%s: %s" % (k, e)) k = 'SRC_URI' - v = self.metadata.get(k) + v = self._metadata.get(k) if v: try: use_reduce(v, is_src_uri=True, eapi=eapi, matchall=True, @@ -205,43 +302,52 @@ class Package(Task): onlydeps=self.onlydeps, operation=self.operation, root_config=self.root_config, type_name=self.type_name) - def _masks(self): + def _eval_masks(self): masks = {} settings = self.root_config.settings - if self.invalid is not None: + if self.invalid is not False: masks['invalid'] = self.invalid - if not settings._accept_chost(self.cpv, self.metadata): - masks['CHOST'] = self.metadata['CHOST'] + if not settings._accept_chost(self.cpv, self._metadata): + masks['CHOST'] = self._metadata['CHOST'] - eapi = self.metadata["EAPI"] + eapi = self.eapi if not portage.eapi_is_supported(eapi): masks['EAPI.unsupported'] = eapi if portage._eapi_is_deprecated(eapi): masks['EAPI.deprecated'] = eapi missing_keywords = settings._getMissingKeywords( - self.cpv, self.metadata) + self.cpv, self._metadata) if missing_keywords: masks['KEYWORDS'] = missing_keywords try: missing_properties = settings._getMissingProperties( - self.cpv, self.metadata) + self.cpv, self._metadata) if missing_properties: masks['PROPERTIES'] = missing_properties except InvalidDependString: # already recorded as 'invalid' pass - mask_atom = settings._getMaskAtom(self.cpv, self.metadata) + try: + missing_restricts = settings._getMissingRestrict( + self.cpv, self._metadata) + if missing_restricts: + masks['RESTRICT'] = missing_restricts + except InvalidDependString: + # already recorded as 'invalid' + pass + + mask_atom = settings._getMaskAtom(self.cpv, self._metadata) if mask_atom is not None: masks['package.mask'] = mask_atom try: missing_licenses = settings._getMissingLicenses( - self.cpv, self.metadata) + self.cpv, self._metadata) if missing_licenses: masks['LICENSE'] = missing_licenses except InvalidDependString: @@ -249,13 +355,13 @@ class Package(Task): pass if not masks: - masks = None + masks = False return masks - def _visible(self, masks): + def _eval_visiblity(self, masks): - if masks is not None: + if masks is not False: if 'EAPI.unsupported' in masks: return False @@ -267,7 +373,8 @@ class Package(Task): 'CHOST' in masks or \ 'EAPI.deprecated' in masks or \ 'KEYWORDS' in masks or \ - 'PROPERTIES' in masks): + 'PROPERTIES' in masks or \ + 'RESTRICT' in masks): return False if 'package.mask' in masks or \ @@ -280,7 +387,7 @@ class Package(Task): """returns None, 'missing', or 'unstable'.""" missing = self.root_config.settings._getRawMissingKeywords( - self.cpv, self.metadata) + self.cpv, self._metadata) if not missing: return None @@ -301,17 +408,22 @@ class Package(Task): """returns a bool if the cpv is in the list of expanded pmaskdict[cp] available ebuilds""" pmask = self.root_config.settings._getRawMaskAtom( - self.cpv, self.metadata) + self.cpv, self._metadata) return pmask is not None def _metadata_exception(self, k, e): + if k.endswith('DEPEND'): + qacat = 'dependency.syntax' + else: + qacat = k + ".syntax" + # For unicode safety with python-2.x we need to avoid # using the string format operator with a non-unicode # format string, since that will result in the # PortageException.__str__() method being invoked, # followed by unsafe decoding that may result in a - # UnicodeDecodeError. Therefore, use _unicode_decode() + # UnicodeDecodeError. Therefore, use unicode_literals # to ensure that format strings are unicode, so that # PortageException.__unicode__() is used when necessary # in python-2.x. @@ -323,27 +435,25 @@ class Package(Task): continue categorized_error = True self._invalid_metadata(error.category, - _unicode_decode("%s: %s") % (k, error)) + "%s: %s" % (k, error)) if not categorized_error: - self._invalid_metadata(k + ".syntax", - _unicode_decode("%s: %s") % (k, e)) + self._invalid_metadata(qacat,"%s: %s" % (k, e)) else: # For installed packages, show the path of the file # containing the invalid metadata, since the user may # want to fix the deps by hand. vardb = self.root_config.trees['vartree'].dbapi path = vardb.getpath(self.cpv, filename=k) - self._invalid_metadata(k + ".syntax", - _unicode_decode("%s: %s in '%s'") % (k, e, path)) + self._invalid_metadata(qacat, "%s: %s in '%s'" % (k, e, path)) def _invalid_metadata(self, msg_type, msg): - if self.invalid is None: - self.invalid = {} - msgs = self.invalid.get(msg_type) + if self._invalid is None: + self._invalid = {} + msgs = self._invalid.get(msg_type) if msgs is None: msgs = [] - self.invalid[msg_type] = msgs + self._invalid[msg_type] = msgs msgs.append(msg) def __str__(self): @@ -389,13 +499,16 @@ class Package(Task): # Share identical frozenset instances when available. _frozensets = {} - def __init__(self, pkg, use_str): + def __init__(self, pkg, enabled_flags): self._pkg = pkg self._expand = None self._expand_hidden = None self._force = None self._mask = None - self.enabled = frozenset(use_str.split()) + if eapi_has_use_aliases(pkg.eapi): + for enabled_flag in enabled_flags: + enabled_flags.extend(pkg.iuse.alias_mapping.get(enabled_flag, [])) + self.enabled = frozenset(enabled_flags) if pkg.built: # Use IUSE to validate USE settings for built packages, # in case the package manager that built this package @@ -445,7 +558,7 @@ class Package(Task): @property def repo(self): - return self.metadata['repository'] + return self._metadata['repository'] @property def repo_priority(self): @@ -457,7 +570,7 @@ class Package(Task): @property def use(self): if self._use is None: - self.metadata._init_use() + self._init_use() return self._use def _get_pkgsettings(self): @@ -466,28 +579,81 @@ class Package(Task): pkgsettings.setcpv(self) return pkgsettings + def _init_use(self): + if self.built: + # Use IUSE to validate USE settings for built packages, + # in case the package manager that built this package + # failed to do that for some reason (or in case of + # data corruption). The enabled flags must be consistent + # with implicit IUSE, in order to avoid potential + # inconsistencies in USE dep matching (see bug #453400). + use_str = self._metadata['USE'] + is_valid_flag = self.iuse.is_valid_flag + enabled_flags = [x for x in use_str.split() if is_valid_flag(x)] + use_str = " ".join(enabled_flags) + self._use = self._use_class( + self, enabled_flags) + else: + try: + use_str = _PackageMetadataWrapperBase.__getitem__( + self._metadata, 'USE') + except KeyError: + use_str = None + calculated_use = False + if not use_str: + use_str = self._get_pkgsettings()["PORTAGE_USE"] + calculated_use = True + self._use = self._use_class( + self, use_str.split()) + # Initialize these now, since USE access has just triggered + # setcpv, and we want to cache the result of the force/mask + # calculations that were done. + if calculated_use: + self._use._init_force_mask() + + _PackageMetadataWrapperBase.__setitem__( + self._metadata, 'USE', use_str) + + return use_str + class _iuse(object): - __slots__ = ("__weakref__", "all", "enabled", "disabled", - "tokens") + ("_iuse_implicit_match",) + __slots__ = ("__weakref__", "_iuse_implicit_match", "_pkg", "alias_mapping", + "all", "all_aliases", "enabled", "disabled", "tokens") - def __init__(self, tokens, iuse_implicit_match): + def __init__(self, pkg, tokens, iuse_implicit_match, aliases, eapi): + self._pkg = pkg self.tokens = tuple(tokens) self._iuse_implicit_match = iuse_implicit_match enabled = [] disabled = [] other = [] + enabled_aliases = [] + disabled_aliases = [] + other_aliases = [] + aliases_supported = eapi_has_use_aliases(eapi) + self.alias_mapping = {} for x in tokens: prefix = x[:1] if prefix == "+": enabled.append(x[1:]) + if aliases_supported: + self.alias_mapping[x[1:]] = aliases.get(x[1:], []) + enabled_aliases.extend(self.alias_mapping[x[1:]]) elif prefix == "-": disabled.append(x[1:]) + if aliases_supported: + self.alias_mapping[x[1:]] = aliases.get(x[1:], []) + disabled_aliases.extend(self.alias_mapping[x[1:]]) else: other.append(x) - self.enabled = frozenset(enabled) - self.disabled = frozenset(disabled) + if aliases_supported: + self.alias_mapping[x] = aliases.get(x, []) + other_aliases.extend(self.alias_mapping[x]) + self.enabled = frozenset(chain(enabled, enabled_aliases)) + self.disabled = frozenset(chain(disabled, disabled_aliases)) self.all = frozenset(chain(enabled, disabled, other)) + self.all_aliases = frozenset(chain(enabled_aliases, disabled_aliases, other_aliases)) def is_valid_flag(self, flags): """ @@ -498,7 +664,7 @@ class Package(Task): flags = [flags] for flag in flags: - if not flag in self.all and \ + if not flag in self.all and not flag in self.all_aliases and \ not self._iuse_implicit_match(flag): return False return True @@ -511,11 +677,28 @@ class Package(Task): flags = [flags] missing_iuse = [] for flag in flags: - if not flag in self.all and \ + if not flag in self.all and not flag in self.all_aliases and \ not self._iuse_implicit_match(flag): missing_iuse.append(flag) return missing_iuse + def get_real_flag(self, flag): + """ + Returns the flag's name within the scope of this package + (accounting for aliases), or None if the flag is unknown. + """ + if flag in self.all: + return flag + elif flag in self.all_aliases: + for k, v in self.alias_mapping.items(): + if flag in v: + return k + + if self._iuse_implicit_match(flag): + return flag + + return None + def __len__(self): return 4 @@ -568,7 +751,7 @@ class _PackageMetadataWrapper(_PackageMetadataWrapperBase): __slots__ = ("_pkg",) _wrapped_keys = frozenset( - ["COUNTER", "INHERITED", "IUSE", "SLOT", "USE", "_mtime_"]) + ["COUNTER", "INHERITED", "USE", "_mtime_"]) _use_conditional_keys = frozenset( ['LICENSE', 'PROPERTIES', 'PROVIDE', 'RESTRICT',]) @@ -581,31 +764,6 @@ class _PackageMetadataWrapper(_PackageMetadataWrapperBase): self.update(metadata) - def _init_use(self): - if self._pkg.built: - use_str = self['USE'] - self._pkg._use = self._pkg._use_class( - self._pkg, use_str) - else: - try: - use_str = _PackageMetadataWrapperBase.__getitem__(self, 'USE') - except KeyError: - use_str = None - calculated_use = False - if not use_str: - use_str = self._pkg._get_pkgsettings()["PORTAGE_USE"] - calculated_use = True - _PackageMetadataWrapperBase.__setitem__(self, 'USE', use_str) - self._pkg._use = self._pkg._use_class( - self._pkg, use_str) - # Initialize these now, since USE access has just triggered - # setcpv, and we want to cache the result of the force/mask - # calculations that were done. - if calculated_use: - self._pkg._use._init_force_mask() - - return use_str - def __getitem__(self, k): v = _PackageMetadataWrapperBase.__getitem__(self, k) if k in self._use_conditional_keys: @@ -623,7 +781,7 @@ class _PackageMetadataWrapper(_PackageMetadataWrapperBase): elif k == 'USE' and not self._pkg.built: if not v: # This is lazy because it's expensive. - v = self._init_use() + v = self._pkg._init_use() return v @@ -637,13 +795,6 @@ class _PackageMetadataWrapper(_PackageMetadataWrapperBase): v = frozenset(v.split()) self._pkg.inherited = v - def _set_iuse(self, k, v): - self._pkg.iuse = self._pkg._iuse( - v.split(), self._pkg.root_config.settings._iuse_implicit_match) - - def _set_slot(self, k, v): - self._pkg.slot = v - def _set_counter(self, k, v): if isinstance(v, basestring): try: diff --git a/portage_with_autodep/pym/_emerge/Package.pyo b/portage_with_autodep/pym/_emerge/Package.pyo Binary files differindex 3d37317..6bec2fe 100644 --- a/portage_with_autodep/pym/_emerge/Package.pyo +++ b/portage_with_autodep/pym/_emerge/Package.pyo diff --git a/portage_with_autodep/pym/_emerge/PackageArg.pyo b/portage_with_autodep/pym/_emerge/PackageArg.pyo Binary files differindex c50e145..2367aee 100644 --- a/portage_with_autodep/pym/_emerge/PackageArg.pyo +++ b/portage_with_autodep/pym/_emerge/PackageArg.pyo diff --git a/portage_with_autodep/pym/_emerge/PackageMerge.py b/portage_with_autodep/pym/_emerge/PackageMerge.py index eed34e9..ef298ca 100644 --- a/portage_with_autodep/pym/_emerge/PackageMerge.py +++ b/portage_with_autodep/pym/_emerge/PackageMerge.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.CompositeTask import CompositeTask @@ -11,6 +11,9 @@ class PackageMerge(CompositeTask): self.scheduler = self.merge.scheduler pkg = self.merge.pkg pkg_count = self.merge.pkg_count + pkg_color = "PKG_MERGE" + if pkg.type_name == "binary": + pkg_color = "PKG_BINARY_MERGE" if pkg.installed: action_desc = "Uninstalling" @@ -26,7 +29,7 @@ class PackageMerge(CompositeTask): msg = "%s %s%s" % \ (action_desc, counter_str, - colorize("GOOD", pkg.cpv)) + colorize(pkg_color, pkg.cpv)) if pkg.root_config.settings["ROOT"] != "/": msg += " %s %s" % (preposition, pkg.root) diff --git a/portage_with_autodep/pym/_emerge/PackageMerge.pyo b/portage_with_autodep/pym/_emerge/PackageMerge.pyo Binary files differindex 6403d1b..171b38c 100644 --- a/portage_with_autodep/pym/_emerge/PackageMerge.pyo +++ b/portage_with_autodep/pym/_emerge/PackageMerge.pyo diff --git a/portage_with_autodep/pym/_emerge/PackageUninstall.py b/portage_with_autodep/pym/_emerge/PackageUninstall.py index eb6a947..16c2f74 100644 --- a/portage_with_autodep/pym/_emerge/PackageUninstall.py +++ b/portage_with_autodep/pym/_emerge/PackageUninstall.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import logging @@ -33,7 +33,7 @@ class PackageUninstall(CompositeTask): # Apparently the package got uninstalled # already, so we can safely return early. self.returncode = os.EX_OK - self.wait() + self._async_wait() return self.settings.setcpv(self.pkg) @@ -67,7 +67,7 @@ class PackageUninstall(CompositeTask): if retval != os.EX_OK: self._builddir_lock.unlock() self.returncode = retval - self.wait() + self._async_wait() return self._writemsg_level(">>> Unmerging %s...\n" % (self.pkg.cpv,), diff --git a/portage_with_autodep/pym/_emerge/PackageUninstall.pyo b/portage_with_autodep/pym/_emerge/PackageUninstall.pyo Binary files differindex 847c749..e70bec2 100644 --- a/portage_with_autodep/pym/_emerge/PackageUninstall.pyo +++ b/portage_with_autodep/pym/_emerge/PackageUninstall.pyo diff --git a/portage_with_autodep/pym/_emerge/PackageVirtualDbapi.py b/portage_with_autodep/pym/_emerge/PackageVirtualDbapi.py index 0f7be44..56a5576 100644 --- a/portage_with_autodep/pym/_emerge/PackageVirtualDbapi.py +++ b/portage_with_autodep/pym/_emerge/PackageVirtualDbapi.py @@ -140,10 +140,10 @@ class PackageVirtualDbapi(dbapi): self._clear_cache() def aux_get(self, cpv, wants, myrepo=None): - metadata = self._cpv_map[cpv].metadata + metadata = self._cpv_map[cpv]._metadata return [metadata.get(x, "") for x in wants] def aux_update(self, cpv, values): - self._cpv_map[cpv].metadata.update(values) + self._cpv_map[cpv]._metadata.update(values) self._clear_cache() diff --git a/portage_with_autodep/pym/_emerge/PackageVirtualDbapi.pyo b/portage_with_autodep/pym/_emerge/PackageVirtualDbapi.pyo Binary files differindex a1a850f..f9c6641 100644 --- a/portage_with_autodep/pym/_emerge/PackageVirtualDbapi.pyo +++ b/portage_with_autodep/pym/_emerge/PackageVirtualDbapi.pyo diff --git a/portage_with_autodep/pym/_emerge/PipeReader.py b/portage_with_autodep/pym/_emerge/PipeReader.py index 90febdf..a8392c3 100644 --- a/portage_with_autodep/pym/_emerge/PipeReader.py +++ b/portage_with_autodep/pym/_emerge/PipeReader.py @@ -1,9 +1,11 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +import fcntl +import sys + from portage import os from _emerge.AbstractPollTask import AbstractPollTask -import fcntl class PipeReader(AbstractPollTask): @@ -27,18 +29,28 @@ class PipeReader(AbstractPollTask): output_handler = self._output_handler for f in self.input_files.values(): - fcntl.fcntl(f.fileno(), fcntl.F_SETFL, - fcntl.fcntl(f.fileno(), fcntl.F_GETFL) | os.O_NONBLOCK) - self._reg_ids.add(self.scheduler.register(f.fileno(), + fd = isinstance(f, int) and f or f.fileno() + fcntl.fcntl(fd, fcntl.F_SETFL, + fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK) + + # FD_CLOEXEC is enabled by default in Python >=3.4. + if sys.hexversion < 0x3040000: + try: + fcntl.FD_CLOEXEC + except AttributeError: + pass + else: + fcntl.fcntl(fd, fcntl.F_SETFD, + fcntl.fcntl(fd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC) + + self._reg_ids.add(self.scheduler.io_add_watch(fd, self._registered_events, output_handler)) self._registered = True - def isAlive(self): - return self._registered - def _cancel(self): + self._unregister() if self.returncode is None: - self.returncode = 1 + self.returncode = self._cancelled_returncode def _wait(self): if self.returncode is not None: @@ -102,11 +114,14 @@ class PipeReader(AbstractPollTask): if self._reg_ids is not None: for reg_id in self._reg_ids: - self.scheduler.unregister(reg_id) + self.scheduler.source_remove(reg_id) self._reg_ids = None if self.input_files is not None: for f in self.input_files.values(): - f.close() + if isinstance(f, int): + os.close(f) + else: + f.close() self.input_files = None diff --git a/portage_with_autodep/pym/_emerge/PipeReader.pyo b/portage_with_autodep/pym/_emerge/PipeReader.pyo Binary files differindex 2f53e7d..1ad40c2 100644 --- a/portage_with_autodep/pym/_emerge/PipeReader.pyo +++ b/portage_with_autodep/pym/_emerge/PipeReader.pyo diff --git a/portage_with_autodep/pym/_emerge/PollScheduler.py b/portage_with_autodep/pym/_emerge/PollScheduler.py index 965dc20..b118ac1 100644 --- a/portage_with_autodep/pym/_emerge/PollScheduler.py +++ b/portage_with_autodep/pym/_emerge/PollScheduler.py @@ -1,18 +1,13 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -import gzip -import errno - try: import threading except ImportError: import dummy_threading as threading -from portage import _encodings -from portage import _unicode_encode -from portage.util import writemsg_level -from portage.util.SlotObject import SlotObject +import portage +from portage.util._async.SchedulerInterface import SchedulerInterface from portage.util._eventloop.EventLoop import EventLoop from portage.util._eventloop.global_event_loop import global_event_loop @@ -20,14 +15,10 @@ from _emerge.getloadavg import getloadavg class PollScheduler(object): - class _sched_iface_class(SlotObject): - __slots__ = ("IO_ERR", "IO_HUP", "IO_IN", "IO_NVAL", "IO_OUT", - "IO_PRI", "child_watch_add", - "idle_add", "io_add_watch", "iteration", - "output", "register", "run", - "source_remove", "timeout_add", "unregister") + # max time between loadavg checks (milliseconds) + _loadavg_latency = None - def __init__(self, main=False): + def __init__(self, main=False, event_loop=None): """ @param main: If True then use global_event_loop(), otherwise use a local EventLoop instance (default is False, for safe use in @@ -38,29 +29,20 @@ class PollScheduler(object): self._terminated_tasks = False self._max_jobs = 1 self._max_load = None - self._jobs = 0 self._scheduling = False self._background = False - if main: + if event_loop is not None: + self._event_loop = event_loop + elif main: self._event_loop = global_event_loop() else: - self._event_loop = EventLoop(main=False) - self.sched_iface = self._sched_iface_class( - IO_ERR=self._event_loop.IO_ERR, - IO_HUP=self._event_loop.IO_HUP, - IO_IN=self._event_loop.IO_IN, - IO_NVAL=self._event_loop.IO_NVAL, - IO_OUT=self._event_loop.IO_OUT, - IO_PRI=self._event_loop.IO_PRI, - child_watch_add=self._event_loop.child_watch_add, - idle_add=self._event_loop.idle_add, - io_add_watch=self._event_loop.io_add_watch, - iteration=self._event_loop.iteration, - output=self._task_output, - register=self._event_loop.io_add_watch, - source_remove=self._event_loop.source_remove, - timeout_add=self._event_loop.timeout_add, - unregister=self._event_loop.source_remove) + self._event_loop = (portage._internal_caller and + global_event_loop() or EventLoop(main=False)) + self._sched_iface = SchedulerInterface(self._event_loop, + is_background=self._is_background) + + def _is_background(self): + return self._background def terminate(self): """ @@ -135,41 +117,23 @@ class PollScheduler(object): Calls _schedule_tasks() and automatically returns early from any recursive calls to this method that the _schedule_tasks() call might trigger. This makes _schedule() safe to call from - inside exit listeners. + inside exit listeners. This method always returns True, so that + it may be scheduled continuously via EventLoop.timeout_add(). """ if self._scheduling: - return False + return True self._scheduling = True try: self._schedule_tasks() finally: self._scheduling = False - - def _main_loop(self): - term_check_id = self.sched_iface.idle_add(self._termination_check) - try: - # Populate initial event sources. We only need to do - # this once here, since it can be called during the - # loop from within event handlers. - self._schedule() - - # Loop while there are jobs to be scheduled. - while self._keep_scheduling(): - self.sched_iface.iteration() - - # Clean shutdown of previously scheduled jobs. In the - # case of termination, this allows for basic cleanup - # such as flushing of buffered output to logs. - while self._is_work_scheduled(): - self.sched_iface.iteration() - finally: - self.sched_iface.source_remove(term_check_id) + return True def _is_work_scheduled(self): return bool(self._running_job_count()) def _running_job_count(self): - return self._jobs + raise NotImplementedError(self) def _can_add_job(self): if self._terminated_tasks: @@ -194,47 +158,3 @@ class PollScheduler(object): return False return True - - def _task_output(self, msg, log_path=None, background=None, - level=0, noiselevel=-1): - """ - Output msg to stdout if not self._background. If log_path - is not None then append msg to the log (appends with - compression if the filename extension of log_path - corresponds to a supported compression type). - """ - - if background is None: - # If the task does not have a local background value - # (like for parallel-fetch), then use the global value. - background = self._background - - msg_shown = False - if not background: - writemsg_level(msg, level=level, noiselevel=noiselevel) - msg_shown = True - - if log_path is not None: - try: - f = open(_unicode_encode(log_path, - encoding=_encodings['fs'], errors='strict'), - mode='ab') - f_real = f - except IOError as e: - if e.errno not in (errno.ENOENT, errno.ESTALE): - raise - if not msg_shown: - writemsg_level(msg, level=level, noiselevel=noiselevel) - else: - - if log_path.endswith('.gz'): - # NOTE: The empty filename argument prevents us from - # triggering a bug in python3 which causes GzipFile - # to raise AttributeError if fileobj.name is bytes - # instead of unicode. - f = gzip.GzipFile(filename='', mode='ab', fileobj=f) - - f.write(_unicode_encode(msg)) - f.close() - if f_real is not f: - f_real.close() diff --git a/portage_with_autodep/pym/_emerge/PollScheduler.pyo b/portage_with_autodep/pym/_emerge/PollScheduler.pyo Binary files differindex b7e52be..d714e01 100644 --- a/portage_with_autodep/pym/_emerge/PollScheduler.pyo +++ b/portage_with_autodep/pym/_emerge/PollScheduler.pyo diff --git a/portage_with_autodep/pym/_emerge/ProgressHandler.pyo b/portage_with_autodep/pym/_emerge/ProgressHandler.pyo Binary files differindex 83e2f7f..fd0c5bb 100644 --- a/portage_with_autodep/pym/_emerge/ProgressHandler.pyo +++ b/portage_with_autodep/pym/_emerge/ProgressHandler.pyo diff --git a/portage_with_autodep/pym/_emerge/QueueScheduler.py b/portage_with_autodep/pym/_emerge/QueueScheduler.py deleted file mode 100644 index 206087c..0000000 --- a/portage_with_autodep/pym/_emerge/QueueScheduler.py +++ /dev/null @@ -1,105 +0,0 @@ -# Copyright 1999-2012 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from _emerge.PollScheduler import PollScheduler - -class QueueScheduler(PollScheduler): - - """ - Add instances of SequentialTaskQueue and then call run(). The - run() method returns when no tasks remain. - """ - - def __init__(self, main=True, max_jobs=None, max_load=None): - PollScheduler.__init__(self, main=main) - - if max_jobs is None: - max_jobs = 1 - - self._max_jobs = max_jobs - self._max_load = max_load - - self._queues = [] - self._schedule_listeners = [] - - def add(self, q): - self._queues.append(q) - - def remove(self, q): - self._queues.remove(q) - - def clear(self): - for q in self._queues: - q.clear() - - def run(self, timeout=None): - - timeout_callback = None - if timeout is not None: - def timeout_callback(): - timeout_callback.timed_out = True - return False - timeout_callback.timed_out = False - timeout_callback.timeout_id = self.sched_iface.timeout_add( - timeout, timeout_callback) - - term_check_id = self.sched_iface.idle_add(self._termination_check) - try: - while not (timeout_callback is not None and - timeout_callback.timed_out): - # We don't have any callbacks to trigger _schedule(), - # so we have to call it explicitly here. - self._schedule() - if self._keep_scheduling(): - self.sched_iface.iteration() - else: - break - - while self._is_work_scheduled() and \ - not (timeout_callback is not None and - timeout_callback.timed_out): - self.sched_iface.iteration() - finally: - self.sched_iface.source_remove(term_check_id) - if timeout_callback is not None: - self.sched_iface.unregister(timeout_callback.timeout_id) - - def _schedule_tasks(self): - """ - @rtype: bool - @return: True if there may be remaining tasks to schedule, - False otherwise. - """ - if self._terminated_tasks: - return - - while self._can_add_job(): - n = self._max_jobs - self._running_job_count() - if n < 1: - break - - if not self._start_next_job(n): - return - - def _keep_scheduling(self): - return not self._terminated_tasks and any(self._queues) - - def _running_job_count(self): - job_count = 0 - for q in self._queues: - job_count += len(q.running_tasks) - self._jobs = job_count - return job_count - - def _start_next_job(self, n=1): - started_count = 0 - for q in self._queues: - initial_job_count = len(q.running_tasks) - q.schedule() - final_job_count = len(q.running_tasks) - if final_job_count > initial_job_count: - started_count += (final_job_count - initial_job_count) - if started_count >= n: - break - return started_count - diff --git a/portage_with_autodep/pym/_emerge/QueueScheduler.pyo b/portage_with_autodep/pym/_emerge/QueueScheduler.pyo Binary files differdeleted file mode 100644 index 88de3ea..0000000 --- a/portage_with_autodep/pym/_emerge/QueueScheduler.pyo +++ /dev/null diff --git a/portage_with_autodep/pym/_emerge/RootConfig.py b/portage_with_autodep/pym/_emerge/RootConfig.py index bb0d768..3648d01 100644 --- a/portage_with_autodep/pym/_emerge/RootConfig.py +++ b/portage_with_autodep/pym/_emerge/RootConfig.py @@ -1,10 +1,10 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 class RootConfig(object): """This is used internally by depgraph to track information about a particular $ROOT.""" - __slots__ = ("root", "setconfig", "sets", "settings", "trees") + __slots__ = ("mtimedb", "root", "setconfig", "sets", "settings", "trees") pkg_tree_map = { "ebuild" : "porttree", @@ -31,4 +31,11 @@ class RootConfig(object): Shallow copy all attributes from another instance. """ for k in self.__slots__: - setattr(self, k, getattr(other, k)) + try: + setattr(self, k, getattr(other, k)) + except AttributeError: + # mtimedb is currently not a required attribute + try: + delattr(self, k) + except AttributeError: + pass diff --git a/portage_with_autodep/pym/_emerge/RootConfig.pyo b/portage_with_autodep/pym/_emerge/RootConfig.pyo Binary files differindex fad3022..d071898 100644 --- a/portage_with_autodep/pym/_emerge/RootConfig.pyo +++ b/portage_with_autodep/pym/_emerge/RootConfig.pyo diff --git a/portage_with_autodep/pym/_emerge/Scheduler.py b/portage_with_autodep/pym/_emerge/Scheduler.py index 30a7e10..d663e97 100644 --- a/portage_with_autodep/pym/_emerge/Scheduler.py +++ b/portage_with_autodep/pym/_emerge/Scheduler.py @@ -1,7 +1,7 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import print_function +from __future__ import print_function, unicode_literals from collections import deque import gc @@ -18,7 +18,7 @@ import zlib import portage from portage import os from portage import _encodings -from portage import _unicode_decode, _unicode_encode +from portage import _unicode_encode from portage.cache.mappings import slot_dict_class from portage.elog.messages import eerror from portage.localization import _ @@ -28,6 +28,8 @@ from portage._sets import SETPREFIX from portage._sets.base import InternalPackageSet from portage.util import ensure_dirs, writemsg, writemsg_level from portage.util.SlotObject import SlotObject +from portage.util._async.SchedulerInterface import SchedulerInterface +from portage.util._eventloop.EventLoop import EventLoop from portage.package.ebuild.digestcheck import digestcheck from portage.package.ebuild.digestgen import digestgen from portage.package.ebuild.doebuild import (_check_temp_dir, @@ -50,6 +52,7 @@ from _emerge.EbuildFetcher import EbuildFetcher from _emerge.EbuildPhase import EbuildPhase from _emerge.emergelog import emergelog from _emerge.FakeVartree import FakeVartree +from _emerge.getloadavg import getloadavg from _emerge._find_deep_system_runtime_deps import _find_deep_system_runtime_deps from _emerge._flush_elog_mod_echo import _flush_elog_mod_echo from _emerge.JobStatusDisplay import JobStatusDisplay @@ -64,6 +67,9 @@ if sys.hexversion >= 0x3000000: class Scheduler(PollScheduler): + # max time between loadavg checks (milliseconds) + _loadavg_latency = 30000 + # max time between display status updates (milliseconds) _max_display_latency = 3000 @@ -79,7 +85,7 @@ class Scheduler(PollScheduler): _opts_no_self_update = frozenset(["--buildpkgonly", "--fetchonly", "--fetch-all-uri", "--pretend"]) - class _iface_class(PollScheduler._sched_iface_class): + class _iface_class(SchedulerInterface): __slots__ = ("fetch", "scheduleSetup", "scheduleUnpack") @@ -135,8 +141,7 @@ class Scheduler(PollScheduler): portage.exception.PortageException.__init__(self, value) def __init__(self, settings, trees, mtimedb, myopts, - spinner, mergelist=None, favorites=None, graph_config=None, - uninstall_only=False): + spinner, mergelist=None, favorites=None, graph_config=None): PollScheduler.__init__(self, main=True) if mergelist is not None: @@ -152,7 +157,6 @@ class Scheduler(PollScheduler): self._spinner = spinner self._mtimedb = mtimedb self._favorites = favorites - self._uninstall_only = uninstall_only self._args_set = InternalPackageSet(favorites, allow_repo=True) self._build_opts = self._build_opts_class() @@ -217,14 +221,15 @@ class Scheduler(PollScheduler): fetch_iface = self._fetch_iface_class(log_file=self._fetch_log, schedule=self._schedule_fetch) self._sched_iface = self._iface_class( + self._event_loop, + is_background=self._is_background, fetch=fetch_iface, scheduleSetup=self._schedule_setup, - scheduleUnpack=self._schedule_unpack, - **dict((k, getattr(self.sched_iface, k)) - for k in self.sched_iface.__slots__)) + scheduleUnpack=self._schedule_unpack) self._prefetchers = weakref.WeakValueDictionary() self._pkg_queue = [] + self._jobs = 0 self._running_tasks = {} self._completed_tasks = set() @@ -243,10 +248,15 @@ class Scheduler(PollScheduler): # The load average takes some time to respond when new # jobs are added, so we need to limit the rate of adding # new jobs. - self._job_delay_max = 10 - self._job_delay_factor = 1.0 - self._job_delay_exp = 1.5 + self._job_delay_max = 5 self._previous_job_start_time = None + self._job_delay_timeout_id = None + + # The load average takes some time to respond when after + # a SIGSTOP/SIGCONT cycle, so delay scheduling for some + # time after SIGCONT is received. + self._sigcont_delay = 5 + self._sigcont_time = None # This is used to memoize the _choose_pkg() result when # no packages can be chosen until one of the existing @@ -300,15 +310,10 @@ class Scheduler(PollScheduler): if not portage.dep.match_from_list( portage.const.PORTAGE_PACKAGE_ATOM, [x]): continue - if self._running_portage is None or \ - self._running_portage.cpv != x.cpv or \ - '9999' in x.cpv or \ - 'git' in x.inherited or \ - 'git-2' in x.inherited: - rval = _check_temp_dir(self.settings) - if rval != os.EX_OK: - return rval - _prepare_self_update(self.settings) + rval = _check_temp_dir(self.settings) + if rval != os.EX_OK: + return rval + _prepare_self_update(self.settings) break return os.EX_OK @@ -328,12 +333,13 @@ class Scheduler(PollScheduler): self._set_graph_config(graph_config) self._blocker_db = {} dynamic_deps = self.myopts.get("--dynamic-deps", "y") != "n" + ignore_built_slot_operator_deps = self.myopts.get( + "--ignore-built-slot-operator-deps", "n") == "y" for root in self.trees: - if self._uninstall_only: - continue if graph_config is None: fake_vartree = FakeVartree(self.trees[root]["root_config"], - pkg_cache=self._pkg_cache, dynamic_deps=dynamic_deps) + pkg_cache=self._pkg_cache, dynamic_deps=dynamic_deps, + ignore_built_slot_operator_deps=ignore_built_slot_operator_deps) fake_vartree.sync() else: fake_vartree = graph_config.trees[root]['vartree'] @@ -410,7 +416,7 @@ class Scheduler(PollScheduler): if not (isinstance(task, Package) and \ task.operation == "merge"): continue - if 'interactive' in task.metadata.properties: + if 'interactive' in task.properties: interactive_tasks.append(task) return interactive_tasks @@ -655,10 +661,11 @@ class Scheduler(PollScheduler): if value and value.strip(): continue msg = _("%(var)s is not set... " - "Are you missing the '%(configroot)setc/make.profile' symlink? " + "Are you missing the '%(configroot)s%(profile_path)s' symlink? " "Is the symlink correct? " "Is your portage tree complete?") % \ - {"var": var, "configroot": settings["PORTAGE_CONFIGROOT"]} + {"var": var, "configroot": settings["PORTAGE_CONFIGROOT"], + "profile_path": portage.const.PROFILE_PATH} out = portage.output.EOutput() for line in textwrap.wrap(msg, 70): @@ -718,7 +725,6 @@ class Scheduler(PollScheduler): return if self._parallel_fetch: - self._status_msg("Starting parallel fetch") prefetchers = self._prefetchers @@ -771,10 +777,10 @@ class Scheduler(PollScheduler): failures = 0 - # Use a local PollScheduler instance here, since we don't + # Use a local EventLoop instance here, since we don't # want tasks here to trigger the usual Scheduler callbacks # that handle job scheduling and status display. - sched_iface = PollScheduler().sched_iface + sched_iface = SchedulerInterface(EventLoop(main=False)) for x in self._mergelist: if not isinstance(x, Package): @@ -783,10 +789,10 @@ class Scheduler(PollScheduler): if x.operation == "uninstall": continue - if x.metadata["EAPI"] in ("0", "1", "2", "3"): + if x.eapi in ("0", "1", "2", "3"): continue - if "pretend" not in x.metadata.defined_phases: + if "pretend" not in x.defined_phases: continue out_str =">>> Running pre-merge checks for " + colorize("INFORM", x.cpv) + "\n" @@ -805,7 +811,7 @@ class Scheduler(PollScheduler): build_dir_path = os.path.join( os.path.realpath(settings["PORTAGE_TMPDIR"]), "portage", x.category, x.pf) - existing_buildir = os.path.isdir(build_dir_path) + existing_builddir = os.path.isdir(build_dir_path) settings["PORTAGE_BUILDDIR"] = build_dir_path build_dir = EbuildBuildDir(scheduler=sched_iface, settings=settings) @@ -816,7 +822,7 @@ class Scheduler(PollScheduler): # Clean up the existing build dir, in case pkg_pretend # checks for available space (bug #390711). - if existing_buildir: + if existing_builddir: if x.built: tree = "bintree" infloc = os.path.join(build_dir_path, "build-info") @@ -905,13 +911,18 @@ class Scheduler(PollScheduler): failures += 1 portage.elog.elog_process(x.cpv, settings) finally: - if current_task is not None and current_task.isAlive(): - current_task.cancel() - current_task.wait() - clean_phase = EbuildPhase(background=False, - phase='clean', scheduler=sched_iface, settings=settings) - clean_phase.start() - clean_phase.wait() + + if current_task is not None: + if current_task.isAlive(): + current_task.cancel() + current_task.wait() + if current_task.returncode == os.EX_OK: + clean_phase = EbuildPhase(background=False, + phase='clean', scheduler=sched_iface, + settings=settings) + clean_phase.start() + clean_phase.wait() + build_dir.unlock() if failures: @@ -1001,6 +1012,8 @@ class Scheduler(PollScheduler): earlier_sigint_handler = signal.signal(signal.SIGINT, sighandler) earlier_sigterm_handler = signal.signal(signal.SIGTERM, sighandler) + earlier_sigcont_handler = \ + signal.signal(signal.SIGCONT, self._sigcont_handler) try: rval = self._merge() @@ -1014,6 +1027,10 @@ class Scheduler(PollScheduler): signal.signal(signal.SIGTERM, earlier_sigterm_handler) else: signal.signal(signal.SIGTERM, signal.SIG_DFL) + if earlier_sigcont_handler is not None: + signal.signal(signal.SIGCONT, earlier_sigcont_handler) + else: + signal.signal(signal.SIGCONT, signal.SIG_DFL) if received_signal: sys.exit(received_signal[0]) @@ -1060,7 +1077,8 @@ class Scheduler(PollScheduler): printer = portage.output.EOutput() background = self._background failure_log_shown = False - if background and len(self._failed_pkgs_all) == 1: + if background and len(self._failed_pkgs_all) == 1 and \ + self.myopts.get('--quiet-fail', 'n') != 'y': # If only one package failed then just show it's # whole log for easy viewing. failed_pkg = self._failed_pkgs_all[-1] @@ -1139,9 +1157,9 @@ class Scheduler(PollScheduler): printer.eerror(line) printer.eerror("") for failed_pkg in self._failed_pkgs_all: - # Use _unicode_decode() to force unicode format string so + # Use unicode_literals to force unicode format string so # that Package.__unicode__() is called in python2. - msg = _unicode_decode(" %s") % (failed_pkg.pkg,) + msg = " %s" % (failed_pkg.pkg,) log_path = self._locate_failure_log(failed_pkg) if log_path is not None: msg += ", Log file:" @@ -1338,6 +1356,38 @@ class Scheduler(PollScheduler): blocker_db = self._blocker_db[pkg.root] blocker_db.discardBlocker(pkg) + def _main_loop(self): + term_check_id = self._event_loop.idle_add(self._termination_check) + loadavg_check_id = None + if self._max_load is not None and \ + self._loadavg_latency is not None and \ + (self._max_jobs is True or self._max_jobs > 1): + # We have to schedule periodically, in case the load + # average has changed since the last call. + loadavg_check_id = self._event_loop.timeout_add( + self._loadavg_latency, self._schedule) + + try: + # Populate initial event sources. Unless we're scheduling + # based on load average, we only need to do this once + # here, since it can be called during the loop from within + # event handlers. + self._schedule() + + # Loop while there are jobs to be scheduled. + while self._keep_scheduling(): + self._event_loop.iteration() + + # Clean shutdown of previously scheduled jobs. In the + # case of termination, this allows for basic cleanup + # such as flushing of buffered output to logs. + while self._is_work_scheduled(): + self._event_loop.iteration() + finally: + self._event_loop.source_remove(term_check_id) + if loadavg_check_id is not None: + self._event_loop.source_remove(loadavg_check_id) + def _merge(self): if self._opts_no_background.intersection(self.myopts): @@ -1348,8 +1398,10 @@ class Scheduler(PollScheduler): failed_pkgs = self._failed_pkgs portage.locks._quiet = self._background portage.elog.add_listener(self._elog_listener) - display_timeout_id = self.sched_iface.timeout_add( - self._max_display_latency, self._status_display.display) + display_timeout_id = None + if self._status_display._isatty and not self._status_display.quiet: + display_timeout_id = self._event_loop.timeout_add( + self._max_display_latency, self._status_display.display) rval = os.EX_OK try: @@ -1358,7 +1410,8 @@ class Scheduler(PollScheduler): self._main_loop_cleanup() portage.locks._quiet = False portage.elog.remove_listener(self._elog_listener) - self.sched_iface.source_remove(display_timeout_id) + if display_timeout_id is not None: + self._event_loop.source_remove(display_timeout_id) if failed_pkgs: rval = failed_pkgs[-1].returncode @@ -1490,12 +1543,15 @@ class Scheduler(PollScheduler): self._config_pool[settings['EROOT']].append(settings) def _keep_scheduling(self): - return bool(not self._terminated_tasks and self._pkg_queue and \ + return bool(not self._terminated.is_set() and self._pkg_queue and \ not (self._failed_pkgs and not self._build_opts.fetchonly)) def _is_work_scheduled(self): return bool(self._running_tasks) + def _running_job_count(self): + return self._jobs + def _schedule_tasks(self): while True: @@ -1536,6 +1592,9 @@ class Scheduler(PollScheduler): not self._task_queues.merge)): break + def _sigcont_handler(self, signum, frame): + self._sigcont_time = time.time() + def _job_delay(self): """ @rtype: bool @@ -1546,14 +1605,53 @@ class Scheduler(PollScheduler): current_time = time.time() - delay = self._job_delay_factor * self._jobs ** self._job_delay_exp + if self._sigcont_time is not None: + + elapsed_seconds = current_time - self._sigcont_time + # elapsed_seconds < 0 means the system clock has been adjusted + if elapsed_seconds > 0 and \ + elapsed_seconds < self._sigcont_delay: + + if self._job_delay_timeout_id is not None: + self._event_loop.source_remove( + self._job_delay_timeout_id) + + self._job_delay_timeout_id = self._event_loop.timeout_add( + 1000 * (self._sigcont_delay - elapsed_seconds), + self._schedule_once) + return True + + # Only set this to None after the delay has expired, + # since this method may be called again before the + # delay has expired. + self._sigcont_time = None + + try: + avg1, avg5, avg15 = getloadavg() + except OSError: + return False + + delay = self._job_delay_max * avg1 / self._max_load if delay > self._job_delay_max: delay = self._job_delay_max - if (current_time - self._previous_job_start_time) < delay: + elapsed_seconds = current_time - self._previous_job_start_time + # elapsed_seconds < 0 means the system clock has been adjusted + if elapsed_seconds > 0 and elapsed_seconds < delay: + + if self._job_delay_timeout_id is not None: + self._event_loop.source_remove( + self._job_delay_timeout_id) + + self._job_delay_timeout_id = self._event_loop.timeout_add( + 1000 * (delay - elapsed_seconds), self._schedule_once) return True return False + def _schedule_once(self): + self._schedule() + return False + def _schedule_tasks_imp(self): """ @rtype: bool @@ -1735,7 +1833,7 @@ class Scheduler(PollScheduler): # scope e = exc mydepgraph = e.depgraph - dropped_tasks = set() + dropped_tasks = {} if e is not None: def unsatisfied_resume_dep_msg(): @@ -1772,11 +1870,7 @@ class Scheduler(PollScheduler): return False if success and self._show_list(): - mylist = mydepgraph.altlist() - if mylist: - if "--tree" in self.myopts: - mylist.reverse() - mydepgraph.display(mylist, favorites=self._favorites) + mydepgraph.display(mydepgraph.altlist(), favorites=self._favorites) if not success: self._post_mod_echo_msgs.append(mydepgraph.display_problems) @@ -1785,7 +1879,7 @@ class Scheduler(PollScheduler): self._init_graph(mydepgraph.schedulerGraph()) msg_width = 75 - for task in dropped_tasks: + for task, atoms in dropped_tasks.items(): if not (isinstance(task, Package) and task.operation == "merge"): continue pkg = task @@ -1793,7 +1887,10 @@ class Scheduler(PollScheduler): " %s" % (pkg.cpv,) if pkg.root_config.settings["ROOT"] != "/": msg += " for %s" % (pkg.root,) - msg += " dropped due to unsatisfied dependency." + if not atoms: + msg += " dropped because it is masked or unavailable" + else: + msg += " dropped because it requires %s" % ", ".join(atoms) for line in textwrap.wrap(msg, msg_width): eerror(line, phase="other", key=pkg.cpv) settings = self.pkgsettings[pkg.root] @@ -1838,11 +1935,21 @@ class Scheduler(PollScheduler): root_config = pkg.root_config world_set = root_config.sets["selected"] world_locked = False - if hasattr(world_set, "lock"): - world_set.lock() - world_locked = True + atom = None + + if pkg.operation != "uninstall": + # Do this before acquiring the lock, since it queries the + # portdbapi which can call the global event loop, triggering + # a concurrent call to this method or something else that + # needs an exclusive (non-reentrant) lock on the world file. + atom = create_world_atom(pkg, args_set, root_config) try: + + if hasattr(world_set, "lock"): + world_set.lock() + world_locked = True + if hasattr(world_set, "load"): world_set.load() # maybe it's changed on disk @@ -1854,8 +1961,7 @@ class Scheduler(PollScheduler): for s in pkg.root_config.setconfig.active: world_set.remove(SETPREFIX+s) else: - atom = create_world_atom(pkg, args_set, root_config) - if atom: + if atom is not None: if hasattr(world_set, "add"): self._status_msg(('Recording %s in "world" ' + \ 'favorites file...') % atom) diff --git a/portage_with_autodep/pym/_emerge/Scheduler.pyo b/portage_with_autodep/pym/_emerge/Scheduler.pyo Binary files differindex 5555703..e61506d 100644 --- a/portage_with_autodep/pym/_emerge/Scheduler.pyo +++ b/portage_with_autodep/pym/_emerge/Scheduler.pyo diff --git a/portage_with_autodep/pym/_emerge/SequentialTaskQueue.pyo b/portage_with_autodep/pym/_emerge/SequentialTaskQueue.pyo Binary files differindex 3ab65c9..abf4b59 100644 --- a/portage_with_autodep/pym/_emerge/SequentialTaskQueue.pyo +++ b/portage_with_autodep/pym/_emerge/SequentialTaskQueue.pyo diff --git a/portage_with_autodep/pym/_emerge/SetArg.py b/portage_with_autodep/pym/_emerge/SetArg.py index 94cf0a6..5c82975 100644 --- a/portage_with_autodep/pym/_emerge/SetArg.py +++ b/portage_with_autodep/pym/_emerge/SetArg.py @@ -1,9 +1,12 @@ -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.DependencyArg import DependencyArg from portage._sets import SETPREFIX class SetArg(DependencyArg): + + __slots__ = ('name', 'pset') + def __init__(self, pset=None, **kwargs): DependencyArg.__init__(self, **kwargs) self.pset = pset diff --git a/portage_with_autodep/pym/_emerge/SetArg.pyo b/portage_with_autodep/pym/_emerge/SetArg.pyo Binary files differindex 5a3d9d9..bc86247 100644 --- a/portage_with_autodep/pym/_emerge/SetArg.pyo +++ b/portage_with_autodep/pym/_emerge/SetArg.pyo diff --git a/portage_with_autodep/pym/_emerge/SpawnProcess.py b/portage_with_autodep/pym/_emerge/SpawnProcess.py index 9fbc964..15d3dc5 100644 --- a/portage_with_autodep/pym/_emerge/SpawnProcess.py +++ b/portage_with_autodep/pym/_emerge/SpawnProcess.py @@ -1,17 +1,23 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 2008-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from _emerge.SubProcess import SubProcess +try: + import fcntl +except ImportError: + # http://bugs.jython.org/issue1074 + fcntl = None + +import errno +import logging +import signal import sys -from portage.cache.mappings import slot_dict_class + +from _emerge.SubProcess import SubProcess import portage -from portage import _encodings -from portage import _unicode_encode from portage import os from portage.const import BASH_BINARY -import fcntl -import errno -import gzip +from portage.util import writemsg_level +from portage.util._async.PipeLogger import PipeLogger class SpawnProcess(SubProcess): @@ -23,31 +29,27 @@ class SpawnProcess(SubProcess): _spawn_kwarg_names = ("env", "opt_name", "fd_pipes", "uid", "gid", "groups", "umask", "logfile", - "path_lookup", "pre_exec") + "path_lookup", "pre_exec", "close_fds", "cgroup", + "unshare_ipc", "unshare_net") __slots__ = ("args",) + \ - _spawn_kwarg_names + ("_log_file_real", "_selinux_type",) - - _file_names = ("log", "process", "stdout") - _files_dict = slot_dict_class(_file_names, prefix="") + _spawn_kwarg_names + ("_pipe_logger", "_selinux_type",) def _start(self): if self.fd_pipes is None: self.fd_pipes = {} + else: + self.fd_pipes = self.fd_pipes.copy() fd_pipes = self.fd_pipes - self._files = self._files_dict() - files = self._files - master_fd, slave_fd = self._pipe(fd_pipes) - fcntl.fcntl(master_fd, fcntl.F_SETFL, - fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK) - files.process = master_fd - logfile = None - if self._can_log(slave_fd): - logfile = self.logfile + can_log = self._can_log(slave_fd) + if can_log: + log_file_path = self.logfile + else: + log_file_path = None null_input = None if not self.background or 0 in fd_pipes: @@ -62,48 +64,34 @@ class SpawnProcess(SubProcess): null_input = os.open('/dev/null', os.O_RDWR) fd_pipes[0] = null_input - fd_pipes.setdefault(0, sys.stdin.fileno()) - fd_pipes.setdefault(1, sys.stdout.fileno()) - fd_pipes.setdefault(2, sys.stderr.fileno()) + fd_pipes.setdefault(0, portage._get_stdin().fileno()) + fd_pipes.setdefault(1, sys.__stdout__.fileno()) + fd_pipes.setdefault(2, sys.__stderr__.fileno()) # flush any pending output + stdout_filenos = (sys.__stdout__.fileno(), sys.__stderr__.fileno()) for fd in fd_pipes.values(): - if fd == sys.stdout.fileno(): - sys.stdout.flush() - if fd == sys.stderr.fileno(): - sys.stderr.flush() + if fd in stdout_filenos: + sys.__stdout__.flush() + sys.__stderr__.flush() + break - if logfile is not None: + fd_pipes_orig = fd_pipes.copy() - fd_pipes_orig = fd_pipes.copy() + if log_file_path is not None or self.background: fd_pipes[1] = slave_fd fd_pipes[2] = slave_fd - files.log = open(_unicode_encode(logfile, - encoding=_encodings['fs'], errors='strict'), mode='ab') - if logfile.endswith('.gz'): - self._log_file_real = files.log - files.log = gzip.GzipFile(filename='', mode='ab', - fileobj=files.log) - - portage.util.apply_secpass_permissions(logfile, - uid=portage.portage_uid, gid=portage.portage_gid, - mode=0o660) - - if not self.background: - files.stdout = os.dup(fd_pipes_orig[1]) - - output_handler = self._output_handler - else: - - # Create a dummy pipe so the scheduler can monitor - # the process from inside a poll() loop. - fd_pipes[self._dummy_pipe_fd] = slave_fd - if self.background: - fd_pipes[1] = slave_fd - fd_pipes[2] = slave_fd - output_handler = self._dummy_handler + # Create a dummy pipe that PipeLogger uses to efficiently + # monitor for process exit by listening for the EOF event. + # Re-use of the allocated fd number for the key in fd_pipes + # guarantees that the keys will not collide for similarly + # allocated pipes which are used by callers such as + # FileDigester and MergeProcess. See the _setup_pipes + # docstring for more benefits of this allocation approach. + self._dummy_pipe_fd = slave_fd + fd_pipes[slave_fd] = slave_fd kwargs = {} for k in self._spawn_kwarg_names: @@ -115,10 +103,6 @@ class SpawnProcess(SubProcess): kwargs["returnpid"] = True kwargs.pop("logfile", None) - self._reg_id = self.scheduler.register(files.process, - self._registered_events, output_handler) - self._registered = True - retval = self._spawn(self.args, **kwargs) os.close(slave_fd) @@ -129,11 +113,32 @@ class SpawnProcess(SubProcess): # spawn failed self._unregister() self._set_returncode((self.pid, retval)) - self.wait() + self._async_wait() return self.pid = retval[0] - portage.process.spawned_pids.remove(self.pid) + + stdout_fd = None + if can_log and not self.background: + stdout_fd = os.dup(fd_pipes_orig[1]) + # FD_CLOEXEC is enabled by default in Python >=3.4. + if sys.hexversion < 0x3040000 and fcntl is not None: + try: + fcntl.FD_CLOEXEC + except AttributeError: + pass + else: + fcntl.fcntl(stdout_fd, fcntl.F_SETFD, + fcntl.fcntl(stdout_fd, + fcntl.F_GETFD) | fcntl.FD_CLOEXEC) + + self._pipe_logger = PipeLogger(background=self.background, + scheduler=self.scheduler, input_fd=master_fd, + log_file_path=log_file_path, + stdout_fd=stdout_fd) + self._pipe_logger.addExitListener(self._pipe_logger_exit) + self._pipe_logger.start() + self._registered = True def _can_log(self, slave_fd): return True @@ -157,92 +162,56 @@ class SpawnProcess(SubProcess): return spawn_func(args, **kwargs) - def _output_handler(self, fd, event): - - files = self._files - while True: - buf = self._read_buf(fd, event) - - if buf is None: - # not a POLLIN event, EAGAIN, etc... - break - - if not buf: - # EOF - self._unregister() - self.wait() - break - - else: - if not self.background: - write_successful = False - failures = 0 - while True: - try: - if not write_successful: - os.write(files.stdout, buf) - write_successful = True - break - except OSError as e: - if e.errno != errno.EAGAIN: - raise - del e - failures += 1 - if failures > 50: - # Avoid a potentially infinite loop. In - # most cases, the failure count is zero - # and it's unlikely to exceed 1. - raise - - # This means that a subprocess has put an inherited - # stdio file descriptor (typically stdin) into - # O_NONBLOCK mode. This is not acceptable (see bug - # #264435), so revert it. We need to use a loop - # here since there's a race condition due to - # parallel processes being able to change the - # flags on the inherited file descriptor. - # TODO: When possible, avoid having child processes - # inherit stdio file descriptors from portage - # (maybe it can't be avoided with - # PROPERTIES=interactive). - fcntl.fcntl(files.stdout, fcntl.F_SETFL, - fcntl.fcntl(files.stdout, - fcntl.F_GETFL) ^ os.O_NONBLOCK) - - files.log.write(buf) - files.log.flush() - - self._unregister_if_appropriate(event) - - return True - - def _dummy_handler(self, fd, event): - """ - This method is mainly interested in detecting EOF, since - the only purpose of the pipe is to allow the scheduler to - monitor the process from inside a poll() loop. - """ - - while True: - buf = self._read_buf(fd, event) - - if buf is None: - # not a POLLIN event, EAGAIN, etc... - break - - if not buf: - # EOF - self._unregister() - self.wait() - break - - self._unregister_if_appropriate(event) - - return True - - def _unregister(self): - super(SpawnProcess, self)._unregister() - if self._log_file_real is not None: - # Avoid "ResourceWarning: unclosed file" since python 3.2. - self._log_file_real.close() - self._log_file_real = None + def _pipe_logger_exit(self, pipe_logger): + self._pipe_logger = None + self._unregister() + self.wait() + + def _waitpid_loop(self): + SubProcess._waitpid_loop(self) + + pipe_logger = self._pipe_logger + if pipe_logger is not None: + self._pipe_logger = None + pipe_logger.removeExitListener(self._pipe_logger_exit) + pipe_logger.cancel() + pipe_logger.wait() + + def _set_returncode(self, wait_retval): + SubProcess._set_returncode(self, wait_retval) + + if self.cgroup: + def get_pids(cgroup): + try: + with open(os.path.join(cgroup, 'cgroup.procs'), 'r') as f: + return [int(p) for p in f.read().split()] + except OSError: + # cgroup removed already? + return [] + + def kill_all(pids, sig): + for p in pids: + try: + os.kill(p, sig) + except OSError as e: + if e.errno == errno.EPERM: + # Reported with hardened kernel (bug #358211). + writemsg_level( + "!!! kill: (%i) - Operation not permitted\n" % + (p,), level=logging.ERROR, + noiselevel=-1) + elif e.errno != errno.ESRCH: + raise + + # step 1: kill all orphans + pids = get_pids(self.cgroup) + if pids: + kill_all(pids, signal.SIGKILL) + + # step 2: remove the cgroup + try: + os.rmdir(self.cgroup) + except OSError: + # it may be removed already, or busy + # we can't do anything good about it + pass diff --git a/portage_with_autodep/pym/_emerge/SpawnProcess.pyo b/portage_with_autodep/pym/_emerge/SpawnProcess.pyo Binary files differindex 7a6142e..b32e588 100644 --- a/portage_with_autodep/pym/_emerge/SpawnProcess.pyo +++ b/portage_with_autodep/pym/_emerge/SpawnProcess.pyo diff --git a/portage_with_autodep/pym/_emerge/SubProcess.py b/portage_with_autodep/pym/_emerge/SubProcess.py index 76b313f..13d9382 100644 --- a/portage_with_autodep/pym/_emerge/SubProcess.py +++ b/portage_with_autodep/pym/_emerge/SubProcess.py @@ -1,7 +1,10 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +import logging + from portage import os +from portage.util import writemsg_level from _emerge.AbstractPollTask import AbstractPollTask import signal import errno @@ -9,12 +12,7 @@ import errno class SubProcess(AbstractPollTask): __slots__ = ("pid",) + \ - ("_files", "_reg_id") - - # A file descriptor is required for the scheduler to monitor changes from - # inside a poll() loop. When logging is not enabled, create a pipe just to - # serve this purpose alone. - _dummy_pipe_fd = 9 + ("_dummy_pipe_fd", "_files", "_reg_id") # This is how much time we allow for waitpid to succeed after # we've sent a kill signal to our subprocess. @@ -50,7 +48,13 @@ class SubProcess(AbstractPollTask): try: os.kill(self.pid, signal.SIGTERM) except OSError as e: - if e.errno != errno.ESRCH: + if e.errno == errno.EPERM: + # Reported with hardened kernel (bug #358211). + writemsg_level( + "!!! kill: (%i) - Operation not permitted\n" % + (self.pid,), level=logging.ERROR, + noiselevel=-1) + elif e.errno != errno.ESRCH: raise def isAlive(self): @@ -69,7 +73,13 @@ class SubProcess(AbstractPollTask): try: os.kill(self.pid, signal.SIGKILL) except OSError as e: - if e.errno != errno.ESRCH: + if e.errno == errno.EPERM: + # Reported with hardened kernel (bug #358211). + writemsg_level( + "!!! kill: (%i) - Operation not permitted\n" % + (self.pid,), level=logging.ERROR, + noiselevel=-1) + elif e.errno != errno.ESRCH: raise del e self._wait_loop(timeout=self._cancel_timeout) @@ -116,7 +126,7 @@ class SubProcess(AbstractPollTask): self._registered = False if self._reg_id is not None: - self.scheduler.unregister(self._reg_id) + self.scheduler.source_remove(self._reg_id) self._reg_id = None if self._files is not None: diff --git a/portage_with_autodep/pym/_emerge/SubProcess.pyo b/portage_with_autodep/pym/_emerge/SubProcess.pyo Binary files differindex 26e13e1..0fb99fb 100644 --- a/portage_with_autodep/pym/_emerge/SubProcess.pyo +++ b/portage_with_autodep/pym/_emerge/SubProcess.pyo diff --git a/portage_with_autodep/pym/_emerge/Task.py b/portage_with_autodep/pym/_emerge/Task.py index 40f5066..250d458 100644 --- a/portage_with_autodep/pym/_emerge/Task.py +++ b/portage_with_autodep/pym/_emerge/Task.py @@ -1,4 +1,4 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from portage.util.SlotObject import SlotObject @@ -41,3 +41,10 @@ class Task(SlotObject): strings. """ return "(%s)" % ", ".join(("'%s'" % x for x in self._hash_key)) + + def __repr__(self): + if self._hash_key is None: + # triggered by python-trace + return SlotObject.__repr__(self) + return "<%s (%s)>" % (self.__class__.__name__, + ", ".join(("'%s'" % x for x in self._hash_key))) diff --git a/portage_with_autodep/pym/_emerge/Task.pyo b/portage_with_autodep/pym/_emerge/Task.pyo Binary files differindex 2958cb1..eb71dac 100644 --- a/portage_with_autodep/pym/_emerge/Task.pyo +++ b/portage_with_autodep/pym/_emerge/Task.pyo diff --git a/portage_with_autodep/pym/_emerge/TaskScheduler.py b/portage_with_autodep/pym/_emerge/TaskScheduler.py deleted file mode 100644 index 583bfe3..0000000 --- a/portage_with_autodep/pym/_emerge/TaskScheduler.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 1999-2012 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from _emerge.QueueScheduler import QueueScheduler -from _emerge.SequentialTaskQueue import SequentialTaskQueue - -class TaskScheduler(object): - - """ - A simple way to handle scheduling of AsynchrousTask instances. Simply - add tasks and call run(). The run() method returns when no tasks remain. - """ - - def __init__(self, main=True, max_jobs=None, max_load=None): - self._queue = SequentialTaskQueue(max_jobs=max_jobs) - self._scheduler = QueueScheduler(main=main, - max_jobs=max_jobs, max_load=max_load) - self.sched_iface = self._scheduler.sched_iface - self.run = self._scheduler.run - self.clear = self._scheduler.clear - self.wait = self._queue.wait - self._scheduler.add(self._queue) - - def add(self, task): - self._queue.add(task) - diff --git a/portage_with_autodep/pym/_emerge/TaskScheduler.pyo b/portage_with_autodep/pym/_emerge/TaskScheduler.pyo Binary files differdeleted file mode 100644 index 8b84de7..0000000 --- a/portage_with_autodep/pym/_emerge/TaskScheduler.pyo +++ /dev/null diff --git a/portage_with_autodep/pym/_emerge/TaskSequence.pyo b/portage_with_autodep/pym/_emerge/TaskSequence.pyo Binary files differindex b98196e..0257937 100644 --- a/portage_with_autodep/pym/_emerge/TaskSequence.pyo +++ b/portage_with_autodep/pym/_emerge/TaskSequence.pyo diff --git a/portage_with_autodep/pym/_emerge/UninstallFailure.pyo b/portage_with_autodep/pym/_emerge/UninstallFailure.pyo Binary files differindex 9f1c88b..50fba3d 100644 --- a/portage_with_autodep/pym/_emerge/UninstallFailure.pyo +++ b/portage_with_autodep/pym/_emerge/UninstallFailure.pyo diff --git a/portage_with_autodep/pym/_emerge/UnmergeDepPriority.py b/portage_with_autodep/pym/_emerge/UnmergeDepPriority.py index 4316600..ec44a67 100644 --- a/portage_with_autodep/pym/_emerge/UnmergeDepPriority.py +++ b/portage_with_autodep/pym/_emerge/UnmergeDepPriority.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.AbstractDepPriority import AbstractDepPriority @@ -7,15 +7,16 @@ class UnmergeDepPriority(AbstractDepPriority): """ Combination of properties Priority Category - runtime 0 HARD - runtime_post -1 HARD - buildtime -2 SOFT - (none of the above) -2 SOFT + runtime_slot_op 0 HARD + runtime -1 HARD + runtime_post -2 HARD + buildtime -3 SOFT + (none of the above) -3 SOFT """ MAX = 0 - SOFT = -2 - MIN = -2 + SOFT = -3 + MIN = -3 def __init__(self, **kwargs): AbstractDepPriority.__init__(self, **kwargs) @@ -23,17 +24,21 @@ class UnmergeDepPriority(AbstractDepPriority): self.optional = True def __int__(self): - if self.runtime: + if self.runtime_slot_op: return 0 - if self.runtime_post: + if self.runtime: return -1 - if self.buildtime: + if self.runtime_post: return -2 - return -2 + if self.buildtime: + return -3 + return -3 def __str__(self): if self.ignored: return "ignored" + if self.runtime_slot_op: + return "hard slot op" myvalue = self.__int__() if myvalue > self.SOFT: return "hard" diff --git a/portage_with_autodep/pym/_emerge/UnmergeDepPriority.pyo b/portage_with_autodep/pym/_emerge/UnmergeDepPriority.pyo Binary files differindex b163ed7..8f0e204 100644 --- a/portage_with_autodep/pym/_emerge/UnmergeDepPriority.pyo +++ b/portage_with_autodep/pym/_emerge/UnmergeDepPriority.pyo diff --git a/portage_with_autodep/pym/_emerge/UseFlagDisplay.py b/portage_with_autodep/pym/_emerge/UseFlagDisplay.py index 3daca19..f460474 100644 --- a/portage_with_autodep/pym/_emerge/UseFlagDisplay.py +++ b/portage_with_autodep/pym/_emerge/UseFlagDisplay.py @@ -1,10 +1,12 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + from itertools import chain import sys -from portage import _encodings, _unicode_decode, _unicode_encode +from portage import _encodings, _unicode_encode from portage.output import red from portage.util import cmp_sort_key from portage.output import blue @@ -114,9 +116,9 @@ def pkg_use_display(pkg, opts, modified_use=None): flags.sort(key=UseFlagDisplay.sort_combined) else: flags.sort(key=UseFlagDisplay.sort_separated) - # Use _unicode_decode() to force unicode format string so + # Use unicode_literals to force unicode format string so # that UseFlagDisplay.__unicode__() is called in python2. flag_displays.append('%s="%s"' % (varname, - ' '.join(_unicode_decode("%s") % (f,) for f in flags))) + ' '.join("%s" % (f,) for f in flags))) return ' '.join(flag_displays) diff --git a/portage_with_autodep/pym/_emerge/UseFlagDisplay.pyo b/portage_with_autodep/pym/_emerge/UseFlagDisplay.pyo Binary files differindex 005b007..570ef73 100644 --- a/portage_with_autodep/pym/_emerge/UseFlagDisplay.pyo +++ b/portage_with_autodep/pym/_emerge/UseFlagDisplay.pyo diff --git a/portage_with_autodep/pym/_emerge/__init__.pyo b/portage_with_autodep/pym/_emerge/__init__.pyo Binary files differindex fba4ca5..007c6ae 100644 --- a/portage_with_autodep/pym/_emerge/__init__.pyo +++ b/portage_with_autodep/pym/_emerge/__init__.pyo diff --git a/portage_with_autodep/pym/_emerge/_find_deep_system_runtime_deps.pyo b/portage_with_autodep/pym/_emerge/_find_deep_system_runtime_deps.pyo Binary files differindex 8ad61b2..0b21a14 100644 --- a/portage_with_autodep/pym/_emerge/_find_deep_system_runtime_deps.pyo +++ b/portage_with_autodep/pym/_emerge/_find_deep_system_runtime_deps.pyo diff --git a/portage_with_autodep/pym/_emerge/_flush_elog_mod_echo.pyo b/portage_with_autodep/pym/_emerge/_flush_elog_mod_echo.pyo Binary files differindex f211d41..16bdc11 100644 --- a/portage_with_autodep/pym/_emerge/_flush_elog_mod_echo.pyo +++ b/portage_with_autodep/pym/_emerge/_flush_elog_mod_echo.pyo diff --git a/portage_with_autodep/pym/_emerge/actions.py b/portage_with_autodep/pym/_emerge/actions.py index eaf5a15..9bb4774 100644 --- a/portage_with_autodep/pym/_emerge/actions.py +++ b/portage_with_autodep/pym/_emerge/actions.py @@ -1,7 +1,7 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import print_function +from __future__ import print_function, unicode_literals import errno import logging @@ -18,27 +18,35 @@ import sys import tempfile import textwrap import time +import warnings from itertools import chain import portage portage.proxy.lazyimport.lazyimport(globals(), + 'portage.dbapi._similar_name_search:similar_name_search', + 'portage.debug', 'portage.news:count_unread_news,display_news_notifications', + 'portage.util._get_vm_info:get_vm_info', + '_emerge.chk_updated_cfg_files:chk_updated_cfg_files', + '_emerge.help:help@emerge_help', + '_emerge.post_emerge:display_news_notification,post_emerge', + '_emerge.stdout_spinner:stdout_spinner', ) from portage.localization import _ from portage import os from portage import shutil -from portage import eapi_is_supported, _unicode_decode +from portage import eapi_is_supported, _encodings, _unicode_decode from portage.cache.cache_errors import CacheError -from portage.const import GLOBAL_CONFIG_PATH -from portage.const import _ENABLE_DYN_LINK_MAP, _ENABLE_SET_CONFIG +from portage.const import GLOBAL_CONFIG_PATH, VCS_DIRS, _DEPCLEAN_LIB_CHECK_DEFAULT +from portage.const import SUPPORTED_BINPKG_FORMATS, TIMESTAMP_FORMAT from portage.dbapi.dep_expand import dep_expand from portage.dbapi._expand_new_virt import expand_new_virt -from portage.dep import Atom, extended_cp_match +from portage.dep import Atom from portage.eclass_cache import hashed_path -from portage.exception import InvalidAtom +from portage.exception import InvalidAtom, InvalidData from portage.output import blue, bold, colorize, create_color_func, darkgreen, \ - red, yellow + red, xtermTitle, xtermTitleReset, yellow good = create_color_func("GOOD") bad = create_color_func("BAD") warn = create_color_func("WARN") @@ -46,9 +54,13 @@ from portage.package.ebuild._ipc.QueryCommand import QueryCommand from portage.package.ebuild.doebuild import _check_temp_dir from portage._sets import load_default_config, SETPREFIX from portage._sets.base import InternalPackageSet -from portage.util import cmp_sort_key, writemsg, \ +from portage.util import cmp_sort_key, writemsg, varexpand, \ writemsg_level, writemsg_stdout from portage.util.digraph import digraph +from portage.util.SlotObject import SlotObject +from portage.util._async.run_main_scheduler import run_main_scheduler +from portage.util._async.SchedulerInterface import SchedulerInterface +from portage.util._eventloop.global_event_loop import global_event_loop from portage._global_updates import _global_updates from _emerge.clear_caches import clear_caches @@ -76,6 +88,9 @@ from _emerge.userquery import userquery if sys.hexversion >= 0x3000000: long = int + _unicode = str +else: + _unicode = unicode def action_build(settings, trees, mtimedb, myopts, myaction, myfiles, spinner): @@ -172,6 +187,7 @@ def action_build(settings, trees, mtimedb, verbose = "--verbose" in myopts quiet = "--quiet" in myopts myparams = create_depgraph_params(myopts, myaction) + mergelist_shown = False if pretend or fetchonly: # make the mtimedb readonly @@ -273,8 +289,14 @@ def action_build(settings, trees, mtimedb, "dropped due to\n" + \ "!!! masking or unsatisfied dependencies:\n\n", noiselevel=-1) - for task in dropped_tasks: - portage.writemsg(" " + str(task) + "\n", noiselevel=-1) + for task, atoms in dropped_tasks.items(): + if not atoms: + writemsg(" %s is masked or unavailable\n" % + (task,), noiselevel=-1) + else: + writemsg(" %s requires %s\n" % + (task, ", ".join(atoms)), noiselevel=-1) + portage.writemsg("\n", noiselevel=-1) del dropped_tasks else: @@ -305,6 +327,7 @@ def action_build(settings, trees, mtimedb, mydepgraph.display_problems() return 1 + mergecount = None if "--pretend" not in myopts and \ ("--ask" in myopts or "--tree" in myopts or \ "--verbose" in myopts) and \ @@ -316,17 +339,19 @@ def action_build(settings, trees, mtimedb, return os.EX_OK favorites = mtimedb["resume"]["favorites"] retval = mydepgraph.display( - mydepgraph.altlist(reversed=tree), + mydepgraph.altlist(), favorites=favorites) mydepgraph.display_problems() + mergelist_shown = True if retval != os.EX_OK: return retval prompt="Would you like to resume merging these packages?" else: retval = mydepgraph.display( - mydepgraph.altlist(reversed=("--tree" in myopts)), + mydepgraph.altlist(), favorites=favorites) mydepgraph.display_problems() + mergelist_shown = True if retval != os.EX_OK: return retval mergecount=0 @@ -334,6 +359,7 @@ def action_build(settings, trees, mtimedb, if isinstance(x, Package) and x.operation == "merge": mergecount += 1 + prompt = None if mergecount==0: sets = trees[settings['EROOT']]['root_config'].sets world_candidates = None @@ -346,14 +372,11 @@ def action_build(settings, trees, mtimedb, world_candidates = [x for x in favorites \ if not (x.startswith(SETPREFIX) and \ not sets[x[1:]].world_candidate)] + if "selective" in myparams and \ not oneshot and world_candidates: - print() - for x in world_candidates: - print(" %s %s" % (good("*"), x)) - prompt="Would you like to add these packages to your world favorites?" - elif settings["AUTOCLEAN"] and "yes"==settings["AUTOCLEAN"]: - prompt="Nothing to merge; would you like to auto-clean packages?" + # Prompt later, inside saveNomergeFavorites. + prompt = None else: print() print("Nothing to merge; quitting.") @@ -364,13 +387,15 @@ def action_build(settings, trees, mtimedb, else: prompt="Would you like to merge these packages?" print() - if "--ask" in myopts and userquery(prompt, enter_invalid) == "No": + if prompt is not None and "--ask" in myopts and \ + userquery(prompt, enter_invalid) == "No": print() print("Quitting.") print() return 128 + signal.SIGINT # Don't ask again (e.g. when auto-cleaning packages after merge) - myopts.pop("--ask", None) + if mergecount != 0: + myopts.pop("--ask", None) if ("--pretend" in myopts) and not ("--fetchonly" in myopts or "--fetch-all-uri" in myopts): if ("--resume" in myopts): @@ -380,45 +405,27 @@ def action_build(settings, trees, mtimedb, return os.EX_OK favorites = mtimedb["resume"]["favorites"] retval = mydepgraph.display( - mydepgraph.altlist(reversed=tree), + mydepgraph.altlist(), favorites=favorites) mydepgraph.display_problems() + mergelist_shown = True if retval != os.EX_OK: return retval else: retval = mydepgraph.display( - mydepgraph.altlist(reversed=("--tree" in myopts)), + mydepgraph.altlist(), favorites=favorites) mydepgraph.display_problems() + mergelist_shown = True if retval != os.EX_OK: return retval - if "--buildpkgonly" in myopts: - graph_copy = mydepgraph._dynamic_config.digraph.copy() - removed_nodes = set() - for node in graph_copy: - if not isinstance(node, Package) or \ - node.operation == "nomerge": - removed_nodes.add(node) - graph_copy.difference_update(removed_nodes) - if not graph_copy.hasallzeros(ignore_priority = \ - DepPrioritySatisfiedRange.ignore_medium): - print("\n!!! --buildpkgonly requires all dependencies to be merged.") - print("!!! You have to merge the dependencies before you can build this package.\n") - return 1 + else: - if "--buildpkgonly" in myopts: - graph_copy = mydepgraph._dynamic_config.digraph.copy() - removed_nodes = set() - for node in graph_copy: - if not isinstance(node, Package) or \ - node.operation == "nomerge": - removed_nodes.add(node) - graph_copy.difference_update(removed_nodes) - if not graph_copy.hasallzeros(ignore_priority = \ - DepPrioritySatisfiedRange.ignore_medium): - print("\n!!! --buildpkgonly requires all dependencies to be merged.") - print("!!! Cannot merge requested packages. Merge deps and try again.\n") - return 1 + + if not mergelist_shown: + # If we haven't already shown the merge list above, at + # least show warnings about missed updates and such. + mydepgraph.display_problems() if ("--resume" in myopts): favorites=mtimedb["resume"]["favorites"] @@ -433,25 +440,29 @@ def action_build(settings, trees, mtimedb, mydepgraph.saveNomergeFavorites() - mergetask = Scheduler(settings, trees, mtimedb, myopts, - spinner, favorites=favorites, - graph_config=mydepgraph.schedulerGraph()) - - del mydepgraph - clear_caches(trees) - - retval = mergetask.merge() - - if retval == os.EX_OK and not (buildpkgonly or fetchonly or pretend): - if "yes" == settings.get("AUTOCLEAN"): - portage.writemsg_stdout(">>> Auto-cleaning packages...\n") - unmerge(trees[settings['EROOT']]['root_config'], - myopts, "clean", [], - ldpath_mtimes, autoclean=1) - else: - portage.writemsg_stdout(colorize("WARN", "WARNING:") - + " AUTOCLEAN is disabled. This can cause serious" - + " problems due to overlapping packages.\n") + if mergecount == 0: + retval = os.EX_OK + else: + mergetask = Scheduler(settings, trees, mtimedb, myopts, + spinner, favorites=favorites, + graph_config=mydepgraph.schedulerGraph()) + + del mydepgraph + clear_caches(trees) + + retval = mergetask.merge() + + if retval == os.EX_OK and \ + not (buildpkgonly or fetchonly or pretend): + if "yes" == settings.get("AUTOCLEAN"): + portage.writemsg_stdout(">>> Auto-cleaning packages...\n") + unmerge(trees[settings['EROOT']]['root_config'], + myopts, "clean", [], + ldpath_mtimes, autoclean=1) + else: + portage.writemsg_stdout(colorize("WARN", "WARNING:") + + " AUTOCLEAN is disabled. This can cause serious" + + " problems due to overlapping packages.\n") return retval @@ -531,7 +542,8 @@ def action_depclean(settings, trees, ldpath_mtimes, # specific packages. msg = [] - if not _ENABLE_DYN_LINK_MAP: + if "preserve-libs" not in settings.features and \ + not myopts.get("--depclean-lib-check", _DEPCLEAN_LIB_CHECK_DEFAULT) != "n": msg.append("Depclean may break link level dependencies. Thus, it is\n") msg.append("recommended to use a tool such as " + good("`revdep-rebuild`") + " (from\n") msg.append("app-portage/gentoolkit) in order to detect such breakage.\n") @@ -597,11 +609,17 @@ def action_depclean(settings, trees, ldpath_mtimes, if not cleanlist and "--quiet" in myopts: return rval + set_atoms = {} + for k in ("system", "selected"): + try: + set_atoms[k] = root_config.setconfig.getSetAtoms(k) + except portage.exception.PackageSetNotFound: + # A nested set could not be resolved, so ignore nested sets. + set_atoms[k] = root_config.sets[k].getAtoms() + print("Packages installed: " + str(len(vardb.cpv_all()))) - print("Packages in world: " + \ - str(len(root_config.sets["selected"].getAtoms()))) - print("Packages in system: " + \ - str(len(root_config.sets["system"].getAtoms()))) + print("Packages in world: %d" % len(set_atoms["selected"])) + print("Packages in system: %d" % len(set_atoms["system"])) print("Required packages: "+str(req_pkg_count)) if "--pretend" in myopts: print("Number to remove: "+str(len(cleanlist))) @@ -634,13 +652,21 @@ def calc_depclean(settings, trees, ldpath_mtimes, required_sets[protected_set_name] = protected_set system_set = psets["system"] - if not system_set or not selected_set: + set_atoms = {} + for k in ("system", "selected"): + try: + set_atoms[k] = root_config.setconfig.getSetAtoms(k) + except portage.exception.PackageSetNotFound: + # A nested set could not be resolved, so ignore nested sets. + set_atoms[k] = root_config.sets[k].getAtoms() - if not system_set: + if not set_atoms["system"] or not set_atoms["selected"]: + + if not set_atoms["system"]: writemsg_level("!!! You have no system list.\n", level=logging.ERROR, noiselevel=-1) - if not selected_set: + if not set_atoms["selected"]: writemsg_level("!!! You have no world file.\n", level=logging.WARNING, noiselevel=-1) @@ -684,7 +710,7 @@ def calc_depclean(settings, trees, ldpath_mtimes, continue except portage.exception.InvalidDependString as e: show_invalid_depstring_notice(pkg, - pkg.metadata["PROVIDE"], str(e)) + pkg._metadata["PROVIDE"], _unicode(e)) del e protected_set.add("=" + pkg.cpv) continue @@ -738,7 +764,7 @@ def calc_depclean(settings, trees, ldpath_mtimes, continue except portage.exception.InvalidDependString as e: show_invalid_depstring_notice(pkg, - pkg.metadata["PROVIDE"], str(e)) + pkg._metadata["PROVIDE"], _unicode(e)) del e protected_set.add("=" + pkg.cpv) continue @@ -756,7 +782,7 @@ def calc_depclean(settings, trees, ldpath_mtimes, required_sets['__excluded__'].add("=" + pkg.cpv) except portage.exception.InvalidDependString as e: show_invalid_depstring_notice(pkg, - pkg.metadata["PROVIDE"], str(e)) + pkg._metadata["PROVIDE"], _unicode(e)) del e required_sets['__excluded__'].add("=" + pkg.cpv) @@ -792,7 +818,12 @@ def calc_depclean(settings, trees, ldpath_mtimes, msg.append("the following required packages not being installed:") msg.append("") for atom, parent in unresolvable: - msg.append(" %s pulled in by:" % (atom,)) + if atom != atom.unevaluated_atom and \ + vardb.match(_unicode(atom)): + msg.append(" %s (%s) pulled in by:" % + (atom.unevaluated_atom, atom)) + else: + msg.append(" %s pulled in by:" % (atom,)) msg.append(" %s" % (parent,)) msg.append("") msg.extend(textwrap.wrap( @@ -835,15 +866,27 @@ def calc_depclean(settings, trees, ldpath_mtimes, required_pkgs_total += 1 def show_parents(child_node): - parent_nodes = graph.parent_nodes(child_node) - if not parent_nodes: + parent_atoms = \ + resolver._dynamic_config._parent_atoms.get(child_node, []) + + # Never display the special internal protected_set. + parent_atoms = [parent_atom for parent_atom in parent_atoms + if not (isinstance(parent_atom[0], SetArg) and + parent_atom[0].name == protected_set_name)] + + if not parent_atoms: # With --prune, the highest version can be pulled in without any # real parent since all installed packages are pulled in. In that # case there's nothing to show here. return + parent_atom_dict = {} + for parent, atom in parent_atoms: + parent_atom_dict.setdefault(parent, []).append(atom) + parent_strs = [] - for node in parent_nodes: - parent_strs.append(str(getattr(node, "cpv", node))) + for parent, atoms in parent_atom_dict.items(): + parent_strs.append("%s requires %s" % + (getattr(parent, "cpv", parent), ", ".join(atoms))) parent_strs.sort() msg = [] msg.append(" %s pulled in by:\n" % (child_node.cpv,)) @@ -868,12 +911,6 @@ def calc_depclean(settings, trees, ldpath_mtimes, graph.debug_print() writemsg("\n", noiselevel=-1) - # Never display the special internal protected_set. - for node in graph: - if isinstance(node, SetArg) and node.name == protected_set_name: - graph.remove(node) - break - pkgs_to_remove = [] if action == "depclean": @@ -926,10 +963,19 @@ def calc_depclean(settings, trees, ldpath_mtimes, cleanlist = create_cleanlist() clean_set = set(cleanlist) - if cleanlist and \ - real_vardb._linkmap is not None and \ - myopts.get("--depclean-lib-check") != "n" and \ - "preserve-libs" not in settings.features: + depclean_lib_check = cleanlist and real_vardb._linkmap is not None and \ + myopts.get("--depclean-lib-check", _DEPCLEAN_LIB_CHECK_DEFAULT) != "n" + preserve_libs = "preserve-libs" in settings.features + preserve_libs_restrict = False + + if depclean_lib_check and preserve_libs: + for pkg in cleanlist: + if "preserve-libs" in pkg.restrict: + preserve_libs_restrict = True + break + + if depclean_lib_check and \ + (preserve_libs_restrict or not preserve_libs): # Check if any of these packages are the sole providers of libraries # with consumers that have not been selected for removal. If so, these @@ -942,6 +988,13 @@ def calc_depclean(settings, trees, ldpath_mtimes, writemsg_level(">>> Checking for lib consumers...\n") for pkg in cleanlist: + + if preserve_libs and "preserve-libs" not in pkg.restrict: + # Any needed libraries will be preserved + # when this package is unmerged, so there's + # no need to account for it here. + continue + pkg_dblink = real_vardb._dblink(pkg.cpv) consumers = {} @@ -1096,7 +1149,8 @@ def calc_depclean(settings, trees, ldpath_mtimes, "installed", root_config, installed=True) if not resolver._add_pkg(pkg, Dependency(parent=consumer_pkg, - priority=UnmergeDepPriority(runtime=True), + priority=UnmergeDepPriority(runtime=True, + runtime_slot_op=True), root=pkg.root)): resolver.display_problems() return 1, [], False, 0 @@ -1133,30 +1187,30 @@ def calc_depclean(settings, trees, ldpath_mtimes, graph = digraph() del cleanlist[:] - dep_keys = ["DEPEND", "RDEPEND", "PDEPEND"] runtime = UnmergeDepPriority(runtime=True) runtime_post = UnmergeDepPriority(runtime_post=True) buildtime = UnmergeDepPriority(buildtime=True) priority_map = { "RDEPEND": runtime, "PDEPEND": runtime_post, + "HDEPEND": buildtime, "DEPEND": buildtime, } for node in clean_set: graph.add(node, None) - for dep_type in dep_keys: - depstr = node.metadata[dep_type] + for dep_type in Package._dep_keys: + depstr = node._metadata[dep_type] if not depstr: continue priority = priority_map[dep_type] if debug: - writemsg_level(_unicode_decode("\nParent: %s\n") \ + writemsg_level("\nParent: %s\n" % (node,), noiselevel=-1, level=logging.DEBUG) - writemsg_level(_unicode_decode( "Depstring: %s\n") \ + writemsg_level( "Depstring: %s\n" % (depstr,), noiselevel=-1, level=logging.DEBUG) - writemsg_level(_unicode_decode( "Priority: %s\n") \ + writemsg_level( "Priority: %s\n" % (priority,), noiselevel=-1, level=logging.DEBUG) try: @@ -1170,7 +1224,7 @@ def calc_depclean(settings, trees, ldpath_mtimes, if debug: writemsg_level("Candidates: [%s]\n" % \ - ', '.join(_unicode_decode("'%s'") % (x,) for x in atoms), + ', '.join("'%s'" % (x,) for x in atoms), noiselevel=-1, level=logging.DEBUG) for atom in atoms: @@ -1184,7 +1238,15 @@ def calc_depclean(settings, trees, ldpath_mtimes, continue for child_node in matches: if child_node in clean_set: - graph.add(child_node, node, priority=priority) + + mypriority = priority.copy() + if atom.slot_operator_built: + if mypriority.buildtime: + mypriority.buildtime_slot_op = True + if mypriority.runtime: + mypriority.runtime_slot_op = True + + graph.add(child_node, node, priority=mypriority) if debug: writemsg_level("\nunmerge digraph:\n\n", @@ -1264,11 +1326,8 @@ def action_deselect(settings, trees, opts, atoms): allow_repo=True, allow_wildcard=True)) for cpv in vardb.match(atom): - slot, = vardb.aux_get(cpv, ["SLOT"]) - if not slot: - slot = "0" - expanded_atoms.add(Atom("%s:%s" % \ - (portage.cpv_getkey(cpv), slot))) + pkg = vardb._pkg_str(cpv, None) + expanded_atoms.add(Atom("%s:%s" % (pkg.cp, pkg.slot))) discard_atoms = set() for atom in world_set: @@ -1287,12 +1346,21 @@ def action_deselect(settings, trees, opts, atoms): break if discard_atoms: for atom in sorted(discard_atoms): + if pretend: - print(">>> Would remove %s from \"world\" favorites file..." % \ - colorize("INFORM", str(atom))) + action_desc = "Would remove" + else: + action_desc = "Removing" + + if atom.startswith(SETPREFIX): + filename = "world_sets" else: - print(">>> Removing %s from \"world\" favorites file..." % \ - colorize("INFORM", str(atom))) + filename = "world" + + writemsg_stdout( + ">>> %s %s from \"%s\" favorites file...\n" % + (action_desc, colorize("INFORM", _unicode(atom)), + filename), noiselevel=-1) if '--ask' in opts: prompt = "Would you like to remove these " + \ @@ -1330,10 +1398,90 @@ class _info_pkgs_ver(object): def action_info(settings, trees, myopts, myfiles): + # See if we can find any packages installed matching the strings + # passed on the command line + mypkgs = [] + eroot = settings['EROOT'] + vardb = trees[eroot]["vartree"].dbapi + portdb = trees[eroot]['porttree'].dbapi + bindb = trees[eroot]["bintree"].dbapi + for x in myfiles: + any_match = False + cp_exists = bool(vardb.match(x.cp)) + installed_match = vardb.match(x) + for installed in installed_match: + mypkgs.append((installed, "installed")) + any_match = True + + if any_match: + continue + + for db, pkg_type in ((portdb, "ebuild"), (bindb, "binary")): + if pkg_type == "binary" and "--usepkg" not in myopts: + continue + + # Use match instead of cp_list, to account for old-style virtuals. + if not cp_exists and db.match(x.cp): + cp_exists = True + # Search for masked packages too. + if not cp_exists and hasattr(db, "xmatch") and \ + db.xmatch("match-all", x.cp): + cp_exists = True + + matches = db.match(x) + matches.reverse() + for match in matches: + if pkg_type == "binary": + if db.bintree.isremote(match): + continue + auxkeys = ["EAPI", "DEFINED_PHASES"] + metadata = dict(zip(auxkeys, db.aux_get(match, auxkeys))) + if metadata["EAPI"] not in ("0", "1", "2", "3") and \ + "info" in metadata["DEFINED_PHASES"].split(): + mypkgs.append((match, pkg_type)) + break + + if not cp_exists: + xinfo = '"%s"' % x.unevaluated_atom + # Discard null/ from failed cpv_expand category expansion. + xinfo = xinfo.replace("null/", "") + if settings["ROOT"] != "/": + xinfo = "%s for %s" % (xinfo, eroot) + writemsg("\nemerge: there are no ebuilds to satisfy %s.\n" % + colorize("INFORM", xinfo), noiselevel=-1) + + if myopts.get("--misspell-suggestions", "y") != "n": + + writemsg("\nemerge: searching for similar names..." + , noiselevel=-1) + + dbs = [vardb] + #if "--usepkgonly" not in myopts: + dbs.append(portdb) + if "--usepkg" in myopts: + dbs.append(bindb) + + matches = similar_name_search(dbs, x) + + if len(matches) == 1: + writemsg("\nemerge: Maybe you meant " + matches[0] + "?\n" + , noiselevel=-1) + elif len(matches) > 1: + writemsg( + "\nemerge: Maybe you meant any of these: %s?\n" % \ + (", ".join(matches),), noiselevel=-1) + else: + # Generally, this would only happen if + # all dbapis are empty. + writemsg(" nothing similar found.\n" + , noiselevel=-1) + + return 1 + output_buffer = [] append = output_buffer.append root_config = trees[settings['EROOT']]['root_config'] - running_eroot = trees._running_eroot + chost = settings.get("CHOST") append(getportageversion(settings["PORTDIR"], None, settings.profile_path, settings["CHOST"], @@ -1347,6 +1495,18 @@ def action_info(settings, trees, myopts, myfiles): append(header_width * "=") append("System uname: %s" % (platform.platform(aliased=1),)) + vm_info = get_vm_info() + if "ram.total" in vm_info: + line = "%-9s %10d total" % ("KiB Mem:", vm_info["ram.total"] / 1024) + if "ram.free" in vm_info: + line += ",%10d free" % (vm_info["ram.free"] / 1024,) + append(line) + if "swap.total" in vm_info: + line = "%-9s %10d total" % ("KiB Swap:", vm_info["swap.total"] / 1024) + if "swap.free" in vm_info: + line += ",%10d free" % (vm_info["swap.free"] / 1024,) + append(line) + lastSync = portage.grabfile(os.path.join( settings["PORTDIR"], "metadata", "timestamp.chk")) if lastSync: @@ -1355,6 +1515,23 @@ def action_info(settings, trees, myopts, myfiles): lastSync = "Unknown" append("Timestamp of tree: %s" % (lastSync,)) + ld_names = [] + if chost: + ld_names.append(chost + "-ld") + ld_names.append("ld") + for name in ld_names: + try: + proc = subprocess.Popen([name, "--version"], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + except OSError: + pass + else: + output = _unicode_decode(proc.communicate()[0]).splitlines() + proc.wait() + if proc.wait() == os.EX_OK and output: + append("ld %s" % (output[0])) + break + try: proc = subprocess.Popen(["distcc", "--version"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) @@ -1391,7 +1568,6 @@ def action_info(settings, trees, myopts, myfiles): "sys-devel/binutils", "sys-devel/libtool", "dev-lang/python"] myvars += portage.util.grabfile(settings["PORTDIR"]+"/profiles/info_pkgs") atoms = [] - vardb = trees[running_eroot]['vartree'].dbapi for x in myvars: try: x = Atom(x) @@ -1404,7 +1580,6 @@ def action_info(settings, trees, myopts, myfiles): myvars = sorted(set(atoms)) - portdb = trees[running_eroot]['porttree'].dbapi main_repo = portdb.getRepositoryName(portdb.porttree_root) cp_map = {} cp_max_len = 0 @@ -1456,11 +1631,11 @@ def action_info(settings, trees, myopts, myfiles): append("Repositories: %s" % \ " ".join(repo.name for repo in repos)) - if _ENABLE_SET_CONFIG: + installed_sets = sorted(s for s in + root_config.sets['selected'].getNonAtoms() if s.startswith(SETPREFIX)) + if installed_sets: sets_line = "Installed sets: " - sets_line += ", ".join(s for s in \ - sorted(root_config.sets['selected'].getNonAtoms()) \ - if s.startswith(SETPREFIX)) + sets_line += ", ".join(installed_sets) append(sets_line) if "--verbose" in myopts: @@ -1471,7 +1646,7 @@ def action_info(settings, trees, myopts, myfiles): 'PORTDIR_OVERLAY', 'PORTAGE_BUNZIP2_COMMAND', 'PORTAGE_BZIP2_COMMAND', 'USE', 'CHOST', 'CFLAGS', 'CXXFLAGS', - 'ACCEPT_KEYWORDS', 'ACCEPT_LICENSE', 'SYNC', 'FEATURES', + 'ACCEPT_KEYWORDS', 'ACCEPT_LICENSE', 'FEATURES', 'EMERGE_DEFAULT_OPTS'] myvars.extend(portage.util.grabfile(settings["PORTDIR"]+"/profiles/info_vars")) @@ -1517,40 +1692,7 @@ def action_info(settings, trees, myopts, myfiles): append("") writemsg_stdout("\n".join(output_buffer), noiselevel=-1) - - # See if we can find any packages installed matching the strings - # passed on the command line - mypkgs = [] - eroot = settings['EROOT'] - vardb = trees[eroot]["vartree"].dbapi - portdb = trees[eroot]['porttree'].dbapi - bindb = trees[eroot]["bintree"].dbapi - for x in myfiles: - match_found = False - installed_match = vardb.match(x) - for installed in installed_match: - mypkgs.append((installed, "installed")) - match_found = True - - if match_found: - continue - - for db, pkg_type in ((portdb, "ebuild"), (bindb, "binary")): - if pkg_type == "binary" and "--usepkg" not in myopts: - continue - - matches = db.match(x) - matches.reverse() - for match in matches: - if pkg_type == "binary": - if db.bintree.isremote(match): - continue - auxkeys = ["EAPI", "DEFINED_PHASES"] - metadata = dict(zip(auxkeys, db.aux_get(match, auxkeys))) - if metadata["EAPI"] not in ("0", "1", "2", "3") and \ - "info" in metadata["DEFINED_PHASES"].split(): - mypkgs.append((match, pkg_type)) - break + del output_buffer[:] # If some packages were found... if mypkgs: @@ -1564,11 +1706,15 @@ def action_info(settings, trees, myopts, myfiles): # Loop through each package # Only print settings if they differ from global settings header_title = "Package Settings" - print(header_width * "=") - print(header_title.rjust(int(header_width/2 + len(header_title)/2))) - print(header_width * "=") - from portage.output import EOutput - out = EOutput() + append(header_width * "=") + append(header_title.rjust(int(header_width/2 + len(header_title)/2))) + append(header_width * "=") + append("") + writemsg_stdout("\n".join(output_buffer), + noiselevel=-1) + del output_buffer[:] + + out = portage.output.EOutput() for mypkg in mypkgs: cpv = mypkg[0] pkg_type = mypkg[1] @@ -1586,28 +1732,32 @@ def action_info(settings, trees, myopts, myfiles): root_config=root_config, type_name=pkg_type) if pkg_type == "installed": - print("\n%s was built with the following:" % \ + append("\n%s was built with the following:" % \ colorize("INFORM", str(pkg.cpv))) elif pkg_type == "ebuild": - print("\n%s would be build with the following:" % \ + append("\n%s would be build with the following:" % \ colorize("INFORM", str(pkg.cpv))) elif pkg_type == "binary": - print("\n%s (non-installed binary) was built with the following:" % \ + append("\n%s (non-installed binary) was built with the following:" % \ colorize("INFORM", str(pkg.cpv))) - writemsg_stdout('%s\n' % pkg_use_display(pkg, myopts), - noiselevel=-1) + append('%s' % pkg_use_display(pkg, myopts)) if pkg_type == "installed": for myvar in mydesiredvars: if metadata[myvar].split() != settings.get(myvar, '').split(): - print("%s=\"%s\"" % (myvar, metadata[myvar])) - print() + append("%s=\"%s\"" % (myvar, metadata[myvar])) + append("") + append("") + writemsg_stdout("\n".join(output_buffer), + noiselevel=-1) + del output_buffer[:] if metadata['DEFINED_PHASES']: if 'info' not in metadata['DEFINED_PHASES'].split(): continue - print(">>> Attempting to run pkg_info() for '%s'" % pkg.cpv) + writemsg_stdout(">>> Attempting to run pkg_info() for '%s'\n" + % pkg.cpv, noiselevel=-1) if pkg_type == "installed": ebuildpath = vardb.findname(pkg.cpv) @@ -1834,6 +1984,7 @@ def action_metadata(settings, portdb, myopts, porttrees=None): print() signal.signal(signal.SIGWINCH, signal.SIG_DFL) + portdb.flush_cache() sys.stdout.flush() os.umask(old_umask) @@ -1843,35 +1994,12 @@ def action_regen(settings, portdb, max_jobs, max_load): #regenerate cache entries sys.stdout.flush() - regen = MetadataRegen(portdb, max_jobs=max_jobs, max_load=max_load) - received_signal = [] + regen = MetadataRegen(portdb, max_jobs=max_jobs, + max_load=max_load, main=True) - def emergeexitsig(signum, frame): - signal.signal(signal.SIGINT, signal.SIG_IGN) - signal.signal(signal.SIGTERM, signal.SIG_IGN) - portage.util.writemsg("\n\nExiting on signal %(signal)s\n" % \ - {"signal":signum}) - regen.terminate() - received_signal.append(128 + signum) - - earlier_sigint_handler = signal.signal(signal.SIGINT, emergeexitsig) - earlier_sigterm_handler = signal.signal(signal.SIGTERM, emergeexitsig) - - try: - regen.run() - finally: - # Restore previous handlers - if earlier_sigint_handler is not None: - signal.signal(signal.SIGINT, earlier_sigint_handler) - else: - signal.signal(signal.SIGINT, signal.SIG_DFL) - if earlier_sigterm_handler is not None: - signal.signal(signal.SIGTERM, earlier_sigterm_handler) - else: - signal.signal(signal.SIGTERM, signal.SIG_DFL) - - if received_signal: - sys.exit(received_signal[0]) + signum = run_main_scheduler(regen) + if signum is not None: + sys.exit(128 + signum) portage.writemsg_stdout("done!\n") return regen.returncode @@ -1892,37 +2020,110 @@ def action_search(root_config, myopts, myfiles, spinner): sys.exit(1) searchinstance.output() -def action_sync(settings, trees, mtimedb, myopts, myaction): +def action_sync(emerge_config, trees=DeprecationWarning, + mtimedb=DeprecationWarning, opts=DeprecationWarning, + action=DeprecationWarning): + + if not isinstance(emerge_config, _emerge_config): + warnings.warn("_emerge.actions.action_sync() now expects " + "an _emerge_config instance as the first parameter", + DeprecationWarning, stacklevel=2) + emerge_config = load_emerge_config( + action=action, args=[], trees=trees, opts=opts) + + xterm_titles = "notitles" not in \ + emerge_config.target_config.settings.features + emergelog(xterm_titles, " === sync") + + selected_repos = [] + unknown_repo_names = [] + missing_sync_type = [] + if emerge_config.args: + for repo_name in emerge_config.args: + try: + repo = emerge_config.target_config.settings.repositories[repo_name] + except KeyError: + unknown_repo_names.append(repo_name) + else: + selected_repos.append(repo) + if repo.sync_type is None: + missing_sync_type.append(repo) + + if unknown_repo_names: + writemsg_level("!!! %s\n" % _("Unknown repo(s): %s") % + " ".join(unknown_repo_names), + level=logging.ERROR, noiselevel=-1) + + if missing_sync_type: + writemsg_level("!!! %s\n" % + _("Missing sync-type for repo(s): %s") % + " ".join(repo.name for repo in missing_sync_type), + level=logging.ERROR, noiselevel=-1) + + if unknown_repo_names or missing_sync_type: + return 1 + + else: + selected_repos.extend(emerge_config.target_config.settings.repositories) + + for repo in selected_repos: + if repo.sync_type is not None: + returncode = _sync_repo(emerge_config, repo) + if returncode != os.EX_OK: + return returncode + + # Reload the whole config from scratch. + portage._sync_disabled_warnings = False + load_emerge_config(emerge_config=emerge_config) + adjust_configs(emerge_config.opts, emerge_config.trees) + + if emerge_config.opts.get('--package-moves') != 'n' and \ + _global_updates(emerge_config.trees, + emerge_config.target_config.mtimedb["updates"], + quiet=("--quiet" in emerge_config.opts)): + emerge_config.target_config.mtimedb.commit() + # Reload the whole config from scratch. + load_emerge_config(emerge_config=emerge_config) + adjust_configs(emerge_config.opts, emerge_config.trees) + + mybestpv = emerge_config.target_config.trees['porttree'].dbapi.xmatch( + "bestmatch-visible", portage.const.PORTAGE_PACKAGE_ATOM) + mypvs = portage.best( + emerge_config.target_config.trees['vartree'].dbapi.match( + portage.const.PORTAGE_PACKAGE_ATOM)) + + chk_updated_cfg_files(emerge_config.target_config.root, + portage.util.shlex_split( + emerge_config.target_config.settings.get("CONFIG_PROTECT", ""))) + + if mybestpv != mypvs and "--quiet" not in emerge_config.opts: + print() + print(warn(" * ")+bold("An update to portage is available.")+" It is _highly_ recommended") + print(warn(" * ")+"that you update portage now, before any other packages are updated.") + print() + print(warn(" * ")+"To update portage, run 'emerge --oneshot portage' now.") + print() + + display_news_notification(emerge_config.target_config, emerge_config.opts) + return os.EX_OK + +def _sync_repo(emerge_config, repo): + settings, trees, mtimedb = emerge_config + myopts = emerge_config.opts enter_invalid = '--ask-enter-invalid' in myopts xterm_titles = "notitles" not in settings.features - emergelog(xterm_titles, " === sync") - portdb = trees[settings['EROOT']]['porttree'].dbapi - myportdir = portdb.porttree_root - if not myportdir: - myportdir = settings.get('PORTDIR', '') - if myportdir and myportdir.strip(): - myportdir = os.path.realpath(myportdir) - else: - myportdir = None + msg = ">>> Synchronization of repository '%s' located in '%s'..." % (repo.name, repo.location) + emergelog(xterm_titles, msg) + writemsg_level(msg + "\n") out = portage.output.EOutput() - global_config_path = GLOBAL_CONFIG_PATH - if settings['EPREFIX']: - global_config_path = os.path.join(settings['EPREFIX'], - GLOBAL_CONFIG_PATH.lstrip(os.sep)) - if not myportdir: - sys.stderr.write("!!! PORTDIR is undefined. " + \ - "Is %s/make.globals missing?\n" % global_config_path) - sys.exit(1) - if myportdir[-1]=="/": - myportdir=myportdir[:-1] try: - st = os.stat(myportdir) + st = os.stat(repo.location) except OSError: st = None if st is None: - print(">>>",myportdir,"not found, creating it.") - portage.util.ensure_dirs(myportdir, mode=0o755) - st = os.stat(myportdir) + print(">>> '%s' not found, creating it." % repo.location) + portage.util.ensure_dirs(repo.location, mode=0o755) + st = os.stat(repo.location) usersync_uid = None spawn_kwargs = {} @@ -1955,59 +2156,51 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): if rval != os.EX_OK: return rval - syncuri = settings.get("SYNC", "").strip() - if not syncuri: - writemsg_level("!!! SYNC is undefined. " + \ - "Is %s/make.globals missing?\n" % global_config_path, - noiselevel=-1, level=logging.ERROR) - return 1 + syncuri = repo.sync_uri - vcs_dirs = frozenset([".git", ".svn", "CVS", ".hg"]) - vcs_dirs = vcs_dirs.intersection(os.listdir(myportdir)) + vcs_dirs = frozenset(VCS_DIRS) + vcs_dirs = vcs_dirs.intersection(os.listdir(repo.location)) os.umask(0o022) dosyncuri = syncuri updatecache_flg = False - git = False - if myaction == "metadata": - print("skipping sync") - updatecache_flg = True - elif ".git" in vcs_dirs: + if repo.sync_type == "git": # Update existing git repository, and ignore the syncuri. We are # going to trust the user and assume that the user is in the branch # that he/she wants updated. We'll let the user manage branches with # git directly. if portage.process.find_binary("git") is None: msg = ["Command not found: git", - "Type \"emerge dev-util/git\" to enable git support."] + "Type \"emerge %s\" to enable git support." % portage.const.GIT_PACKAGE_ATOM] for l in msg: writemsg_level("!!! %s\n" % l, level=logging.ERROR, noiselevel=-1) return 1 - msg = ">>> Starting git pull in %s..." % myportdir + msg = ">>> Starting git pull in %s..." % repo.location emergelog(xterm_titles, msg ) writemsg_level(msg + "\n") exitcode = portage.process.spawn_bash("cd %s ; git pull" % \ - (portage._shell_quote(myportdir),), **spawn_kwargs) + (portage._shell_quote(repo.location),), + **portage._native_kwargs(spawn_kwargs)) if exitcode != os.EX_OK: - msg = "!!! git pull error in %s." % myportdir + msg = "!!! git pull error in %s." % repo.location emergelog(xterm_titles, msg) writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1) return exitcode - msg = ">>> Git pull in %s successful" % myportdir + msg = ">>> Git pull in %s successful" % repo.location emergelog(xterm_titles, msg) writemsg_level(msg + "\n") - git = True - elif syncuri[:8]=="rsync://" or syncuri[:6]=="ssh://": + elif repo.sync_type == "rsync": for vcs_dir in vcs_dirs: writemsg_level(("!!! %s appears to be under revision " + \ "control (contains %s).\n!!! Aborting rsync sync.\n") % \ - (myportdir, vcs_dir), level=logging.ERROR, noiselevel=-1) + (repo.location, vcs_dir), level=logging.ERROR, noiselevel=-1) return 1 - if not os.path.exists("/usr/bin/rsync"): + rsync_binary = portage.process.find_binary("rsync") + if rsync_binary is None: print("!!! /usr/bin/rsync does not exist, so rsync support is disabled.") - print("!!! Type \"emerge net-misc/rsync\" to enable rsync support.") - sys.exit(1) + print("!!! Type \"emerge %s\" to enable rsync support." % portage.const.RSYNC_PACKAGE_ATOM) + return os.EX_UNAVAILABLE mytimeout=180 rsync_opts = [] @@ -2019,6 +2212,7 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): "--safe-links", # Ignore links outside of tree "--perms", # Preserve permissions "--times", # Preserive mod times + "--omit-dir-times", "--compress", # Compress the data transmitted "--force", # Force deletion on non-empty dirs "--whole-file", # Don't do block transfers, only entire files @@ -2081,14 +2275,14 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): # Real local timestamp file. servertimestampfile = os.path.join( - myportdir, "metadata", "timestamp.chk") + repo.location, "metadata", "timestamp.chk") content = portage.util.grabfile(servertimestampfile) mytimestamp = 0 if content: try: mytimestamp = time.mktime(time.strptime(content[0], - "%a, %d %b %Y %H:%M:%S +0000")) + TIMESTAMP_FORMAT)) except (OverflowError, ValueError): pass del content @@ -2112,9 +2306,12 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?", syncuri, maxsplit=4)[1:5] except ValueError: - writemsg_level("!!! SYNC is invalid: %s\n" % syncuri, + writemsg_level("!!! sync-uri is invalid: %s\n" % syncuri, noiselevel=-1, level=logging.ERROR) return 1 + + ssh_opts = settings.get("PORTAGE_SSH_OPTS") + if port is None: port="" if user_name is None: @@ -2230,7 +2427,10 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): if mytimestamp != 0 and "--quiet" not in myopts: print(">>> Checking server timestamp ...") - rsynccommand = ["/usr/bin/rsync"] + rsync_opts + extra_rsync_opts + rsynccommand = [rsync_binary] + rsync_opts + extra_rsync_opts + + if proto == 'ssh' and ssh_opts: + rsynccommand.append("--rsh=ssh " + ssh_opts) if "--debug" in myopts: print(rsynccommand) @@ -2276,7 +2476,8 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): rsync_initial_timeout) mypids.extend(portage.process.spawn( - mycommand, returnpid=True, **spawn_kwargs)) + mycommand, returnpid=True, + **portage._native_kwargs(spawn_kwargs))) exitcode = os.waitpid(mypids[0], 0)[1] if usersync_uid is not None: portage.util.apply_permissions(tmpservertimestampfile, @@ -2306,12 +2507,11 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): exitcode = (exitcode & 0xff) << 8 else: exitcode = exitcode >> 8 - if mypids: - portage.process.spawned_pids.remove(mypids[0]) + if content: try: servertimestamp = time.mktime(time.strptime( - content[0], "%a, %d %b %Y %H:%M:%S +0000")) + content[0], TIMESTAMP_FORMAT)) except (OverflowError, ValueError): pass del mycommand, mypids, content @@ -2327,7 +2527,7 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): print(">>> In order to force sync, remove '%s'." % servertimestampfile) print(">>>") print() - sys.exit(0) + return os.EX_OK elif (servertimestamp != 0) and (servertimestamp < mytimestamp): emergelog(xterm_titles, ">>> Server out of date: %s" % dosyncuri) @@ -2341,8 +2541,33 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): exitcode = SERVER_OUT_OF_DATE elif (servertimestamp == 0) or (servertimestamp > mytimestamp): # actual sync - mycommand = rsynccommand + [dosyncuri+"/", myportdir] - exitcode = portage.process.spawn(mycommand, **spawn_kwargs) + mycommand = rsynccommand + [dosyncuri+"/", repo.location] + exitcode = None + try: + exitcode = portage.process.spawn(mycommand, + **portage._native_kwargs(spawn_kwargs)) + finally: + if exitcode is None: + # interrupted + exitcode = 128 + signal.SIGINT + + # 0 Success + # 1 Syntax or usage error + # 2 Protocol incompatibility + # 5 Error starting client-server protocol + # 35 Timeout waiting for daemon connection + if exitcode not in (0, 1, 2, 5, 35): + # If the exit code is not among those listed above, + # then we may have a partial/inconsistent sync + # state, so our previously read timestamp as well + # as the corresponding file can no longer be + # trusted. + mytimestamp = 0 + try: + os.unlink(servertimestampfile) + except OSError: + pass + if exitcode in [0,1,3,4,11,14,20,21]: break elif exitcode in [1,3,4,11,14,20,21]: @@ -2368,23 +2593,23 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): if (exitcode==0): emergelog(xterm_titles, "=== Sync completed with %s" % dosyncuri) elif exitcode == SERVER_OUT_OF_DATE: - sys.exit(1) + return 1 elif exitcode == EXCEEDED_MAX_RETRIES: sys.stderr.write( ">>> Exceeded PORTAGE_RSYNC_RETRIES: %s\n" % maxretries) - sys.exit(1) + return 1 elif (exitcode>0): msg = [] if exitcode==1: msg.append("Rsync has reported that there is a syntax error. Please ensure") - msg.append("that your SYNC statement is proper.") - msg.append("SYNC=" + settings["SYNC"]) + msg.append("that sync-uri attribute for repository '%s' is proper." % repo.name) + msg.append("sync-uri: '%s'" % repo.sync_uri) elif exitcode==11: msg.append("Rsync has reported that there is a File IO error. Normally") msg.append("this means your disk is full, but can be caused by corruption") - msg.append("on the filesystem that contains PORTDIR. Please investigate") + msg.append("on the filesystem that contains repository '%s'. Please investigate" % repo.name) msg.append("and try again after the problem has been fixed.") - msg.append("PORTDIR=" + settings["PORTDIR"]) + msg.append("Location of repository: '%s'" % repo.location) elif exitcode==20: msg.append("Rsync was killed before it finished.") else: @@ -2395,115 +2620,76 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): msg.append("(and possibly your system's filesystem) configuration.") for line in msg: out.eerror(line) - sys.exit(exitcode) - elif syncuri[:6]=="cvs://": + return exitcode + elif repo.sync_type == "cvs": if not os.path.exists("/usr/bin/cvs"): print("!!! /usr/bin/cvs does not exist, so CVS support is disabled.") - print("!!! Type \"emerge dev-vcs/cvs\" to enable CVS support.") - sys.exit(1) - cvsroot=syncuri[6:] - cvsdir=os.path.dirname(myportdir) - if not os.path.exists(myportdir+"/CVS"): + print("!!! Type \"emerge %s\" to enable CVS support." % portage.const.CVS_PACKAGE_ATOM) + return os.EX_UNAVAILABLE + cvs_root = syncuri + if cvs_root.startswith("cvs://"): + cvs_root = cvs_root[6:] + if not os.path.exists(os.path.join(repo.location, "CVS")): #initial checkout print(">>> Starting initial cvs checkout with "+syncuri+"...") - if os.path.exists(cvsdir+"/gentoo-x86"): - print("!!! existing",cvsdir+"/gentoo-x86 directory; exiting.") - sys.exit(1) try: - os.rmdir(myportdir) + os.rmdir(repo.location) except OSError as e: if e.errno != errno.ENOENT: sys.stderr.write( - "!!! existing '%s' directory; exiting.\n" % myportdir) - sys.exit(1) + "!!! existing '%s' directory; exiting.\n" % repo.location) + return 1 del e if portage.process.spawn_bash( - "cd %s; exec cvs -z0 -d %s co -P gentoo-x86" % \ - (portage._shell_quote(cvsdir), portage._shell_quote(cvsroot)), - **spawn_kwargs) != os.EX_OK: + "cd %s; exec cvs -z0 -d %s co -P -d %s %s" % + (portage._shell_quote(os.path.dirname(repo.location)), portage._shell_quote(cvs_root), + portage._shell_quote(os.path.basename(repo.location)), portage._shell_quote(repo.sync_cvs_repo)), + **portage._native_kwargs(spawn_kwargs)) != os.EX_OK: print("!!! cvs checkout error; exiting.") - sys.exit(1) - os.rename(os.path.join(cvsdir, "gentoo-x86"), myportdir) + return 1 else: #cvs update print(">>> Starting cvs update with "+syncuri+"...") retval = portage.process.spawn_bash( "cd %s; exec cvs -z0 -q update -dP" % \ - (portage._shell_quote(myportdir),), **spawn_kwargs) + (portage._shell_quote(repo.location),), + **portage._native_kwargs(spawn_kwargs)) if retval != os.EX_OK: writemsg_level("!!! cvs update error; exiting.\n", noiselevel=-1, level=logging.ERROR) - sys.exit(retval) + return retval dosyncuri = syncuri - else: - writemsg_level("!!! Unrecognized protocol: SYNC='%s'\n" % (syncuri,), - noiselevel=-1, level=logging.ERROR) - return 1 # Reload the whole config from scratch. - settings, trees, mtimedb = load_emerge_config(trees=trees) - adjust_configs(myopts, trees) - root_config = trees[settings['EROOT']]['root_config'] + settings, trees, mtimedb = load_emerge_config(emerge_config=emerge_config) + adjust_configs(emerge_config.opts, emerge_config.trees) portdb = trees[settings['EROOT']]['porttree'].dbapi - if git: + if repo.sync_type == "git": # NOTE: Do this after reloading the config, in case # it did not exist prior to sync, so that the config # and portdb properly account for its existence. - exitcode = git_sync_timestamps(portdb, myportdir) + exitcode = git_sync_timestamps(portdb, repo.location) if exitcode == os.EX_OK: updatecache_flg = True - if updatecache_flg and \ - myaction != "metadata" and \ - "metadata-transfer" not in settings.features: + if updatecache_flg and "metadata-transfer" not in settings.features: updatecache_flg = False if updatecache_flg and \ - os.path.exists(os.path.join(myportdir, 'metadata', 'cache')): + os.path.exists(os.path.join(repo.location, 'metadata', 'cache')): - # Only update cache for myportdir since that's + # Only update cache for repo.location since that's # the only one that's been synced here. - action_metadata(settings, portdb, myopts, porttrees=[myportdir]) + action_metadata(settings, portdb, myopts, porttrees=[repo.location]) - if myopts.get('--package-moves') != 'n' and \ - _global_updates(trees, mtimedb["updates"], quiet=("--quiet" in myopts)): - mtimedb.commit() - # Reload the whole config from scratch. - settings, trees, mtimedb = load_emerge_config(trees=trees) - adjust_configs(myopts, trees) - portdb = trees[settings['EROOT']]['porttree'].dbapi - root_config = trees[settings['EROOT']]['root_config'] - - mybestpv = portdb.xmatch("bestmatch-visible", - portage.const.PORTAGE_PACKAGE_ATOM) - mypvs = portage.best( - trees[settings['EROOT']]['vartree'].dbapi.match( - portage.const.PORTAGE_PACKAGE_ATOM)) - - chk_updated_cfg_files(settings["EROOT"], - portage.util.shlex_split(settings.get("CONFIG_PROTECT", ""))) - - if myaction != "metadata": - postsync = os.path.join(settings["PORTAGE_CONFIGROOT"], - portage.USER_CONFIG_PATH, "bin", "post_sync") - if os.access(postsync, os.X_OK): - retval = portage.process.spawn( - [postsync, dosyncuri], env=settings.environ()) - if retval != os.EX_OK: - writemsg_level( - " %s spawn failed of %s\n" % (bad("*"), postsync,), - level=logging.ERROR, noiselevel=-1) - - if(mybestpv != mypvs) and not "--quiet" in myopts: - print() - print(warn(" * ")+bold("An update to portage is available.")+" It is _highly_ recommended") - print(warn(" * ")+"that you update portage now, before any other packages are updated.") - print() - print(warn(" * ")+"To update portage, run 'emerge portage' now.") - print() + postsync = os.path.join(settings["PORTAGE_CONFIGROOT"], portage.USER_CONFIG_PATH, "bin", "post_sync") + if os.access(postsync, os.X_OK): + retval = portage.process.spawn([postsync, dosyncuri], env=settings.environ()) + if retval != os.EX_OK: + writemsg_level(" %s spawn failed of %s\n" % (bad("*"), postsync,), + level=logging.ERROR, noiselevel=-1) - display_news_notification(root_config, myopts) return os.EX_OK def action_uninstall(settings, trees, ldpath_mtimes, @@ -2572,16 +2758,30 @@ def action_uninstall(settings, trees, ldpath_mtimes, level=logging.ERROR, noiselevel=-1) return 1 - for cp in vardb.cp_all(): - if extended_cp_match(ext_atom.cp, cp): - atom = cp + for cpv in vardb.cpv_all(): + if portage.match_from_list(ext_atom, [cpv]): + require_metadata = False + atom = portage.cpv_getkey(cpv) + if ext_atom.operator == '=*': + atom = "=" + atom + "-" + \ + portage.versions.cpv_getversion(cpv) if ext_atom.slot: atom += ":" + ext_atom.slot + require_metadata = True if ext_atom.repo: atom += "::" + ext_atom.repo + require_metadata = True + + atom = Atom(atom, allow_repo=True) + if require_metadata: + try: + cpv = vardb._pkg_str(cpv, ext_atom.repo) + except (KeyError, InvalidData): + continue + if not portage.match_from_list(atom, [cpv]): + continue - if vardb.match(atom): - valid_atoms.append(Atom(atom, allow_repo=True)) + valid_atoms.append(atom) else: msg = [] @@ -2611,13 +2811,8 @@ def action_uninstall(settings, trees, ldpath_mtimes, if owners: for cpv in owners: - slot = vardb.aux_get(cpv, ['SLOT'])[0] - if not slot: - # portage now masks packages with missing slot, but it's - # possible that one was installed by an older version - atom = portage.cpv_getkey(cpv) - else: - atom = '%s:%s' % (portage.cpv_getkey(cpv), slot) + pkg = vardb._pkg_str(cpv, None) + atom = '%s:%s' % (pkg.cp, pkg.slot) valid_atoms.append(portage.dep.Atom(atom)) else: writemsg_level(("!!! '%s' is not claimed " + \ @@ -2641,20 +2836,20 @@ def action_uninstall(settings, trees, ldpath_mtimes, if action == 'deselect': return action_deselect(settings, trees, opts, valid_atoms) - # Create a Scheduler for calls to unmerge(), in order to cause - # redirection of ebuild phase output to logs as required for - # options such as --quiet. - sched = Scheduler(settings, trees, None, opts, - spinner, uninstall_only=True) - sched._background = sched._background_mode() - sched._status_display.quiet = True - - if sched._background: - sched.settings.unlock() - sched.settings["PORTAGE_BACKGROUND"] = "1" - sched.settings.backup_changes("PORTAGE_BACKGROUND") - sched.settings.lock() - sched.pkgsettings[eroot] = portage.config(clone=sched.settings) + # Use the same logic as the Scheduler class to trigger redirection + # of ebuild pkg_prerm/postrm phase output to logs as appropriate + # for options such as --jobs, --quiet and --quiet-build. + max_jobs = opts.get("--jobs", 1) + background = (max_jobs is True or max_jobs > 1 or + "--quiet" in opts or opts.get("--quiet-build") == "y") + sched_iface = SchedulerInterface(global_event_loop(), + is_background=lambda: background) + + if background: + settings.unlock() + settings["PORTAGE_BACKGROUND"] = "1" + settings.backup_changes("PORTAGE_BACKGROUND") + settings.lock() if action in ('clean', 'unmerge') or \ (action == 'prune' and "--nodeps" in opts): @@ -2662,10 +2857,11 @@ def action_uninstall(settings, trees, ldpath_mtimes, ordered = action == 'unmerge' rval = unmerge(trees[settings['EROOT']]['root_config'], opts, action, valid_atoms, ldpath_mtimes, ordered=ordered, - scheduler=sched._sched_iface) + scheduler=sched_iface) else: rval = action_depclean(settings, trees, ldpath_mtimes, - opts, action, valid_atoms, spinner, scheduler=sched._sched_iface) + opts, action, valid_atoms, spinner, + scheduler=sched_iface) return rval @@ -2771,6 +2967,10 @@ def adjust_config(myopts, settings): settings["NOCOLOR"] = "true" settings.backup_changes("NOCOLOR") + if "--pkg-format" in myopts: + settings["PORTAGE_BINPKG_FORMAT"] = myopts["--pkg-format"] + settings.backup_changes("PORTAGE_BINPKG_FORMAT") + def display_missing_pkg_set(root_config, set_name): msg = [] @@ -2797,6 +2997,7 @@ def relative_profile_path(portdir, abs_profile): def getportageversion(portdir, _unused, profile, chost, vardb): profilever = None + repositories = vardb.settings.repositories if profile: profilever = relative_profile_path(portdir, profile) if profilever is None: @@ -2807,6 +3008,20 @@ def getportageversion(portdir, _unused, profile, chost, vardb): os.path.join(profile, parent)) if profilever is not None: break + colon = parent.find(":") + if colon != -1: + p_repo_name = parent[:colon] + try: + p_repo_loc = \ + repositories.get_location_for_name(p_repo_name) + except KeyError: + pass + else: + profilever = relative_profile_path(p_repo_loc, + os.path.join(p_repo_loc, 'profiles', + parent[colon+1:])) + if profilever is not None: + break except portage.exception.PortageException: pass @@ -2979,61 +3194,53 @@ def git_sync_timestamps(portdb, portdir): return os.EX_OK -def load_emerge_config(trees=None): +class _emerge_config(SlotObject): + + __slots__ = ('action', 'args', 'opts', + 'running_config', 'target_config', 'trees') + + # Support unpack as tuple, for load_emerge_config backward compatibility. + def __iter__(self): + yield self.target_config.settings + yield self.trees + yield self.target_config.mtimedb + + def __getitem__(self, index): + return list(self)[index] + + def __len__(self): + return 3 + +def load_emerge_config(emerge_config=None, **kargs): + + if emerge_config is None: + emerge_config = _emerge_config(**kargs) + kwargs = {} - for k, envvar in (("config_root", "PORTAGE_CONFIGROOT"), ("target_root", "ROOT")): + for k, envvar in (("config_root", "PORTAGE_CONFIGROOT"), ("target_root", "ROOT"), + ("eprefix", "EPREFIX")): v = os.environ.get(envvar, None) if v and v.strip(): kwargs[k] = v - trees = portage.create_trees(trees=trees, **kwargs) + emerge_config.trees = portage.create_trees(trees=emerge_config.trees, + **portage._native_kwargs(kwargs)) - for root_trees in trees.values(): + for root_trees in emerge_config.trees.values(): settings = root_trees["vartree"].settings settings._init_dirs() setconfig = load_default_config(settings, root_trees) root_trees["root_config"] = RootConfig(settings, root_trees, setconfig) - settings = trees[trees._target_eroot]['vartree'].settings - mtimedbfile = os.path.join(settings['EROOT'], portage.CACHE_PATH, "mtimedb") - mtimedb = portage.MtimeDB(mtimedbfile) - QueryCommand._db = trees - return settings, trees, mtimedb - -def chk_updated_cfg_files(eroot, config_protect): - target_root = eroot - result = list( - portage.util.find_updated_config_files(target_root, config_protect)) - - for x in result: - writemsg_level("\n %s " % (colorize("WARN", "* " + _("IMPORTANT:"))), - level=logging.INFO, noiselevel=-1) - if not x[1]: # it's a protected file - writemsg_level( _("config file '%s' needs updating.\n") % x[0], - level=logging.INFO, noiselevel=-1) - else: # it's a protected dir - if len(x[1]) == 1: - head, tail = os.path.split(x[1][0]) - tail = tail[len("._cfg0000_"):] - fpath = os.path.join(head, tail) - writemsg_level(_("config file '%s' needs updating.\n") % fpath, - level=logging.INFO, noiselevel=-1) - else: - writemsg_level( _("%d config files in '%s' need updating.\n") % \ - (len(x[1]), x[0]), level=logging.INFO, noiselevel=-1) + target_eroot = emerge_config.trees._target_eroot + emerge_config.target_config = \ + emerge_config.trees[target_eroot]['root_config'] + emerge_config.target_config.mtimedb = portage.MtimeDB( + os.path.join(target_eroot, portage.CACHE_PATH, "mtimedb")) + emerge_config.running_config = emerge_config.trees[ + emerge_config.trees._running_eroot]['root_config'] + QueryCommand._db = emerge_config.trees - if result: - print(" "+yellow("*")+ " See the "+colorize("INFORM", _("CONFIGURATION FILES"))\ - + " " + _("section of the") + " " + bold("emerge")) - print(" "+yellow("*")+ " " + _("man page to learn how to update config files.")) - - -def display_news_notification(root_config, myopts): - if "news" not in root_config.settings.features: - return - portdb = root_config.trees["porttree"].dbapi - vardb = root_config.trees["vartree"].dbapi - news_counts = count_unread_news(portdb, vardb) - display_news_notifications(news_counts) + return emerge_config def getgccversion(chost): """ @@ -3089,3 +3296,772 @@ def getgccversion(chost): portage.writemsg(gcc_not_found_error, noiselevel=-1) return "[unavailable]" + +# Warn about features that may confuse users and +# lead them to report invalid bugs. +_emerge_features_warn = frozenset(['keeptemp', 'keepwork']) + +def validate_ebuild_environment(trees): + features_warn = set() + for myroot in trees: + settings = trees[myroot]["vartree"].settings + settings.validate() + features_warn.update( + _emerge_features_warn.intersection(settings.features)) + + if features_warn: + msg = "WARNING: The FEATURES variable contains one " + \ + "or more values that should be disabled under " + \ + "normal circumstances: %s" % " ".join(features_warn) + out = portage.output.EOutput() + for line in textwrap.wrap(msg, 65): + out.ewarn(line) + +def check_procfs(): + procfs_path = '/proc' + if platform.system() not in ("Linux",) or \ + os.path.ismount(procfs_path): + return os.EX_OK + msg = "It seems that %s is not mounted. You have been warned." % procfs_path + writemsg_level("".join("!!! %s\n" % l for l in textwrap.wrap(msg, 70)), + level=logging.ERROR, noiselevel=-1) + return 1 + +def config_protect_check(trees): + for root, root_trees in trees.items(): + settings = root_trees["root_config"].settings + if not settings.get("CONFIG_PROTECT"): + msg = "!!! CONFIG_PROTECT is empty" + if settings["ROOT"] != "/": + msg += " for '%s'" % root + msg += "\n" + writemsg_level(msg, level=logging.WARN, noiselevel=-1) + +def apply_priorities(settings): + ionice(settings) + nice(settings) + +def nice(settings): + try: + os.nice(int(settings.get("PORTAGE_NICENESS", "0"))) + except (OSError, ValueError) as e: + out = portage.output.EOutput() + out.eerror("Failed to change nice value to '%s'" % \ + settings["PORTAGE_NICENESS"]) + out.eerror("%s\n" % str(e)) + +def ionice(settings): + + ionice_cmd = settings.get("PORTAGE_IONICE_COMMAND") + if ionice_cmd: + ionice_cmd = portage.util.shlex_split(ionice_cmd) + if not ionice_cmd: + return + + variables = {"PID" : str(os.getpid())} + cmd = [varexpand(x, mydict=variables) for x in ionice_cmd] + + try: + rval = portage.process.spawn(cmd, env=os.environ) + except portage.exception.CommandNotFound: + # The OS kernel probably doesn't support ionice, + # so return silently. + return + + if rval != os.EX_OK: + out = portage.output.EOutput() + out.eerror("PORTAGE_IONICE_COMMAND returned %d" % (rval,)) + out.eerror("See the make.conf(5) man page for PORTAGE_IONICE_COMMAND usage instructions.") + +def setconfig_fallback(root_config): + setconfig = root_config.setconfig + setconfig._create_default_config() + setconfig._parse(update=True) + root_config.sets = setconfig.getSets() + +def get_missing_sets(root_config): + # emerge requires existence of "world", "selected", and "system" + missing_sets = [] + + for s in ("selected", "system", "world",): + if s not in root_config.sets: + missing_sets.append(s) + + return missing_sets + +def missing_sets_warning(root_config, missing_sets): + if len(missing_sets) > 2: + missing_sets_str = ", ".join('"%s"' % s for s in missing_sets[:-1]) + missing_sets_str += ', and "%s"' % missing_sets[-1] + elif len(missing_sets) == 2: + missing_sets_str = '"%s" and "%s"' % tuple(missing_sets) + else: + missing_sets_str = '"%s"' % missing_sets[-1] + msg = ["emerge: incomplete set configuration, " + \ + "missing set(s): %s" % missing_sets_str] + if root_config.sets: + msg.append(" sets defined: %s" % ", ".join(root_config.sets)) + global_config_path = portage.const.GLOBAL_CONFIG_PATH + if portage.const.EPREFIX: + global_config_path = os.path.join(portage.const.EPREFIX, + portage.const.GLOBAL_CONFIG_PATH.lstrip(os.sep)) + msg.append(" This usually means that '%s'" % \ + (os.path.join(global_config_path, "sets/portage.conf"),)) + msg.append(" is missing or corrupt.") + msg.append(" Falling back to default world and system set configuration!!!") + for line in msg: + writemsg_level(line + "\n", level=logging.ERROR, noiselevel=-1) + +def ensure_required_sets(trees): + warning_shown = False + for root_trees in trees.values(): + missing_sets = get_missing_sets(root_trees["root_config"]) + if missing_sets and not warning_shown: + warning_shown = True + missing_sets_warning(root_trees["root_config"], missing_sets) + if missing_sets: + setconfig_fallback(root_trees["root_config"]) + +def expand_set_arguments(myfiles, myaction, root_config): + retval = os.EX_OK + setconfig = root_config.setconfig + + sets = setconfig.getSets() + + # In order to know exactly which atoms/sets should be added to the + # world file, the depgraph performs set expansion later. It will get + # confused about where the atoms came from if it's not allowed to + # expand them itself. + do_not_expand = myaction is None + newargs = [] + for a in myfiles: + if a in ("system", "world"): + newargs.append(SETPREFIX+a) + else: + newargs.append(a) + myfiles = newargs + del newargs + newargs = [] + + # separators for set arguments + ARG_START = "{" + ARG_END = "}" + + for i in range(0, len(myfiles)): + if myfiles[i].startswith(SETPREFIX): + start = 0 + end = 0 + x = myfiles[i][len(SETPREFIX):] + newset = "" + while x: + start = x.find(ARG_START) + end = x.find(ARG_END) + if start > 0 and start < end: + namepart = x[:start] + argpart = x[start+1:end] + + # TODO: implement proper quoting + args = argpart.split(",") + options = {} + for a in args: + if "=" in a: + k, v = a.split("=", 1) + options[k] = v + else: + options[a] = "True" + setconfig.update(namepart, options) + newset += (x[:start-len(namepart)]+namepart) + x = x[end+len(ARG_END):] + else: + newset += x + x = "" + myfiles[i] = SETPREFIX+newset + + sets = setconfig.getSets() + + # display errors that occurred while loading the SetConfig instance + for e in setconfig.errors: + print(colorize("BAD", "Error during set creation: %s" % e)) + + unmerge_actions = ("unmerge", "prune", "clean", "depclean") + + for a in myfiles: + if a.startswith(SETPREFIX): + s = a[len(SETPREFIX):] + if s not in sets: + display_missing_pkg_set(root_config, s) + return (None, 1) + if s == "installed": + msg = ("The @installed set is deprecated and will soon be " + "removed. Please refer to bug #387059 for details.") + out = portage.output.EOutput() + for line in textwrap.wrap(msg, 50): + out.ewarn(line) + setconfig.active.append(s) + + if do_not_expand: + # Loading sets can be slow, so skip it here, in order + # to allow the depgraph to indicate progress with the + # spinner while sets are loading (bug #461412). + newargs.append(a) + continue + + try: + set_atoms = setconfig.getSetAtoms(s) + except portage.exception.PackageSetNotFound as e: + writemsg_level(("emerge: the given set '%s' " + \ + "contains a non-existent set named '%s'.\n") % \ + (s, e), level=logging.ERROR, noiselevel=-1) + if s in ('world', 'selected') and \ + SETPREFIX + e.value in sets['selected']: + writemsg_level(("Use `emerge --deselect %s%s` to " + "remove this set from world_sets.\n") % + (SETPREFIX, e,), level=logging.ERROR, + noiselevel=-1) + return (None, 1) + if myaction in unmerge_actions and \ + not sets[s].supportsOperation("unmerge"): + writemsg_level("emerge: the given set '%s' does " % s + \ + "not support unmerge operations\n", + level=logging.ERROR, noiselevel=-1) + retval = 1 + elif not set_atoms: + writemsg_level("emerge: '%s' is an empty set\n" % s, + level=logging.INFO, noiselevel=-1) + else: + newargs.extend(set_atoms) + for error_msg in sets[s].errors: + writemsg_level("%s\n" % (error_msg,), + level=logging.ERROR, noiselevel=-1) + else: + newargs.append(a) + return (newargs, retval) + +def repo_name_check(trees): + missing_repo_names = set() + for root_trees in trees.values(): + porttree = root_trees.get("porttree") + if porttree: + portdb = porttree.dbapi + missing_repo_names.update(portdb.getMissingRepoNames()) + if portdb.porttree_root in missing_repo_names and \ + not os.path.exists(os.path.join( + portdb.porttree_root, "profiles")): + # This is normal if $PORTDIR happens to be empty, + # so don't warn about it. + missing_repo_names.remove(portdb.porttree_root) + + # Skip warnings about missing repo_name entries for + # /usr/local/portage (see bug #248603). + try: + missing_repo_names.remove('/usr/local/portage') + except KeyError: + pass + + if missing_repo_names: + msg = [] + msg.append("WARNING: One or more repositories " + \ + "have missing repo_name entries:") + msg.append("") + for p in missing_repo_names: + msg.append("\t%s/profiles/repo_name" % (p,)) + msg.append("") + msg.extend(textwrap.wrap("NOTE: Each repo_name entry " + \ + "should be a plain text file containing a unique " + \ + "name for the repository on the first line.", 70)) + msg.append("\n") + writemsg_level("".join("%s\n" % l for l in msg), + level=logging.WARNING, noiselevel=-1) + + return bool(missing_repo_names) + +def repo_name_duplicate_check(trees): + ignored_repos = {} + for root, root_trees in trees.items(): + if 'porttree' in root_trees: + portdb = root_trees['porttree'].dbapi + if portdb.settings.get('PORTAGE_REPO_DUPLICATE_WARN') != '0': + for repo_name, paths in portdb.getIgnoredRepos(): + k = (root, repo_name, portdb.getRepositoryPath(repo_name)) + ignored_repos.setdefault(k, []).extend(paths) + + if ignored_repos: + msg = [] + msg.append('WARNING: One or more repositories ' + \ + 'have been ignored due to duplicate') + msg.append(' profiles/repo_name entries:') + msg.append('') + for k in sorted(ignored_repos): + msg.append(' %s overrides' % ", ".join(k)) + for path in ignored_repos[k]: + msg.append(' %s' % (path,)) + msg.append('') + msg.extend(' ' + x for x in textwrap.wrap( + "All profiles/repo_name entries must be unique in order " + \ + "to avoid having duplicates ignored. " + \ + "Set PORTAGE_REPO_DUPLICATE_WARN=\"0\" in " + \ + "/etc/portage/make.conf if you would like to disable this warning.")) + msg.append("\n") + writemsg_level(''.join('%s\n' % l for l in msg), + level=logging.WARNING, noiselevel=-1) + + return bool(ignored_repos) + +def run_action(emerge_config): + + # skip global updates prior to sync, since it's called after sync + if emerge_config.action not in ('help', 'info', 'sync', 'version') and \ + emerge_config.opts.get('--package-moves') != 'n' and \ + _global_updates(emerge_config.trees, + emerge_config.target_config.mtimedb["updates"], + quiet=("--quiet" in emerge_config.opts)): + emerge_config.target_config.mtimedb.commit() + # Reload the whole config from scratch. + load_emerge_config(emerge_config=emerge_config) + + xterm_titles = "notitles" not in \ + emerge_config.target_config.settings.features + if xterm_titles: + xtermTitle("emerge") + + if "--digest" in emerge_config.opts: + os.environ["FEATURES"] = os.environ.get("FEATURES","") + " digest" + # Reload the whole config from scratch so that the portdbapi internal + # config is updated with new FEATURES. + load_emerge_config(emerge_config=emerge_config) + + # NOTE: adjust_configs() can map options to FEATURES, so any relevant + # options adjustments should be made prior to calling adjust_configs(). + if "--buildpkgonly" in emerge_config.opts: + emerge_config.opts["--buildpkg"] = True + + if "getbinpkg" in emerge_config.target_config.settings.features: + emerge_config.opts["--getbinpkg"] = True + + if "--getbinpkgonly" in emerge_config.opts: + emerge_config.opts["--getbinpkg"] = True + + if "--getbinpkgonly" in emerge_config.opts: + emerge_config.opts["--usepkgonly"] = True + + if "--getbinpkg" in emerge_config.opts: + emerge_config.opts["--usepkg"] = True + + if "--usepkgonly" in emerge_config.opts: + emerge_config.opts["--usepkg"] = True + + if "--buildpkgonly" in emerge_config.opts: + # --buildpkgonly will not merge anything, so + # it cancels all binary package options. + for opt in ("--getbinpkg", "--getbinpkgonly", + "--usepkg", "--usepkgonly"): + emerge_config.opts.pop(opt, None) + + adjust_configs(emerge_config.opts, emerge_config.trees) + apply_priorities(emerge_config.target_config.settings) + + for fmt in emerge_config.target_config.settings["PORTAGE_BINPKG_FORMAT"].split(): + if not fmt in portage.const.SUPPORTED_BINPKG_FORMATS: + if "--pkg-format" in emerge_config.opts: + problematic="--pkg-format" + else: + problematic="PORTAGE_BINPKG_FORMAT" + + writemsg_level(("emerge: %s is not set correctly. Format " + \ + "'%s' is not supported.\n") % (problematic, fmt), + level=logging.ERROR, noiselevel=-1) + return 1 + + if emerge_config.action == 'version': + writemsg_stdout(getportageversion( + emerge_config.target_config.settings["PORTDIR"], + None, + emerge_config.target_config.settings.profile_path, + emerge_config.target_config.settings["CHOST"], + emerge_config.target_config.trees['vartree'].dbapi) + '\n', + noiselevel=-1) + return 0 + elif emerge_config.action == 'help': + emerge_help() + return 0 + + spinner = stdout_spinner() + if "candy" in emerge_config.target_config.settings.features: + spinner.update = spinner.update_scroll + + if "--quiet" not in emerge_config.opts: + portage.deprecated_profile_check( + settings=emerge_config.target_config.settings) + repo_name_check(emerge_config.trees) + repo_name_duplicate_check(emerge_config.trees) + config_protect_check(emerge_config.trees) + check_procfs() + + for mytrees in emerge_config.trees.values(): + mydb = mytrees["porttree"].dbapi + # Freeze the portdbapi for performance (memoize all xmatch results). + mydb.freeze() + + if emerge_config.action in ('search', None) and \ + "--usepkg" in emerge_config.opts: + # Populate the bintree with current --getbinpkg setting. + # This needs to happen before expand_set_arguments(), in case + # any sets use the bintree. + mytrees["bintree"].populate( + getbinpkgs="--getbinpkg" in emerge_config.opts) + + del mytrees, mydb + + for x in emerge_config.args: + if x.endswith((".ebuild", ".tbz2")) and \ + os.path.exists(os.path.abspath(x)): + print(colorize("BAD", "\n*** emerging by path is broken " + "and may not always work!!!\n")) + break + + if emerge_config.action == "list-sets": + writemsg_stdout("".join("%s\n" % s for s in + sorted(emerge_config.target_config.sets))) + return os.EX_OK + elif emerge_config.action == "check-news": + news_counts = count_unread_news( + emerge_config.target_config.trees["porttree"].dbapi, + emerge_config.target_config.trees["vartree"].dbapi) + if any(news_counts.values()): + display_news_notifications(news_counts) + elif "--quiet" not in emerge_config.opts: + print("", colorize("GOOD", "*"), "No news items were found.") + return os.EX_OK + + ensure_required_sets(emerge_config.trees) + + if emerge_config.action is None and \ + "--resume" in emerge_config.opts and emerge_config.args: + writemsg("emerge: unexpected argument(s) for --resume: %s\n" % + " ".join(emerge_config.args), noiselevel=-1) + return 1 + + # only expand sets for actions taking package arguments + oldargs = emerge_config.args[:] + if emerge_config.action in ("clean", "config", "depclean", + "info", "prune", "unmerge", None): + newargs, retval = expand_set_arguments( + emerge_config.args, emerge_config.action, + emerge_config.target_config) + if retval != os.EX_OK: + return retval + + # Need to handle empty sets specially, otherwise emerge will react + # with the help message for empty argument lists + if oldargs and not newargs: + print("emerge: no targets left after set expansion") + return 0 + + emerge_config.args = newargs + + if "--tree" in emerge_config.opts and \ + "--columns" in emerge_config.opts: + print("emerge: can't specify both of \"--tree\" and \"--columns\".") + return 1 + + if '--emptytree' in emerge_config.opts and \ + '--noreplace' in emerge_config.opts: + writemsg_level("emerge: can't specify both of " + \ + "\"--emptytree\" and \"--noreplace\".\n", + level=logging.ERROR, noiselevel=-1) + return 1 + + if ("--quiet" in emerge_config.opts): + spinner.update = spinner.update_quiet + portage.util.noiselimit = -1 + + if "--fetch-all-uri" in emerge_config.opts: + emerge_config.opts["--fetchonly"] = True + + if "--skipfirst" in emerge_config.opts and \ + "--resume" not in emerge_config.opts: + emerge_config.opts["--resume"] = True + + # Allow -p to remove --ask + if "--pretend" in emerge_config.opts: + emerge_config.opts.pop("--ask", None) + + # forbid --ask when not in a terminal + # note: this breaks `emerge --ask | tee logfile`, but that doesn't work anyway. + if ("--ask" in emerge_config.opts) and (not sys.stdin.isatty()): + portage.writemsg("!!! \"--ask\" should only be used in a terminal. Exiting.\n", + noiselevel=-1) + return 1 + + if emerge_config.target_config.settings.get("PORTAGE_DEBUG", "") == "1": + spinner.update = spinner.update_quiet + portage.util.noiselimit = 0 + if "python-trace" in emerge_config.target_config.settings.features: + portage.debug.set_trace(True) + + if not ("--quiet" in emerge_config.opts): + if '--nospinner' in emerge_config.opts or \ + emerge_config.target_config.settings.get('TERM') == 'dumb' or \ + not sys.stdout.isatty(): + spinner.update = spinner.update_basic + + if "--debug" in emerge_config.opts: + print("myaction", emerge_config.action) + print("myopts", emerge_config.opts) + + if not emerge_config.action and not emerge_config.args and \ + "--resume" not in emerge_config.opts: + emerge_help() + return 1 + + pretend = "--pretend" in emerge_config.opts + fetchonly = "--fetchonly" in emerge_config.opts or \ + "--fetch-all-uri" in emerge_config.opts + buildpkgonly = "--buildpkgonly" in emerge_config.opts + + # check if root user is the current user for the actions where emerge needs this + if portage.data.secpass < 2: + # We've already allowed "--version" and "--help" above. + if "--pretend" not in emerge_config.opts and \ + emerge_config.action not in ("search", "info"): + need_superuser = emerge_config.action in ('clean', 'depclean', + 'deselect', 'prune', 'unmerge') or not \ + (fetchonly or \ + (buildpkgonly and portage.data.secpass >= 1) or \ + emerge_config.action in ("metadata", "regen", "sync")) + if portage.data.secpass < 1 or \ + need_superuser: + if need_superuser: + access_desc = "superuser" + else: + access_desc = "portage group" + # Always show portage_group_warning() when only portage group + # access is required but the user is not in the portage group. + if "--ask" in emerge_config.opts: + writemsg_stdout("This action requires %s access...\n" % \ + (access_desc,), noiselevel=-1) + if portage.data.secpass < 1 and not need_superuser: + portage.data.portage_group_warning() + if userquery("Would you like to add --pretend to options?", + "--ask-enter-invalid" in emerge_config.opts) == "No": + return 128 + signal.SIGINT + emerge_config.opts["--pretend"] = True + emerge_config.opts.pop("--ask") + else: + sys.stderr.write(("emerge: %s access is required\n") \ + % access_desc) + if portage.data.secpass < 1 and not need_superuser: + portage.data.portage_group_warning() + return 1 + + # Disable emergelog for everything except build or unmerge operations. + # This helps minimize parallel emerge.log entries that can confuse log + # parsers like genlop. + disable_emergelog = False + for x in ("--pretend", "--fetchonly", "--fetch-all-uri"): + if x in emerge_config.opts: + disable_emergelog = True + break + if disable_emergelog: + pass + elif emerge_config.action in ("search", "info"): + disable_emergelog = True + elif portage.data.secpass < 1: + disable_emergelog = True + + import _emerge.emergelog + _emerge.emergelog._disable = disable_emergelog + + if not disable_emergelog: + emerge_log_dir = \ + emerge_config.target_config.settings.get('EMERGE_LOG_DIR') + if emerge_log_dir: + try: + # At least the parent needs to exist for the lock file. + portage.util.ensure_dirs(emerge_log_dir) + except portage.exception.PortageException as e: + writemsg_level("!!! Error creating directory for " + \ + "EMERGE_LOG_DIR='%s':\n!!! %s\n" % \ + (emerge_log_dir, e), + noiselevel=-1, level=logging.ERROR) + portage.util.ensure_dirs(_emerge.emergelog._emerge_log_dir) + else: + _emerge.emergelog._emerge_log_dir = emerge_log_dir + else: + _emerge.emergelog._emerge_log_dir = os.path.join(os.sep, + portage.const.EPREFIX.lstrip(os.sep), "var", "log") + portage.util.ensure_dirs(_emerge.emergelog._emerge_log_dir) + + if not "--pretend" in emerge_config.opts: + time_fmt = "%b %d, %Y %H:%M:%S" + if sys.hexversion < 0x3000000: + time_fmt = portage._unicode_encode(time_fmt) + time_str = time.strftime(time_fmt, time.localtime(time.time())) + # Avoid potential UnicodeDecodeError in Python 2, since strftime + # returns bytes in Python 2, and %b may contain non-ascii chars. + time_str = _unicode_decode(time_str, + encoding=_encodings['content'], errors='replace') + emergelog(xterm_titles, "Started emerge on: %s" % time_str) + myelogstr="" + if emerge_config.opts: + opt_list = [] + for opt, arg in emerge_config.opts.items(): + if arg is True: + opt_list.append(opt) + elif isinstance(arg, list): + # arguments like --exclude that use 'append' action + for x in arg: + opt_list.append("%s=%s" % (opt, x)) + else: + opt_list.append("%s=%s" % (opt, arg)) + myelogstr=" ".join(opt_list) + if emerge_config.action: + myelogstr += " --" + emerge_config.action + if oldargs: + myelogstr += " " + " ".join(oldargs) + emergelog(xterm_titles, " *** emerge " + myelogstr) + + oldargs = None + + def emergeexitsig(signum, frame): + signal.signal(signal.SIGTERM, signal.SIG_IGN) + portage.util.writemsg( + "\n\nExiting on signal %(signal)s\n" % {"signal":signum}) + sys.exit(128 + signum) + + signal.signal(signal.SIGTERM, emergeexitsig) + + def emergeexit(): + """This gets out final log message in before we quit.""" + if "--pretend" not in emerge_config.opts: + emergelog(xterm_titles, " *** terminating.") + if xterm_titles: + xtermTitleReset() + portage.atexit_register(emergeexit) + + if emerge_config.action in ("config", "metadata", "regen", "sync"): + if "--pretend" in emerge_config.opts: + sys.stderr.write(("emerge: The '%s' action does " + \ + "not support '--pretend'.\n") % emerge_config.action) + return 1 + + if "sync" == emerge_config.action: + return action_sync(emerge_config) + elif "metadata" == emerge_config.action: + action_metadata(emerge_config.target_config.settings, + emerge_config.target_config.trees['porttree'].dbapi, + emerge_config.opts) + elif emerge_config.action=="regen": + validate_ebuild_environment(emerge_config.trees) + return action_regen(emerge_config.target_config.settings, + emerge_config.target_config.trees['porttree'].dbapi, + emerge_config.opts.get("--jobs"), + emerge_config.opts.get("--load-average")) + # HELP action + elif "config" == emerge_config.action: + validate_ebuild_environment(emerge_config.trees) + action_config(emerge_config.target_config.settings, + emerge_config.trees, emerge_config.opts, emerge_config.args) + + # SEARCH action + elif "search" == emerge_config.action: + validate_ebuild_environment(emerge_config.trees) + action_search(emerge_config.target_config, + emerge_config.opts, emerge_config.args, spinner) + + elif emerge_config.action in \ + ('clean', 'depclean', 'deselect', 'prune', 'unmerge'): + validate_ebuild_environment(emerge_config.trees) + rval = action_uninstall(emerge_config.target_config.settings, + emerge_config.trees, emerge_config.target_config.mtimedb["ldpath"], + emerge_config.opts, emerge_config.action, + emerge_config.args, spinner) + if not (emerge_config.action == 'deselect' or + buildpkgonly or fetchonly or pretend): + post_emerge(emerge_config.action, emerge_config.opts, + emerge_config.args, emerge_config.target_config.root, + emerge_config.trees, emerge_config.target_config.mtimedb, rval) + return rval + + elif emerge_config.action == 'info': + + # Ensure atoms are valid before calling unmerge(). + vardb = emerge_config.target_config.trees['vartree'].dbapi + portdb = emerge_config.target_config.trees['porttree'].dbapi + bindb = emerge_config.target_config.trees['bintree'].dbapi + valid_atoms = [] + for x in emerge_config.args: + if is_valid_package_atom(x, allow_repo=True): + try: + #look at the installed files first, if there is no match + #look at the ebuilds, since EAPI 4 allows running pkg_info + #on non-installed packages + valid_atom = dep_expand(x, mydb=vardb) + if valid_atom.cp.split("/")[0] == "null": + valid_atom = dep_expand(x, mydb=portdb) + + if valid_atom.cp.split("/")[0] == "null" and \ + "--usepkg" in emerge_config.opts: + valid_atom = dep_expand(x, mydb=bindb) + + valid_atoms.append(valid_atom) + + except portage.exception.AmbiguousPackageName as e: + msg = "The short ebuild name \"" + x + \ + "\" is ambiguous. Please specify " + \ + "one of the following " + \ + "fully-qualified ebuild names instead:" + for line in textwrap.wrap(msg, 70): + writemsg_level("!!! %s\n" % (line,), + level=logging.ERROR, noiselevel=-1) + for i in e.args[0]: + writemsg_level(" %s\n" % colorize("INFORM", i), + level=logging.ERROR, noiselevel=-1) + writemsg_level("\n", level=logging.ERROR, noiselevel=-1) + return 1 + continue + msg = [] + msg.append("'%s' is not a valid package atom." % (x,)) + msg.append("Please check ebuild(5) for full details.") + writemsg_level("".join("!!! %s\n" % line for line in msg), + level=logging.ERROR, noiselevel=-1) + return 1 + + return action_info(emerge_config.target_config.settings, + emerge_config.trees, emerge_config.opts, valid_atoms) + + # "update", "system", or just process files: + else: + validate_ebuild_environment(emerge_config.trees) + + for x in emerge_config.args: + if x.startswith(SETPREFIX) or \ + is_valid_package_atom(x, allow_repo=True): + continue + if x[:1] == os.sep: + continue + try: + os.lstat(x) + continue + except OSError: + pass + msg = [] + msg.append("'%s' is not a valid package atom." % (x,)) + msg.append("Please check ebuild(5) for full details.") + writemsg_level("".join("!!! %s\n" % line for line in msg), + level=logging.ERROR, noiselevel=-1) + return 1 + + # GLEP 42 says to display news *after* an emerge --pretend + if "--pretend" not in emerge_config.opts: + display_news_notification( + emerge_config.target_config, emerge_config.opts) + retval = action_build(emerge_config.target_config.settings, + emerge_config.trees, emerge_config.target_config.mtimedb, + emerge_config.opts, emerge_config.action, + emerge_config.args, spinner) + post_emerge(emerge_config.action, emerge_config.opts, + emerge_config.args, emerge_config.target_config.root, + emerge_config.trees, emerge_config.target_config.mtimedb, retval) + + return retval diff --git a/portage_with_autodep/pym/_emerge/actions.pyo b/portage_with_autodep/pym/_emerge/actions.pyo Binary files differindex 4fbda01..42060b5 100644 --- a/portage_with_autodep/pym/_emerge/actions.pyo +++ b/portage_with_autodep/pym/_emerge/actions.pyo diff --git a/portage_with_autodep/pym/_emerge/clear_caches.py b/portage_with_autodep/pym/_emerge/clear_caches.py index 7b7c5ec..513df62 100644 --- a/portage_with_autodep/pym/_emerge/clear_caches.py +++ b/portage_with_autodep/pym/_emerge/clear_caches.py @@ -1,8 +1,7 @@ -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import gc -from portage.util.listdir import dircache def clear_caches(trees): for d in trees.values(): @@ -15,5 +14,4 @@ def clear_caches(trees): pass else: d["vartree"].dbapi._linkmap._clear_cache() - dircache.clear() gc.collect() diff --git a/portage_with_autodep/pym/_emerge/clear_caches.pyo b/portage_with_autodep/pym/_emerge/clear_caches.pyo Binary files differindex 2e6f010..d07fbeb 100644 --- a/portage_with_autodep/pym/_emerge/clear_caches.pyo +++ b/portage_with_autodep/pym/_emerge/clear_caches.pyo diff --git a/portage_with_autodep/pym/_emerge/countdown.py b/portage_with_autodep/pym/_emerge/countdown.py index 5abdc8a..62e3c8d 100644 --- a/portage_with_autodep/pym/_emerge/countdown.py +++ b/portage_with_autodep/pym/_emerge/countdown.py @@ -1,4 +1,4 @@ -# Copyright 1999-2009 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -8,15 +8,15 @@ import time from portage.output import colorize -def countdown(secs=5, doing="Starting"): + +def countdown(secs=5, doing='Starting'): if secs: - print(">>> Waiting",secs,"seconds before starting...") - print(">>> (Control-C to abort)...\n"+doing+" in: ", end=' ') - ticks=list(range(secs)) - ticks.reverse() - for sec in ticks: - sys.stdout.write(colorize("UNMERGE_WARN", str(sec+1)+" ")) + print( + '>>> Waiting %s seconds before starting...\n' + '>>> (Control-C to abort)...\n' + '%s in:' % (secs, doing), end='') + for sec in range(secs, 0, -1): + sys.stdout.write(colorize('UNMERGE_WARN', ' %i' % sec)) sys.stdout.flush() time.sleep(1) print() - diff --git a/portage_with_autodep/pym/_emerge/countdown.pyo b/portage_with_autodep/pym/_emerge/countdown.pyo Binary files differindex 537dd27..37341fd 100644 --- a/portage_with_autodep/pym/_emerge/countdown.pyo +++ b/portage_with_autodep/pym/_emerge/countdown.pyo diff --git a/portage_with_autodep/pym/_emerge/create_depgraph_params.py b/portage_with_autodep/pym/_emerge/create_depgraph_params.py index 8f15c68..98a7646 100644 --- a/portage_with_autodep/pym/_emerge/create_depgraph_params.py +++ b/portage_with_autodep/pym/_emerge/create_depgraph_params.py @@ -15,12 +15,22 @@ def create_depgraph_params(myopts, myaction): # complete: completely account for all known dependencies # remove: build graph for use in removing packages # rebuilt_binaries: replace installed packages with rebuilt binaries + # rebuild_if_new_slot: rebuild or reinstall packages when + # slot/sub-slot := operator dependencies can be satisfied by a newer + # slot/sub-slot, so that older packages slots will become eligible for + # removal by the --depclean action as soon as possible + # ignore_built_slot_operator_deps: ignore the slot/sub-slot := operator parts + # of dependencies that have been recorded when packages where built myparams = {"recurse" : True} bdeps = myopts.get("--with-bdeps") if bdeps is not None: myparams["bdeps"] = bdeps + ignore_built_slot_operator_deps = myopts.get("--ignore-built-slot-operator-deps") + if ignore_built_slot_operator_deps is not None: + myparams["ignore_built_slot_operator_deps"] = ignore_built_slot_operator_deps + dynamic_deps = myopts.get("--dynamic-deps") if dynamic_deps is not None: myparams["dynamic_deps"] = dynamic_deps @@ -31,6 +41,10 @@ def create_depgraph_params(myopts, myaction): myparams["selective"] = True return myparams + rebuild_if_new_slot = myopts.get('--rebuild-if-new-slot') + if rebuild_if_new_slot is not None: + myparams['rebuild_if_new_slot'] = rebuild_if_new_slot + if "--update" in myopts or \ "--newuse" in myopts or \ "--reinstall" in myopts or \ @@ -42,6 +56,11 @@ def create_depgraph_params(myopts, myaction): if deep is not None and deep != 0: myparams["deep"] = deep + complete_if_new_use = \ + myopts.get("--complete-graph-if-new-use") + if complete_if_new_use is not None: + myparams["complete_if_new_use"] = complete_if_new_use + complete_if_new_ver = \ myopts.get("--complete-graph-if-new-ver") if complete_if_new_ver is not None: diff --git a/portage_with_autodep/pym/_emerge/create_depgraph_params.pyo b/portage_with_autodep/pym/_emerge/create_depgraph_params.pyo Binary files differindex 834580a..d9625b8 100644 --- a/portage_with_autodep/pym/_emerge/create_depgraph_params.pyo +++ b/portage_with_autodep/pym/_emerge/create_depgraph_params.pyo diff --git a/portage_with_autodep/pym/_emerge/create_world_atom.py b/portage_with_autodep/pym/_emerge/create_world_atom.py index 35fb7c4..ac994cc 100644 --- a/portage_with_autodep/pym/_emerge/create_world_atom.py +++ b/portage_with_autodep/pym/_emerge/create_world_atom.py @@ -1,7 +1,15 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +import sys + from portage.dep import _repo_separator +from portage.exception import InvalidData + +if sys.hexversion >= 0x3000000: + _unicode = str +else: + _unicode = unicode def create_world_atom(pkg, args_set, root_config): """Create a new atom for the world file if one does not exist. If the @@ -35,16 +43,15 @@ def create_world_atom(pkg, args_set, root_config): for cpv in portdb.match(cp): for repo in repos: try: - available_slots.add(portdb.aux_get(cpv, ["SLOT"], - myrepo=repo)[0]) - except KeyError: + available_slots.add(portdb._pkg_str(_unicode(cpv), repo).slot) + except (KeyError, InvalidData): pass slotted = len(available_slots) > 1 or \ (len(available_slots) == 1 and "0" not in available_slots) if not slotted: # check the vdb in case this is multislot - available_slots = set(vardb.aux_get(cpv, ["SLOT"])[0] \ + available_slots = set(vardb._pkg_str(cpv, None).slot \ for cpv in vardb.match(cp)) slotted = len(available_slots) > 1 or \ (len(available_slots) == 1 and "0" not in available_slots) @@ -83,14 +90,14 @@ def create_world_atom(pkg, args_set, root_config): matched_slots = set() if mydb is vardb: for cpv in matches: - matched_slots.add(mydb.aux_get(cpv, ["SLOT"])[0]) + matched_slots.add(mydb._pkg_str(cpv, None).slot) else: for cpv in matches: for repo in repos: try: - matched_slots.add(portdb.aux_get(cpv, ["SLOT"], - myrepo=repo)[0]) - except KeyError: + matched_slots.add( + portdb._pkg_str(_unicode(cpv), repo).slot) + except (KeyError, InvalidData): pass if len(matched_slots) == 1: diff --git a/portage_with_autodep/pym/_emerge/create_world_atom.pyo b/portage_with_autodep/pym/_emerge/create_world_atom.pyo Binary files differindex ac3fb5d..523c5a0 100644 --- a/portage_with_autodep/pym/_emerge/create_world_atom.pyo +++ b/portage_with_autodep/pym/_emerge/create_world_atom.pyo diff --git a/portage_with_autodep/pym/_emerge/depgraph.py b/portage_with_autodep/pym/_emerge/depgraph.py index 572cea7..763f3fd 100644 --- a/portage_with_autodep/pym/_emerge/depgraph.py +++ b/portage_with_autodep/pym/_emerge/depgraph.py @@ -1,32 +1,37 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import print_function +from __future__ import print_function, unicode_literals -import difflib import errno import io import logging import stat import sys import textwrap +import warnings from collections import deque from itertools import chain import portage from portage import os, OrderedDict from portage import _unicode_decode, _unicode_encode, _encodings -from portage.const import PORTAGE_PACKAGE_ATOM, USER_CONFIG_PATH +from portage.const import PORTAGE_PACKAGE_ATOM, USER_CONFIG_PATH, VCS_DIRS from portage.dbapi import dbapi from portage.dbapi.dep_expand import dep_expand +from portage.dbapi._similar_name_search import similar_name_search from portage.dep import Atom, best_match_to_list, extract_affecting_use, \ check_required_use, human_readable_required_use, match_from_list, \ _repo_separator -from portage.eapi import eapi_has_strong_blocks, eapi_has_required_use -from portage.exception import InvalidAtom, InvalidDependString, PortageException +from portage.dep._slot_operator import ignore_built_slot_operator_deps +from portage.eapi import eapi_has_strong_blocks, eapi_has_required_use, \ + _get_eapi_attrs +from portage.exception import (InvalidAtom, InvalidData, InvalidDependString, + PackageNotFound, PortageException) from portage.output import colorize, create_color_func, \ darkgreen, green bad = create_color_func("BAD") +from portage.package.ebuild.config import _get_feature_flags from portage.package.ebuild.getmaskingstatus import \ _getmaskingstatus, _MaskReason from portage._sets import SETPREFIX @@ -36,13 +41,16 @@ from portage.util import cmp_sort_key, writemsg, writemsg_stdout from portage.util import ensure_dirs from portage.util import writemsg_level, write_atomic from portage.util.digraph import digraph -from portage.util.listdir import _ignorecvs_dirs +from portage.util._async.TaskScheduler import TaskScheduler +from portage.util._eventloop.EventLoop import EventLoop +from portage.util._eventloop.global_event_loop import global_event_loop from portage.versions import catpkgsplit from _emerge.AtomArg import AtomArg from _emerge.Blocker import Blocker from _emerge.BlockerCache import BlockerCache from _emerge.BlockerDepPriority import BlockerDepPriority +from .chk_updated_cfg_files import chk_updated_cfg_files from _emerge.countdown import countdown from _emerge.create_world_atom import create_world_atom from _emerge.Dependency import Dependency @@ -50,6 +58,7 @@ from _emerge.DependencyArg import DependencyArg from _emerge.DepPriority import DepPriority from _emerge.DepPriorityNormalRange import DepPriorityNormalRange from _emerge.DepPrioritySatisfiedRange import DepPrioritySatisfiedRange +from _emerge.EbuildMetadataPhase import EbuildMetadataPhase from _emerge.FakeVartree import FakeVartree from _emerge._find_deep_system_runtime_deps import _find_deep_system_runtime_deps from _emerge.is_valid_package_atom import insert_category_into_atom, \ @@ -73,6 +82,9 @@ from _emerge.resolver.output import Display if sys.hexversion >= 0x3000000: basestring = str long = int + _unicode = str +else: + _unicode = unicode class _scheduler_graph_config(object): def __init__(self, trees, pkg_cache, graph, mergelist): @@ -85,9 +97,9 @@ def _wildcard_set(atoms): pkgs = InternalPackageSet(allow_wildcard=True) for x in atoms: try: - x = Atom(x, allow_wildcard=True) + x = Atom(x, allow_wildcard=True, allow_repo=False) except portage.exception.InvalidAtom: - x = Atom("*/" + x, allow_wildcard=True) + x = Atom("*/" + x, allow_wildcard=True, allow_repo=False) pkgs.add(x) return pkgs @@ -110,6 +122,8 @@ class _frozen_depgraph_config(object): self._pkg_cache = {} self._highest_license_masked = {} dynamic_deps = myopts.get("--dynamic-deps", "y") != "n" + ignore_built_slot_operator_deps = myopts.get( + "--ignore-built-slot-operator-deps", "n") == "y" for myroot in trees: self.trees[myroot] = {} # Create a RootConfig instance that references @@ -124,7 +138,8 @@ class _frozen_depgraph_config(object): FakeVartree(trees[myroot]["root_config"], pkg_cache=self._pkg_cache, pkg_root_config=self.roots[myroot], - dynamic_deps=dynamic_deps) + dynamic_deps=dynamic_deps, + ignore_built_slot_operator_deps=ignore_built_slot_operator_deps) self.pkgsettings[myroot] = portage.config( clone=self.trees[myroot]["vartree"].settings) @@ -259,13 +274,12 @@ class _rebuild_config(object): return True elif (parent.installed and root_slot not in self.reinstall_list): - inst_build_time = parent.metadata.get("BUILD_TIME") try: bin_build_time, = bindb.aux_get(parent.cpv, ["BUILD_TIME"]) except KeyError: continue - if bin_build_time != inst_build_time: + if bin_build_time != _unicode(parent.build_time): # 2) Remote binary package is valid, and local package # is not up to date. Force reinstall. reinstall = True @@ -366,12 +380,15 @@ class _dynamic_depgraph_config(object): # This use used to check if we have accounted for blockers # relevant to a package. self._traversed_pkg_deps = set() - self._slot_collision_info = {} + # This should be ordered such that the backtracker will + # attempt to solve conflicts which occurred earlier first, + # since an earlier conflict can be the cause of a conflict + # which occurs later. + self._slot_collision_info = OrderedDict() # Slot collision nodes are not allowed to block other packages since # blocker validation is only able to account for one package per slot. self._slot_collision_nodes = set() self._parent_atoms = {} - self._slot_conflict_parent_atoms = set() self._slot_conflict_handler = None self._circular_dependency_handler = None self._serialized_tasks_cache = None @@ -401,15 +418,20 @@ class _dynamic_depgraph_config(object): self._needed_license_changes = backtrack_parameters.needed_license_changes self._needed_use_config_changes = backtrack_parameters.needed_use_config_changes self._runtime_pkg_mask = backtrack_parameters.runtime_pkg_mask + self._slot_operator_replace_installed = backtrack_parameters.slot_operator_replace_installed + self._prune_rebuilds = backtrack_parameters.prune_rebuilds self._need_restart = False # For conditions that always require user intervention, such as # unsatisfied REQUIRED_USE (currently has no autounmask support). self._skip_restart = False self._backtrack_infos = {} + self._buildpkgonly_deps_unsatisfied = False self._autounmask = depgraph._frozen_config.myopts.get('--autounmask') != 'n' self._success_without_autounmask = False self._traverse_ignored_deps = False + self._complete_mode = False + self._slot_operator_deps = {} for myroot in depgraph._frozen_config.trees: self.sets[myroot] = _depgraph_sets() @@ -432,6 +454,7 @@ class _dynamic_depgraph_config(object): self._graph_trees[myroot]["vartree"] = graph_tree self._graph_trees[myroot]["graph_db"] = graph_tree.dbapi self._graph_trees[myroot]["graph"] = self.digraph + self._graph_trees[myroot]["want_update_pkg"] = depgraph._want_update_pkg def filtered_tree(): pass filtered_tree.dbapi = _dep_check_composite_db(depgraph, myroot) @@ -458,6 +481,7 @@ class _dynamic_depgraph_config(object): self._filtered_trees[myroot]["graph"] = self.digraph self._filtered_trees[myroot]["vartree"] = \ depgraph._frozen_config.trees[myroot]["vartree"] + self._filtered_trees[myroot]["want_update_pkg"] = depgraph._want_update_pkg dbs = [] # (db, pkg_type, built, installed, db_keys) @@ -488,8 +512,6 @@ class depgraph(object): pkg_tree_map = RootConfig.pkg_tree_map - _dep_keys = ["DEPEND", "RDEPEND", "PDEPEND"] - def __init__(self, settings, trees, myopts, myparams, spinner, frozen_config=None, backtrack_parameters=BacktrackParameter(), allow_backtracking=False): if frozen_config is None: @@ -503,6 +525,9 @@ class depgraph(object): self._select_atoms = self._select_atoms_highest_available self._select_package = self._select_pkg_highest_available + self._event_loop = (portage._internal_caller and + global_event_loop() or EventLoop(main=False)) + def _load_vdb(self): """ Load installed package metadata if appropriate. This used to be called @@ -521,10 +546,6 @@ class depgraph(object): preload_installed_pkgs = \ "--nodeps" not in self._frozen_config.myopts - if self._frozen_config.myopts.get("--root-deps") is not None and \ - myroot != self._frozen_config.target_root: - continue - fake_vartree = self._frozen_config.trees[myroot]["vartree"] if not fake_vartree.dbapi: # This needs to be called for the first depgraph, but not for @@ -541,21 +562,156 @@ class depgraph(object): fakedb = self._dynamic_config._graph_trees[ myroot]["vartree"].dbapi - for pkg in vardb: - self._spinner_update() - if dynamic_deps: - # This causes FakeVartree to update the - # Package instance dependencies via - # PackageVirtualDbapi.aux_update() - vardb.aux_get(pkg.cpv, []) - fakedb.cpv_inject(pkg) + if not dynamic_deps: + for pkg in vardb: + fakedb.cpv_inject(pkg) + else: + max_jobs = self._frozen_config.myopts.get("--jobs") + max_load = self._frozen_config.myopts.get("--load-average") + scheduler = TaskScheduler( + self._dynamic_deps_preload(fake_vartree, fakedb), + max_jobs=max_jobs, + max_load=max_load, + event_loop=fake_vartree._portdb._event_loop) + scheduler.start() + scheduler.wait() self._dynamic_config._vdb_loaded = True + def _dynamic_deps_preload(self, fake_vartree, fakedb): + portdb = fake_vartree._portdb + for pkg in fake_vartree.dbapi: + self._spinner_update() + fakedb.cpv_inject(pkg) + ebuild_path, repo_path = \ + portdb.findname2(pkg.cpv, myrepo=pkg.repo) + if ebuild_path is None: + fake_vartree.dynamic_deps_preload(pkg, None) + continue + metadata, ebuild_hash = portdb._pull_valid_cache( + pkg.cpv, ebuild_path, repo_path) + if metadata is not None: + fake_vartree.dynamic_deps_preload(pkg, metadata) + else: + proc = EbuildMetadataPhase(cpv=pkg.cpv, + ebuild_hash=ebuild_hash, + portdb=portdb, repo_path=repo_path, + settings=portdb.doebuild_settings) + proc.addExitListener( + self._dynamic_deps_proc_exit(pkg, fake_vartree)) + yield proc + + class _dynamic_deps_proc_exit(object): + + __slots__ = ('_pkg', '_fake_vartree') + + def __init__(self, pkg, fake_vartree): + self._pkg = pkg + self._fake_vartree = fake_vartree + + def __call__(self, proc): + metadata = None + if proc.returncode == os.EX_OK: + metadata = proc.metadata + self._fake_vartree.dynamic_deps_preload(self._pkg, metadata) + def _spinner_update(self): if self._frozen_config.spinner: self._frozen_config.spinner.update() + def _compute_abi_rebuild_info(self): + """ + Fill self._forced_rebuilds with packages that cause rebuilds. + """ + + debug = "--debug" in self._frozen_config.myopts + + # Get all atoms that might have caused a forced rebuild. + atoms = {} + for s in self._dynamic_config._initial_arg_list: + if s.force_reinstall: + root = s.root_config.root + atoms.setdefault(root, set()).update(s.pset) + + if debug: + writemsg_level("forced reinstall atoms:\n", + level=logging.DEBUG, noiselevel=-1) + + for root in atoms: + writemsg_level(" root: %s\n" % root, + level=logging.DEBUG, noiselevel=-1) + for atom in atoms[root]: + writemsg_level(" atom: %s\n" % atom, + level=logging.DEBUG, noiselevel=-1) + writemsg_level("\n\n", + level=logging.DEBUG, noiselevel=-1) + + # Go through all slot operator deps and check if one of these deps + # has a parent that is matched by one of the atoms from above. + forced_rebuilds = {} + for (root, slot_atom), deps in self._dynamic_config._slot_operator_deps.items(): + rebuild_atoms = atoms.get(root, set()) + + for dep in deps: + if dep.parent.installed or dep.child.installed or \ + dep.parent.slot_atom not in rebuild_atoms: + continue + + # Make sure the child's slot/subslot has changed. If it hasn't, + # then another child has forced this rebuild. + installed_pkg = self._select_pkg_from_installed(root, dep.child.slot_atom)[0] + if installed_pkg and installed_pkg.slot == dep.child.slot and \ + installed_pkg.sub_slot == dep.child.sub_slot: + continue + + # The child has forced a rebuild of the parent + forced_rebuilds.setdefault(root, {}).setdefault(dep.child, set()).add(dep.parent) + + if debug: + writemsg_level("slot operator dependencies:\n", + level=logging.DEBUG, noiselevel=-1) + + for (root, slot_atom), deps in self._dynamic_config._slot_operator_deps.items(): + writemsg_level(" (%s, %s)\n" % \ + (root, slot_atom), level=logging.DEBUG, noiselevel=-1) + for dep in deps: + writemsg_level(" parent: %s\n" % dep.parent, level=logging.DEBUG, noiselevel=-1) + writemsg_level(" child: %s (%s)\n" % (dep.child, dep.priority), level=logging.DEBUG, noiselevel=-1) + + writemsg_level("\n\n", + level=logging.DEBUG, noiselevel=-1) + + + writemsg_level("forced rebuilds:\n", + level=logging.DEBUG, noiselevel=-1) + + for root in forced_rebuilds: + writemsg_level(" root: %s\n" % root, + level=logging.DEBUG, noiselevel=-1) + for child in forced_rebuilds[root]: + writemsg_level(" child: %s\n" % child, + level=logging.DEBUG, noiselevel=-1) + for parent in forced_rebuilds[root][child]: + writemsg_level(" parent: %s\n" % parent, + level=logging.DEBUG, noiselevel=-1) + writemsg_level("\n\n", + level=logging.DEBUG, noiselevel=-1) + + self._forced_rebuilds = forced_rebuilds + + def _show_abi_rebuild_info(self): + + if not self._forced_rebuilds: + return + + writemsg_stdout("\nThe following packages are causing rebuilds:\n\n", noiselevel=-1) + + for root in self._forced_rebuilds: + for child in self._forced_rebuilds[root]: + writemsg_stdout(" %s causes rebuilds for:\n" % (child,), noiselevel=-1) + for parent in self._forced_rebuilds[root][child]: + writemsg_stdout(" %s\n" % (parent,), noiselevel=-1) + def _show_ignored_binaries(self): """ Show binaries that have been ignored because their USE didn't @@ -582,8 +738,7 @@ class depgraph(object): if selected_pkg.installed and \ selected_pkg.cpv == pkg.cpv and \ - selected_pkg.metadata.get('BUILD_TIME') == \ - pkg.metadata.get('BUILD_TIME'): + selected_pkg.build_time == pkg.build_time: # We don't care about ignored binaries when an # identical installed instance is selected to # fill the slot. @@ -599,11 +754,17 @@ class depgraph(object): "due to non matching USE:\n\n", noiselevel=-1) for pkg, flags in self._dynamic_config.ignored_binaries.items(): - writemsg(" =%s" % pkg.cpv, noiselevel=-1) + flag_display = [] + for flag in sorted(flags): + if flag not in pkg.use.enabled: + flag = "-" + flag + flag_display.append(flag) + flag_display = " ".join(flag_display) + # The user can paste this line into package.use + writemsg(" =%s %s" % (pkg.cpv, flag_display), noiselevel=-1) if pkg.root_config.settings["ROOT"] != "/": - writemsg(" for %s" % (pkg.root,), noiselevel=-1) - writemsg("\n use flag(s): %s\n" % ", ".join(sorted(flags)), - noiselevel=-1) + writemsg(" # for %s" % (pkg.root,), noiselevel=-1) + writemsg("\n", noiselevel=-1) msg = [ "", @@ -617,7 +778,7 @@ class depgraph(object): line = colorize("INFORM", line) writemsg(line + "\n", noiselevel=-1) - def _show_missed_update(self): + def _get_missed_updates(self): # In order to minimize noise, show only the highest # missed update from each SLOT. @@ -643,6 +804,12 @@ class depgraph(object): missed_updates[k] = (pkg, mask_type, parent_atoms) break + return missed_updates + + def _show_missed_update(self): + + missed_updates = self._get_missed_updates() + if not missed_updates: return @@ -760,7 +927,7 @@ class depgraph(object): conflict = handler.get_conflict() writemsg(conflict, noiselevel=-1) - + explanation = handler.get_explanation() if explanation: writemsg(explanation, noiselevel=-1) @@ -801,37 +968,727 @@ class depgraph(object): def _process_slot_conflicts(self): """ + If there are any slot conflicts and backtracking is enabled, + _complete_graph should complete the graph before this method + is called, so that all relevant reverse dependencies are + available for use in backtracking decisions. + """ + for (slot_atom, root), slot_nodes in \ + self._dynamic_config._slot_collision_info.items(): + self._process_slot_conflict(root, slot_atom, slot_nodes) + + def _process_slot_conflict(self, root, slot_atom, slot_nodes): + """ Process slot conflict data to identify specific atoms which lead to conflict. These atoms only match a subset of the packages that have been pulled into a given slot. """ - for (slot_atom, root), slot_nodes \ - in self._dynamic_config._slot_collision_info.items(): - all_parent_atoms = set() - for pkg in slot_nodes: - parent_atoms = self._dynamic_config._parent_atoms.get(pkg) - if not parent_atoms: + debug = "--debug" in self._frozen_config.myopts + + slot_parent_atoms = set() + for pkg in slot_nodes: + parent_atoms = self._dynamic_config._parent_atoms.get(pkg) + if not parent_atoms: + continue + slot_parent_atoms.update(parent_atoms) + + conflict_pkgs = [] + conflict_atoms = {} + for pkg in slot_nodes: + + if self._dynamic_config._allow_backtracking and \ + pkg in self._dynamic_config._runtime_pkg_mask: + if debug: + writemsg_level( + "!!! backtracking loop detected: %s %s\n" % \ + (pkg, + self._dynamic_config._runtime_pkg_mask[pkg]), + level=logging.DEBUG, noiselevel=-1) + + parent_atoms = self._dynamic_config._parent_atoms.get(pkg) + if parent_atoms is None: + parent_atoms = set() + self._dynamic_config._parent_atoms[pkg] = parent_atoms + + all_match = True + for parent_atom in slot_parent_atoms: + if parent_atom in parent_atoms: continue - all_parent_atoms.update(parent_atoms) + # Use package set for matching since it will match via + # PROVIDE when necessary, while match_from_list does not. + parent, atom = parent_atom + atom_set = InternalPackageSet( + initial_atoms=(atom,), allow_repo=True) + if atom_set.findAtomForPackage(pkg, + modified_use=self._pkg_use_enabled(pkg)): + parent_atoms.add(parent_atom) + else: + all_match = False + conflict_atoms.setdefault(parent_atom, set()).add(pkg) - for pkg in slot_nodes: - parent_atoms = self._dynamic_config._parent_atoms.get(pkg) - if parent_atoms is None: - parent_atoms = set() - self._dynamic_config._parent_atoms[pkg] = parent_atoms - for parent_atom in all_parent_atoms: - if parent_atom in parent_atoms: + if not all_match: + conflict_pkgs.append(pkg) + + if conflict_pkgs and \ + self._dynamic_config._allow_backtracking and \ + not self._accept_blocker_conflicts(): + remaining = [] + for pkg in conflict_pkgs: + if self._slot_conflict_backtrack_abi(pkg, + slot_nodes, conflict_atoms): + backtrack_infos = self._dynamic_config._backtrack_infos + config = backtrack_infos.setdefault("config", {}) + config.setdefault("slot_conflict_abi", set()).add(pkg) + else: + remaining.append(pkg) + if remaining: + self._slot_confict_backtrack(root, slot_atom, + slot_parent_atoms, remaining) + + def _slot_confict_backtrack(self, root, slot_atom, + all_parents, conflict_pkgs): + + debug = "--debug" in self._frozen_config.myopts + existing_node = self._dynamic_config._slot_pkg_map[root][slot_atom] + # In order to avoid a missed update, first mask lower versions + # that conflict with higher versions (the backtracker visits + # these in reverse order). + conflict_pkgs.sort(reverse=True) + backtrack_data = [] + for to_be_masked in conflict_pkgs: + # For missed update messages, find out which + # atoms matched to_be_selected that did not + # match to_be_masked. + parent_atoms = \ + self._dynamic_config._parent_atoms.get(to_be_masked, set()) + conflict_atoms = set(parent_atom for parent_atom in all_parents \ + if parent_atom not in parent_atoms) + backtrack_data.append((to_be_masked, conflict_atoms)) + + to_be_masked = backtrack_data[-1][0] + + self._dynamic_config._backtrack_infos.setdefault( + "slot conflict", []).append(backtrack_data) + self._dynamic_config._need_restart = True + if debug: + msg = [] + msg.append("") + msg.append("") + msg.append("backtracking due to slot conflict:") + msg.append(" first package: %s" % existing_node) + msg.append(" package to mask: %s" % to_be_masked) + msg.append(" slot: %s" % slot_atom) + msg.append(" parents: %s" % ", ".join( \ + "(%s, '%s')" % (ppkg, atom) for ppkg, atom in all_parents)) + msg.append("") + writemsg_level("".join("%s\n" % l for l in msg), + noiselevel=-1, level=logging.DEBUG) + + def _slot_conflict_backtrack_abi(self, pkg, slot_nodes, conflict_atoms): + """ + If one or more conflict atoms have a slot/sub-slot dep that can be resolved + by rebuilding the parent package, then schedule the rebuild via + backtracking, and return True. Otherwise, return False. + """ + + found_update = False + for parent_atom, conflict_pkgs in conflict_atoms.items(): + parent, atom = parent_atom + if atom.slot_operator != "=" or not parent.built: + continue + + if pkg not in conflict_pkgs: + continue + + for other_pkg in slot_nodes: + if other_pkg in conflict_pkgs: + continue + + dep = Dependency(atom=atom, child=other_pkg, + parent=parent, root=pkg.root) + + new_dep = \ + self._slot_operator_update_probe_slot_conflict(dep) + if new_dep is not None: + self._slot_operator_update_backtrack(dep, + new_dep=new_dep) + found_update = True + + return found_update + + def _slot_change_probe(self, dep): + """ + @rtype: bool + @return: True if dep.child should be rebuilt due to a change + in sub-slot (without revbump, as in bug #456208). + """ + if not (isinstance(dep.parent, Package) and \ + not dep.parent.built and dep.child.built): + return None + + root_config = self._frozen_config.roots[dep.root] + matches = [] + try: + matches.append(self._pkg(dep.child.cpv, "ebuild", + root_config, myrepo=dep.child.repo)) + except PackageNotFound: + pass + + for unbuilt_child in chain(matches, + self._iter_match_pkgs(root_config, "ebuild", + Atom("=%s" % (dep.child.cpv,)))): + if unbuilt_child in self._dynamic_config._runtime_pkg_mask: + continue + if self._frozen_config.excluded_pkgs.findAtomForPackage( + unbuilt_child, + modified_use=self._pkg_use_enabled(unbuilt_child)): + continue + if not self._pkg_visibility_check(unbuilt_child): + continue + break + else: + return None + + if unbuilt_child.slot == dep.child.slot and \ + unbuilt_child.sub_slot == dep.child.sub_slot: + return None + + return unbuilt_child + + def _slot_change_backtrack(self, dep, new_child_slot): + child = dep.child + if "--debug" in self._frozen_config.myopts: + msg = [] + msg.append("") + msg.append("") + msg.append("backtracking due to slot/sub-slot change:") + msg.append(" child package: %s" % child) + msg.append(" child slot: %s/%s" % + (child.slot, child.sub_slot)) + msg.append(" new child: %s" % new_child_slot) + msg.append(" new child slot: %s/%s" % + (new_child_slot.slot, new_child_slot.sub_slot)) + msg.append(" parent package: %s" % dep.parent) + msg.append(" atom: %s" % dep.atom) + msg.append("") + writemsg_level("\n".join(msg), + noiselevel=-1, level=logging.DEBUG) + backtrack_infos = self._dynamic_config._backtrack_infos + config = backtrack_infos.setdefault("config", {}) + + # mask unwanted binary packages if necessary + masks = {} + if not child.installed: + masks.setdefault(dep.child, {})["slot_operator_mask_built"] = None + if masks: + config.setdefault("slot_operator_mask_built", {}).update(masks) + + # trigger replacement of installed packages if necessary + reinstalls = set() + if child.installed: + replacement_atom = self._replace_installed_atom(child) + if replacement_atom is not None: + reinstalls.add((child.root, replacement_atom)) + if reinstalls: + config.setdefault("slot_operator_replace_installed", + set()).update(reinstalls) + + self._dynamic_config._need_restart = True + + def _slot_operator_update_backtrack(self, dep, new_child_slot=None, + new_dep=None): + if new_child_slot is None: + child = dep.child + else: + child = new_child_slot + if "--debug" in self._frozen_config.myopts: + msg = [] + msg.append("") + msg.append("") + msg.append("backtracking due to missed slot abi update:") + msg.append(" child package: %s" % child) + if new_child_slot is not None: + msg.append(" new child slot package: %s" % new_child_slot) + msg.append(" parent package: %s" % dep.parent) + if new_dep is not None: + msg.append(" new parent pkg: %s" % new_dep.parent) + msg.append(" atom: %s" % dep.atom) + msg.append("") + writemsg_level("\n".join(msg), + noiselevel=-1, level=logging.DEBUG) + backtrack_infos = self._dynamic_config._backtrack_infos + config = backtrack_infos.setdefault("config", {}) + + # mask unwanted binary packages if necessary + abi_masks = {} + if new_child_slot is None: + if not child.installed: + abi_masks.setdefault(child, {})["slot_operator_mask_built"] = None + if not dep.parent.installed: + abi_masks.setdefault(dep.parent, {})["slot_operator_mask_built"] = None + if abi_masks: + config.setdefault("slot_operator_mask_built", {}).update(abi_masks) + + # trigger replacement of installed packages if necessary + abi_reinstalls = set() + if dep.parent.installed: + if new_dep is not None: + replacement_atom = new_dep.parent.slot_atom + else: + replacement_atom = self._replace_installed_atom(dep.parent) + if replacement_atom is not None: + abi_reinstalls.add((dep.parent.root, replacement_atom)) + if new_child_slot is None and child.installed: + replacement_atom = self._replace_installed_atom(child) + if replacement_atom is not None: + abi_reinstalls.add((child.root, replacement_atom)) + if abi_reinstalls: + config.setdefault("slot_operator_replace_installed", + set()).update(abi_reinstalls) + + self._dynamic_config._need_restart = True + + def _slot_operator_update_probe_slot_conflict(self, dep): + new_dep = self._slot_operator_update_probe(dep, slot_conflict=True) + + if new_dep is not None: + return new_dep + + if self._dynamic_config._autounmask is True: + + for autounmask_level in self._autounmask_levels(): + + new_dep = self._slot_operator_update_probe(dep, + slot_conflict=True, autounmask_level=autounmask_level) + + if new_dep is not None: + return new_dep + + return None + + def _slot_operator_update_probe(self, dep, new_child_slot=False, + slot_conflict=False, autounmask_level=None): + """ + slot/sub-slot := operators tend to prevent updates from getting pulled in, + since installed packages pull in packages with the slot/sub-slot that they + were built against. Detect this case so that we can schedule rebuilds + and reinstalls when appropriate. + NOTE: This function only searches for updates that involve upgrades + to higher versions, since the logic required to detect when a + downgrade would be desirable is not implemented. + """ + + if dep.child.installed and \ + self._frozen_config.excluded_pkgs.findAtomForPackage(dep.child, + modified_use=self._pkg_use_enabled(dep.child)): + return None + + if dep.parent.installed and \ + self._frozen_config.excluded_pkgs.findAtomForPackage(dep.parent, + modified_use=self._pkg_use_enabled(dep.parent)): + return None + + debug = "--debug" in self._frozen_config.myopts + selective = "selective" in self._dynamic_config.myparams + want_downgrade = None + + for replacement_parent in self._iter_similar_available(dep.parent, + dep.parent.slot_atom, autounmask_level=autounmask_level): + + selected_atoms = None + + atoms = set() + invalid_metadata = False + for dep_key in ("DEPEND", "HDEPEND", "RDEPEND", "PDEPEND"): + dep_string = replacement_parent._metadata[dep_key] + if not dep_string: + continue + + try: + dep_string = portage.dep.use_reduce(dep_string, + uselist=self._pkg_use_enabled(replacement_parent), + is_valid_flag=replacement_parent.iuse.is_valid_flag, + flat=True, token_class=Atom, + eapi=replacement_parent.eapi) + except portage.exception.InvalidDependString: + invalid_metadata = True + break + + atoms.update(token for token in dep_string if isinstance(token, Atom)) + + if invalid_metadata: + continue + + # List of list of child,atom pairs for each atom. + replacement_candidates = [] + # Set of all packages all atoms can agree on. + all_candidate_pkgs = None + + for atom in atoms: + if atom.blocker or \ + atom.cp != dep.atom.cp: + continue + + # Discard USE deps, we're only searching for an approximate + # pattern, and dealing with USE states is too complex for + # this purpose. + unevaluated_atom = atom.unevaluated_atom + atom = atom.without_use + + if replacement_parent.built and \ + portage.dep._match_slot(atom, dep.child): + # Our selected replacement_parent appears to be built + # for the existing child selection. So, discard this + # parent and search for another. + break + + candidate_pkg_atoms = [] + candidate_pkgs = [] + for pkg in self._iter_similar_available( + dep.child, atom): + if pkg.slot == dep.child.slot and \ + pkg.sub_slot == dep.child.sub_slot: + # If slot/sub-slot is identical, then there's + # no point in updating. continue - # Use package set for matching since it will match via - # PROVIDE when necessary, while match_from_list does not. - parent, atom = parent_atom - atom_set = InternalPackageSet( - initial_atoms=(atom,), allow_repo=True) - if atom_set.findAtomForPackage(pkg, modified_use=self._pkg_use_enabled(pkg)): - parent_atoms.add(parent_atom) + if new_child_slot: + if pkg.slot == dep.child.slot: + continue + if pkg < dep.child: + # the new slot only matters if the + # package version is higher + continue else: - self._dynamic_config._slot_conflict_parent_atoms.add(parent_atom) + if pkg.slot != dep.child.slot: + continue + if pkg < dep.child: + if want_downgrade is None: + want_downgrade = self._downgrade_probe(dep.child) + # be careful not to trigger a rebuild when + # the only version available with a + # different slot_operator is an older version + if not want_downgrade: + continue + + insignificant = False + if not slot_conflict and \ + selective and \ + dep.parent.installed and \ + dep.child.installed and \ + dep.parent.cpv == replacement_parent.cpv and \ + dep.child.cpv == pkg.cpv: + # Then can happen if the child's sub-slot changed + # without a revision bump. The sub-slot change is + # considered insignificant until one of its parent + # packages needs to be rebuilt (which may trigger a + # slot conflict). + insignificant = True + + if not insignificant: + # Evaluate USE conditionals and || deps, in order + # to see if this atom is really desirable, since + # otherwise we may trigger an undesirable rebuild + # as in bug #460304. + if selected_atoms is None: + selected_atoms = self._select_atoms_probe( + dep.child.root, replacement_parent) + if unevaluated_atom not in selected_atoms: + continue + + if not insignificant: + candidate_pkg_atoms.append((pkg, unevaluated_atom)) + candidate_pkgs.append(pkg) + + replacement_candidates.append(candidate_pkg_atoms) + if all_candidate_pkgs is None: + all_candidate_pkgs = set(candidate_pkgs) + else: + all_candidate_pkgs.intersection_update(candidate_pkgs) + + if not all_candidate_pkgs: + # If the atoms that connect parent and child can't agree on + # any replacement child, we can't do anything. + continue + + # Now select one of the pkgs as replacement. This is as easy as + # selecting the highest version. + # The more complicated part is to choose an atom for the + # new Dependency object. Choose the one which ranked the selected + # parent highest. + selected = None + for candidate_pkg_atoms in replacement_candidates: + for i, (pkg, atom) in enumerate(candidate_pkg_atoms): + if pkg not in all_candidate_pkgs: + continue + if selected is None or \ + selected[0] < pkg or \ + (selected[0] is pkg and i < selected[2]): + selected = (pkg, atom, i) + + if debug: + msg = [] + msg.append("") + msg.append("") + msg.append("slot_operator_update_probe:") + msg.append(" existing child package: %s" % dep.child) + msg.append(" existing parent package: %s" % dep.parent) + msg.append(" new child package: %s" % selected[0]) + msg.append(" new parent package: %s" % replacement_parent) + msg.append("") + writemsg_level("\n".join(msg), + noiselevel=-1, level=logging.DEBUG) + + return Dependency(parent=replacement_parent, + child=selected[0], atom=selected[1]) + + if debug: + msg = [] + msg.append("") + msg.append("") + msg.append("slot_operator_update_probe:") + msg.append(" existing child package: %s" % dep.child) + msg.append(" existing parent package: %s" % dep.parent) + msg.append(" new child package: %s" % None) + msg.append(" new parent package: %s" % None) + msg.append("") + writemsg_level("\n".join(msg), + noiselevel=-1, level=logging.DEBUG) + + return None + + def _slot_operator_unsatisfied_probe(self, dep): + + if dep.parent.installed and \ + self._frozen_config.excluded_pkgs.findAtomForPackage(dep.parent, + modified_use=self._pkg_use_enabled(dep.parent)): + return False + + debug = "--debug" in self._frozen_config.myopts + + for replacement_parent in self._iter_similar_available(dep.parent, + dep.parent.slot_atom): + + for atom in replacement_parent.validated_atoms: + if not atom.slot_operator == "=" or \ + atom.blocker or \ + atom.cp != dep.atom.cp: + continue + + # Discard USE deps, we're only searching for an approximate + # pattern, and dealing with USE states is too complex for + # this purpose. + atom = atom.without_use + + pkg, existing_node = self._select_package(dep.root, atom, + onlydeps=dep.onlydeps) + + if pkg is not None: + + if debug: + msg = [] + msg.append("") + msg.append("") + msg.append("slot_operator_unsatisfied_probe:") + msg.append(" existing parent package: %s" % dep.parent) + msg.append(" existing parent atom: %s" % dep.atom) + msg.append(" new parent package: %s" % replacement_parent) + msg.append(" new child package: %s" % pkg) + msg.append("") + writemsg_level("\n".join(msg), + noiselevel=-1, level=logging.DEBUG) + + return True + + if debug: + msg = [] + msg.append("") + msg.append("") + msg.append("slot_operator_unsatisfied_probe:") + msg.append(" existing parent package: %s" % dep.parent) + msg.append(" existing parent atom: %s" % dep.atom) + msg.append(" new parent package: %s" % None) + msg.append(" new child package: %s" % None) + msg.append("") + writemsg_level("\n".join(msg), + noiselevel=-1, level=logging.DEBUG) + + return False + + def _slot_operator_unsatisfied_backtrack(self, dep): + + parent = dep.parent + + if "--debug" in self._frozen_config.myopts: + msg = [] + msg.append("") + msg.append("") + msg.append("backtracking due to unsatisfied " + "built slot-operator dep:") + msg.append(" parent package: %s" % parent) + msg.append(" atom: %s" % dep.atom) + msg.append("") + writemsg_level("\n".join(msg), + noiselevel=-1, level=logging.DEBUG) + + backtrack_infos = self._dynamic_config._backtrack_infos + config = backtrack_infos.setdefault("config", {}) + + # mask unwanted binary packages if necessary + masks = {} + if not parent.installed: + masks.setdefault(parent, {})["slot_operator_mask_built"] = None + if masks: + config.setdefault("slot_operator_mask_built", {}).update(masks) + + # trigger replacement of installed packages if necessary + reinstalls = set() + if parent.installed: + replacement_atom = self._replace_installed_atom(parent) + if replacement_atom is not None: + reinstalls.add((parent.root, replacement_atom)) + if reinstalls: + config.setdefault("slot_operator_replace_installed", + set()).update(reinstalls) + + self._dynamic_config._need_restart = True + + def _downgrade_probe(self, pkg): + """ + Detect cases where a downgrade of the given package is considered + desirable due to the current version being masked or unavailable. + """ + available_pkg = None + for available_pkg in self._iter_similar_available(pkg, + pkg.slot_atom): + if available_pkg >= pkg: + # There's an available package of the same or higher + # version, so downgrade seems undesirable. + return False + + return available_pkg is not None + + def _select_atoms_probe(self, root, pkg): + selected_atoms = [] + use = self._pkg_use_enabled(pkg) + for k in pkg._dep_keys: + v = pkg._metadata.get(k) + if not v: + continue + selected_atoms.extend(self._select_atoms( + root, v, myuse=use, parent=pkg)[pkg]) + return frozenset(x.unevaluated_atom for + x in selected_atoms) + + def _iter_similar_available(self, graph_pkg, atom, autounmask_level=None): + """ + Given a package that's in the graph, do a rough check to + see if a similar package is available to install. The given + graph_pkg itself may be yielded only if it's not installed. + """ + + usepkgonly = "--usepkgonly" in self._frozen_config.myopts + useoldpkg_atoms = self._frozen_config.useoldpkg_atoms + use_ebuild_visibility = self._frozen_config.myopts.get( + '--use-ebuild-visibility', 'n') != 'n' + + for pkg in self._iter_match_pkgs_any( + graph_pkg.root_config, atom): + if pkg.cp != graph_pkg.cp: + # discard old-style virtual match + continue + if pkg.installed: + continue + if pkg in self._dynamic_config._runtime_pkg_mask: + continue + if self._frozen_config.excluded_pkgs.findAtomForPackage(pkg, + modified_use=self._pkg_use_enabled(pkg)): + continue + if pkg.built: + if self._equiv_binary_installed(pkg): + continue + if not (not use_ebuild_visibility and + (usepkgonly or useoldpkg_atoms.findAtomForPackage( + pkg, modified_use=self._pkg_use_enabled(pkg)))) and \ + not self._equiv_ebuild_visible(pkg, + autounmask_level=autounmask_level): + continue + if not self._pkg_visibility_check(pkg, + autounmask_level=autounmask_level): + continue + yield pkg + + def _replace_installed_atom(self, inst_pkg): + """ + Given an installed package, generate an atom suitable for + slot_operator_replace_installed backtracking info. The replacement + SLOT may differ from the installed SLOT, so first search by cpv. + """ + built_pkgs = [] + for pkg in self._iter_similar_available(inst_pkg, + Atom("=%s" % inst_pkg.cpv)): + if not pkg.built: + return pkg.slot_atom + elif not pkg.installed: + # avoid using SLOT from a built instance + built_pkgs.append(pkg) + + for pkg in self._iter_similar_available(inst_pkg, inst_pkg.slot_atom): + if not pkg.built: + return pkg.slot_atom + elif not pkg.installed: + # avoid using SLOT from a built instance + built_pkgs.append(pkg) + + if built_pkgs: + best_version = None + for pkg in built_pkgs: + if best_version is None or pkg > best_version: + best_version = pkg + return best_version.slot_atom + + return None + + def _slot_operator_trigger_reinstalls(self): + """ + Search for packages with slot-operator deps on older slots, and schedule + rebuilds if they can link to a newer slot that's in the graph. + """ + + rebuild_if_new_slot = self._dynamic_config.myparams.get( + "rebuild_if_new_slot", "y") == "y" + + for slot_key, slot_info in self._dynamic_config._slot_operator_deps.items(): + + for dep in slot_info: + + atom = dep.atom + if atom.slot_operator is None: + continue + + if not atom.slot_operator_built: + new_child_slot = self._slot_change_probe(dep) + if new_child_slot is not None: + self._slot_change_backtrack(dep, new_child_slot) + continue + + if not (dep.parent and + isinstance(dep.parent, Package) and dep.parent.built): + continue + + # Check for slot update first, since we don't want to + # trigger reinstall of the child package when a newer + # slot will be used instead. + if rebuild_if_new_slot: + new_dep = self._slot_operator_update_probe(dep, + new_child_slot=True) + if new_dep is not None: + self._slot_operator_update_backtrack(dep, + new_child_slot=new_dep.child) + + if dep.want_update: + if self._slot_operator_update_probe(dep): + self._slot_operator_update_backtrack(dep) def _reinstall_for_flags(self, pkg, forced_flags, orig_use, orig_iuse, cur_use, cur_iuse): @@ -845,18 +1702,22 @@ class depgraph(object): in ("y", "auto")) newuse = "--newuse" in self._frozen_config.myopts changed_use = "changed-use" == self._frozen_config.myopts.get("--reinstall") + feature_flags = _get_feature_flags( + _get_eapi_attrs(pkg.eapi)) if newuse or (binpkg_respect_use and not changed_use): flags = set(orig_iuse.symmetric_difference( cur_iuse).difference(forced_flags)) flags.update(orig_iuse.intersection(orig_use).symmetric_difference( cur_iuse.intersection(cur_use))) + flags.difference_update(feature_flags) if flags: return flags elif changed_use or binpkg_respect_use: - flags = orig_iuse.intersection(orig_use).symmetric_difference( - cur_iuse.intersection(cur_use)) + flags = set(orig_iuse.intersection(orig_use).symmetric_difference( + cur_iuse.intersection(cur_use))) + flags.difference_update(feature_flags) if flags: return flags return None @@ -954,7 +1815,7 @@ class depgraph(object): # The blocker applies to the root where # the parent is or will be installed. blocker = Blocker(atom=dep.atom, - eapi=dep.parent.metadata["EAPI"], + eapi=dep.parent.eapi, priority=dep.priority, root=dep.parent.root) self._dynamic_config._blocker_parents.add(blocker, dep.parent) return 1 @@ -991,9 +1852,17 @@ class depgraph(object): (dep.parent, self._dynamic_config._runtime_pkg_mask[ dep.parent]), noiselevel=-1) - elif not self.need_restart(): + elif dep.atom.slot_operator_built and \ + self._slot_operator_unsatisfied_probe(dep): + self._slot_operator_unsatisfied_backtrack(dep) + return 1 + else: # Do not backtrack if only USE have to be changed in - # order to satisfy the dependency. + # order to satisfy the dependency. Note that when + # want_restart_for_use_change sets the need_restart + # flag, it causes _select_pkg_highest_available to + # return None, and eventually we come through here + # and skip the "missing dependency" backtracking path. dep_pkg, existing_node = \ self._select_package(dep.root, dep.atom.without_use, onlydeps=dep.onlydeps) @@ -1100,12 +1969,13 @@ class depgraph(object): # package selection, since we want to prompt the user # for USE adjustment rather than have REQUIRED_USE # affect package selection and || dep choices. - if not pkg.built and pkg.metadata.get("REQUIRED_USE") and \ - eapi_has_required_use(pkg.metadata["EAPI"]): + if not pkg.built and pkg._metadata.get("REQUIRED_USE") and \ + eapi_has_required_use(pkg.eapi): required_use_is_sat = check_required_use( - pkg.metadata["REQUIRED_USE"], + pkg._metadata["REQUIRED_USE"], self._pkg_use_enabled(pkg), - pkg.iuse.is_valid_flag) + pkg.iuse.is_valid_flag, + eapi=pkg.eapi) if not required_use_is_sat: if dep.atom is not None and dep.parent is not None: self._add_parent_atom(pkg, (dep.parent, dep.atom)) @@ -1132,140 +2002,27 @@ class depgraph(object): if existing_node: if existing_node_matches: # The existing node can be reused. - if arg_atoms: - for parent_atom in arg_atoms: - parent, atom = parent_atom - self._dynamic_config.digraph.add(existing_node, parent, - priority=priority) - self._add_parent_atom(existing_node, parent_atom) - # If a direct circular dependency is not an unsatisfied - # buildtime dependency then drop it here since otherwise - # it can skew the merge order calculation in an unwanted - # way. - if existing_node != myparent or \ - (priority.buildtime and not priority.satisfied): - self._dynamic_config.digraph.addnode(existing_node, myparent, - priority=priority) - if dep.atom is not None and dep.parent is not None: - self._add_parent_atom(existing_node, - (dep.parent, dep.atom)) - return 1 - else: - # A slot conflict has occurred. - # The existing node should not already be in - # runtime_pkg_mask, since that would trigger an - # infinite backtracking loop. - if self._dynamic_config._allow_backtracking and \ - existing_node in \ - self._dynamic_config._runtime_pkg_mask: - if "--debug" in self._frozen_config.myopts: - writemsg( - "!!! backtracking loop detected: %s %s\n" % \ - (existing_node, - self._dynamic_config._runtime_pkg_mask[ - existing_node]), noiselevel=-1) - elif self._dynamic_config._allow_backtracking and \ - not self._accept_blocker_conflicts() and \ - not self.need_restart(): - - self._add_slot_conflict(pkg) - if dep.atom is not None and dep.parent is not None: - self._add_parent_atom(pkg, (dep.parent, dep.atom)) - - if arg_atoms: - for parent_atom in arg_atoms: - parent, atom = parent_atom - self._add_parent_atom(pkg, parent_atom) - self._process_slot_conflicts() - - backtrack_data = [] - fallback_data = [] - all_parents = set() - # The ordering of backtrack_data can make - # a difference here, because both mask actions may lead - # to valid, but different, solutions and the one with - # 'existing_node' masked is usually the better one. Because - # of that, we choose an order such that - # the backtracker will first explore the choice with - # existing_node masked. The backtracker reverses the - # order, so the order it uses is the reverse of the - # order shown here. See bug #339606. - for to_be_selected, to_be_masked in (existing_node, pkg), (pkg, existing_node): - # For missed update messages, find out which - # atoms matched to_be_selected that did not - # match to_be_masked. - parent_atoms = \ - self._dynamic_config._parent_atoms.get(to_be_selected, set()) - if parent_atoms: - conflict_atoms = self._dynamic_config._slot_conflict_parent_atoms.intersection(parent_atoms) - if conflict_atoms: - parent_atoms = conflict_atoms - - all_parents.update(parent_atoms) - - all_match = True - for parent, atom in parent_atoms: - i = InternalPackageSet(initial_atoms=(atom,), - allow_repo=True) - if not i.findAtomForPackage(to_be_masked): - all_match = False - break - - fallback_data.append((to_be_masked, parent_atoms)) - - if all_match: - # 'to_be_masked' does not violate any parent atom, which means - # there is no point in masking it. - pass - else: - backtrack_data.append((to_be_masked, parent_atoms)) - - if not backtrack_data: - # This shouldn't happen, but fall back to the old - # behavior if this gets triggered somehow. - backtrack_data = fallback_data - - if len(backtrack_data) > 1: - # NOTE: Generally, we prefer to mask the higher - # version since this solves common cases in which a - # lower version is needed so that all dependencies - # will be satisfied (bug #337178). However, if - # existing_node happens to be installed then we - # mask that since this is a common case that is - # triggered when --update is not enabled. - if existing_node.installed: - pass - elif pkg > existing_node: - backtrack_data.reverse() - - to_be_masked = backtrack_data[-1][0] + if pkg != existing_node: + pkg = existing_node + previously_added = True + try: + arg_atoms = list(self._iter_atoms_for_pkg(pkg)) + except InvalidDependString as e: + if not pkg.installed: + # should have been masked before + # it was selected + raise - self._dynamic_config._backtrack_infos["slot conflict"] = backtrack_data - self._dynamic_config._need_restart = True - if "--debug" in self._frozen_config.myopts: - msg = [] - msg.append("") - msg.append("") - msg.append("backtracking due to slot conflict:") - if backtrack_data is fallback_data: - msg.append("!!! backtrack_data fallback") - msg.append(" first package: %s" % existing_node) - msg.append(" second package: %s" % pkg) - msg.append(" package to mask: %s" % to_be_masked) - msg.append(" slot: %s" % pkg.slot_atom) - msg.append(" parents: %s" % ", ".join( \ - "(%s, '%s')" % (ppkg, atom) for ppkg, atom in all_parents)) - msg.append("") - writemsg_level("".join("%s\n" % l for l in msg), - noiselevel=-1, level=logging.DEBUG) - return 0 + if debug: + writemsg_level( + "%s%s %s\n" % ("Re-used Child:".ljust(15), + pkg, pkg_use_display(pkg, + self._frozen_config.myopts, + modified_use=self._pkg_use_enabled(pkg))), + level=logging.DEBUG, noiselevel=-1) - # A slot collision has occurred. Sometimes this coincides - # with unresolvable blockers, so the slot collision will be - # shown later if there are no unresolvable blockers. + else: self._add_slot_conflict(pkg) - slot_collision = True - if debug: writemsg_level( "%s%s %s\n" % ("Slot Conflict:".ljust(15), @@ -1274,6 +2031,8 @@ class depgraph(object): modified_use=self._pkg_use_enabled(existing_node))), level=logging.DEBUG, noiselevel=-1) + slot_collision = True + if slot_collision: # Now add this node to the graph so that self.display() # can show use flags and --tree portage.output. This node is @@ -1298,11 +2057,11 @@ class depgraph(object): # doesn't already. Any pre-existing providers will be preferred # over this one. try: - pkgsettings.setinst(pkg.cpv, pkg.metadata) + pkgsettings.setinst(pkg.cpv, pkg._metadata) # For consistency, also update the global virtuals. settings = self._frozen_config.roots[pkg.root].settings settings.unlock() - settings.setinst(pkg.cpv, pkg.metadata) + settings.setinst(pkg.cpv, pkg._metadata) settings.lock() except portage.exception.InvalidDependString: if not pkg.installed: @@ -1312,12 +2071,19 @@ class depgraph(object): if arg_atoms: self._dynamic_config._set_nodes.add(pkg) - # Do this even when addme is False (--onlydeps) so that the + # Do this even for onlydeps, so that the # parent/child relationship is always known in case # self._show_slot_collision_notice() needs to be called later. - self._dynamic_config.digraph.add(pkg, myparent, priority=priority) - if dep.atom is not None and dep.parent is not None: - self._add_parent_atom(pkg, (dep.parent, dep.atom)) + # If a direct circular dependency is not an unsatisfied + # buildtime dependency then drop it here since otherwise + # it can skew the merge order calculation in an unwanted + # way. + if pkg != dep.parent or \ + (priority.buildtime and not priority.satisfied): + self._dynamic_config.digraph.add(pkg, + dep.parent, priority=priority) + if dep.atom is not None and dep.parent is not None: + self._add_parent_atom(pkg, (dep.parent, dep.atom)) if arg_atoms: for parent_atom in arg_atoms: @@ -1330,10 +2096,27 @@ class depgraph(object): # Installing package A, we need to make sure package A's deps are met. # emerge --deep <pkgspec>; we need to recursively check dependencies of pkgspec # If we are in --nodeps (no recursion) mode, we obviously only check 1 level of dependencies. - if arg_atoms: - depth = 0 + if arg_atoms and depth > 0: + for parent, atom in arg_atoms: + if parent.reset_depth: + depth = 0 + break + + if previously_added and pkg.depth is not None: + depth = min(pkg.depth, depth) pkg.depth = depth deep = self._dynamic_config.myparams.get("deep", 0) + update = "--update" in self._frozen_config.myopts + + dep.want_update = (not self._dynamic_config._complete_mode and + (arg_atoms or update) and + not (deep is not True and depth > deep)) + + dep.child = pkg + if (not pkg.onlydeps and + dep.atom and dep.atom.slot_operator is not None): + self._add_slot_operator_dep(dep) + recurse = deep is True or depth + 1 <= deep dep_stack = self._dynamic_config._dep_stack if "recurse" not in self._dynamic_config.myparams: @@ -1365,6 +2148,14 @@ class depgraph(object): self._dynamic_config._parent_atoms[pkg] = parent_atoms parent_atoms.add(parent_atom) + def _add_slot_operator_dep(self, dep): + slot_key = (dep.root, dep.child.slot_atom) + slot_info = self._dynamic_config._slot_operator_deps.get(slot_key) + if slot_info is None: + slot_info = [] + self._dynamic_config._slot_operator_deps[slot_key] = slot_info + slot_info.append(dep) + def _add_slot_conflict(self, pkg): self._dynamic_config._slot_collision_nodes.add(pkg) slot_key = (pkg.slot_atom, pkg.root) @@ -1378,12 +2169,12 @@ class depgraph(object): def _add_pkg_deps(self, pkg, allow_unsatisfied=False): myroot = pkg.root - metadata = pkg.metadata + metadata = pkg._metadata removal_action = "remove" in self._dynamic_config.myparams + eapi_attrs = _get_eapi_attrs(pkg.eapi) edepend={} - depkeys = ["DEPEND","RDEPEND","PDEPEND"] - for k in depkeys: + for k in Package._dep_keys: edepend[k] = metadata[k] if not pkg.built and \ @@ -1410,31 +2201,44 @@ class depgraph(object): # Removal actions never traverse ignored buildtime # dependencies, so it's safe to discard them early. edepend["DEPEND"] = "" + edepend["HDEPEND"] = "" ignore_build_time_deps = True + ignore_depend_deps = ignore_build_time_deps + ignore_hdepend_deps = ignore_build_time_deps + if removal_action: depend_root = myroot else: - depend_root = self._frozen_config._running_root.root - root_deps = self._frozen_config.myopts.get("--root-deps") - if root_deps is not None: - if root_deps is True: - depend_root = myroot - elif root_deps == "rdeps": - ignore_build_time_deps = True + if eapi_attrs.hdepend: + depend_root = myroot + else: + depend_root = self._frozen_config._running_root.root + root_deps = self._frozen_config.myopts.get("--root-deps") + if root_deps is not None: + if root_deps is True: + depend_root = myroot + elif root_deps == "rdeps": + ignore_depend_deps = True # If rebuild mode is not enabled, it's safe to discard ignored # build-time dependencies. If you want these deps to be traversed # in "complete" mode then you need to specify --with-bdeps=y. - if ignore_build_time_deps and \ - not self._rebuild.rebuild: - edepend["DEPEND"] = "" + if not self._rebuild.rebuild: + if ignore_depend_deps: + edepend["DEPEND"] = "" + if ignore_hdepend_deps: + edepend["HDEPEND"] = "" deps = ( (depend_root, edepend["DEPEND"], self._priority(buildtime=True, - optional=(pkg.built or ignore_build_time_deps), - ignored=ignore_build_time_deps)), + optional=(pkg.built or ignore_depend_deps), + ignored=ignore_depend_deps)), + (self._frozen_config._running_root.root, edepend["HDEPEND"], + self._priority(buildtime=True, + optional=(pkg.built or ignore_hdepend_deps), + ignored=ignore_hdepend_deps)), (myroot, edepend["RDEPEND"], self._priority(runtime=True)), (myroot, edepend["PDEPEND"], @@ -1456,7 +2260,10 @@ class depgraph(object): try: dep_string = portage.dep.use_reduce(dep_string, - uselist=self._pkg_use_enabled(pkg), is_valid_flag=pkg.iuse.is_valid_flag) + uselist=self._pkg_use_enabled(pkg), + is_valid_flag=pkg.iuse.is_valid_flag, + opconvert=True, token_class=Atom, + eapi=pkg.eapi) except portage.exception.InvalidDependString as e: if not pkg.installed: # should have been masked before it was selected @@ -1468,7 +2275,9 @@ class depgraph(object): # practical to ignore this issue for installed packages. try: dep_string = portage.dep.use_reduce(dep_string, - uselist=self._pkg_use_enabled(pkg)) + uselist=self._pkg_use_enabled(pkg), + opconvert=True, token_class=Atom, + eapi=pkg.eapi) except portage.exception.InvalidDependString as e: self._dynamic_config._masked_installed.add(pkg) del e @@ -1489,9 +2298,6 @@ class depgraph(object): if not dep_string: continue - dep_string = portage.dep.paren_enclose(dep_string, - unevaluated_atom=True) - if not self._add_pkg_dep_string( pkg, dep_root, dep_priority, dep_string, allow_unsatisfied): @@ -1514,6 +2320,37 @@ class depgraph(object): finally: self._dynamic_config._autounmask = _autounmask_backup + def _ignore_dependency(self, atom, pkg, child, dep, mypriority, recurse_satisfied): + """ + In some cases, dep_check will return deps that shouldn't + be processed any further, so they are identified and + discarded here. Try to discard as few as possible since + discarded dependencies reduce the amount of information + available for optimization of merge order. + Don't ignore dependencies if pkg has a slot operator dependency on the child + and the child has changed slot/sub_slot. + """ + if not mypriority.satisfied: + return False + slot_operator_rebuild = False + if atom.slot_operator == '=' and \ + (pkg.root, pkg.slot_atom) in self._dynamic_config._slot_operator_replace_installed and \ + mypriority.satisfied is not child and \ + mypriority.satisfied.installed and \ + child and \ + not child.installed and \ + (child.slot != mypriority.satisfied.slot or child.sub_slot != mypriority.satisfied.sub_slot): + slot_operator_rebuild = True + + return not atom.blocker and \ + not recurse_satisfied and \ + mypriority.satisfied.visible and \ + dep.child is not None and \ + not dep.child.installed and \ + self._dynamic_config._slot_pkg_map[dep.child.root].get( + dep.child.slot_atom) is None and \ + not slot_operator_rebuild + def _wrapped_add_pkg_dep_string(self, pkg, dep_root, dep_priority, dep_string, allow_unsatisfied): depth = pkg.depth + 1 @@ -1525,7 +2362,9 @@ class depgraph(object): if debug: writemsg_level("\nParent: %s\n" % (pkg,), noiselevel=-1, level=logging.DEBUG) - writemsg_level("Depstring: %s\n" % (dep_string,), + dep_repr = portage.dep.paren_enclose(dep_string, + unevaluated_atom=True, opconvert=True) + writemsg_level("Depstring: %s\n" % (dep_repr,), noiselevel=-1, level=logging.DEBUG) writemsg_level("Priority: %s\n" % (dep_priority,), noiselevel=-1, level=logging.DEBUG) @@ -1570,6 +2409,13 @@ class depgraph(object): mypriority = dep_priority.copy() if not atom.blocker: + + if atom.slot_operator == "=": + if mypriority.buildtime: + mypriority.buildtime_slot_op = True + if mypriority.runtime: + mypriority.runtime_slot_op = True + inst_pkgs = [inst_pkg for inst_pkg in reversed(vardb.match_pkgs(atom)) if not reinstall_atoms.findAtomForPackage(inst_pkg, @@ -1589,30 +2435,18 @@ class depgraph(object): priority=mypriority, root=dep_root) # In some cases, dep_check will return deps that shouldn't - # be proccessed any further, so they are identified and + # be processed any further, so they are identified and # discarded here. Try to discard as few as possible since # discarded dependencies reduce the amount of information # available for optimization of merge order. ignored = False - if not atom.blocker and \ - not recurse_satisfied and \ - mypriority.satisfied and \ - mypriority.satisfied.visible and \ - dep.child is not None and \ - not dep.child.installed and \ - self._dynamic_config._slot_pkg_map[dep.child.root].get( - dep.child.slot_atom) is None: + if self._ignore_dependency(atom, pkg, child, dep, mypriority, recurse_satisfied): myarg = None - if dep.root == self._frozen_config.target_root: - try: - myarg = next(self._iter_atoms_for_pkg(dep.child)) - except StopIteration: - pass - except InvalidDependString: - if not dep.child.installed: - # This shouldn't happen since the package - # should have been masked. - raise + try: + myarg = next(self._iter_atoms_for_pkg(dep.child), None) + except InvalidDependString: + if not dep.child.installed: + raise if myarg is None: # Existing child selection may not be valid unless @@ -1709,23 +2543,13 @@ class depgraph(object): collapsed_parent=pkg, collapsed_priority=dep_priority) ignored = False - if not atom.blocker and \ - not recurse_satisfied and \ - mypriority.satisfied and \ - mypriority.satisfied.visible and \ - dep.child is not None and \ - not dep.child.installed and \ - self._dynamic_config._slot_pkg_map[dep.child.root].get( - dep.child.slot_atom) is None: + if self._ignore_dependency(atom, pkg, child, dep, mypriority, recurse_satisfied): myarg = None - if dep.root == self._frozen_config.target_root: - try: - myarg = next(self._iter_atoms_for_pkg(dep.child)) - except StopIteration: - pass - except InvalidDependString: - if not dep.child.installed: - raise + try: + myarg = next(self._iter_atoms_for_pkg(dep.child), None) + except InvalidDependString: + if not dep.child.installed: + raise if myarg is None: ignored = True @@ -1767,7 +2591,7 @@ class depgraph(object): yield (atom, None) continue dep_pkg, existing_node = self._select_package( - root_config.root, atom) + root_config.root, atom, parent=parent) if dep_pkg is None: yield (atom, None) continue @@ -1819,9 +2643,14 @@ class depgraph(object): # Yield ~, =*, < and <= atoms first, since those are more likely to # cause slot conflicts, and we want those atoms to be displayed # in the resulting slot conflict message (see bug #291142). + # Give similar treatment to slot/sub-slot atoms. conflict_atoms = [] normal_atoms = [] + abi_atoms = [] for atom in cp_atoms: + if atom.slot_operator_built: + abi_atoms.append(atom) + continue conflict = False for child_pkg in atom_pkg_graph.child_nodes(atom): existing_node, matches = \ @@ -1834,7 +2663,7 @@ class depgraph(object): else: normal_atoms.append(atom) - for atom in chain(conflict_atoms, normal_atoms): + for atom in chain(abi_atoms, conflict_atoms, normal_atoms): child_pkgs = atom_pkg_graph.child_nodes(atom) # if more than one child, yield highest version if len(child_pkgs) > 1: @@ -1844,37 +2673,25 @@ class depgraph(object): def _queue_disjunctive_deps(self, pkg, dep_root, dep_priority, dep_struct): """ Queue disjunctive (virtual and ||) deps in self._dynamic_config._dep_disjunctive_stack. - Yields non-disjunctive deps. Raises InvalidDependString when + Yields non-disjunctive deps. Raises InvalidDependString when necessary. """ - i = 0 - while i < len(dep_struct): - x = dep_struct[i] + for x in dep_struct: if isinstance(x, list): - for y in self._queue_disjunctive_deps( - pkg, dep_root, dep_priority, x): - yield y - elif x == "||": - self._queue_disjunction(pkg, dep_root, dep_priority, - [ x, dep_struct[ i + 1 ] ] ) - i += 1 + if x and x[0] == "||": + self._queue_disjunction(pkg, dep_root, dep_priority, [x]) + else: + for y in self._queue_disjunctive_deps( + pkg, dep_root, dep_priority, x): + yield y else: - try: - x = portage.dep.Atom(x, eapi=pkg.metadata["EAPI"]) - except portage.exception.InvalidAtom: - if not pkg.installed: - raise portage.exception.InvalidDependString( - "invalid atom: '%s'" % x) + # Note: Eventually this will check for PROPERTIES=virtual + # or whatever other metadata gets implemented for this + # purpose. + if x.cp.startswith('virtual/'): + self._queue_disjunction(pkg, dep_root, dep_priority, [x]) else: - # Note: Eventually this will check for PROPERTIES=virtual - # or whatever other metadata gets implemented for this - # purpose. - if x.cp.startswith('virtual/'): - self._queue_disjunction( pkg, dep_root, - dep_priority, [ str(x) ] ) - else: - yield str(x) - i += 1 + yield x def _queue_disjunction(self, pkg, dep_root, dep_priority, dep_struct): self._dynamic_config._dep_disjunctive_stack.append( @@ -1887,10 +2704,8 @@ class depgraph(object): """ pkg, dep_root, dep_priority, dep_struct = \ self._dynamic_config._dep_disjunctive_stack.pop() - dep_string = portage.dep.paren_enclose(dep_struct, - unevaluated_atom=True) if not self._add_pkg_dep_string( - pkg, dep_root, dep_priority, dep_string, allow_unsatisfied): + pkg, dep_root, dep_priority, dep_struct, allow_unsatisfied): return 0 return 1 @@ -1965,9 +2780,24 @@ class depgraph(object): continue yield arg, atom - def select_files(self, myfiles): + def select_files(self, args): + # Use the global event loop for spinner progress + # indication during file owner lookups (bug #461412). + spinner_id = None + try: + spinner = self._frozen_config.spinner + if spinner is not None and \ + spinner.update is not spinner.update_quiet: + spinner_id = self._event_loop.idle_add( + self._frozen_config.spinner.update) + return self._select_files(args) + finally: + if spinner_id is not None: + self._event_loop.source_remove(spinner_id) + + def _select_files(self, myfiles): """Given a list of .tbz2s, .ebuilds sets, and deps, populate - self._dynamic_config._initial_arg_list and call self._resolve to create the + self._dynamic_config._initial_arg_list and call self._resolve to create the appropriate depgraph and return a favorite list.""" self._load_vdb() debug = "--debug" in self._frozen_config.myopts @@ -2000,8 +2830,18 @@ class depgraph(object): writemsg("!!! Please ensure the tbz2 exists as specified.\n\n", noiselevel=-1) return 0, myfavorites mytbz2=portage.xpak.tbz2(x) - mykey=mytbz2.getelements("CATEGORY")[0]+"/"+os.path.splitext(os.path.basename(x))[0] - if os.path.realpath(x) != \ + mykey = None + cat = mytbz2.getfile("CATEGORY") + if cat is not None: + cat = _unicode_decode(cat.strip(), + encoding=_encodings['repo.content']) + mykey = cat + "/" + os.path.basename(x)[:-5] + + if mykey is None: + writemsg(colorize("BAD", "\n*** Package is missing CATEGORY metadata: %s.\n\n" % x), noiselevel=-1) + self._dynamic_config._skip_restart = True + return 0, myfavorites + elif os.path.realpath(x) != \ os.path.realpath(bindb.bintree.getname(mykey)): writemsg(colorize("BAD", "\n*** You need to adjust PKGDIR to emerge this package.\n\n"), noiselevel=-1) self._dynamic_config._skip_restart = True @@ -2016,15 +2856,16 @@ class depgraph(object): pkgdir = os.path.dirname(ebuild_path) tree_root = os.path.dirname(os.path.dirname(pkgdir)) cp = pkgdir[len(tree_root)+1:] - e = portage.exception.PackageNotFound( - ("%s is not in a valid portage tree " + \ - "hierarchy or does not exist") % x) + error_msg = ("\n\n!!! '%s' is not in a valid portage tree " + "hierarchy or does not exist\n") % x if not portage.isvalidatom(cp): - raise e + writemsg(error_msg, noiselevel=-1) + return 0, myfavorites cat = portage.catsplit(cp)[0] mykey = cat + "/" + os.path.basename(ebuild_path[:-7]) if not portage.isvalidatom("="+mykey): - raise e + writemsg(error_msg, noiselevel=-1) + return 0, myfavorites ebuild_path = portdb.findname(mykey) if ebuild_path: if ebuild_path != os.path.join(os.path.realpath(tree_root), @@ -2040,8 +2881,8 @@ class depgraph(object): countdown(int(self._frozen_config.settings["EMERGE_WARNING_DELAY"]), "Continuing...") else: - raise portage.exception.PackageNotFound( - "%s is not in a valid portage tree hierarchy or does not exist" % x) + writemsg(error_msg, noiselevel=-1) + return 0, myfavorites pkg = self._pkg(mykey, "ebuild", root_config, onlydeps=onlydeps, myrepo=portdb.getRepositoryName( os.path.dirname(os.path.dirname(os.path.dirname(ebuild_path))))) @@ -2074,6 +2915,30 @@ class depgraph(object): raise portage.exception.PackageSetNotFound(s) if s in depgraph_sets.sets: continue + + try: + set_atoms = root_config.setconfig.getSetAtoms(s) + except portage.exception.PackageSetNotFound as e: + writemsg_level("\n\n", level=logging.ERROR, + noiselevel=-1) + for pset in list(depgraph_sets.sets.values()) + [sets[s]]: + for error_msg in pset.errors: + writemsg_level("%s\n" % (error_msg,), + level=logging.ERROR, noiselevel=-1) + + writemsg_level(("emerge: the given set '%s' " + "contains a non-existent set named '%s'.\n") % \ + (s, e), level=logging.ERROR, noiselevel=-1) + if s in ('world', 'selected') and \ + SETPREFIX + e.value in sets['selected']: + writemsg_level(("Use `emerge --deselect %s%s` to " + "remove this set from world_sets.\n") % + (SETPREFIX, e,), level=logging.ERROR, + noiselevel=-1) + writemsg_level("\n", level=logging.ERROR, + noiselevel=-1) + return False, myfavorites + pset = sets[s] depgraph_sets.sets[s] = pset args.append(SetArg(arg=x, pset=pset, @@ -2093,7 +2958,7 @@ class depgraph(object): # came from, if any. # 2) It takes away freedom from the resolver to choose other # possible expansions when necessary. - if "/" in x: + if "/" in x.split(":")[0]: args.append(AtomArg(arg=x, atom=Atom(x, allow_repo=True), root_config=root_config)) continue @@ -2194,13 +3059,8 @@ class depgraph(object): return 0, [] for cpv in owners: - slot = vardb.aux_get(cpv, ["SLOT"])[0] - if not slot: - # portage now masks packages with missing slot, but it's - # possible that one was installed by an older version - atom = Atom(portage.cpv_getkey(cpv)) - else: - atom = Atom("%s:%s" % (portage.cpv_getkey(cpv), slot)) + pkg = vardb._pkg_str(cpv, None) + atom = Atom("%s:%s" % (pkg.cp, pkg.slot)) args.append(AtomArg(arg=atom, atom=atom, root_config=root_config)) @@ -2248,6 +3108,7 @@ class depgraph(object): args = revised_greedy_args del revised_greedy_args + args.extend(self._gen_reinstall_sets()) self._set_args(args) myfavorites = set(myfavorites) @@ -2255,7 +3116,8 @@ class depgraph(object): if isinstance(arg, (AtomArg, PackageArg)): myfavorites.add(arg.atom) elif isinstance(arg, SetArg): - myfavorites.add(arg.arg) + if not arg.internal: + myfavorites.add(arg.arg) myfavorites = list(myfavorites) if debug: @@ -2263,12 +3125,38 @@ class depgraph(object): # Order needs to be preserved since a feature of --nodeps # is to allow the user to force a specific merge order. self._dynamic_config._initial_arg_list = args[:] - + return self._resolve(myfavorites) - + + def _gen_reinstall_sets(self): + + atom_list = [] + for root, atom in self._rebuild.rebuild_list: + atom_list.append((root, '__auto_rebuild__', atom)) + for root, atom in self._rebuild.reinstall_list: + atom_list.append((root, '__auto_reinstall__', atom)) + for root, atom in self._dynamic_config._slot_operator_replace_installed: + atom_list.append((root, '__auto_slot_operator_replace_installed__', atom)) + + set_dict = {} + for root, set_name, atom in atom_list: + set_dict.setdefault((root, set_name), []).append(atom) + + for (root, set_name), atoms in set_dict.items(): + yield SetArg(arg=(SETPREFIX + set_name), + # Set reset_depth=False here, since we don't want these + # special sets to interact with depth calculations (see + # the emerge --deep=DEPTH option), though we want them + # to behave like normal arguments in most other respects. + pset=InternalPackageSet(initial_atoms=atoms), + force_reinstall=True, + internal=True, + reset_depth=False, + root_config=self._frozen_config.roots[root]) + def _resolve(self, myfavorites): - """Given self._dynamic_config._initial_arg_list, pull in the root nodes, - call self._creategraph to process theier deps and return + """Given self._dynamic_config._initial_arg_list, pull in the root nodes, + call self._creategraph to process theier deps and return a favorite list.""" debug = "--debug" in self._frozen_config.myopts onlydeps = "--onlydeps" in self._frozen_config.myopts @@ -2277,10 +3165,7 @@ class depgraph(object): pprovideddict = pkgsettings.pprovideddict virtuals = pkgsettings.getvirtuals() args = self._dynamic_config._initial_arg_list[:] - for root, atom in chain(self._rebuild.rebuild_list, - self._rebuild.reinstall_list): - args.append(AtomArg(arg=atom, atom=atom, - root_config=self._frozen_config.roots[root])) + for arg in self._expand_set_args(args, add_to_digraph=True): for atom in arg.pset.getAtoms(): self._spinner_update() @@ -2322,6 +3207,16 @@ class depgraph(object): if pprovided_match: continue + excluded = False + for any_match in self._iter_match_pkgs_any( + self._frozen_config.roots[myroot], atom): + if self._frozen_config.excluded_pkgs.findAtomForPackage( + any_match, modified_use=self._pkg_use_enabled(any_match)): + excluded = True + break + if excluded: + continue + if not (isinstance(arg, SetArg) and \ arg.name in ("selected", "system", "world")): self._dynamic_config._unsatisfied_deps_for_display.append( @@ -2390,22 +3285,12 @@ class depgraph(object): except self._unknown_internal_error: return False, myfavorites - digraph_set = frozenset(self._dynamic_config.digraph) - - if digraph_set.intersection( - self._dynamic_config._needed_unstable_keywords) or \ - digraph_set.intersection( - self._dynamic_config._needed_p_mask_changes) or \ - digraph_set.intersection( - self._dynamic_config._needed_use_config_changes) or \ - digraph_set.intersection( - self._dynamic_config._needed_license_changes) : - #We failed if the user needs to change the configuration - self._dynamic_config._success_without_autounmask = True + if (self._dynamic_config._slot_collision_info and + not self._accept_blocker_conflicts()) or \ + (self._dynamic_config._allow_backtracking and + "slot conflict" in self._dynamic_config._backtrack_infos): return False, myfavorites - digraph_set = None - if self._rebuild.trigger_rebuilds(): backtrack_infos = self._dynamic_config._backtrack_infos config = backtrack_infos.setdefault("config", {}) @@ -2414,6 +3299,68 @@ class depgraph(object): self._dynamic_config._need_restart = True return False, myfavorites + if "config" in self._dynamic_config._backtrack_infos and \ + ("slot_operator_mask_built" in self._dynamic_config._backtrack_infos["config"] or + "slot_operator_replace_installed" in self._dynamic_config._backtrack_infos["config"]) and \ + self.need_restart(): + return False, myfavorites + + if not self._dynamic_config._prune_rebuilds and \ + self._dynamic_config._slot_operator_replace_installed and \ + self._get_missed_updates(): + # When there are missed updates, we might have triggered + # some unnecessary rebuilds (see bug #439688). So, prune + # all the rebuilds and backtrack with the problematic + # updates masked. The next backtrack run should pull in + # any rebuilds that are really needed, and this + # prune_rebuilds path should never be entered more than + # once in a series of backtracking nodes (in order to + # avoid a backtracking loop). + backtrack_infos = self._dynamic_config._backtrack_infos + config = backtrack_infos.setdefault("config", {}) + config["prune_rebuilds"] = True + self._dynamic_config._need_restart = True + return False, myfavorites + + if self.need_restart(): + # want_restart_for_use_change triggers this + return False, myfavorites + + if "--fetchonly" not in self._frozen_config.myopts and \ + "--buildpkgonly" in self._frozen_config.myopts: + graph_copy = self._dynamic_config.digraph.copy() + removed_nodes = set() + for node in graph_copy: + if not isinstance(node, Package) or \ + node.operation == "nomerge": + removed_nodes.add(node) + graph_copy.difference_update(removed_nodes) + if not graph_copy.hasallzeros(ignore_priority = \ + DepPrioritySatisfiedRange.ignore_medium): + self._dynamic_config._buildpkgonly_deps_unsatisfied = True + self._dynamic_config._skip_restart = True + return False, myfavorites + + # Any failures except those due to autounmask *alone* should return + # before this point, since the success_without_autounmask flag that's + # set below is reserved for cases where there are *zero* other + # problems. For reference, see backtrack_depgraph, where it skips the + # get_best_run() call when success_without_autounmask is True. + + digraph_nodes = self._dynamic_config.digraph.nodes + + if any(x in digraph_nodes for x in + self._dynamic_config._needed_unstable_keywords) or \ + any(x in digraph_nodes for x in + self._dynamic_config._needed_p_mask_changes) or \ + any(x in digraph_nodes for x in + self._dynamic_config._needed_use_config_changes) or \ + any(x in digraph_nodes for x in + self._dynamic_config._needed_license_changes) : + #We failed if the user needs to change the configuration + self._dynamic_config._success_without_autounmask = True + return False, myfavorites + # We're true here unless we are missing binaries. return (True, myfavorites) @@ -2455,8 +3402,8 @@ class depgraph(object): if refs is None: refs = [] atom_arg_map[atom_key] = refs - if arg not in refs: - refs.append(arg) + if arg not in refs: + refs.append(arg) for root in self._dynamic_config.sets: depgraph_sets = self._dynamic_config.sets[root] @@ -2486,14 +3433,15 @@ class depgraph(object): slots = set() for cpv in vardb.match(atom): # don't mix new virtuals with old virtuals - if portage.cpv_getkey(cpv) == highest_pkg.cp: - slots.add(vardb.aux_get(cpv, ["SLOT"])[0]) + pkg = vardb._pkg_str(cpv, None) + if pkg.cp == highest_pkg.cp: + slots.add(pkg.slot) - slots.add(highest_pkg.metadata["SLOT"]) + slots.add(highest_pkg.slot) if len(slots) == 1: return [] greedy_pkgs = [] - slots.remove(highest_pkg.metadata["SLOT"]) + slots.remove(highest_pkg.slot) while slots: slot = slots.pop() slot_atom = portage.dep.Atom("%s:%s" % (highest_pkg.cp, slot)) @@ -2507,9 +3455,9 @@ class depgraph(object): return [pkg.slot_atom for pkg in greedy_pkgs] blockers = {} - blocker_dep_keys = ["DEPEND", "PDEPEND", "RDEPEND"] + blocker_dep_keys = Package._dep_keys for pkg in greedy_pkgs + [highest_pkg]: - dep_str = " ".join(pkg.metadata[k] for k in blocker_dep_keys) + dep_str = " ".join(pkg._metadata[k] for k in blocker_dep_keys) try: selected_atoms = self._select_atoms( pkg.root, dep_str, self._pkg_use_enabled(pkg), @@ -2561,13 +3509,30 @@ class depgraph(object): not been scheduled for replacement. """ kwargs["trees"] = self._dynamic_config._graph_trees - return self._select_atoms_highest_available(*pargs, **kwargs) + return self._select_atoms_highest_available(*pargs, + **portage._native_kwargs(kwargs)) def _select_atoms_highest_available(self, root, depstring, myuse=None, parent=None, strict=True, trees=None, priority=None): """This will raise InvalidDependString if necessary. If trees is None then self._dynamic_config._filtered_trees is used.""" + if not isinstance(depstring, list): + eapi = None + is_valid_flag = None + if parent is not None: + eapi = parent.eapi + if not parent.installed: + is_valid_flag = parent.iuse.is_valid_flag + depstring = portage.dep.use_reduce(depstring, + uselist=myuse, opconvert=True, token_class=Atom, + is_valid_flag=is_valid_flag, eapi=eapi) + + if (self._dynamic_config.myparams.get( + "ignore_built_slot_operator_deps", "n") == "y" and + parent and parent.built): + ignore_built_slot_operator_deps(depstring) + pkgsettings = self._frozen_config.pkgsettings[root] if trees is None: trees = self._dynamic_config._filtered_trees @@ -2682,7 +3647,7 @@ class depgraph(object): return try: rdepend = self._select_atoms_from_graph( - pkg.root, pkg.metadata.get("RDEPEND", ""), + pkg.root, pkg._metadata.get("RDEPEND", ""), myuse=self._pkg_use_enabled(pkg), parent=pkg, strict=False) except InvalidDependString as e: @@ -2710,7 +3675,7 @@ class depgraph(object): """ try: rdepend = self._select_atoms( - pkg.root, pkg.metadata.get("RDEPEND", ""), + pkg.root, pkg._metadata.get("RDEPEND", ""), myuse=self._pkg_use_enabled(pkg), parent=pkg, priority=self._priority(runtime=True)) except InvalidDependString as e: @@ -2749,19 +3714,29 @@ class depgraph(object): child = None all_parents = self._dynamic_config._parent_atoms graph = self._dynamic_config.digraph + verbose_main_repo_display = "--verbose-main-repo-display" in \ + self._frozen_config.myopts + + def format_pkg(pkg): + pkg_name = "%s" % (pkg.cpv,) + if verbose_main_repo_display or pkg.repo != \ + pkg.root_config.settings.repositories.mainRepo().name: + pkg_name += _repo_separator + pkg.repo + return pkg_name if target_atom is not None and isinstance(node, Package): affecting_use = set() - for dep_str in "DEPEND", "RDEPEND", "PDEPEND": + for dep_str in Package._dep_keys: try: affecting_use.update(extract_affecting_use( - node.metadata[dep_str], target_atom, - eapi=node.metadata["EAPI"])) + node._metadata[dep_str], target_atom, + eapi=node.eapi)) except InvalidDependString: if not node.installed: raise affecting_use.difference_update(node.use.mask, node.use.force) - pkg_name = _unicode_decode("%s") % (node.cpv,) + pkg_name = format_pkg(node) + if affecting_use: usedep = [] for flag in affecting_use: @@ -2816,7 +3791,7 @@ class depgraph(object): node_type = "set" else: node_type = "argument" - dep_chain.append((_unicode_decode("%s") % (node,), node_type)) + dep_chain.append(("%s" % (node,), node_type)) elif node is not start_node: for ppkg, patom in all_parents[child]: @@ -2833,23 +3808,23 @@ class depgraph(object): if priorities is None: # This edge comes from _parent_atoms and was not added to # the graph, and _parent_atoms does not contain priorities. - dep_strings.add(node.metadata["DEPEND"]) - dep_strings.add(node.metadata["RDEPEND"]) - dep_strings.add(node.metadata["PDEPEND"]) + for k in Package._dep_keys: + dep_strings.add(node._metadata[k]) else: for priority in priorities: if priority.buildtime: - dep_strings.add(node.metadata["DEPEND"]) + for k in Package._buildtime_keys: + dep_strings.add(node._metadata[k]) if priority.runtime: - dep_strings.add(node.metadata["RDEPEND"]) + dep_strings.add(node._metadata["RDEPEND"]) if priority.runtime_post: - dep_strings.add(node.metadata["PDEPEND"]) + dep_strings.add(node._metadata["PDEPEND"]) affecting_use = set() for dep_str in dep_strings: try: affecting_use.update(extract_affecting_use( - dep_str, atom, eapi=node.metadata["EAPI"])) + dep_str, atom, eapi=node.eapi)) except InvalidDependString: if not node.installed: raise @@ -2858,7 +3833,7 @@ class depgraph(object): affecting_use.difference_update(node.use.mask, \ node.use.force) - pkg_name = _unicode_decode("%s") % (node.cpv,) + pkg_name = format_pkg(node) if affecting_use: usedep = [] for flag in affecting_use: @@ -2910,8 +3885,7 @@ class depgraph(object): if self._dynamic_config.digraph.parent_nodes(parent_arg): selected_parent = parent_arg else: - dep_chain.append( - (_unicode_decode("%s") % (parent_arg,), "argument")) + dep_chain.append(("%s" % (parent_arg,), "argument")) selected_parent = None node = selected_parent @@ -2926,7 +3900,7 @@ class depgraph(object): else: display_list.append("required by %s" % node) - msg = "#" + ", ".join(display_list) + "\n" + msg = "# " + "\n# ".join(display_list) + "\n" return msg @@ -2947,7 +3921,7 @@ class depgraph(object): if arg: xinfo='"%s"' % arg if isinstance(myparent, AtomArg): - xinfo = _unicode_decode('"%s"') % (myparent,) + xinfo = '"%s"' % (myparent,) # Discard null/ from failed cpv_expand category expansion. xinfo = xinfo.replace("null/", "") if root != self._frozen_config._running_root.root: @@ -2992,9 +3966,9 @@ class depgraph(object): repo = metadata.get('repository') pkg = self._pkg(cpv, pkg_type, root_config, installed=installed, myrepo=repo) - # pkg.metadata contains calculated USE for ebuilds, + # pkg._metadata contains calculated USE for ebuilds, # required later for getMissingLicenses. - metadata = pkg.metadata + metadata = pkg._metadata if pkg.invalid: # Avoid doing any operations with packages that # have invalid metadata. It would be unsafe at @@ -3033,12 +4007,13 @@ class depgraph(object): raise if not mreasons and \ not pkg.built and \ - pkg.metadata.get("REQUIRED_USE") and \ - eapi_has_required_use(pkg.metadata["EAPI"]): + pkg._metadata.get("REQUIRED_USE") and \ + eapi_has_required_use(pkg.eapi): if not check_required_use( - pkg.metadata["REQUIRED_USE"], + pkg._metadata["REQUIRED_USE"], self._pkg_use_enabled(pkg), - pkg.iuse.is_valid_flag): + pkg.iuse.is_valid_flag, + eapi=pkg.eapi): required_use_unsatisfied.append(pkg) continue root_slot = (pkg.root, pkg.slot_atom) @@ -3083,12 +4058,12 @@ class depgraph(object): untouchable_flags = \ frozenset(chain(pkg.use.mask, pkg.use.force)) - if untouchable_flags.intersection( + if any(x in untouchable_flags for x in chain(need_enable, need_disable)): continue missing_use_adjustable.add(pkg) - required_use = pkg.metadata.get("REQUIRED_USE") + required_use = pkg._metadata.get("REQUIRED_USE") required_use_warning = "" if required_use: old_use = self._pkg_use_enabled(pkg) @@ -3097,8 +4072,10 @@ class depgraph(object): new_use.add(flag) for flag in need_disable: new_use.discard(flag) - if check_required_use(required_use, old_use, pkg.iuse.is_valid_flag) and \ - not check_required_use(required_use, new_use, pkg.iuse.is_valid_flag): + if check_required_use(required_use, old_use, + pkg.iuse.is_valid_flag, eapi=pkg.eapi) \ + and not check_required_use(required_use, new_use, + pkg.iuse.is_valid_flag, eapi=pkg.eapi): required_use_warning = ", this change violates use flag constraints " + \ "defined by %s: '%s'" % (pkg.cpv, human_readable_required_use(required_use)) @@ -3133,10 +4110,10 @@ class depgraph(object): untouchable_flags = \ frozenset(chain(myparent.use.mask, myparent.use.force)) - if untouchable_flags.intersection(involved_flags): + if any(x in untouchable_flags for x in involved_flags): continue - required_use = myparent.metadata.get("REQUIRED_USE") + required_use = myparent._metadata.get("REQUIRED_USE") required_use_warning = "" if required_use: old_use = self._pkg_use_enabled(myparent) @@ -3146,8 +4123,12 @@ class depgraph(object): new_use.discard(flag) else: new_use.add(flag) - if check_required_use(required_use, old_use, myparent.iuse.is_valid_flag) and \ - not check_required_use(required_use, new_use, myparent.iuse.is_valid_flag): + if check_required_use(required_use, old_use, + myparent.iuse.is_valid_flag, + eapi=myparent.eapi) and \ + not check_required_use(required_use, new_use, + myparent.iuse.is_valid_flag, + eapi=myparent.eapi): required_use_warning = ", this change violates use flag constraints " + \ "defined by %s: '%s'" % (myparent.cpv, \ human_readable_required_use(required_use)) @@ -3234,14 +4215,15 @@ class depgraph(object): writemsg("\n The following REQUIRED_USE flag constraints " + \ "are unsatisfied:\n", noiselevel=-1) reduced_noise = check_required_use( - pkg.metadata["REQUIRED_USE"], + pkg._metadata["REQUIRED_USE"], self._pkg_use_enabled(pkg), - pkg.iuse.is_valid_flag).tounicode() + pkg.iuse.is_valid_flag, + eapi=pkg.eapi).tounicode() writemsg(" %s\n" % \ human_readable_required_use(reduced_noise), noiselevel=-1) normalized_required_use = \ - " ".join(pkg.metadata["REQUIRED_USE"].split()) + " ".join(pkg._metadata["REQUIRED_USE"].split()) if reduced_noise != normalized_required_use: writemsg("\n The above constraints " + \ "are a subset of the following complete expression:\n", @@ -3286,57 +4268,17 @@ class depgraph(object): not cp_exists and \ self._frozen_config.myopts.get( "--misspell-suggestions", "y") != "n": - cp = myparent.atom.cp.lower() - cat, pkg = portage.catsplit(cp) - if cat == "null": - cat = None writemsg("\nemerge: searching for similar names..." , noiselevel=-1) - all_cp = set() - all_cp.update(vardb.cp_all()) + dbs = [vardb] if "--usepkgonly" not in self._frozen_config.myopts: - all_cp.update(portdb.cp_all()) + dbs.append(portdb) if "--usepkg" in self._frozen_config.myopts: - all_cp.update(bindb.cp_all()) - # discard dir containing no ebuilds - all_cp.discard(cp) - - orig_cp_map = {} - for cp_orig in all_cp: - orig_cp_map.setdefault(cp_orig.lower(), []).append(cp_orig) - all_cp = set(orig_cp_map) - - if cat: - matches = difflib.get_close_matches(cp, all_cp) - else: - pkg_to_cp = {} - for other_cp in list(all_cp): - other_pkg = portage.catsplit(other_cp)[1] - if other_pkg == pkg: - # Check for non-identical package that - # differs only by upper/lower case. - identical = True - for cp_orig in orig_cp_map[other_cp]: - if portage.catsplit(cp_orig)[1] != \ - portage.catsplit(atom.cp)[1]: - identical = False - break - if identical: - # discard dir containing no ebuilds - all_cp.discard(other_cp) - continue - pkg_to_cp.setdefault(other_pkg, set()).add(other_cp) - pkg_matches = difflib.get_close_matches(pkg, pkg_to_cp) - matches = [] - for pkg_match in pkg_matches: - matches.extend(pkg_to_cp[pkg_match]) + dbs.append(bindb) - matches_orig_case = [] - for cp in matches: - matches_orig_case.extend(orig_cp_map[cp]) - matches = matches_orig_case + matches = similar_name_search(dbs, atom) if len(matches) == 1: writemsg("\nemerge: Maybe you meant " + matches[0] + "?\n" @@ -3357,8 +4299,7 @@ class depgraph(object): dep_chain = self._get_dep_chain(myparent, atom) for node, node_type in dep_chain: msg.append('(dependency required by "%s" [%s])' % \ - (colorize('INFORM', _unicode_decode("%s") % \ - (node)), node_type)) + (colorize('INFORM', "%s" % (node)), node_type)) if msg: writemsg("\n".join(msg), noiselevel=-1) @@ -3436,7 +4377,8 @@ class depgraph(object): # the newly built package still won't have the expected slot. # Therefore, assume that such SLOT dependencies are already # satisfied rather than forcing a rebuild. - if not matched_something and installed and atom.slot is not None: + if not matched_something and installed and \ + atom.slot is not None and not atom.slot_operator_built: if "remove" in self._dynamic_config.myparams: # We need to search the portdbapi, which is not in our @@ -3460,11 +4402,11 @@ class depgraph(object): for other_db, other_type, other_built, \ other_installed, other_keys in dbs: try: - if atom.slot == \ - other_db.aux_get(cpv, ["SLOT"])[0]: + if portage.dep._match_slot(atom, + other_db._pkg_str(_unicode(cpv), None)): slot_available = True break - except KeyError: + except (KeyError, InvalidData): pass if not slot_available: continue @@ -3476,20 +4418,12 @@ class depgraph(object): yield inst_pkg return - def _select_pkg_highest_available(self, root, atom, onlydeps=False): - cache_key = (root, atom, atom.unevaluated_atom, onlydeps) + def _select_pkg_highest_available(self, root, atom, onlydeps=False, parent=None): + cache_key = (root, atom, atom.unevaluated_atom, onlydeps, self._dynamic_config._autounmask) ret = self._dynamic_config._highest_pkg_cache.get(cache_key) if ret is not None: - pkg, existing = ret - if pkg and not existing: - existing = self._dynamic_config._slot_pkg_map[root].get(pkg.slot_atom) - if existing and existing == pkg: - # Update the cache to reflect that the - # package has been added to the graph. - ret = pkg, pkg - self._dynamic_config._highest_pkg_cache[cache_key] = ret return ret - ret = self._select_pkg_highest_available_imp(root, atom, onlydeps=onlydeps) + ret = self._select_pkg_highest_available_imp(root, atom, onlydeps=onlydeps, parent=parent) self._dynamic_config._highest_pkg_cache[cache_key] = ret pkg, existing = ret if pkg is not None: @@ -3504,21 +4438,85 @@ class depgraph(object): True if the user has not explicitly requested for this package to be replaced (typically via an atom on the command line). """ - if "selective" not in self._dynamic_config.myparams and \ - pkg.root == self._frozen_config.target_root: - if self._frozen_config.excluded_pkgs.findAtomForPackage(pkg, - modified_use=self._pkg_use_enabled(pkg)): - return True - try: - next(self._iter_atoms_for_pkg(pkg)) - except StopIteration: - pass - except portage.exception.InvalidDependString: - pass - else: + if self._frozen_config.excluded_pkgs.findAtomForPackage(pkg, + modified_use=self._pkg_use_enabled(pkg)): + return True + + arg = False + try: + for arg, atom in self._iter_atoms_for_pkg(pkg): + if arg.force_reinstall: + return False + except InvalidDependString: + pass + + if "selective" in self._dynamic_config.myparams: + return True + + return not arg + + def _want_update_pkg(self, parent, pkg): + + if self._frozen_config.excluded_pkgs.findAtomForPackage(pkg, + modified_use=self._pkg_use_enabled(pkg)): + return False + + arg_atoms = None + try: + arg_atoms = list(self._iter_atoms_for_pkg(pkg)) + except InvalidDependString: + if not pkg.installed: + # should have been masked before it was selected + raise + + depth = parent.depth or 0 + depth += 1 + + if arg_atoms: + for arg, atom in arg_atoms: + if arg.reset_depth: + depth = 0 + break + + deep = self._dynamic_config.myparams.get("deep", 0) + update = "--update" in self._frozen_config.myopts + + return (not self._dynamic_config._complete_mode and + (arg_atoms or update) and + not (deep is not True and depth > deep)) + + def _equiv_ebuild_visible(self, pkg, autounmask_level=None): + try: + pkg_eb = self._pkg( + pkg.cpv, "ebuild", pkg.root_config, myrepo=pkg.repo) + except portage.exception.PackageNotFound: + pkg_eb_visible = False + for pkg_eb in self._iter_match_pkgs(pkg.root_config, + "ebuild", Atom("=%s" % (pkg.cpv,))): + if self._pkg_visibility_check(pkg_eb, autounmask_level): + pkg_eb_visible = True + break + if not pkg_eb_visible: return False + else: + if not self._pkg_visibility_check(pkg_eb, autounmask_level): + return False + return True + def _equiv_binary_installed(self, pkg): + build_time = pkg.build_time + if not build_time: + return False + + try: + inst_pkg = self._pkg(pkg.cpv, "installed", + pkg.root_config, installed=True) + except PackageNotFound: + return False + + return build_time == inst_pkg.build_time + class _AutounmaskLevel(object): __slots__ = ("allow_use_changes", "allow_unstable_keywords", "allow_license_changes", \ "allow_missing_keywords", "allow_unmasks") @@ -3534,11 +4532,13 @@ class depgraph(object): """ Iterate over the different allowed things to unmask. - 1. USE + 0. USE + 1. USE + license 2. USE + ~arch + license 3. USE + ~arch + license + missing keywords - 4. USE + ~arch + license + masks - 5. USE + ~arch + license + missing keywords + masks + 4. USE + license + masks + 5. USE + ~arch + license + masks + 6. USE + ~arch + license + missing keywords + masks Some thoughts: * Do least invasive changes first. @@ -3553,16 +4553,30 @@ class depgraph(object): autounmask_level = self._AutounmaskLevel() autounmask_level.allow_use_changes = True + yield autounmask_level - for only_use_changes in (True, False): + autounmask_level.allow_license_changes = True + yield autounmask_level - autounmask_level.allow_unstable_keywords = (not only_use_changes) - autounmask_level.allow_license_changes = (not only_use_changes) + autounmask_level.allow_unstable_keywords = True + yield autounmask_level - for missing_keyword, unmask in ((False,False), (True, False), (False, True), (True, True)): + if not autounmask_keep_masks: - if (only_use_changes or autounmask_keep_masks) and (missing_keyword or unmask): - break + autounmask_level.allow_missing_keywords = True + yield autounmask_level + + # 4. USE + license + masks + # Try to respect keywords while discarding + # package.mask (see bug #463394). + autounmask_level.allow_unstable_keywords = False + autounmask_level.allow_missing_keywords = False + autounmask_level.allow_unmasks = True + yield autounmask_level + + autounmask_level.allow_unstable_keywords = True + + for missing_keyword, unmask in ((False, True), (True, True)): autounmask_level.allow_missing_keywords = missing_keyword autounmask_level.allow_unmasks = unmask @@ -3570,33 +4584,42 @@ class depgraph(object): yield autounmask_level - def _select_pkg_highest_available_imp(self, root, atom, onlydeps=False): - pkg, existing = self._wrapped_select_pkg_highest_available_imp(root, atom, onlydeps=onlydeps) + def _select_pkg_highest_available_imp(self, root, atom, onlydeps=False, parent=None): + pkg, existing = self._wrapped_select_pkg_highest_available_imp( + root, atom, onlydeps=onlydeps, parent=parent) default_selection = (pkg, existing) - def reset_pkg(pkg): + if self._dynamic_config._autounmask is True: if pkg is not None and \ pkg.installed and \ not self._want_installed_pkg(pkg): pkg = None - if self._dynamic_config._autounmask is True: - reset_pkg(pkg) + # Temporarily reset _need_restart state, in order to + # avoid interference as reported in bug #459832. + earlier_need_restart = self._dynamic_config._need_restart + self._dynamic_config._need_restart = False + try: + for autounmask_level in self._autounmask_levels(): + if pkg is not None: + break - for autounmask_level in self._autounmask_levels(): - if pkg is not None: - break + pkg, existing = \ + self._wrapped_select_pkg_highest_available_imp( + root, atom, onlydeps=onlydeps, + autounmask_level=autounmask_level, parent=parent) - pkg, existing = \ - self._wrapped_select_pkg_highest_available_imp( - root, atom, onlydeps=onlydeps, - autounmask_level=autounmask_level) + if pkg is not None and \ + pkg.installed and \ + not self._want_installed_pkg(pkg): + pkg = None - reset_pkg(pkg) - - if self._dynamic_config._need_restart: - return None, None + if self._dynamic_config._need_restart: + return None, None + finally: + if earlier_need_restart: + self._dynamic_config._need_restart = True if pkg is None: # This ensures that we can fall back to an installed package @@ -3726,25 +4749,29 @@ class depgraph(object): new_changes = {} for flag, state in target_use.items(): + real_flag = pkg.iuse.get_real_flag(flag) + if real_flag is None: + # Triggered by use-dep defaults. + continue if state: - if flag not in old_use: - if new_changes.get(flag) == False: + if real_flag not in old_use: + if new_changes.get(real_flag) == False: return old_use - new_changes[flag] = True + new_changes[real_flag] = True new_use.add(flag) else: - if flag in old_use: - if new_changes.get(flag) == True: + if real_flag in old_use: + if new_changes.get(real_flag) == True: return old_use - new_changes[flag] = False + new_changes[real_flag] = False new_use.update(old_use.difference(target_use)) def want_restart_for_use_change(pkg, new_use): if pkg not in self._dynamic_config.digraph.nodes: return False - for key in "DEPEND", "RDEPEND", "PDEPEND", "LICENSE": - dep = pkg.metadata[key] + for key in Package._dep_keys + ("LICENSE",): + dep = pkg._metadata[key] old_val = set(portage.dep.use_reduce(dep, pkg.use.enabled, is_valid_flag=pkg.iuse.is_valid_flag, flat=True)) new_val = set(portage.dep.use_reduce(dep, new_use, is_valid_flag=pkg.iuse.is_valid_flag, flat=True)) @@ -3758,7 +4785,7 @@ class depgraph(object): new_use, changes = self._dynamic_config._needed_use_config_changes.get(pkg) for ppkg, atom in parent_atoms: if not atom.use or \ - not atom.use.required.intersection(changes): + not any(x in atom.use.required for x in changes): continue else: return True @@ -3767,13 +4794,15 @@ class depgraph(object): if new_changes != old_changes: #Don't do the change if it violates REQUIRED_USE. - required_use = pkg.metadata.get("REQUIRED_USE") - if required_use and check_required_use(required_use, old_use, pkg.iuse.is_valid_flag) and \ - not check_required_use(required_use, new_use, pkg.iuse.is_valid_flag): + required_use = pkg._metadata.get("REQUIRED_USE") + if required_use and check_required_use(required_use, old_use, + pkg.iuse.is_valid_flag, eapi=pkg.eapi) and \ + not check_required_use(required_use, new_use, + pkg.iuse.is_valid_flag, eapi=pkg.eapi): return old_use - if pkg.use.mask.intersection(new_changes) or \ - pkg.use.force.intersection(new_changes): + if any(x in pkg.use.mask for x in new_changes) or \ + any(x in pkg.use.force for x in new_changes): return old_use self._dynamic_config._needed_use_config_changes[pkg] = (new_use, new_changes) @@ -3785,14 +4814,13 @@ class depgraph(object): self._dynamic_config._need_restart = True return new_use - def _wrapped_select_pkg_highest_available_imp(self, root, atom, onlydeps=False, autounmask_level=None): + def _wrapped_select_pkg_highest_available_imp(self, root, atom, onlydeps=False, autounmask_level=None, parent=None): root_config = self._frozen_config.roots[root] pkgsettings = self._frozen_config.pkgsettings[root] dbs = self._dynamic_config._filtered_trees[root]["dbs"] vardb = self._frozen_config.roots[root].trees["vartree"].dbapi # List of acceptable packages, ordered by type preference. matched_packages = [] - matched_pkgs_ignore_use = [] highest_version = None if not isinstance(atom, portage.dep.Atom): atom = portage.dep.Atom(atom) @@ -3844,7 +4872,7 @@ class depgraph(object): # Ignore USE deps for the initial match since we want to # ensure that updates aren't missed solely due to the user's # USE configuration. - for pkg in self._iter_match_pkgs(root_config, pkg_type, atom.without_use, + for pkg in self._iter_match_pkgs(root_config, pkg_type, atom.without_use, onlydeps=onlydeps): if pkg.cp != atom_cp and have_new_virt: # pull in a new-style virtual instead @@ -3930,8 +4958,8 @@ class depgraph(object): for selected_pkg in matched_packages: if selected_pkg.type_name == "binary" and \ selected_pkg.cpv == pkg.cpv and \ - selected_pkg.metadata.get('BUILD_TIME') == \ - pkg.metadata.get('BUILD_TIME'): + selected_pkg.build_time == \ + pkg.build_time: identical_binary = True break @@ -3944,37 +4972,24 @@ class depgraph(object): if not use_ebuild_visibility and (usepkgonly or useoldpkg): if pkg.installed and pkg.masks: continue - else: - try: - pkg_eb = self._pkg( - pkg.cpv, "ebuild", root_config, myrepo=pkg.repo) - except portage.exception.PackageNotFound: - pkg_eb_visible = False - for pkg_eb in self._iter_match_pkgs(pkg.root_config, - "ebuild", Atom("=%s" % (pkg.cpv,))): - if self._pkg_visibility_check(pkg_eb, autounmask_level): - pkg_eb_visible = True - break - if not pkg_eb_visible: - continue - else: - if not self._pkg_visibility_check(pkg_eb, autounmask_level): - continue + elif not self._equiv_ebuild_visible(pkg, + autounmask_level=autounmask_level): + continue # Calculation of USE for unbuilt ebuilds is relatively # expensive, so it is only performed lazily, after the # above visibility checks are complete. myarg = None - if root == self._frozen_config.target_root: - try: - myarg = next(self._iter_atoms_for_pkg(pkg)) - except StopIteration: - pass - except portage.exception.InvalidDependString: - if not installed: - # masked by corruption - continue + try: + for myarg, myarg_atom in self._iter_atoms_for_pkg(pkg): + if myarg.force_reinstall: + reinstall = True + break + except InvalidDependString: + if not installed: + # masked by corruption + continue if not installed and myarg: found_available_arg = True @@ -3987,7 +5002,6 @@ class depgraph(object): if atom.use: - matched_pkgs_ignore_use.append(pkg) if autounmask_level and autounmask_level.allow_use_changes and not pkg.built: target_use = {} for flag in atom.use.enabled: @@ -4000,11 +5014,14 @@ class depgraph(object): use_match = True can_adjust_use = not pkg.built - missing_enabled = atom.use.missing_enabled.difference(pkg.iuse.all) - missing_disabled = atom.use.missing_disabled.difference(pkg.iuse.all) + is_valid_flag = pkg.iuse.is_valid_flag + missing_enabled = frozenset(x for x in + atom.use.missing_enabled if not is_valid_flag(x)) + missing_disabled = frozenset(x for x in + atom.use.missing_disabled if not is_valid_flag(x)) if atom.use.enabled: - if atom.use.enabled.intersection(missing_disabled): + if any(x in atom.use.enabled for x in missing_disabled): use_match = False can_adjust_use = False need_enabled = atom.use.enabled.difference(use) @@ -4013,11 +5030,11 @@ class depgraph(object): if need_enabled: use_match = False if can_adjust_use: - if pkg.use.mask.intersection(need_enabled): + if any(x in pkg.use.mask for x in need_enabled): can_adjust_use = False if atom.use.disabled: - if atom.use.disabled.intersection(missing_enabled): + if any(x in atom.use.disabled for x in missing_enabled): use_match = False can_adjust_use = False need_disabled = atom.use.disabled.intersection(use) @@ -4026,8 +5043,8 @@ class depgraph(object): if need_disabled: use_match = False if can_adjust_use: - if pkg.use.force.difference( - pkg.use.mask).intersection(need_disabled): + if any(x in pkg.use.force and x not in + pkg.use.mask for x in need_disabled): can_adjust_use = False if not use_match: @@ -4075,7 +5092,11 @@ class depgraph(object): break # Compare built package to current config and # reject the built package if necessary. - if built and not useoldpkg and (not installed or matched_pkgs_ignore_use) and \ + if built and not useoldpkg and \ + (not installed or matched_packages) and \ + not (installed and + self._frozen_config.excluded_pkgs.findAtomForPackage(pkg, + modified_use=self._pkg_use_enabled(pkg))) and \ ("--newuse" in self._frozen_config.myopts or \ "--reinstall" in self._frozen_config.myopts or \ (not installed and self._dynamic_config.myparams.get( @@ -4160,6 +5181,26 @@ class depgraph(object): return existing_node, existing_node if len(matched_packages) > 1: + if parent is not None and \ + (parent.root, parent.slot_atom) in self._dynamic_config._slot_operator_replace_installed: + # We're forcing a rebuild of the parent because we missed some + # update because of a slot operator dep. + if atom.slot_operator == "=" and atom.sub_slot is None: + # This one is a slot operator dep. Exclude the installed packages if a newer non-installed + # pkg exists. + highest_installed = None + for pkg in matched_packages: + if pkg.installed: + if highest_installed is None or pkg.version > highest_installed.version: + highest_installed = pkg + + if highest_installed: + non_installed = [pkg for pkg in matched_packages \ + if not pkg.installed and pkg.version > highest_installed.version] + + if non_installed: + matched_packages = non_installed + if rebuilt_binaries: inst_pkg = None built_pkg = None @@ -4177,15 +5218,8 @@ class depgraph(object): # non-empty, in order to avoid cases like to # bug #306659 where BUILD_TIME fields are missing # in local and/or remote Packages file. - try: - built_timestamp = int(built_pkg.metadata['BUILD_TIME']) - except (KeyError, ValueError): - built_timestamp = 0 - - try: - installed_timestamp = int(inst_pkg.metadata['BUILD_TIME']) - except (KeyError, ValueError): - installed_timestamp = 0 + built_timestamp = built_pkg.build_time + installed_timestamp = inst_pkg.build_time if unbuilt_pkg is not None and unbuilt_pkg > built_pkg: pass @@ -4232,7 +5266,7 @@ class depgraph(object): # ordered by type preference ("ebuild" type is the last resort) return matched_packages[-1], existing_node - def _select_pkg_from_graph(self, root, atom, onlydeps=False): + def _select_pkg_from_graph(self, root, atom, onlydeps=False, parent=None): """ Select packages that have already been added to the graph or those that are installed and have not been scheduled for @@ -4242,11 +5276,18 @@ class depgraph(object): matches = graph_db.match_pkgs(atom) if not matches: return None, None - pkg = matches[-1] # highest match - in_graph = self._dynamic_config._slot_pkg_map[root].get(pkg.slot_atom) - return pkg, in_graph - def _select_pkg_from_installed(self, root, atom, onlydeps=False): + # There may be multiple matches, and they may + # conflict with eachother, so choose the highest + # version that has already been added to the graph. + for pkg in reversed(matches): + if pkg in self._dynamic_config.digraph: + return pkg, pkg + + # Fall back to installed packages + return self._select_pkg_from_installed(root, atom, onlydeps=onlydeps, parent=parent) + + def _select_pkg_from_installed(self, root, atom, onlydeps=False, parent=None): """ Select packages that are installed. """ @@ -4269,6 +5310,14 @@ class depgraph(object): unmasked = [pkg for pkg in matches if not pkg.masks] if unmasked: matches = unmasked + if len(matches) > 1: + # Now account for packages for which existing + # ebuilds are masked or unavailable (bug #445506). + unmasked = [pkg for pkg in matches if + self._equiv_ebuild_visible(pkg)] + if unmasked: + matches = unmasked + pkg = matches[-1] # highest match in_graph = self._dynamic_config._slot_pkg_map[root].get(pkg.slot_atom) return pkg, in_graph @@ -4293,9 +5342,19 @@ class depgraph(object): "recurse" not in self._dynamic_config.myparams: return 1 + complete_if_new_use = self._dynamic_config.myparams.get( + "complete_if_new_use", "y") == "y" + complete_if_new_ver = self._dynamic_config.myparams.get( + "complete_if_new_ver", "y") == "y" + rebuild_if_new_slot = self._dynamic_config.myparams.get( + "rebuild_if_new_slot", "y") == "y" + complete_if_new_slot = rebuild_if_new_slot + if "complete" not in self._dynamic_config.myparams and \ - self._dynamic_config.myparams.get("complete_if_new_ver", "y") == "y": - # Enable complete mode if an installed package version will change. + (complete_if_new_use or + complete_if_new_ver or complete_if_new_slot): + # Enable complete mode if an installed package will change somehow. + use_change = False version_change = False for node in self._dynamic_config.digraph: if not isinstance(node, Package) or \ @@ -4303,12 +5362,42 @@ class depgraph(object): continue vardb = self._frozen_config.roots[ node.root].trees["vartree"].dbapi - inst_pkg = vardb.match_pkgs(node.slot_atom) - if inst_pkg and (inst_pkg[0] > node or inst_pkg[0] < node): - version_change = True - break - if version_change: + if complete_if_new_use or complete_if_new_ver: + inst_pkg = vardb.match_pkgs(node.slot_atom) + if inst_pkg and inst_pkg[0].cp == node.cp: + inst_pkg = inst_pkg[0] + if complete_if_new_ver: + if inst_pkg < node or node < inst_pkg: + version_change = True + break + elif not (inst_pkg.slot == node.slot and + inst_pkg.sub_slot == node.sub_slot): + # slot/sub-slot change without revbump gets + # similar treatment to a version change + version_change = True + break + + # Intersect enabled USE with IUSE, in order to + # ignore forced USE from implicit IUSE flags, since + # they're probably irrelevant and they are sensitive + # to use.mask/force changes in the profile. + if complete_if_new_use and \ + (node.iuse.all != inst_pkg.iuse.all or + self._pkg_use_enabled(node).intersection(node.iuse.all) != + self._pkg_use_enabled(inst_pkg).intersection(inst_pkg.iuse.all)): + use_change = True + break + + if complete_if_new_slot: + cp_list = vardb.match_pkgs(Atom(node.cp)) + if (cp_list and cp_list[0].cp == node.cp and + not any(node.slot == pkg.slot and + node.sub_slot == pkg.sub_slot for pkg in cp_list)): + version_change = True + break + + if use_change or version_change: self._dynamic_config.myparams["complete"] = True if "complete" not in self._dynamic_config.myparams: @@ -4322,6 +5411,7 @@ class depgraph(object): # scheduled for replacement. Also, toggle the "deep" # parameter so that all dependencies are traversed and # accounted for. + self._dynamic_config._complete_mode = True self._select_atoms = self._select_atoms_from_graph if "remove" in self._dynamic_config.myparams: self._select_package = self._select_pkg_from_installed @@ -4409,7 +5499,7 @@ class depgraph(object): return 0 return 1 - def _pkg(self, cpv, type_name, root_config, installed=False, + def _pkg(self, cpv, type_name, root_config, installed=False, onlydeps=False, myrepo = None): """ Get a package instance from the cache, or create a new @@ -4480,7 +5570,7 @@ class depgraph(object): # For installed packages, always ignore blockers from DEPEND since # only runtime dependencies should be relevant for packages that # are already built. - dep_keys = ["RDEPEND", "PDEPEND"] + dep_keys = Package._runtime_keys for myroot in self._frozen_config.trees: if self._frozen_config.myopts.get("--root-deps") is not None and \ @@ -4542,7 +5632,7 @@ class depgraph(object): self._spinner_update() blocker_data = blocker_cache.get(cpv) if blocker_data is not None and \ - blocker_data.counter != long(pkg.metadata["COUNTER"]): + blocker_data.counter != pkg.counter: blocker_data = None # If blocker data from the graph is available, use @@ -4559,9 +5649,8 @@ class depgraph(object): blockers is not None: # Re-use the blockers from the graph. blocker_atoms = sorted(blockers) - counter = long(pkg.metadata["COUNTER"]) blocker_data = \ - blocker_cache.BlockerData(counter, blocker_atoms) + blocker_cache.BlockerData(pkg.counter, blocker_atoms) blocker_cache[pkg.cpv] = blocker_data continue @@ -4586,7 +5675,7 @@ class depgraph(object): # matches (this can happen if an atom lacks a # category). show_invalid_depstring_notice( - pkg, depstr, _unicode_decode("%s") % (e,)) + pkg, depstr, "%s" % (e,)) del e raise if not success: @@ -4603,22 +5692,20 @@ class depgraph(object): blocker_atoms = [myatom for myatom in atoms \ if myatom.blocker] blocker_atoms.sort() - counter = long(pkg.metadata["COUNTER"]) blocker_cache[cpv] = \ - blocker_cache.BlockerData(counter, blocker_atoms) + blocker_cache.BlockerData(pkg.counter, blocker_atoms) if blocker_atoms: try: for atom in blocker_atoms: blocker = Blocker(atom=atom, - eapi=pkg.metadata["EAPI"], + eapi=pkg.eapi, priority=self._priority(runtime=True), root=myroot) self._dynamic_config._blocker_parents.add(blocker, pkg) except portage.exception.InvalidAtom as e: depstr = " ".join(vardb.aux_get(pkg.cpv, dep_keys)) show_invalid_depstring_notice( - pkg, depstr, - _unicode_decode("Invalid Atom: %s") % (e,)) + pkg, depstr, "Invalid Atom: %s" % (e,)) return False for cpv in stale_cache: del blocker_cache[cpv] @@ -4640,7 +5727,7 @@ class depgraph(object): myroot = blocker.root initial_db = self._frozen_config.trees[myroot]["vartree"].dbapi final_db = self._dynamic_config.mydbapi[myroot] - + provider_virtual = False if blocker.cp in virtuals and \ not self._have_new_virt(blocker.root, blocker.cp): @@ -4751,7 +5838,7 @@ class depgraph(object): for inst_pkg, inst_task in depends_on_order: uninst_task = Package(built=inst_pkg.built, cpv=inst_pkg.cpv, installed=inst_pkg.installed, - metadata=inst_pkg.metadata, + metadata=inst_pkg._metadata, operation="uninstall", root_config=inst_pkg.root_config, type_name=inst_pkg.type_name) @@ -4817,7 +5904,12 @@ class depgraph(object): mygraph.order.sort(key=cmp_sort_key(cmp_merge_preference)) - def altlist(self, reversed=False): + def altlist(self, reversed=DeprecationWarning): + + if reversed is not DeprecationWarning: + warnings.warn("The reversed parameter of " + "_emerge.depgraph.depgraph.altlist() is deprecated", + DeprecationWarning, stacklevel=2) while self._dynamic_config._serialized_tasks_cache is None: self._resolve_conflicts() @@ -4827,9 +5919,13 @@ class depgraph(object): except self._serialize_tasks_retry: pass - retlist = self._dynamic_config._serialized_tasks_cache[:] - if reversed: + retlist = self._dynamic_config._serialized_tasks_cache + if reversed is not DeprecationWarning and reversed: + # TODO: remove the "reversed" parameter (builtin name collision) + retlist = list(retlist) retlist.reverse() + retlist = tuple(retlist) + return retlist def _implicit_libc_deps(self, mergelist, graph): @@ -4937,16 +6033,27 @@ class depgraph(object): root_config.root]["root_config"] = root_config def _resolve_conflicts(self): + + if "complete" not in self._dynamic_config.myparams and \ + self._dynamic_config._allow_backtracking and \ + self._dynamic_config._slot_collision_nodes and \ + not self._accept_blocker_conflicts(): + self._dynamic_config.myparams["complete"] = True + if not self._complete_graph(): raise self._unknown_internal_error() + self._process_slot_conflicts() + + if self._dynamic_config._allow_backtracking: + self._slot_operator_trigger_reinstalls() + if not self._validate_blockers(): - self._dynamic_config._skip_restart = True + # Blockers don't trigger the _skip_restart flag, since + # backtracking may solve blockers when it solves slot + # conflicts (or by blind luck). raise self._unknown_internal_error() - if self._dynamic_config._slot_collision_info: - self._process_slot_conflicts() - def _serialize_tasks(self): debug = "--debug" in self._frozen_config.myopts @@ -5061,7 +6168,7 @@ class depgraph(object): if running_portage is not None: try: portage_rdepend = self._select_atoms_highest_available( - running_root, running_portage.metadata["RDEPEND"], + running_root, running_portage._metadata["RDEPEND"], myuse=self._pkg_use_enabled(running_portage), parent=running_portage, strict=False) except portage.exception.InvalidDependString as e: @@ -5241,7 +6348,7 @@ class depgraph(object): for node in nodes: parents = mygraph.parent_nodes(node, ignore_priority=DepPrioritySatisfiedRange.ignore_soft) - if parents and set(parents).intersection(asap_nodes): + if any(x in asap_nodes for x in parents): selected_nodes = [node] break else: @@ -5409,8 +6516,7 @@ class depgraph(object): other_version = None for pkg in vardb.match_pkgs(atom): if pkg.cpv == task.cpv and \ - pkg.metadata["COUNTER"] == \ - task.metadata["COUNTER"]: + pkg.counter == task.counter: continue other_version = pkg break @@ -5617,7 +6723,7 @@ class depgraph(object): inst_pkg = inst_pkg[0] uninst_task = Package(built=inst_pkg.built, cpv=inst_pkg.cpv, installed=inst_pkg.installed, - metadata=inst_pkg.metadata, + metadata=inst_pkg._metadata, operation="uninstall", root_config=inst_pkg.root_config, type_name=inst_pkg.type_name) @@ -5689,17 +6795,21 @@ class depgraph(object): for blocker in unsolvable_blockers: retlist.append(blocker) + retlist = tuple(retlist) + if unsolvable_blockers and \ not self._accept_blocker_conflicts(): self._dynamic_config._unsatisfied_blockers_for_display = unsolvable_blockers - self._dynamic_config._serialized_tasks_cache = retlist[:] + self._dynamic_config._serialized_tasks_cache = retlist self._dynamic_config._scheduler_graph = scheduler_graph - self._dynamic_config._skip_restart = True + # Blockers don't trigger the _skip_restart flag, since + # backtracking may solve blockers when it solves slot + # conflicts (or by blind luck). raise self._unknown_internal_error() if self._dynamic_config._slot_collision_info and \ not self._accept_blocker_conflicts(): - self._dynamic_config._serialized_tasks_cache = retlist[:] + self._dynamic_config._serialized_tasks_cache = retlist self._dynamic_config._scheduler_graph = scheduler_graph raise self._unknown_internal_error() @@ -5753,13 +6863,8 @@ class depgraph(object): def _show_merge_list(self): if self._dynamic_config._serialized_tasks_cache is not None and \ not (self._dynamic_config._displayed_list is not None and \ - (self._dynamic_config._displayed_list == self._dynamic_config._serialized_tasks_cache or \ - self._dynamic_config._displayed_list == \ - list(reversed(self._dynamic_config._serialized_tasks_cache)))): - display_list = self._dynamic_config._serialized_tasks_cache[:] - if "--tree" in self._frozen_config.myopts: - display_list.reverse() - self.display(display_list) + self._dynamic_config._displayed_list is self._dynamic_config._serialized_tasks_cache): + self.display(self._dynamic_config._serialized_tasks_cache) def _show_unsatisfied_blockers(self, blockers): self._show_merge_list() @@ -5777,10 +6882,18 @@ class depgraph(object): # the reasons are not apparent from the normal merge list # display. + slot_collision_info = self._dynamic_config._slot_collision_info + conflict_pkgs = {} for blocker in blockers: for pkg in chain(self._dynamic_config._blocked_pkgs.child_nodes(blocker), \ self._dynamic_config._blocker_parents.parent_nodes(blocker)): + if (pkg.slot_atom, pkg.root) in slot_collision_info: + # The slot conflict display has better noise reduction + # than the unsatisfied blockers display, so skip + # unsatisfied blockers display for packages involved + # directly in slot conflicts (see bug #385391). + continue parent_atoms = self._dynamic_config._parent_atoms.get(pkg) if not parent_atoms: atom = self._dynamic_config._blocked_world_pkgs.get(pkg) @@ -5838,7 +6951,14 @@ class depgraph(object): else: # Display the specific atom from SetArg or # Package types. - msg.append("%s required by %s" % (atom, parent)) + if atom != atom.unevaluated_atom: + # Show the unevaluated atom, since it can reveal + # issues with conditional use-flags missing + # from IUSE. + msg.append("%s (%s) required by %s" % + (atom.unevaluated_atom, atom, parent)) + else: + msg.append("%s required by %s" % (atom, parent)) msg.append("\n") msg.append("\n") @@ -5854,6 +6974,10 @@ class depgraph(object): # redundantly displaying this exact same merge list # again via _show_merge_list(). self._dynamic_config._displayed_list = mylist + + if "--tree" in self._frozen_config.myopts: + mylist = tuple(reversed(mylist)) + display = Display() return display(self, mylist, favorites, verbosity) @@ -5926,7 +7050,7 @@ class depgraph(object): if is_latest: unstable_keyword_msg[root].append(">=%s %s\n" % (pkg.cpv, keyword)) elif is_latest_in_slot: - unstable_keyword_msg[root].append(">=%s:%s %s\n" % (pkg.cpv, pkg.metadata["SLOT"], keyword)) + unstable_keyword_msg[root].append(">=%s:%s %s\n" % (pkg.cpv, pkg.slot, keyword)) else: unstable_keyword_msg[root].append("=%s %s\n" % (pkg.cpv, keyword)) else: @@ -5949,7 +7073,7 @@ class depgraph(object): keyword = reason.unmask_hint.value comment, filename = portage.getmaskingreason( - pkg.cpv, metadata=pkg.metadata, + pkg.cpv, metadata=pkg._metadata, settings=pkgsettings, portdb=pkg.root_config.trees["porttree"].dbapi, return_location=True) @@ -5966,7 +7090,7 @@ class depgraph(object): if is_latest: p_mask_change_msg[root].append(">=%s\n" % pkg.cpv) elif is_latest_in_slot: - p_mask_change_msg[root].append(">=%s:%s\n" % (pkg.cpv, pkg.metadata["SLOT"])) + p_mask_change_msg[root].append(">=%s:%s\n" % (pkg.cpv, pkg.slot)) else: p_mask_change_msg[root].append("=%s\n" % pkg.cpv) else: @@ -5991,7 +7115,7 @@ class depgraph(object): if is_latest: use_changes_msg[root].append(">=%s %s\n" % (pkg.cpv, " ".join(adjustments))) elif is_latest_in_slot: - use_changes_msg[root].append(">=%s:%s %s\n" % (pkg.cpv, pkg.metadata["SLOT"], " ".join(adjustments))) + use_changes_msg[root].append(">=%s:%s %s\n" % (pkg.cpv, pkg.slot, " ".join(adjustments))) else: use_changes_msg[root].append("=%s %s\n" % (pkg.cpv, " ".join(adjustments))) @@ -6008,7 +7132,7 @@ class depgraph(object): if is_latest: license_msg[root].append(">=%s %s\n" % (pkg.cpv, " ".join(sorted(missing_licenses)))) elif is_latest_in_slot: - license_msg[root].append(">=%s:%s %s\n" % (pkg.cpv, pkg.metadata["SLOT"], " ".join(sorted(missing_licenses)))) + license_msg[root].append(">=%s:%s %s\n" % (pkg.cpv, pkg.slot, " ".join(sorted(missing_licenses)))) else: license_msg[root].append("=%s %s\n" % (pkg.cpv, " ".join(sorted(missing_licenses)))) @@ -6048,7 +7172,7 @@ class depgraph(object): if stat.S_ISREG(st.st_mode): last_file_path = p elif stat.S_ISDIR(st.st_mode): - if os.path.basename(p) in _ignorecvs_dirs: + if os.path.basename(p) in VCS_DIRS: continue try: contents = os.listdir(p) @@ -6117,24 +7241,25 @@ class depgraph(object): if len(roots) > 1: writemsg("\nFor %s:\n" % abs_user_config, noiselevel=-1) + def _writemsg(reason, file): + writemsg(('\nThe following %s are necessary to proceed:\n' + ' (see "%s" in the portage(5) man page for more details)\n') + % (colorize('BAD', reason), file), noiselevel=-1) + if root in unstable_keyword_msg: - writemsg("\nThe following " + colorize("BAD", "keyword changes") + \ - " are necessary to proceed:\n", noiselevel=-1) + _writemsg('keyword changes', 'package.accept_keywords') writemsg(format_msg(unstable_keyword_msg[root]), noiselevel=-1) if root in p_mask_change_msg: - writemsg("\nThe following " + colorize("BAD", "mask changes") + \ - " are necessary to proceed:\n", noiselevel=-1) + _writemsg('mask changes', 'package.unmask') writemsg(format_msg(p_mask_change_msg[root]), noiselevel=-1) if root in use_changes_msg: - writemsg("\nThe following " + colorize("BAD", "USE changes") + \ - " are necessary to proceed:\n", noiselevel=-1) + _writemsg('USE changes', 'package.use') writemsg(format_msg(use_changes_msg[root]), noiselevel=-1) if root in license_msg: - writemsg("\nThe following " + colorize("BAD", "license changes") + \ - " are necessary to proceed:\n", noiselevel=-1) + _writemsg('license changes', 'package.license') writemsg(format_msg(license_msg[root]), noiselevel=-1) protect_obj = {} @@ -6148,11 +7273,12 @@ class depgraph(object): def write_changes(root, changes, file_to_write_to): file_contents = None try: - file_contents = io.open( + with io.open( _unicode_encode(file_to_write_to, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['content'], - errors='replace').readlines() + errors='replace') as f: + file_contents = f.readlines() except IOError as e: if e.errno == errno.ENOENT: file_contents = [] @@ -6218,10 +7344,16 @@ class depgraph(object): noiselevel=-1) writemsg("".join(problems), noiselevel=-1) elif write_to_file and roots: - writemsg("\nAutounmask changes successfully written. Remember to run dispatch-conf.\n", \ + writemsg("\nAutounmask changes successfully written.\n", noiselevel=-1) + for root in roots: + chk_updated_cfg_files(root, + [os.path.join(os.sep, USER_CONFIG_PATH)]) elif not pretend and not autounmask_write and roots: - writemsg("\nUse --autounmask-write to write changes to config files (honoring CONFIG_PROTECT).\n", \ + writemsg("\nUse --autounmask-write to write changes to config files (honoring\n" + "CONFIG_PROTECT). Carefully examine the list of proposed changes,\n" + "paying special attention to mask or keyword changes that may expose\n" + "experimental or unstable packages.\n", noiselevel=-1) @@ -6238,21 +7370,33 @@ class depgraph(object): self._show_circular_deps( self._dynamic_config._circular_deps_for_display) - # The slot conflict display has better noise reduction than - # the unsatisfied blockers display, so skip unsatisfied blockers - # display if there are slot conflicts (see bug #385391). + unresolved_conflicts = False if self._dynamic_config._slot_collision_info: + unresolved_conflicts = True self._show_slot_collision_notice() - elif self._dynamic_config._unsatisfied_blockers_for_display is not None: + if self._dynamic_config._unsatisfied_blockers_for_display is not None: + unresolved_conflicts = True self._show_unsatisfied_blockers( self._dynamic_config._unsatisfied_blockers_for_display) - else: + + # Only show missed updates if there are no unresolved conflicts, + # since they may be irrelevant after the conflicts are solved. + if not unresolved_conflicts: self._show_missed_update() + self._compute_abi_rebuild_info() + self._show_abi_rebuild_info() + self._show_ignored_binaries() self._display_autounmask() + for depgraph_sets in self._dynamic_config.sets.values(): + for pset in depgraph_sets.sets.values(): + for error_msg in pset.errors: + writemsg_level("%s\n" % (error_msg,), + level=logging.ERROR, noiselevel=-1) + # TODO: Add generic support for "set problem" handlers so that # the below warnings aren't special cases for world only. @@ -6328,7 +7472,7 @@ class depgraph(object): pkgsettings = self._frozen_config.pkgsettings[pkg.root] mreasons = get_masking_status(pkg, pkgsettings, root_config, use=self._pkg_use_enabled(pkg)) masked_packages.append((root_config, pkgsettings, - pkg.cpv, pkg.repo, pkg.metadata, mreasons)) + pkg.cpv, pkg.repo, pkg._metadata, mreasons)) if masked_packages: writemsg("\n" + colorize("BAD", "!!!") + \ " The following updates are masked by LICENSE changes:\n", @@ -6343,7 +7487,7 @@ class depgraph(object): pkgsettings = self._frozen_config.pkgsettings[pkg.root] mreasons = get_masking_status(pkg, pkgsettings, root_config, use=self._pkg_use_enabled) masked_packages.append((root_config, pkgsettings, - pkg.cpv, pkg.repo, pkg.metadata, mreasons)) + pkg.cpv, pkg.repo, pkg._metadata, mreasons)) if masked_packages: writemsg("\n" + colorize("BAD", "!!!") + \ " The following installed packages are masked:\n", @@ -6353,7 +7497,15 @@ class depgraph(object): writemsg("\n", noiselevel=-1) for pargs, kwargs in self._dynamic_config._unsatisfied_deps_for_display: - self._show_unsatisfied_dep(*pargs, **kwargs) + self._show_unsatisfied_dep(*pargs, + **portage._native_kwargs(kwargs)) + + if self._dynamic_config._buildpkgonly_deps_unsatisfied: + self._show_merge_list() + writemsg("\n!!! --buildpkgonly requires all " + "dependencies to be merged.\n", noiselevel=-1) + writemsg("!!! Cannot merge requested packages. " + "Merge deps and try again.\n\n", noiselevel=-1) def saveNomergeFavorites(self): """Find atoms in favorites that are not in the mergelist and add them @@ -6401,6 +7553,9 @@ class depgraph(object): continue if arg.root_config.root != root_config.root: continue + if arg.internal: + # __auto_* sets + continue k = arg.name if k in ("selected", "world") or \ not root_config.sets[k].world_candidate: @@ -6411,12 +7566,31 @@ class depgraph(object): all_added.append(SETPREFIX + k) all_added.extend(added_favorites) all_added.sort() - for a in all_added: - writemsg_stdout( - ">>> Recording %s in \"world\" favorites file...\n" % \ - colorize("INFORM", str(a)), noiselevel=-1) if all_added: - world_set.update(all_added) + skip = False + if "--ask" in self._frozen_config.myopts: + writemsg_stdout("\n", noiselevel=-1) + for a in all_added: + writemsg_stdout(" %s %s\n" % (colorize("GOOD", "*"), a), + noiselevel=-1) + writemsg_stdout("\n", noiselevel=-1) + prompt = "Would you like to add these packages to your world " \ + "favorites?" + enter_invalid = '--ask-enter-invalid' in \ + self._frozen_config.myopts + if userquery(prompt, enter_invalid) == "No": + skip = True + + if not skip: + for a in all_added: + if a.startswith(SETPREFIX): + filename = "world_sets" + else: + filename = "world" + writemsg_stdout( + ">>> Recording %s in \"%s\" favorites file...\n" % + (colorize("INFORM", _unicode(a)), filename), noiselevel=-1) + world_set.update(all_added) if world_locked: world_set.unlock() @@ -6691,14 +7865,15 @@ class depgraph(object): try: for pargs, kwargs in self._dynamic_config._unsatisfied_deps_for_display: self._show_unsatisfied_dep( - *pargs, check_autounmask_breakage=True, **kwargs) + *pargs, check_autounmask_breakage=True, + **portage._native_kwargs(kwargs)) except self._autounmask_breakage: return True return False def get_backtrack_infos(self): return self._dynamic_config._backtrack_infos - + class _dep_check_composite_db(dbapi): """ @@ -6793,13 +7968,8 @@ class _dep_check_composite_db(dbapi): return ret[:] def _visible(self, pkg): - if pkg.installed and "selective" not in self._depgraph._dynamic_config.myparams: - try: - arg = next(self._depgraph._iter_atoms_for_pkg(pkg)) - except (StopIteration, portage.exception.InvalidDependString): - arg = None - if arg: - return False + if pkg.installed and not self._depgraph._want_installed_pkg(pkg): + return False if pkg.installed and \ (pkg.masks or not self._depgraph._pkg_visibility_check(pkg)): # Account for packages with masks (like KEYWORDS masks) @@ -6815,24 +7985,8 @@ class _dep_check_composite_db(dbapi): if not avoid_update: if not use_ebuild_visibility and usepkgonly: return False - else: - try: - pkg_eb = self._depgraph._pkg( - pkg.cpv, "ebuild", pkg.root_config, - myrepo=pkg.repo) - except portage.exception.PackageNotFound: - pkg_eb_visible = False - for pkg_eb in self._depgraph._iter_match_pkgs( - pkg.root_config, "ebuild", - Atom("=%s" % (pkg.cpv,))): - if self._depgraph._pkg_visibility_check(pkg_eb): - pkg_eb_visible = True - break - if not pkg_eb_visible: - return False - else: - if not self._depgraph._pkg_visibility_check(pkg_eb): - return False + elif not self._depgraph._equiv_ebuild_visible(pkg): + return False in_graph = self._depgraph._dynamic_config._slot_pkg_map[ self._root].get(pkg.slot_atom) @@ -6854,7 +8008,7 @@ class _dep_check_composite_db(dbapi): return True def aux_get(self, cpv, wants): - metadata = self._cpv_pkg_map[cpv].metadata + metadata = self._cpv_pkg_map[cpv]._metadata return [metadata.get(x, "") for x in wants] def match_pkgs(self, atom): @@ -6928,14 +8082,14 @@ def _spinner_stop(spinner): portage.writemsg_stdout("... done!\n") -def backtrack_depgraph(settings, trees, myopts, myparams, +def backtrack_depgraph(settings, trees, myopts, myparams, myaction, myfiles, spinner): """ Raises PackageSetNotFound if myfiles contains a missing package set. """ _spinner_start(spinner, myopts) try: - return _backtrack_depgraph(settings, trees, myopts, myparams, + return _backtrack_depgraph(settings, trees, myopts, myparams, myaction, myfiles, spinner) finally: _spinner_stop(spinner) @@ -7032,7 +8186,7 @@ def _resume_depgraph(settings, trees, mtimedb, myopts, myparams, spinner): skip_masked = True skip_unsatisfied = True mergelist = mtimedb["resume"]["mergelist"] - dropped_tasks = set() + dropped_tasks = {} frozen_config = _frozen_depgraph_config(settings, trees, myopts, spinner) while True: @@ -7046,12 +8200,21 @@ def _resume_depgraph(settings, trees, mtimedb, myopts, myparams, spinner): raise graph = mydepgraph._dynamic_config.digraph - unsatisfied_parents = dict((dep.parent, dep.parent) \ - for dep in e.value) + unsatisfied_parents = {} traversed_nodes = set() - unsatisfied_stack = list(unsatisfied_parents) + unsatisfied_stack = [(dep.parent, dep.atom) for dep in e.value] while unsatisfied_stack: - pkg = unsatisfied_stack.pop() + pkg, atom = unsatisfied_stack.pop() + if atom is not None and \ + mydepgraph._select_pkg_from_installed( + pkg.root, atom)[0] is not None: + continue + atoms = unsatisfied_parents.get(pkg) + if atoms is None: + atoms = [] + unsatisfied_parents[pkg] = atoms + if atom is not None: + atoms.append(atom) if pkg in traversed_nodes: continue traversed_nodes.add(pkg) @@ -7060,7 +8223,8 @@ def _resume_depgraph(settings, trees, mtimedb, myopts, myparams, spinner): # package scheduled for merge, removing this # package may cause the the parent package's # dependency to become unsatisfied. - for parent_node in graph.parent_nodes(pkg): + for parent_node, atom in \ + mydepgraph._dynamic_config._parent_atoms.get(pkg, []): if not isinstance(parent_node, Package) \ or parent_node.operation not in ("merge", "nomerge"): continue @@ -7068,8 +8232,7 @@ def _resume_depgraph(settings, trees, mtimedb, myopts, myparams, spinner): # ensure that a package with an unsatisfied depenedency # won't get pulled in, even indirectly via a soft # dependency. - unsatisfied_parents[parent_node] = parent_node - unsatisfied_stack.append(parent_node) + unsatisfied_stack.append((parent_node, atom)) unsatisfied_tuples = frozenset(tuple(parent_node) for parent_node in unsatisfied_parents @@ -7090,8 +8253,8 @@ def _resume_depgraph(settings, trees, mtimedb, myopts, myparams, spinner): # Exclude installed packages that have been removed from the graph due # to failure to build/install runtime dependencies after the dependent # package has already been installed. - dropped_tasks.update(pkg for pkg in \ - unsatisfied_parents if pkg.operation != "nomerge") + dropped_tasks.update((pkg, atoms) for pkg, atoms in \ + unsatisfied_parents.items() if pkg.operation != "nomerge") del e, graph, traversed_nodes, \ unsatisfied_parents, unsatisfied_stack @@ -7177,9 +8340,11 @@ def show_masked_packages(masked_packages): shown_comments.add(comment) portdb = root_config.trees["porttree"].dbapi for l in missing_licenses: - l_path = portdb.findLicensePath(l) if l in shown_licenses: continue + l_path = portdb.findLicensePath(l) + if l_path is None: + continue msg = ("A copy of the '%s' license" + \ " is located at '%s'.\n\n") % (l, l_path) writemsg(msg, noiselevel=-1) @@ -7206,9 +8371,9 @@ def _get_masking_status(pkg, pkgsettings, root_config, myrepo=None, use=None): portdb=root_config.trees["porttree"].dbapi, myrepo=myrepo) if not pkg.installed: - if not pkgsettings._accept_chost(pkg.cpv, pkg.metadata): + if not pkgsettings._accept_chost(pkg.cpv, pkg._metadata): mreasons.append(_MaskReason("CHOST", "CHOST: %s" % \ - pkg.metadata["CHOST"])) + pkg._metadata["CHOST"])) if pkg.invalid: for msgs in pkg.invalid.values(): @@ -7216,7 +8381,7 @@ def _get_masking_status(pkg, pkgsettings, root_config, myrepo=None, use=None): mreasons.append( _MaskReason("invalid", "invalid: %s" % (msg,))) - if not pkg.metadata["SLOT"]: + if not pkg._metadata["SLOT"]: mreasons.append( _MaskReason("invalid", "SLOT: undefined")) diff --git a/portage_with_autodep/pym/_emerge/depgraph.pyo b/portage_with_autodep/pym/_emerge/depgraph.pyo Binary files differindex ba00a11..db9b676 100644 --- a/portage_with_autodep/pym/_emerge/depgraph.pyo +++ b/portage_with_autodep/pym/_emerge/depgraph.pyo diff --git a/portage_with_autodep/pym/_emerge/emergelog.py b/portage_with_autodep/pym/_emerge/emergelog.py index b1b093f..aea94f7 100644 --- a/portage_with_autodep/pym/_emerge/emergelog.py +++ b/portage_with_autodep/pym/_emerge/emergelog.py @@ -1,7 +1,7 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import print_function +from __future__ import unicode_literals import io import sys @@ -20,10 +20,6 @@ from portage.output import xtermTitle _disable = True _emerge_log_dir = '/var/log' -# Coerce to unicode, in order to prevent TypeError when writing -# raw bytes to TextIOWrapper with python2. -_log_fmt = _unicode_decode("%.0f: %s\n") - def emergelog(xterm_titles, mystr, short_msg=None): if _disable: @@ -51,10 +47,10 @@ def emergelog(xterm_titles, mystr, short_msg=None): mode=0o660) mylock = portage.locks.lockfile(file_path) try: - mylogfile.write(_log_fmt % (time.time(), mystr)) + mylogfile.write("%.0f: %s\n" % (time.time(), mystr)) mylogfile.close() finally: portage.locks.unlockfile(mylock) except (IOError,OSError,portage.exception.PortageException) as e: if secpass >= 1: - print("emergelog():",e, file=sys.stderr) + portage.util.writemsg("emergelog(): %s\n" % (e,), noiselevel=-1) diff --git a/portage_with_autodep/pym/_emerge/emergelog.pyo b/portage_with_autodep/pym/_emerge/emergelog.pyo Binary files differindex 7e67bd3..997a1c0 100644 --- a/portage_with_autodep/pym/_emerge/emergelog.pyo +++ b/portage_with_autodep/pym/_emerge/emergelog.pyo diff --git a/portage_with_autodep/pym/_emerge/getloadavg.py b/portage_with_autodep/pym/_emerge/getloadavg.py index e9babf1..6a2794f 100644 --- a/portage_with_autodep/pym/_emerge/getloadavg.py +++ b/portage_with_autodep/pym/_emerge/getloadavg.py @@ -1,4 +1,4 @@ -# Copyright 1999-2009 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from portage import os @@ -11,7 +11,8 @@ if getloadavg is None: Raises OSError if the load average was unobtainable. """ try: - loadavg_str = open('/proc/loadavg').readline() + with open('/proc/loadavg') as f: + loadavg_str = f.readline() except IOError: # getloadavg() is only supposed to raise OSError, so convert raise OSError('unknown') diff --git a/portage_with_autodep/pym/_emerge/getloadavg.pyo b/portage_with_autodep/pym/_emerge/getloadavg.pyo Binary files differindex 56bda8c..c6c99db 100644 --- a/portage_with_autodep/pym/_emerge/getloadavg.pyo +++ b/portage_with_autodep/pym/_emerge/getloadavg.pyo diff --git a/portage_with_autodep/pym/_emerge/help.py b/portage_with_autodep/pym/_emerge/help.py index a1dbb37..52cfd00 100644 --- a/portage_with_autodep/pym/_emerge/help.py +++ b/portage_with_autodep/pym/_emerge/help.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -9,11 +9,11 @@ def help(): print(bold("emerge:")+" the other white meat (command-line interface to the Portage system)") print(bold("Usage:")) print(" "+turquoise("emerge")+" [ "+green("options")+" ] [ "+green("action")+" ] [ "+turquoise("ebuild")+" | "+turquoise("tbz2")+" | "+turquoise("file")+" | "+turquoise("@set")+" | "+turquoise("atom")+" ] [ ... ]") - print(" "+turquoise("emerge")+" [ "+green("options")+" ] [ "+green("action")+" ] < "+turquoise("system")+" | "+turquoise("world")+" >") + print(" "+turquoise("emerge")+" [ "+green("options")+" ] [ "+green("action")+" ] < "+turquoise("@system")+" | "+turquoise("@world")+" >") print(" "+turquoise("emerge")+" < "+turquoise("--sync")+" | "+turquoise("--metadata")+" | "+turquoise("--info")+" >") print(" "+turquoise("emerge")+" "+turquoise("--resume")+" [ "+green("--pretend")+" | "+green("--ask")+" | "+green("--skipfirst")+" ]") - print(" "+turquoise("emerge")+" "+turquoise("--help")+" [ "+green("--verbose")+" ] ") - print(bold("Options:")+" "+green("-")+"["+green("abBcCdDefgGhjkKlnNoOpPqrsStuvV")+"]") + print(" "+turquoise("emerge")+" "+turquoise("--help")) + print(bold("Options:")+" "+green("-")+"["+green("abBcCdDefgGhjkKlnNoOpPqrsStuvVw")+"]") print(" [ " + green("--color")+" < " + turquoise("y") + " | "+ turquoise("n")+" > ] [ "+green("--columns")+" ]") print(" [ "+green("--complete-graph")+" ] [ "+green("--deep")+" ]") print(" [ "+green("--jobs") + " " + turquoise("JOBS")+" ] [ "+green("--keep-going")+" ] [ " + green("--load-average")+" " + turquoise("LOAD") + " ]") diff --git a/portage_with_autodep/pym/_emerge/help.pyo b/portage_with_autodep/pym/_emerge/help.pyo Binary files differindex f6fea4e..b309ff2 100644 --- a/portage_with_autodep/pym/_emerge/help.pyo +++ b/portage_with_autodep/pym/_emerge/help.pyo diff --git a/portage_with_autodep/pym/_emerge/is_valid_package_atom.py b/portage_with_autodep/pym/_emerge/is_valid_package_atom.py index 7cb2a5b..112afc1 100644 --- a/portage_with_autodep/pym/_emerge/is_valid_package_atom.py +++ b/portage_with_autodep/pym/_emerge/is_valid_package_atom.py @@ -1,11 +1,12 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import re from portage.dep import isvalidatom def insert_category_into_atom(atom, category): - alphanum = re.search(r'\w', atom) + # Handle '*' character for "extended syntax" wildcard support. + alphanum = re.search(r'[\*\w]', atom, re.UNICODE) if alphanum: ret = atom[:alphanum.start()] + "%s/" % category + \ atom[alphanum.start():] @@ -14,7 +15,7 @@ def insert_category_into_atom(atom, category): return ret def is_valid_package_atom(x, allow_repo=False): - if "/" not in x: + if "/" not in x.split(":")[0]: x2 = insert_category_into_atom(x, 'cat') if x2 != None: x = x2 diff --git a/portage_with_autodep/pym/_emerge/is_valid_package_atom.pyo b/portage_with_autodep/pym/_emerge/is_valid_package_atom.pyo Binary files differindex 20edc85..68aaa52 100644 --- a/portage_with_autodep/pym/_emerge/is_valid_package_atom.pyo +++ b/portage_with_autodep/pym/_emerge/is_valid_package_atom.pyo diff --git a/portage_with_autodep/pym/_emerge/main.py b/portage_with_autodep/pym/_emerge/main.py index c52a3ea..89413a9 100644 --- a/portage_with_autodep/pym/_emerge/main.py +++ b/portage_with_autodep/pym/_emerge/main.py @@ -1,52 +1,24 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function -import logging -import signal -import stat -import subprocess -import sys -import textwrap import platform +import sys + import portage portage.proxy.lazyimport.lazyimport(globals(), - 'portage.news:count_unread_news,display_news_notifications', + 'logging', + 'portage.dep:Atom', + 'portage.util:writemsg_level', + 'textwrap', + '_emerge.actions:load_emerge_config,run_action,' + \ + 'validate_ebuild_environment', + '_emerge.help:help@emerge_help', + '_emerge.is_valid_package_atom:insert_category_into_atom' ) from portage import os -from portage import _encodings -from portage import _unicode_decode -import _emerge.help -import portage.xpak, errno, re, time -from portage.output import colorize, xtermTitle, xtermTitleReset -from portage.output import create_color_func -good = create_color_func("GOOD") -bad = create_color_func("BAD") - -from portage.const import _ENABLE_DYN_LINK_MAP -import portage.elog -import portage.util -import portage.locks -import portage.exception -from portage.data import secpass -from portage.dbapi.dep_expand import dep_expand -from portage.util import normalize_path as normpath -from portage.util import (shlex_split, varexpand, - writemsg_level, writemsg_stdout) -from portage._sets import SETPREFIX -from portage._global_updates import _global_updates - -from _emerge.actions import action_config, action_sync, action_metadata, \ - action_regen, action_search, action_uninstall, action_info, action_build, \ - adjust_configs, chk_updated_cfg_files, display_missing_pkg_set, \ - display_news_notification, getportageversion, load_emerge_config -import _emerge -from _emerge.emergelog import emergelog -from _emerge._flush_elog_mod_echo import _flush_elog_mod_echo -from _emerge.is_valid_package_atom import is_valid_package_atom -from _emerge.stdout_spinner import stdout_spinner -from _emerge.userquery import userquery +from portage.util._argparse import ArgumentParser if sys.hexversion >= 0x3000000: long = int @@ -60,6 +32,7 @@ options=[ "--debug", "--digest", "--emptytree", +"--verbose-conflicts", "--fetchonly", "--fetch-all-uri", "--ignore-default-opts", "--noconfmem", @@ -75,7 +48,6 @@ options=[ "--tree", "--unordered-display", "--update", -"--verbose", "--verbose-main-repo-display", ] @@ -96,7 +68,7 @@ shortmapping={ "s":"--search", "S":"--searchdesc", "t":"--tree", "u":"--update", -"v":"--verbose", "V":"--version" +"V":"--version" } COWSAY_MOO = """ @@ -114,325 +86,6 @@ COWSAY_MOO = """ """ -def chk_updated_info_files(root, infodirs, prev_mtimes, retval): - - if os.path.exists("/usr/bin/install-info"): - out = portage.output.EOutput() - regen_infodirs=[] - for z in infodirs: - if z=='': - continue - inforoot=normpath(root+z) - if os.path.isdir(inforoot) and \ - not [x for x in os.listdir(inforoot) \ - if x.startswith('.keepinfodir')]: - infomtime = os.stat(inforoot)[stat.ST_MTIME] - if inforoot not in prev_mtimes or \ - prev_mtimes[inforoot] != infomtime: - regen_infodirs.append(inforoot) - - if not regen_infodirs: - portage.writemsg_stdout("\n") - if portage.util.noiselimit >= 0: - out.einfo("GNU info directory index is up-to-date.") - else: - portage.writemsg_stdout("\n") - if portage.util.noiselimit >= 0: - out.einfo("Regenerating GNU info directory index...") - - dir_extensions = ("", ".gz", ".bz2") - icount=0 - badcount=0 - errmsg = "" - for inforoot in regen_infodirs: - if inforoot=='': - continue - - if not os.path.isdir(inforoot) or \ - not os.access(inforoot, os.W_OK): - continue - - file_list = os.listdir(inforoot) - file_list.sort() - dir_file = os.path.join(inforoot, "dir") - moved_old_dir = False - processed_count = 0 - for x in file_list: - if x.startswith(".") or \ - os.path.isdir(os.path.join(inforoot, x)): - continue - if x.startswith("dir"): - skip = False - for ext in dir_extensions: - if x == "dir" + ext or \ - x == "dir" + ext + ".old": - skip = True - break - if skip: - continue - if processed_count == 0: - for ext in dir_extensions: - try: - os.rename(dir_file + ext, dir_file + ext + ".old") - moved_old_dir = True - except EnvironmentError as e: - if e.errno != errno.ENOENT: - raise - del e - processed_count += 1 - try: - proc = subprocess.Popen( - ['/usr/bin/install-info', - '--dir-file=%s' % os.path.join(inforoot, "dir"), - os.path.join(inforoot, x)], - env=dict(os.environ, LANG="C", LANGUAGE="C"), - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - except OSError: - myso = None - else: - myso = _unicode_decode( - proc.communicate()[0]).rstrip("\n") - proc.wait() - existsstr="already exists, for file `" - if myso: - if re.search(existsstr,myso): - # Already exists... Don't increment the count for this. - pass - elif myso[:44]=="install-info: warning: no info dir entry in ": - # This info file doesn't contain a DIR-header: install-info produces this - # (harmless) warning (the --quiet switch doesn't seem to work). - # Don't increment the count for this. - pass - else: - badcount=badcount+1 - errmsg += myso + "\n" - icount=icount+1 - - if moved_old_dir and not os.path.exists(dir_file): - # We didn't generate a new dir file, so put the old file - # back where it was originally found. - for ext in dir_extensions: - try: - os.rename(dir_file + ext + ".old", dir_file + ext) - except EnvironmentError as e: - if e.errno != errno.ENOENT: - raise - del e - - # Clean dir.old cruft so that they don't prevent - # unmerge of otherwise empty directories. - for ext in dir_extensions: - try: - os.unlink(dir_file + ext + ".old") - except EnvironmentError as e: - if e.errno != errno.ENOENT: - raise - del e - - #update mtime so we can potentially avoid regenerating. - prev_mtimes[inforoot] = os.stat(inforoot)[stat.ST_MTIME] - - if badcount: - out.eerror("Processed %d info files; %d errors." % \ - (icount, badcount)) - writemsg_level(errmsg, level=logging.ERROR, noiselevel=-1) - else: - if icount > 0 and portage.util.noiselimit >= 0: - out.einfo("Processed %d info files." % (icount,)) - -def display_preserved_libs(vardbapi, myopts): - MAX_DISPLAY = 3 - - if vardbapi._linkmap is None or \ - vardbapi._plib_registry is None: - # preserve-libs is entirely disabled - return - - # Explicitly load and prune the PreservedLibsRegistry in order - # to ensure that we do not display stale data. - vardbapi._plib_registry.load() - - if vardbapi._plib_registry.hasEntries(): - if "--quiet" in myopts: - print() - print(colorize("WARN", "!!!") + " existing preserved libs found") - return - else: - print() - print(colorize("WARN", "!!!") + " existing preserved libs:") - - plibdata = vardbapi._plib_registry.getPreservedLibs() - linkmap = vardbapi._linkmap - consumer_map = {} - owners = {} - - try: - linkmap.rebuild() - except portage.exception.CommandNotFound as e: - writemsg_level("!!! Command Not Found: %s\n" % (e,), - level=logging.ERROR, noiselevel=-1) - del e - else: - search_for_owners = set() - for cpv in plibdata: - internal_plib_keys = set(linkmap._obj_key(f) \ - for f in plibdata[cpv]) - for f in plibdata[cpv]: - if f in consumer_map: - continue - consumers = [] - for c in linkmap.findConsumers(f): - # Filter out any consumers that are also preserved libs - # belonging to the same package as the provider. - if linkmap._obj_key(c) not in internal_plib_keys: - consumers.append(c) - consumers.sort() - consumer_map[f] = consumers - search_for_owners.update(consumers[:MAX_DISPLAY+1]) - - owners = {} - for f in search_for_owners: - owner_set = set() - for owner in linkmap.getOwners(f): - owner_dblink = vardbapi._dblink(owner) - if owner_dblink.exists(): - owner_set.add(owner_dblink) - if owner_set: - owners[f] = owner_set - - for cpv in plibdata: - print(colorize("WARN", ">>>") + " package: %s" % cpv) - samefile_map = {} - for f in plibdata[cpv]: - obj_key = linkmap._obj_key(f) - alt_paths = samefile_map.get(obj_key) - if alt_paths is None: - alt_paths = set() - samefile_map[obj_key] = alt_paths - alt_paths.add(f) - - for alt_paths in samefile_map.values(): - alt_paths = sorted(alt_paths) - for p in alt_paths: - print(colorize("WARN", " * ") + " - %s" % (p,)) - f = alt_paths[0] - consumers = consumer_map.get(f, []) - for c in consumers[:MAX_DISPLAY]: - print(colorize("WARN", " * ") + " used by %s (%s)" % \ - (c, ", ".join(x.mycpv for x in owners.get(c, [])))) - if len(consumers) == MAX_DISPLAY + 1: - print(colorize("WARN", " * ") + " used by %s (%s)" % \ - (consumers[MAX_DISPLAY], ", ".join(x.mycpv \ - for x in owners.get(consumers[MAX_DISPLAY], [])))) - elif len(consumers) > MAX_DISPLAY: - print(colorize("WARN", " * ") + " used by %d other files" % (len(consumers) - MAX_DISPLAY)) - print("Use " + colorize("GOOD", "emerge @preserved-rebuild") + " to rebuild packages using these libraries") - -def post_emerge(myaction, myopts, myfiles, - target_root, trees, mtimedb, retval): - """ - Misc. things to run at the end of a merge session. - - Update Info Files - Update Config Files - Update News Items - Commit mtimeDB - Display preserved libs warnings - - @param myaction: The action returned from parse_opts() - @type myaction: String - @param myopts: emerge options - @type myopts: dict - @param myfiles: emerge arguments - @type myfiles: list - @param target_root: The target EROOT for myaction - @type target_root: String - @param trees: A dictionary mapping each ROOT to it's package databases - @type trees: dict - @param mtimedb: The mtimeDB to store data needed across merge invocations - @type mtimedb: MtimeDB class instance - @param retval: Emerge's return value - @type retval: Int - """ - - root_config = trees[target_root]["root_config"] - vardbapi = trees[target_root]['vartree'].dbapi - settings = vardbapi.settings - info_mtimes = mtimedb["info"] - - # Load the most current variables from ${ROOT}/etc/profile.env - settings.unlock() - settings.reload() - settings.regenerate() - settings.lock() - - config_protect = shlex_split(settings.get("CONFIG_PROTECT", "")) - infodirs = settings.get("INFOPATH","").split(":") + \ - settings.get("INFODIR","").split(":") - - os.chdir("/") - - if retval == os.EX_OK: - exit_msg = " *** exiting successfully." - else: - exit_msg = " *** exiting unsuccessfully with status '%s'." % retval - emergelog("notitles" not in settings.features, exit_msg) - - _flush_elog_mod_echo() - - if not vardbapi._pkgs_changed: - # GLEP 42 says to display news *after* an emerge --pretend - if "--pretend" in myopts: - display_news_notification(root_config, myopts) - # If vdb state has not changed then there's nothing else to do. - return - - vdb_path = os.path.join(root_config.settings['EROOT'], portage.VDB_PATH) - portage.util.ensure_dirs(vdb_path) - vdb_lock = None - if os.access(vdb_path, os.W_OK) and not "--pretend" in myopts: - vardbapi.lock() - vdb_lock = True - - if vdb_lock: - try: - if "noinfo" not in settings.features: - chk_updated_info_files(target_root, - infodirs, info_mtimes, retval) - mtimedb.commit() - finally: - if vdb_lock: - vardbapi.unlock() - - display_preserved_libs(vardbapi, myopts) - chk_updated_cfg_files(settings['EROOT'], config_protect) - - display_news_notification(root_config, myopts) - - postemerge = os.path.join(settings["PORTAGE_CONFIGROOT"], - portage.USER_CONFIG_PATH, "bin", "post_emerge") - if os.access(postemerge, os.X_OK): - hook_retval = portage.process.spawn( - [postemerge], env=settings.environ()) - if hook_retval != os.EX_OK: - writemsg_level( - " %s spawn failed of %s\n" % (bad("*"), postemerge,), - level=logging.ERROR, noiselevel=-1) - - clean_logs(settings) - - if "--quiet" not in myopts and \ - myaction is None and "@world" in myfiles: - show_depclean_suggestion() - -def show_depclean_suggestion(): - out = portage.output.EOutput() - msg = "After world updates, it is important to remove " + \ - "obsolete packages with emerge --depclean. Refer " + \ - "to `man emerge` for more information." - for line in textwrap.wrap(msg, 72): - out.ewarn(line) - def multiple_actions(action1, action2): sys.stderr.write("\n!!! Multiple actions requested... Please choose one only.\n") sys.stderr.write("!!! '%s' or '%s'\n\n" % (action1, action2)) @@ -454,6 +107,16 @@ def insert_optional_args(args): return False valid_integers = valid_integers() + + class valid_floats(object): + def __contains__(self, s): + try: + return float(s) >= 0 + except (ValueError, OverflowError): + return False + + valid_floats = valid_floats() + y_or_n = ('y', 'n',) new_args = [] @@ -467,6 +130,7 @@ def insert_optional_args(args): '--buildpkg' : y_or_n, '--complete-graph' : y_or_n, '--deep' : valid_integers, + '--depclean-lib-check' : y_or_n, '--deselect' : y_or_n, '--binpkg-respect-use' : y_or_n, '--fail-clean' : y_or_n, @@ -474,9 +138,12 @@ def insert_optional_args(args): '--getbinpkgonly' : y_or_n, '--jobs' : valid_integers, '--keep-going' : y_or_n, + '--load-average' : valid_floats, '--package-moves' : y_or_n, '--quiet' : y_or_n, '--quiet-build' : y_or_n, + '--quiet-fail' : y_or_n, + '--rebuild-if-new-slot': y_or_n, '--rebuild-if-new-rev' : y_or_n, '--rebuild-if-new-ver' : y_or_n, '--rebuild-if-unbuilt' : y_or_n, @@ -487,11 +154,9 @@ def insert_optional_args(args): "--use-ebuild-visibility": y_or_n, '--usepkg' : y_or_n, '--usepkgonly' : y_or_n, + '--verbose' : y_or_n, } - if _ENABLE_DYN_LINK_MAP: - default_arg_opts['--depclean-lib-check'] = y_or_n - short_arg_opts = { 'D' : valid_integers, 'j' : valid_integers, @@ -507,6 +172,8 @@ def insert_optional_args(args): 'k' : y_or_n, 'K' : y_or_n, 'q' : y_or_n, + 'v' : y_or_n, + 'w' : y_or_n, } arg_stack = args[:] @@ -595,14 +262,17 @@ def _find_bad_atoms(atoms, less_strict=False): """ bad_atoms = [] for x in ' '.join(atoms).split(): + atom = x + if "/" not in x.split(":")[0]: + x_cat = insert_category_into_atom(x, 'dummy-category') + if x_cat is not None: + atom = x_cat + bad_atom = False try: - atom = portage.dep.Atom(x, allow_wildcard=True, allow_repo=less_strict) + atom = Atom(atom, allow_wildcard=True, allow_repo=less_strict) except portage.exception.InvalidAtom: - try: - atom = portage.dep.Atom("*/"+x, allow_wildcard=True, allow_repo=less_strict) - except portage.exception.InvalidAtom: - bad_atom = True + bad_atom = True if bad_atom or (atom.operator and not less_strict) or atom.blocker or atom.use: bad_atoms.append(x) @@ -630,31 +300,26 @@ def parse_opts(tmpcmdline, silent=False): "--ask": { "shortopt" : "-a", "help" : "prompt before performing any actions", - "type" : "choice", "choices" : true_y_or_n }, "--autounmask": { "help" : "automatically unmask packages", - "type" : "choice", "choices" : true_y_or_n }, "--autounmask-unrestricted-atoms": { "help" : "write autounmask changes with >= atoms if possible", - "type" : "choice", "choices" : true_y_or_n }, "--autounmask-keep-masks": { "help" : "don't add package.unmask entries", - "type" : "choice", "choices" : true_y_or_n }, "--autounmask-write": { "help" : "write changes made by --autounmask to disk", - "type" : "choice", "choices" : true_y_or_n }, @@ -663,6 +328,11 @@ def parse_opts(tmpcmdline, silent=False): "action":"store" }, + "--accept-restrict": { + "help":"temporarily override ACCEPT_RESTRICT", + "action":"store" + }, + "--backtrack": { "help" : "Specifies how many times to backtrack if dependency " + \ @@ -674,7 +344,6 @@ def parse_opts(tmpcmdline, silent=False): "--buildpkg": { "shortopt" : "-b", "help" : "build binary packages", - "type" : "choice", "choices" : true_y_or_n }, @@ -692,19 +361,21 @@ def parse_opts(tmpcmdline, silent=False): }, "--color": { "help":"enable or disable color output", - "type":"choice", "choices":("y", "n") }, "--complete-graph": { "help" : "completely account for all known dependencies", - "type" : "choice", "choices" : true_y_or_n }, + "--complete-graph-if-new-use": { + "help" : "trigger --complete-graph behavior if USE or IUSE will change for an installed package", + "choices" : y_or_n + }, + "--complete-graph-if-new-ver": { "help" : "trigger --complete-graph behavior if an installed package version will change (upgrade or downgrade)", - "type" : "choice", "choices" : y_or_n }, @@ -720,15 +391,18 @@ def parse_opts(tmpcmdline, silent=False): "action" : "store" }, + "--depclean-lib-check": { + "help" : "check for consumers of libraries before removing them", + "choices" : true_y_or_n + }, + "--deselect": { "help" : "remove atoms/sets from the world file", - "type" : "choice", "choices" : true_y_or_n }, "--dynamic-deps": { "help": "substitute the dependencies of installed packages with the dependencies of unbuilt ebuilds", - "type": "choice", "choices": y_or_n }, @@ -742,10 +416,18 @@ def parse_opts(tmpcmdline, silent=False): "--fail-clean": { "help" : "clean temp files after build failure", - "type" : "choice", "choices" : true_y_or_n }, + "--ignore-built-slot-operator-deps": { + "help": "Ignore the slot/sub-slot := operator parts of dependencies that have " + "been recorded when packages where built. This option is intended " + "only for debugging purposes, and it only affects built packages " + "that specify slot/sub-slot := operator dependencies using the " + "experimental \"4-slot-abi\" EAPI.", + "choices": y_or_n + }, + "--jobs": { "shortopt" : "-j", @@ -758,7 +440,6 @@ def parse_opts(tmpcmdline, silent=False): "--keep-going": { "help" : "continue as much as possible after an error", - "type" : "choice", "choices" : true_y_or_n }, @@ -773,18 +454,15 @@ def parse_opts(tmpcmdline, silent=False): "--misspell-suggestions": { "help" : "enable package name misspell suggestions", - "type" : "choice", "choices" : ("y", "n") }, "--with-bdeps": { "help":"include unnecessary build time dependencies", - "type":"choice", "choices":("y", "n") }, "--reinstall": { "help":"specify conditions to trigger package reinstallation", - "type":"choice", "choices":["changed-use"] }, @@ -799,21 +477,18 @@ def parse_opts(tmpcmdline, silent=False): "--binpkg-respect-use": { "help" : "discard binary packages if their use flags \ don't match the current configuration", - "type" : "choice", "choices" : true_y_or_n }, "--getbinpkg": { "shortopt" : "-g", "help" : "fetch binary packages", - "type" : "choice", "choices" : true_y_or_n }, "--getbinpkgonly": { "shortopt" : "-G", "help" : "fetch binary packages only", - "type" : "choice", "choices" : true_y_or_n }, @@ -842,29 +517,48 @@ def parse_opts(tmpcmdline, silent=False): "--package-moves": { "help" : "perform package moves when necessary", - "type" : "choice", "choices" : true_y_or_n }, + "--prefix": { + "help" : "specify the installation prefix", + "action" : "store" + }, + + "--pkg-format": { + "help" : "format of result binary package", + "action" : "store", + }, + "--quiet": { "shortopt" : "-q", "help" : "reduced or condensed output", - "type" : "choice", "choices" : true_y_or_n }, "--quiet-build": { "help" : "redirect build output to logs", - "type" : "choice", "choices" : true_y_or_n, }, + "--quiet-fail": { + "help" : "suppresses display of the build log on stdout", + "choices" : true_y_or_n, + }, + + "--rebuild-if-new-slot": { + "help" : ("Automatically rebuild or reinstall packages when slot/sub-slot := " + "operator dependencies can be satisfied by a newer slot, so that " + "older packages slots will become eligible for removal by the " + "--depclean action as soon as possible."), + "choices" : true_y_or_n + }, + "--rebuild-if-new-rev": { "help" : "Rebuild packages when dependencies that are " + \ "used at both build-time and run-time are built, " + \ "if the dependency is not already installed with the " + \ "same version and revision.", - "type" : "choice", "choices" : true_y_or_n }, @@ -873,21 +567,18 @@ def parse_opts(tmpcmdline, silent=False): "used at both build-time and run-time are built, " + \ "if the dependency is not already installed with the " + \ "same version. Revision numbers are ignored.", - "type" : "choice", "choices" : true_y_or_n }, "--rebuild-if-unbuilt": { "help" : "Rebuild packages when dependencies that are " + \ "used at both build-time and run-time are built.", - "type" : "choice", "choices" : true_y_or_n }, "--rebuilt-binaries": { "help" : "replace installed packages with binary " + \ "packages that have been rebuilt", - "type" : "choice", "choices" : true_y_or_n }, @@ -904,26 +595,23 @@ def parse_opts(tmpcmdline, silent=False): "--root-deps": { "help" : "modify interpretation of depedencies", - "type" : "choice", "choices" :("True", "rdeps") }, "--select": { + "shortopt" : "-w", "help" : "add specified packages to the world set " + \ "(inverse of --oneshot)", - "type" : "choice", "choices" : true_y_or_n }, "--selective": { "help" : "identical to --noreplace", - "type" : "choice", "choices" : true_y_or_n }, "--use-ebuild-visibility": { "help" : "use unbuilt ebuild metadata for visibility checks on built packages", - "type" : "choice", "choices" : true_y_or_n }, @@ -937,42 +625,35 @@ def parse_opts(tmpcmdline, silent=False): "--usepkg": { "shortopt" : "-k", "help" : "use binary packages", - "type" : "choice", "choices" : true_y_or_n }, "--usepkgonly": { "shortopt" : "-K", "help" : "use only binary packages", - "type" : "choice", "choices" : true_y_or_n }, + "--verbose": { + "shortopt" : "-v", + "help" : "verbose output", + "choices" : true_y_or_n + }, } - if _ENABLE_DYN_LINK_MAP: - argument_options["--depclean-lib-check"] = { - "help" : "check for consumers of libraries before removing them", - "type" : "choice", - "choices" : true_y_or_n - } - - from optparse import OptionParser - parser = OptionParser() - if parser.has_option("--help"): - parser.remove_option("--help") + parser = ArgumentParser(add_help=False) for action_opt in actions: - parser.add_option("--" + action_opt, action="store_true", + parser.add_argument("--" + action_opt, action="store_true", dest=action_opt.replace("-", "_"), default=False) for myopt in options: - parser.add_option(myopt, action="store_true", + parser.add_argument(myopt, action="store_true", dest=myopt.lstrip("--").replace("-", "_"), default=False) for shortopt, longopt in shortmapping.items(): - parser.add_option("-" + shortopt, action="store_true", + parser.add_argument("-" + shortopt, action="store_true", dest=longopt.lstrip("--").replace("-", "_"), default=False) for myalias, myopt in longopt_aliases.items(): - parser.add_option(myalias, action="store_true", + parser.add_argument(myalias, action="store_true", dest=myopt.lstrip("--").replace("-", "_"), default=False) for myopt, kwargs in argument_options.items(): @@ -980,12 +661,12 @@ def parse_opts(tmpcmdline, silent=False): args = [myopt] if shortopt is not None: args.append(shortopt) - parser.add_option(dest=myopt.lstrip("--").replace("-", "_"), + parser.add_argument(dest=myopt.lstrip("--").replace("-", "_"), *args, **kwargs) tmpcmdline = insert_optional_args(tmpcmdline) - myoptions, myargs = parser.parse_args(args=tmpcmdline) + myoptions, myargs = parser.parse_known_args(args=tmpcmdline) if myoptions.ask in true_y: myoptions.ask = True @@ -1031,9 +712,8 @@ def parse_opts(tmpcmdline, silent=False): else: myoptions.complete_graph = None - if _ENABLE_DYN_LINK_MAP: - if myoptions.depclean_lib_check in true_y: - myoptions.depclean_lib_check = True + if myoptions.depclean_lib_check in true_y: + myoptions.depclean_lib_check = True if myoptions.exclude: bad_atoms = _find_bad_atoms(myoptions.exclude) @@ -1100,6 +780,12 @@ def parse_opts(tmpcmdline, silent=False): if myoptions.quiet_build in true_y: myoptions.quiet_build = 'y' + if myoptions.quiet_fail in true_y: + myoptions.quiet_fail = 'y' + + if myoptions.rebuild_if_new_slot in true_y: + myoptions.rebuild_if_new_slot = 'y' + if myoptions.rebuild_if_new_ver in true_y: myoptions.rebuild_if_new_ver = True else: @@ -1185,6 +871,9 @@ def parse_opts(tmpcmdline, silent=False): myoptions.jobs = jobs + if myoptions.load_average == "True": + myoptions.load_average = None + if myoptions.load_average: try: load_average = float(myoptions.load_average) @@ -1229,6 +918,11 @@ def parse_opts(tmpcmdline, silent=False): else: myoptions.usepkgonly = None + if myoptions.verbose in true_y: + myoptions.verbose = True + else: + myoptions.verbose = None + for myopt in options: v = getattr(myoptions, myopt.lstrip("--").replace("-", "_")) if v: @@ -1253,320 +947,10 @@ def parse_opts(tmpcmdline, silent=False): if myaction is None and myoptions.deselect is True: myaction = 'deselect' - if myargs and isinstance(myargs[0], bytes): - for i in range(len(myargs)): - myargs[i] = portage._unicode_decode(myargs[i]) - myfiles += myargs return myaction, myopts, myfiles -# Warn about features that may confuse users and -# lead them to report invalid bugs. -_emerge_features_warn = frozenset(['keeptemp', 'keepwork']) - -def validate_ebuild_environment(trees): - features_warn = set() - for myroot in trees: - settings = trees[myroot]["vartree"].settings - settings.validate() - features_warn.update( - _emerge_features_warn.intersection(settings.features)) - - if features_warn: - msg = "WARNING: The FEATURES variable contains one " + \ - "or more values that should be disabled under " + \ - "normal circumstances: %s" % " ".join(features_warn) - out = portage.output.EOutput() - for line in textwrap.wrap(msg, 65): - out.ewarn(line) - -def apply_priorities(settings): - ionice(settings) - nice(settings) - -def nice(settings): - try: - os.nice(int(settings.get("PORTAGE_NICENESS", "0"))) - except (OSError, ValueError) as e: - out = portage.output.EOutput() - out.eerror("Failed to change nice value to '%s'" % \ - settings["PORTAGE_NICENESS"]) - out.eerror("%s\n" % str(e)) - -def ionice(settings): - - ionice_cmd = settings.get("PORTAGE_IONICE_COMMAND") - if ionice_cmd: - ionice_cmd = portage.util.shlex_split(ionice_cmd) - if not ionice_cmd: - return - - variables = {"PID" : str(os.getpid())} - cmd = [varexpand(x, mydict=variables) for x in ionice_cmd] - - try: - rval = portage.process.spawn(cmd, env=os.environ) - except portage.exception.CommandNotFound: - # The OS kernel probably doesn't support ionice, - # so return silently. - return - - if rval != os.EX_OK: - out = portage.output.EOutput() - out.eerror("PORTAGE_IONICE_COMMAND returned %d" % (rval,)) - out.eerror("See the make.conf(5) man page for PORTAGE_IONICE_COMMAND usage instructions.") - -def clean_logs(settings): - - if "clean-logs" not in settings.features: - return - - clean_cmd = settings.get("PORT_LOGDIR_CLEAN") - if clean_cmd: - clean_cmd = shlex_split(clean_cmd) - if not clean_cmd: - return - - logdir = settings.get("PORT_LOGDIR") - if logdir is None or not os.path.isdir(logdir): - return - - variables = {"PORT_LOGDIR" : logdir} - cmd = [varexpand(x, mydict=variables) for x in clean_cmd] - - try: - rval = portage.process.spawn(cmd, env=os.environ) - except portage.exception.CommandNotFound: - rval = 127 - - if rval != os.EX_OK: - out = portage.output.EOutput() - out.eerror("PORT_LOGDIR_CLEAN returned %d" % (rval,)) - out.eerror("See the make.conf(5) man page for " - "PORT_LOGDIR_CLEAN usage instructions.") - -def setconfig_fallback(root_config): - from portage._sets.base import DummyPackageSet - from portage._sets.files import WorldSelectedSet - from portage._sets.profiles import PackagesSystemSet - setconfig = root_config.setconfig - setconfig.psets['world'] = DummyPackageSet(atoms=['@selected', '@system']) - setconfig.psets['selected'] = WorldSelectedSet(root_config.settings['EROOT']) - setconfig.psets['system'] = \ - PackagesSystemSet(root_config.settings.profiles) - root_config.sets = setconfig.getSets() - -def get_missing_sets(root_config): - # emerge requires existence of "world", "selected", and "system" - missing_sets = [] - - for s in ("selected", "system", "world",): - if s not in root_config.sets: - missing_sets.append(s) - - return missing_sets - -def missing_sets_warning(root_config, missing_sets): - if len(missing_sets) > 2: - missing_sets_str = ", ".join('"%s"' % s for s in missing_sets[:-1]) - missing_sets_str += ', and "%s"' % missing_sets[-1] - elif len(missing_sets) == 2: - missing_sets_str = '"%s" and "%s"' % tuple(missing_sets) - else: - missing_sets_str = '"%s"' % missing_sets[-1] - msg = ["emerge: incomplete set configuration, " + \ - "missing set(s): %s" % missing_sets_str] - if root_config.sets: - msg.append(" sets defined: %s" % ", ".join(root_config.sets)) - global_config_path = portage.const.GLOBAL_CONFIG_PATH - if root_config.settings['EPREFIX']: - global_config_path = os.path.join(root_config.settings['EPREFIX'], - portage.const.GLOBAL_CONFIG_PATH.lstrip(os.sep)) - msg.append(" This usually means that '%s'" % \ - (os.path.join(global_config_path, "sets/portage.conf"),)) - msg.append(" is missing or corrupt.") - msg.append(" Falling back to default world and system set configuration!!!") - for line in msg: - writemsg_level(line + "\n", level=logging.ERROR, noiselevel=-1) - -def ensure_required_sets(trees): - warning_shown = False - for root_trees in trees.values(): - missing_sets = get_missing_sets(root_trees["root_config"]) - if missing_sets and not warning_shown: - warning_shown = True - missing_sets_warning(root_trees["root_config"], missing_sets) - if missing_sets: - setconfig_fallback(root_trees["root_config"]) - -def expand_set_arguments(myfiles, myaction, root_config): - retval = os.EX_OK - setconfig = root_config.setconfig - - sets = setconfig.getSets() - - # In order to know exactly which atoms/sets should be added to the - # world file, the depgraph performs set expansion later. It will get - # confused about where the atoms came from if it's not allowed to - # expand them itself. - do_not_expand = (None, ) - newargs = [] - for a in myfiles: - if a in ("system", "world"): - newargs.append(SETPREFIX+a) - else: - newargs.append(a) - myfiles = newargs - del newargs - newargs = [] - - # separators for set arguments - ARG_START = "{" - ARG_END = "}" - - for i in range(0, len(myfiles)): - if myfiles[i].startswith(SETPREFIX): - start = 0 - end = 0 - x = myfiles[i][len(SETPREFIX):] - newset = "" - while x: - start = x.find(ARG_START) - end = x.find(ARG_END) - if start > 0 and start < end: - namepart = x[:start] - argpart = x[start+1:end] - - # TODO: implement proper quoting - args = argpart.split(",") - options = {} - for a in args: - if "=" in a: - k, v = a.split("=", 1) - options[k] = v - else: - options[a] = "True" - setconfig.update(namepart, options) - newset += (x[:start-len(namepart)]+namepart) - x = x[end+len(ARG_END):] - else: - newset += x - x = "" - myfiles[i] = SETPREFIX+newset - - sets = setconfig.getSets() - - # display errors that occurred while loading the SetConfig instance - for e in setconfig.errors: - print(colorize("BAD", "Error during set creation: %s" % e)) - - unmerge_actions = ("unmerge", "prune", "clean", "depclean") - - for a in myfiles: - if a.startswith(SETPREFIX): - s = a[len(SETPREFIX):] - if s not in sets: - display_missing_pkg_set(root_config, s) - return (None, 1) - setconfig.active.append(s) - try: - set_atoms = setconfig.getSetAtoms(s) - except portage.exception.PackageSetNotFound as e: - writemsg_level(("emerge: the given set '%s' " + \ - "contains a non-existent set named '%s'.\n") % \ - (s, e), level=logging.ERROR, noiselevel=-1) - return (None, 1) - if myaction in unmerge_actions and \ - not sets[s].supportsOperation("unmerge"): - sys.stderr.write("emerge: the given set '%s' does " % s + \ - "not support unmerge operations\n") - retval = 1 - elif not set_atoms: - print("emerge: '%s' is an empty set" % s) - elif myaction not in do_not_expand: - newargs.extend(set_atoms) - else: - newargs.append(SETPREFIX+s) - for e in sets[s].errors: - print(e) - else: - newargs.append(a) - return (newargs, retval) - -def repo_name_check(trees): - missing_repo_names = set() - for root_trees in trees.values(): - porttree = root_trees.get("porttree") - if porttree: - portdb = porttree.dbapi - missing_repo_names.update(portdb.getMissingRepoNames()) - if portdb.porttree_root in missing_repo_names and \ - not os.path.exists(os.path.join( - portdb.porttree_root, "profiles")): - # This is normal if $PORTDIR happens to be empty, - # so don't warn about it. - missing_repo_names.remove(portdb.porttree_root) - - if missing_repo_names: - msg = [] - msg.append("WARNING: One or more repositories " + \ - "have missing repo_name entries:") - msg.append("") - for p in missing_repo_names: - msg.append("\t%s/profiles/repo_name" % (p,)) - msg.append("") - msg.extend(textwrap.wrap("NOTE: Each repo_name entry " + \ - "should be a plain text file containing a unique " + \ - "name for the repository on the first line.", 70)) - msg.append("\n") - writemsg_level("".join("%s\n" % l for l in msg), - level=logging.WARNING, noiselevel=-1) - - return bool(missing_repo_names) - -def repo_name_duplicate_check(trees): - ignored_repos = {} - for root, root_trees in trees.items(): - if 'porttree' in root_trees: - portdb = root_trees['porttree'].dbapi - if portdb.settings.get('PORTAGE_REPO_DUPLICATE_WARN') != '0': - for repo_name, paths in portdb.getIgnoredRepos(): - k = (root, repo_name, portdb.getRepositoryPath(repo_name)) - ignored_repos.setdefault(k, []).extend(paths) - - if ignored_repos: - msg = [] - msg.append('WARNING: One or more repositories ' + \ - 'have been ignored due to duplicate') - msg.append(' profiles/repo_name entries:') - msg.append('') - for k in sorted(ignored_repos): - msg.append(' %s overrides' % ", ".join(k)) - for path in ignored_repos[k]: - msg.append(' %s' % (path,)) - msg.append('') - msg.extend(' ' + x for x in textwrap.wrap( - "All profiles/repo_name entries must be unique in order " + \ - "to avoid having duplicates ignored. " + \ - "Set PORTAGE_REPO_DUPLICATE_WARN=\"0\" in " + \ - "/etc/make.conf if you would like to disable this warning.")) - msg.append("\n") - writemsg_level(''.join('%s\n' % l for l in msg), - level=logging.WARNING, noiselevel=-1) - - return bool(ignored_repos) - -def config_protect_check(trees): - for root, root_trees in trees.items(): - settings = root_trees["root_config"].settings - if not settings.get("CONFIG_PROTECT"): - msg = "!!! CONFIG_PROTECT is empty" - if settings["ROOT"] != "/": - msg += " for '%s'" % root - msg += "\n" - writemsg_level(msg, level=logging.WARN, noiselevel=-1) - def profile_check(trees, myaction): if myaction in ("help", "info", "search", "sync", "version"): return os.EX_OK @@ -1584,16 +968,6 @@ def profile_check(trees, myaction): return 1 return os.EX_OK -def check_procfs(): - procfs_path = '/proc' - if platform.system() not in ("Linux",) or \ - os.path.ismount(procfs_path): - return os.EX_OK - msg = "It seems that %s is not mounted. You have been warned." % procfs_path - writemsg_level("".join("!!! %s\n" % l for l in textwrap.wrap(msg, 70)), - level=logging.ERROR, noiselevel=-1) - return 1 - def emerge_main(args=None): """ @param args: command arguments (default: sys.argv[1:]) @@ -1602,11 +976,12 @@ def emerge_main(args=None): if args is None: args = sys.argv[1:] - portage._disable_legacy_globals() - portage.dep._internal_warnings = True + args = portage._decode_argv(args) + # Disable color until we're sure that it should be enabled (after # EMERGE_DEFAULT_OPTS has been parsed). portage.output.havecolor = 0 + # This first pass is just for options that need to be known as early as # possible, such as --config-root. They will be parsed again later, # together with EMERGE_DEFAULT_OPTS (which may vary depending on the @@ -1618,428 +993,45 @@ def emerge_main(args=None): os.environ["PORTAGE_CONFIGROOT"] = myopts["--config-root"] if "--root" in myopts: os.environ["ROOT"] = myopts["--root"] + if "--prefix" in myopts: + os.environ["EPREFIX"] = myopts["--prefix"] if "--accept-properties" in myopts: os.environ["ACCEPT_PROPERTIES"] = myopts["--accept-properties"] + if "--accept-restrict" in myopts: + os.environ["ACCEPT_RESTRICT"] = myopts["--accept-restrict"] + + # optimize --help (no need to load config / EMERGE_DEFAULT_OPTS) + if myaction == "help": + emerge_help() + return os.EX_OK + elif myaction == "moo": + print(COWSAY_MOO % platform.system()) + return os.EX_OK # Portage needs to ensure a sane umask for the files it creates. os.umask(0o22) - settings, trees, mtimedb = load_emerge_config() - portdb = trees[settings['EROOT']]['porttree'].dbapi - rval = profile_check(trees, myaction) + if myaction == "sync": + portage._sync_disabled_warnings = True + emerge_config = load_emerge_config( + action=myaction, args=myfiles, opts=myopts) + rval = profile_check(emerge_config.trees, emerge_config.action) if rval != os.EX_OK: return rval tmpcmdline = [] if "--ignore-default-opts" not in myopts: - tmpcmdline.extend(settings["EMERGE_DEFAULT_OPTS"].split()) + tmpcmdline.extend(portage.util.shlex_split( + emerge_config.target_config.settings.get( + "EMERGE_DEFAULT_OPTS", ""))) tmpcmdline.extend(args) - myaction, myopts, myfiles = parse_opts(tmpcmdline) - - # skip global updates prior to sync, since it's called after sync - if myaction not in ('help', 'info', 'sync', 'version') and \ - myopts.get('--package-moves') != 'n' and \ - _global_updates(trees, mtimedb["updates"], quiet=("--quiet" in myopts)): - mtimedb.commit() - # Reload the whole config from scratch. - settings, trees, mtimedb = load_emerge_config(trees=trees) - portdb = trees[settings['EROOT']]['porttree'].dbapi - - xterm_titles = "notitles" not in settings.features - if xterm_titles: - xtermTitle("emerge") - - if "--digest" in myopts: - os.environ["FEATURES"] = os.environ.get("FEATURES","") + " digest" - # Reload the whole config from scratch so that the portdbapi internal - # config is updated with new FEATURES. - settings, trees, mtimedb = load_emerge_config(trees=trees) - portdb = trees[settings['EROOT']]['porttree'].dbapi - - # NOTE: adjust_configs() can map options to FEATURES, so any relevant - # options adjustments should be made prior to calling adjust_configs(). - if "--buildpkgonly" in myopts: - myopts["--buildpkg"] = True - - adjust_configs(myopts, trees) - apply_priorities(settings) - - if myaction == 'version': - writemsg_stdout(getportageversion( - settings["PORTDIR"], None, - settings.profile_path, settings["CHOST"], - trees[settings['EROOT']]['vartree'].dbapi) + '\n', noiselevel=-1) - return 0 - elif myaction == 'help': - _emerge.help.help() - return 0 - - spinner = stdout_spinner() - if "candy" in settings.features: - spinner.update = spinner.update_scroll - - if "--quiet" not in myopts: - portage.deprecated_profile_check(settings=settings) - if portage.const._ENABLE_REPO_NAME_WARN: - # Bug #248603 - Disable warnings about missing - # repo_name entries for stable branch. - repo_name_check(trees) - repo_name_duplicate_check(trees) - config_protect_check(trees) - check_procfs() - - if "getbinpkg" in settings.features: - myopts["--getbinpkg"] = True - - if "--getbinpkgonly" in myopts: - myopts["--getbinpkg"] = True - - if "--getbinpkgonly" in myopts: - myopts["--usepkgonly"] = True - - if "--getbinpkg" in myopts: - myopts["--usepkg"] = True - - if "--usepkgonly" in myopts: - myopts["--usepkg"] = True - - if "--buildpkgonly" in myopts: - # --buildpkgonly will not merge anything, so - # it cancels all binary package options. - for opt in ("--getbinpkg", "--getbinpkgonly", - "--usepkg", "--usepkgonly"): - myopts.pop(opt, None) - - for mytrees in trees.values(): - mydb = mytrees["porttree"].dbapi - # Freeze the portdbapi for performance (memoize all xmatch results). - mydb.freeze() - - if myaction in ('search', None) and \ - "--usepkg" in myopts: - # Populate the bintree with current --getbinpkg setting. - # This needs to happen before expand_set_arguments(), in case - # any sets use the bintree. - mytrees["bintree"].populate( - getbinpkgs="--getbinpkg" in myopts) - - del mytrees, mydb - - if "moo" in myfiles: - print(COWSAY_MOO % platform.system()) - msg = ("The above `emerge moo` display is deprecated. " - "Please use `emerge --moo` instead.") - for line in textwrap.wrap(msg, 50): - print(" %s %s" % (colorize("WARN", "*"), line)) - - for x in myfiles: - ext = os.path.splitext(x)[1] - if (ext == ".ebuild" or ext == ".tbz2") and os.path.exists(os.path.abspath(x)): - print(colorize("BAD", "\n*** emerging by path is broken and may not always work!!!\n")) - break - - root_config = trees[settings['EROOT']]['root_config'] - if myaction == "moo": - print(COWSAY_MOO % platform.system()) - return os.EX_OK - elif myaction == "list-sets": - writemsg_stdout("".join("%s\n" % s for s in sorted(root_config.sets))) - return os.EX_OK - elif myaction == "check-news": - news_counts = count_unread_news( - root_config.trees["porttree"].dbapi, - root_config.trees["vartree"].dbapi) - if any(news_counts.values()): - display_news_notifications(news_counts) - elif "--quiet" not in myopts: - print("", colorize("GOOD", "*"), "No news items were found.") - return os.EX_OK - - ensure_required_sets(trees) - - # only expand sets for actions taking package arguments - oldargs = myfiles[:] - if myaction in ("clean", "config", "depclean", "info", "prune", "unmerge", None): - myfiles, retval = expand_set_arguments(myfiles, myaction, root_config) - if retval != os.EX_OK: - return retval - - # Need to handle empty sets specially, otherwise emerge will react - # with the help message for empty argument lists - if oldargs and not myfiles: - print("emerge: no targets left after set expansion") - return 0 - - if ("--tree" in myopts) and ("--columns" in myopts): - print("emerge: can't specify both of \"--tree\" and \"--columns\".") - return 1 - - if '--emptytree' in myopts and '--noreplace' in myopts: - writemsg_level("emerge: can't specify both of " + \ - "\"--emptytree\" and \"--noreplace\".\n", - level=logging.ERROR, noiselevel=-1) - return 1 + emerge_config.action, emerge_config.opts, emerge_config.args = \ + parse_opts(tmpcmdline) - if ("--quiet" in myopts): - spinner.update = spinner.update_quiet - portage.util.noiselimit = -1 - - if "--fetch-all-uri" in myopts: - myopts["--fetchonly"] = True - - if "--skipfirst" in myopts and "--resume" not in myopts: - myopts["--resume"] = True - - # Allow -p to remove --ask - if "--pretend" in myopts: - myopts.pop("--ask", None) - - # forbid --ask when not in a terminal - # note: this breaks `emerge --ask | tee logfile`, but that doesn't work anyway. - if ("--ask" in myopts) and (not sys.stdin.isatty()): - portage.writemsg("!!! \"--ask\" should only be used in a terminal. Exiting.\n", - noiselevel=-1) - return 1 - - if settings.get("PORTAGE_DEBUG", "") == "1": - spinner.update = spinner.update_quiet - portage.util.noiselimit = 0 - if "python-trace" in settings.features: - import portage.debug as portage_debug - portage_debug.set_trace(True) - - if not ("--quiet" in myopts): - if '--nospinner' in myopts or \ - settings.get('TERM') == 'dumb' or \ - not sys.stdout.isatty(): - spinner.update = spinner.update_basic - - if "--debug" in myopts: - print("myaction", myaction) - print("myopts", myopts) - - if not myaction and not myfiles and "--resume" not in myopts: - _emerge.help.help() - return 1 - - pretend = "--pretend" in myopts - fetchonly = "--fetchonly" in myopts or "--fetch-all-uri" in myopts - buildpkgonly = "--buildpkgonly" in myopts - - # check if root user is the current user for the actions where emerge needs this - if portage.secpass < 2: - # We've already allowed "--version" and "--help" above. - if "--pretend" not in myopts and myaction not in ("search","info"): - need_superuser = myaction in ('clean', 'depclean', 'deselect', - 'prune', 'unmerge') or not \ - (fetchonly or \ - (buildpkgonly and secpass >= 1) or \ - myaction in ("metadata", "regen", "sync")) - if portage.secpass < 1 or \ - need_superuser: - if need_superuser: - access_desc = "superuser" - else: - access_desc = "portage group" - # Always show portage_group_warning() when only portage group - # access is required but the user is not in the portage group. - from portage.data import portage_group_warning - if "--ask" in myopts: - writemsg_stdout("This action requires %s access...\n" % \ - (access_desc,), noiselevel=-1) - if portage.secpass < 1 and not need_superuser: - portage_group_warning() - if userquery("Would you like to add --pretend to options?", - "--ask-enter-invalid" in myopts) == "No": - return 128 + signal.SIGINT - myopts["--pretend"] = True - del myopts["--ask"] - else: - sys.stderr.write(("emerge: %s access is required\n") \ - % access_desc) - if portage.secpass < 1 and not need_superuser: - portage_group_warning() - return 1 - - # Disable emergelog for everything except build or unmerge operations. - # This helps minimize parallel emerge.log entries that can confuse log - # parsers like genlop. - disable_emergelog = False - for x in ("--pretend", "--fetchonly", "--fetch-all-uri"): - if x in myopts: - disable_emergelog = True - break - if disable_emergelog: - pass - elif myaction in ("search", "info"): - disable_emergelog = True - elif portage.data.secpass < 1: - disable_emergelog = True - - _emerge.emergelog._disable = disable_emergelog - - if not disable_emergelog: - if 'EMERGE_LOG_DIR' in settings: - try: - # At least the parent needs to exist for the lock file. - portage.util.ensure_dirs(settings['EMERGE_LOG_DIR']) - except portage.exception.PortageException as e: - writemsg_level("!!! Error creating directory for " + \ - "EMERGE_LOG_DIR='%s':\n!!! %s\n" % \ - (settings['EMERGE_LOG_DIR'], e), - noiselevel=-1, level=logging.ERROR) - portage.util.ensure_dirs(_emerge.emergelog._emerge_log_dir) - else: - _emerge.emergelog._emerge_log_dir = settings["EMERGE_LOG_DIR"] - else: - _emerge.emergelog._emerge_log_dir = os.path.join(os.sep, - settings["EPREFIX"].lstrip(os.sep), "var", "log") - portage.util.ensure_dirs(_emerge.emergelog._emerge_log_dir) - - if not "--pretend" in myopts: - emergelog(xterm_titles, "Started emerge on: "+\ - _unicode_decode( - time.strftime("%b %d, %Y %H:%M:%S", time.localtime()), - encoding=_encodings['content'], errors='replace')) - myelogstr="" - if myopts: - opt_list = [] - for opt, arg in myopts.items(): - if arg is True: - opt_list.append(opt) - elif isinstance(arg, list): - # arguments like --exclude that use 'append' action - for x in arg: - opt_list.append("%s=%s" % (opt, x)) - else: - opt_list.append("%s=%s" % (opt, arg)) - myelogstr=" ".join(opt_list) - if myaction: - myelogstr += " --" + myaction - if myfiles: - myelogstr += " " + " ".join(oldargs) - emergelog(xterm_titles, " *** emerge " + myelogstr) - del oldargs - - def emergeexitsig(signum, frame): - signal.signal(signal.SIGINT, signal.SIG_IGN) - signal.signal(signal.SIGTERM, signal.SIG_IGN) - portage.util.writemsg("\n\nExiting on signal %(signal)s\n" % {"signal":signum}) - sys.exit(128 + signum) - signal.signal(signal.SIGINT, emergeexitsig) - signal.signal(signal.SIGTERM, emergeexitsig) - - def emergeexit(): - """This gets out final log message in before we quit.""" - if "--pretend" not in myopts: - emergelog(xterm_titles, " *** terminating.") - if xterm_titles: - xtermTitleReset() - portage.atexit_register(emergeexit) - - if myaction in ("config", "metadata", "regen", "sync"): - if "--pretend" in myopts: - sys.stderr.write(("emerge: The '%s' action does " + \ - "not support '--pretend'.\n") % myaction) - return 1 - - if "sync" == myaction: - return action_sync(settings, trees, mtimedb, myopts, myaction) - elif "metadata" == myaction: - action_metadata(settings, portdb, myopts) - elif myaction=="regen": - validate_ebuild_environment(trees) - return action_regen(settings, portdb, myopts.get("--jobs"), - myopts.get("--load-average")) - # HELP action - elif "config"==myaction: - validate_ebuild_environment(trees) - action_config(settings, trees, myopts, myfiles) - - # SEARCH action - elif "search"==myaction: - validate_ebuild_environment(trees) - action_search(trees[settings['EROOT']]['root_config'], - myopts, myfiles, spinner) - - elif myaction in ('clean', 'depclean', 'deselect', 'prune', 'unmerge'): - validate_ebuild_environment(trees) - rval = action_uninstall(settings, trees, mtimedb["ldpath"], - myopts, myaction, myfiles, spinner) - if not (myaction == 'deselect' or buildpkgonly or fetchonly or pretend): - post_emerge(myaction, myopts, myfiles, settings['EROOT'], - trees, mtimedb, rval) - return rval - - elif myaction == 'info': - - # Ensure atoms are valid before calling unmerge(). - vardb = trees[settings['EROOT']]['vartree'].dbapi - portdb = trees[settings['EROOT']]['porttree'].dbapi - bindb = trees[settings['EROOT']]["bintree"].dbapi - valid_atoms = [] - for x in myfiles: - if is_valid_package_atom(x, allow_repo=True): - try: - #look at the installed files first, if there is no match - #look at the ebuilds, since EAPI 4 allows running pkg_info - #on non-installed packages - valid_atom = dep_expand(x, mydb=vardb, settings=settings) - if valid_atom.cp.split("/")[0] == "null": - valid_atom = dep_expand(x, mydb=portdb, settings=settings) - if valid_atom.cp.split("/")[0] == "null" and "--usepkg" in myopts: - valid_atom = dep_expand(x, mydb=bindb, settings=settings) - valid_atoms.append(valid_atom) - except portage.exception.AmbiguousPackageName as e: - msg = "The short ebuild name \"" + x + \ - "\" is ambiguous. Please specify " + \ - "one of the following " + \ - "fully-qualified ebuild names instead:" - for line in textwrap.wrap(msg, 70): - writemsg_level("!!! %s\n" % (line,), - level=logging.ERROR, noiselevel=-1) - for i in e.args[0]: - writemsg_level(" %s\n" % colorize("INFORM", i), - level=logging.ERROR, noiselevel=-1) - writemsg_level("\n", level=logging.ERROR, noiselevel=-1) - return 1 - continue - msg = [] - msg.append("'%s' is not a valid package atom." % (x,)) - msg.append("Please check ebuild(5) for full details.") - writemsg_level("".join("!!! %s\n" % line for line in msg), - level=logging.ERROR, noiselevel=-1) - return 1 - - return action_info(settings, trees, myopts, valid_atoms) - - # "update", "system", or just process files: - else: - validate_ebuild_environment(trees) - - for x in myfiles: - if x.startswith(SETPREFIX) or \ - is_valid_package_atom(x, allow_repo=True): - continue - if x[:1] == os.sep: - continue - try: - os.lstat(x) + try: + return run_action(emerge_config) + finally: + # Call destructors for our portdbapi instances. + for x in emerge_config.trees.values(): + if "porttree" in x.lazy_items: continue - except OSError: - pass - msg = [] - msg.append("'%s' is not a valid package atom." % (x,)) - msg.append("Please check ebuild(5) for full details.") - writemsg_level("".join("!!! %s\n" % line for line in msg), - level=logging.ERROR, noiselevel=-1) - return 1 - - # GLEP 42 says to display news *after* an emerge --pretend - if "--pretend" not in myopts: - display_news_notification(root_config, myopts) - retval = action_build(settings, trees, mtimedb, - myopts, myaction, myfiles, spinner) - post_emerge(myaction, myopts, myfiles, settings['EROOT'], - trees, mtimedb, retval) - - return retval + x["porttree"].dbapi.close_caches() diff --git a/portage_with_autodep/pym/_emerge/main.pyo b/portage_with_autodep/pym/_emerge/main.pyo Binary files differindex aaeb5b9..2047458 100644 --- a/portage_with_autodep/pym/_emerge/main.pyo +++ b/portage_with_autodep/pym/_emerge/main.pyo diff --git a/portage_with_autodep/pym/_emerge/resolver/__init__.pyo b/portage_with_autodep/pym/_emerge/resolver/__init__.pyo Binary files differindex 5c1b374..a5d6b7f 100644 --- a/portage_with_autodep/pym/_emerge/resolver/__init__.pyo +++ b/portage_with_autodep/pym/_emerge/resolver/__init__.pyo diff --git a/portage_with_autodep/pym/_emerge/resolver/backtracking.py b/portage_with_autodep/pym/_emerge/resolver/backtracking.py index f2857b0..c29b9d4 100644 --- a/portage_with_autodep/pym/_emerge/resolver/backtracking.py +++ b/portage_with_autodep/pym/_emerge/resolver/backtracking.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import copy @@ -7,7 +7,8 @@ class BacktrackParameter(object): __slots__ = ( "needed_unstable_keywords", "runtime_pkg_mask", "needed_use_config_changes", "needed_license_changes", - "rebuild_list", "reinstall_list", "needed_p_mask_changes" + "prune_rebuilds", "rebuild_list", "reinstall_list", "needed_p_mask_changes", + "slot_operator_mask_built", "slot_operator_replace_installed" ) def __init__(self): @@ -18,6 +19,9 @@ class BacktrackParameter(object): self.needed_license_changes = {} self.rebuild_list = set() self.reinstall_list = set() + self.slot_operator_replace_installed = set() + self.slot_operator_mask_built = set() + self.prune_rebuilds = False def __deepcopy__(self, memo=None): if memo is None: @@ -29,11 +33,18 @@ class BacktrackParameter(object): #to our sets and dicts. The existing content is immutable. result.needed_unstable_keywords = copy.copy(self.needed_unstable_keywords) result.needed_p_mask_changes = copy.copy(self.needed_p_mask_changes) - result.runtime_pkg_mask = copy.copy(self.runtime_pkg_mask) result.needed_use_config_changes = copy.copy(self.needed_use_config_changes) result.needed_license_changes = copy.copy(self.needed_license_changes) result.rebuild_list = copy.copy(self.rebuild_list) result.reinstall_list = copy.copy(self.reinstall_list) + result.slot_operator_replace_installed = copy.copy(self.slot_operator_replace_installed) + result.slot_operator_mask_built = self.slot_operator_mask_built.copy() + result.prune_rebuilds = self.prune_rebuilds + + # runtime_pkg_mask contains nested dicts that must also be copied + result.runtime_pkg_mask = {} + for k, v in self.runtime_pkg_mask.items(): + result.runtime_pkg_mask[k] = copy.copy(v) return result @@ -44,7 +55,10 @@ class BacktrackParameter(object): self.needed_use_config_changes == other.needed_use_config_changes and \ self.needed_license_changes == other.needed_license_changes and \ self.rebuild_list == other.rebuild_list and \ - self.reinstall_list == other.reinstall_list + self.reinstall_list == other.reinstall_list and \ + self.slot_operator_replace_installed == other.slot_operator_replace_installed and \ + self.slot_operator_mask_built == other.slot_operator_mask_built and \ + self.prune_rebuilds == other.prune_rebuilds class _BacktrackNode(object): @@ -114,9 +128,10 @@ class Backtracker(object): before, we revert the mask for other packages (bug 375573). """ - for pkg in runtime_pkg_mask: + for pkg, mask_info in runtime_pkg_mask.items(): - if "missing dependency" in runtime_pkg_mask[pkg]: + if "missing dependency" in mask_info or \ + "slot_operator_mask_built" in mask_info: continue entry_is_valid = False @@ -131,6 +146,13 @@ class Backtracker(object): return True + def _feedback_slot_conflicts(self, conflicts_data): + # Only create BacktrackNode instances for the first + # conflict which occurred, since the conflicts that + # occurred later may have been caused by the first + # conflict. + self._feedback_slot_conflict(conflicts_data[0]) + def _feedback_slot_conflict(self, conflict_data): for pkg, parent_atoms in conflict_data: new_node = copy.deepcopy(self._current_node) @@ -174,10 +196,30 @@ class Backtracker(object): elif change == "needed_use_config_changes": for pkg, (new_use, new_changes) in data: para.needed_use_config_changes[pkg] = (new_use, new_changes) + elif change == "slot_conflict_abi": + new_node.terminal = False + elif change == "slot_operator_mask_built": + para.slot_operator_mask_built.update(data) + for pkg, mask_reasons in data.items(): + para.runtime_pkg_mask.setdefault(pkg, + {}).update(mask_reasons) + elif change == "slot_operator_replace_installed": + para.slot_operator_replace_installed.update(data) elif change == "rebuild_list": para.rebuild_list.update(data) elif change == "reinstall_list": para.reinstall_list.update(data) + elif change == "prune_rebuilds": + para.prune_rebuilds = True + para.slot_operator_replace_installed.clear() + for pkg in para.slot_operator_mask_built: + runtime_masks = para.runtime_pkg_mask.get(pkg) + if runtime_masks is None: + continue + runtime_masks.pop("slot_operator_mask_built", None) + if not runtime_masks: + para.runtime_pkg_mask.pop(pkg) + para.slot_operator_mask_built.clear() self._add(new_node, explore=explore) self._current_node = new_node @@ -196,7 +238,7 @@ class Backtracker(object): #There is at most one of the following types of conflicts for a given restart. if "slot conflict" in infos: - self._feedback_slot_conflict(infos["slot conflict"]) + self._feedback_slot_conflicts(infos["slot conflict"]) elif "missing dependency" in infos: self._feedback_missing_dep(infos["missing dependency"]) diff --git a/portage_with_autodep/pym/_emerge/resolver/backtracking.pyo b/portage_with_autodep/pym/_emerge/resolver/backtracking.pyo Binary files differindex d989c15..bc8338e 100644 --- a/portage_with_autodep/pym/_emerge/resolver/backtracking.pyo +++ b/portage_with_autodep/pym/_emerge/resolver/backtracking.pyo diff --git a/portage_with_autodep/pym/_emerge/resolver/circular_dependency.py b/portage_with_autodep/pym/_emerge/resolver/circular_dependency.py index aca81fa..b710671 100644 --- a/portage_with_autodep/pym/_emerge/resolver/circular_dependency.py +++ b/portage_with_autodep/pym/_emerge/resolver/circular_dependency.py @@ -1,7 +1,7 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import print_function +from __future__ import print_function, unicode_literals from itertools import chain, product import logging @@ -11,6 +11,7 @@ from portage.exception import InvalidDependString from portage.output import colorize from portage.util import writemsg_level from _emerge.DepPrioritySatisfiedRange import DepPrioritySatisfiedRange +from _emerge.Package import Package class circular_dependency_handler(object): @@ -61,8 +62,7 @@ class circular_dependency_handler(object): node = nodes[0] display_order.append(node) tempgraph.remove(node) - display_order.reverse() - return display_order + return tuple(display_order) def _prepare_circular_dep_message(self): """ @@ -113,9 +113,10 @@ class circular_dependency_handler(object): parent_atoms = self.all_parent_atoms.get(pkg) if priorities[-1].buildtime: - dep = parent.metadata["DEPEND"] + dep = " ".join(parent._metadata[k] + for k in Package._buildtime_keys) elif priorities[-1].runtime: - dep = parent.metadata["RDEPEND"] + dep = parent._metadata["RDEPEND"] for ppkg, atom in parent_atoms: if ppkg == parent: @@ -125,7 +126,7 @@ class circular_dependency_handler(object): try: affecting_use = extract_affecting_use(dep, parent_atom, - eapi=parent.metadata["EAPI"]) + eapi=parent.eapi) except InvalidDependString: if not parent.installed: raise @@ -144,7 +145,8 @@ class circular_dependency_handler(object): #If any of the flags we're going to touch is in REQUIRED_USE, add all #other flags in REQUIRED_USE to affecting_use, to not lose any solution. required_use_flags = get_required_use_flags( - parent.metadata.get("REQUIRED_USE", "")) + parent._metadata.get("REQUIRED_USE", ""), + eapi=parent.eapi) if affecting_use.intersection(required_use_flags): # TODO: Find out exactly which REQUIRED_USE flags are @@ -186,9 +188,11 @@ class circular_dependency_handler(object): parent_atom not in reduced_dep: #We found an assignment that removes the atom from 'dep'. #Make sure it doesn't conflict with REQUIRED_USE. - required_use = parent.metadata.get("REQUIRED_USE", "") + required_use = parent._metadata.get("REQUIRED_USE", "") - if check_required_use(required_use, current_use, parent.iuse.is_valid_flag): + if check_required_use(required_use, current_use, + parent.iuse.is_valid_flag, + eapi=parent.eapi): use = self.depgraph._pkg_use_enabled(parent) solution = set() for flag, state in zip(affecting_use, use_state): diff --git a/portage_with_autodep/pym/_emerge/resolver/circular_dependency.pyo b/portage_with_autodep/pym/_emerge/resolver/circular_dependency.pyo Binary files differindex c1f95dc..17fb71d 100644 --- a/portage_with_autodep/pym/_emerge/resolver/circular_dependency.pyo +++ b/portage_with_autodep/pym/_emerge/resolver/circular_dependency.pyo diff --git a/portage_with_autodep/pym/_emerge/resolver/output.py b/portage_with_autodep/pym/_emerge/resolver/output.py index 1208bf9..3e8552f 100644 --- a/portage_with_autodep/pym/_emerge/resolver/output.py +++ b/portage_with_autodep/pym/_emerge/resolver/output.py @@ -1,26 +1,30 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 """Resolver output display operation. """ +from __future__ import unicode_literals + __all__ = ( "Display", ) import sys +import portage from portage import os -from portage import _unicode_decode from portage.dbapi.dep_expand import dep_expand -from portage.dep import cpvequal, _repo_separator +from portage.dep import cpvequal, _repo_separator, _slot_separator +from portage.eapi import _get_eapi_attrs from portage.exception import InvalidDependString, SignatureException +from portage.package.ebuild.config import _get_feature_flags from portage.package.ebuild._spawn_nofetch import spawn_nofetch from portage.output import ( blue, colorize, create_color_func, - darkblue, darkgreen, green, nc_len, red, teal, turquoise, yellow ) + darkblue, darkgreen, green, nc_len, teal) bad = create_color_func("BAD") from portage.util import writemsg_stdout -from portage.versions import best, catpkgsplit +from portage.versions import best from _emerge.Blocker import Blocker from _emerge.create_world_atom import create_world_atom @@ -30,7 +34,9 @@ from _emerge.show_invalid_depstring_notice import show_invalid_depstring_notice if sys.hexversion >= 0x3000000: basestring = str - + _unicode = str +else: + _unicode = unicode class Display(object): """Formats and outputs the depgrah supplied it for merge/re-merge, etc. @@ -54,11 +60,6 @@ class Display(object): self.oldlp = None self.myfetchlist = None self.indent = '' - self.is_new = True - self.cur_use = None - self.cur_iuse = None - self.old_use = '' - self.old_iuse = '' self.use_expand = None self.use_expand_hidden = None self.pkgsettings = None @@ -68,93 +69,54 @@ class Display(object): self.blocker_style = None - def _blockers(self, pkg, fetch_symbol): - """Processes pkg for blockers and adds colorized strings to + def _blockers(self, blocker): + """Adds colorized strings to self.print_msg and self.blockers - @param pkg: _emerge.Package.Package instance - @param fetch_symbol: string + @param blocker: _emerge.Blocker.Blocker instance @rtype: bool Modifies class globals: self.blocker_style, self.resolved, self.print_msg """ - if pkg.satisfied: + if blocker.satisfied: self.blocker_style = "PKG_BLOCKER_SATISFIED" - addl = "%s %s " % (colorize(self.blocker_style, "b"), - fetch_symbol) + addl = "%s " % (colorize(self.blocker_style, "b"),) else: self.blocker_style = "PKG_BLOCKER" - addl = "%s %s " % (colorize(self.blocker_style, "B"), - fetch_symbol) + addl = "%s " % (colorize(self.blocker_style, "B"),) addl += self.empty_space_in_brackets() self.resolved = dep_expand( - str(pkg.atom).lstrip("!"), mydb=self.vardb, + _unicode(blocker.atom).lstrip("!"), mydb=self.vardb, settings=self.pkgsettings ) if self.conf.columns and self.conf.quiet: - addl += " " + colorize(self.blocker_style, str(self.resolved)) + addl += " " + colorize(self.blocker_style, _unicode(self.resolved)) else: addl = "[%s %s] %s%s" % \ (colorize(self.blocker_style, "blocks"), addl, self.indent, - colorize(self.blocker_style, str(self.resolved)) + colorize(self.blocker_style, _unicode(self.resolved)) ) - block_parents = self.conf.blocker_parents.parent_nodes(pkg) - block_parents = set([pnode[2] for pnode in block_parents]) + block_parents = self.conf.blocker_parents.parent_nodes(blocker) + block_parents = set(_unicode(pnode.cpv) for pnode in block_parents) block_parents = ", ".join(block_parents) - if self.resolved != pkg[2]: + if blocker.atom.blocker.overlap.forbid: + blocking_desc = "hard blocking" + else: + blocking_desc = "blocking" + if self.resolved != blocker.atom: addl += colorize(self.blocker_style, - " (\"%s\" is blocking %s)") % \ - (str(pkg.atom).lstrip("!"), block_parents) + " (\"%s\" is %s %s)" % + (_unicode(blocker.atom).lstrip("!"), + blocking_desc, block_parents)) else: addl += colorize(self.blocker_style, - " (is blocking %s)") % block_parents - if isinstance(pkg, Blocker) and pkg.satisfied: - if self.conf.columns: - return True - self.print_msg.append(addl) + " (is %s %s)" % (blocking_desc, block_parents)) + if blocker.satisfied: + if not self.conf.columns: + self.print_msg.append(addl) else: self.blockers.append(addl) - return False - - - def _display_use(self, pkg, myoldbest, myinslotlist): - """ USE flag display - - @param pkg: _emerge.Package.Package instance - @param myoldbest: list of installed versions - @param myinslotlist: list of installed slots - Modifies class globals: self.forced_flags, self.cur_iuse, - self.old_iuse, self.old_use, self.use_expand - """ - - self.forced_flags = set() - self.forced_flags.update(pkg.use.force) - self.forced_flags.update(pkg.use.mask) - - self.cur_use = [flag for flag in self.conf.pkg_use_enabled(pkg) \ - if flag in pkg.iuse.all] - self.cur_iuse = sorted(pkg.iuse.all) - - if myoldbest and myinslotlist: - previous_cpv = myoldbest[0].cpv - else: - previous_cpv = pkg.cpv - if self.vardb.cpv_exists(previous_cpv): - previous_pkg = self.vardb.match_pkgs('=' + previous_cpv)[0] - self.old_iuse = sorted(previous_pkg.iuse.all) - self.old_use = previous_pkg.use.enabled - self.is_new = False - else: - self.old_iuse = [] - self.old_use = [] - self.is_new = True - - self.old_use = [flag for flag in self.old_use if flag in self.old_iuse] - - self.use_expand = pkg.use.expand - self.use_expand_hidden = pkg.use.expand_hidden - return def include_mask_str(self): return self.conf.verbosity > 1 @@ -219,13 +181,40 @@ class Display(object): return ret - def recheck_hidden(self, pkg): - """ Prevent USE_EXPAND_HIDDEN flags from being hidden if they - are the only thing that triggered reinstallation. + def _display_use(self, pkg, pkg_info): + """ USE flag display @param pkg: _emerge.Package.Package instance - Modifies self.use_expand_hidden, self.use_expand, self.verboseadd + @param pkg_info: PkgInfo instance + Modifies self.use_expand_hidden, self.use_expand, self.verboseadd, + self.forced_flags """ + + self.forced_flags = set() + self.forced_flags.update(pkg.use.force) + self.forced_flags.update(pkg.use.mask) + + cur_use = [flag for flag in self.conf.pkg_use_enabled(pkg) \ + if flag in pkg.iuse.all] + cur_iuse = sorted(pkg.iuse.all) + + if pkg_info.previous_pkg is not None: + previous_pkg = pkg_info.previous_pkg + old_iuse = sorted(previous_pkg.iuse.all) + old_use = previous_pkg.use.enabled + is_new = False + else: + old_iuse = [] + old_use = [] + is_new = True + + old_use = [flag for flag in old_use if flag in old_iuse] + + self.use_expand = pkg.use.expand + self.use_expand_hidden = pkg.use.expand_hidden + + # Prevent USE_EXPAND_HIDDEN flags from being hidden if they + # are the only thing that triggered reinstallation. reinst_flags_map = {} reinstall_for_flags = self.conf.reinstall_nodes.get(pkg) reinst_expand_map = None @@ -246,13 +235,14 @@ class Display(object): reinst_expand_map) cur_iuse_map, iuse_forced = \ - self.map_to_use_expand(self.cur_iuse, forced_flags=True) - cur_use_map = self.map_to_use_expand(self.cur_use) - old_iuse_map = self.map_to_use_expand(self.old_iuse) - old_use_map = self.map_to_use_expand(self.old_use) + self.map_to_use_expand(cur_iuse, forced_flags=True) + cur_use_map = self.map_to_use_expand(cur_use) + old_iuse_map = self.map_to_use_expand(old_iuse) + old_use_map = self.map_to_use_expand(old_use) use_expand = sorted(self.use_expand) use_expand.insert(0, "USE") + feature_flags = _get_feature_flags(_get_eapi_attrs(pkg.eapi)) for key in use_expand: if key in self.use_expand_hidden: @@ -260,7 +250,7 @@ class Display(object): self.verboseadd += _create_use_string(self.conf, key.upper(), cur_iuse_map[key], iuse_forced[key], cur_use_map[key], old_iuse_map[key], - old_use_map[key], self.is_new, + old_use_map[key], is_new, feature_flags, reinst_flags_map.get(key)) return @@ -318,13 +308,14 @@ class Display(object): kwargs["myrepo"] = pkg.repo myfilesdict = None try: - myfilesdict = db.getfetchsizes(pkg.cpv, **kwargs) + myfilesdict = db.getfetchsizes(pkg.cpv, + **portage._native_kwargs(kwargs)) except InvalidDependString as e: # FIXME: validate SRC_URI earlier depstr, = db.aux_get(pkg.cpv, ["SRC_URI"], myrepo=pkg.repo) show_invalid_depstring_notice( - pkg, depstr, str(e)) + pkg, depstr, _unicode(e)) raise except SignatureException: # missing/invalid binary package SIZE signature @@ -343,15 +334,13 @@ class Display(object): if self.quiet_repo_display: # overlay verbose # assign index for a previous version in the same slot - slot_matches = self.vardb.match(pkg.slot_atom) - if slot_matches: - repo_name_prev = self.vardb.aux_get(slot_matches[0], - ["repository"])[0] + if pkg_info.previous_pkg is not None: + repo_name_prev = pkg_info.previous_pkg.repo else: repo_name_prev = None # now use the data to generate output - if pkg.installed or not slot_matches: + if pkg.installed or pkg_info.previous_pkg is None: self.repoadd = self.conf.repo_display.repoStr( pkg_info.repo_path_real) else: @@ -370,58 +359,86 @@ class Display(object): repoadd_set.add(self.repoadd) - def convert_myoldbest(self, pkg, myoldbest): + def convert_myoldbest(self, pkg, pkg_info): """converts and colorizes a version list to a string @param pkg: _emerge.Package.Package instance - @param myoldbest: list + @param pkg_info: dictionary @rtype string. """ + myoldbest = pkg_info.oldbest_list # Convert myoldbest from a list to a string. myoldbest_str = "" if myoldbest: versions = [] for pos, old_pkg in enumerate(myoldbest): - key = catpkgsplit(old_pkg.cpv)[2] + "-" + catpkgsplit(old_pkg.cpv)[3] + key = old_pkg.version if key[-3:] == "-r0": key = key[:-3] - if self.conf.verbosity == 3 and not self.quiet_repo_display and (self.verbose_main_repo_display or - any(x.repo != self.portdb.repositories.mainRepo().name for x in myoldbest + [pkg])): - key += _repo_separator + old_pkg.repo + if self.conf.verbosity == 3: + if pkg_info.attr_display.new_slot: + key += _slot_separator + old_pkg.slot + if old_pkg.slot != old_pkg.sub_slot: + key += "/" + old_pkg.sub_slot + elif any(x.slot + "/" + x.sub_slot != "0/0" for x in myoldbest + [pkg]): + key += _slot_separator + old_pkg.slot + if old_pkg.slot != old_pkg.sub_slot or \ + old_pkg.slot == pkg.slot and old_pkg.sub_slot != pkg.sub_slot: + key += "/" + old_pkg.sub_slot + if not self.quiet_repo_display and (self.verbose_main_repo_display or + self.portdb.repositories.mainRepo() is None or + any(x.repo != self.portdb.repositories.mainRepo().name for x in myoldbest + [pkg])): + key += _repo_separator + old_pkg.repo versions.append(key) myoldbest_str = blue("["+", ".join(versions)+"]") return myoldbest_str + def _append_slot(self, pkg_str, pkg, pkg_info): + """Potentially appends slot and subslot to package string. - def set_interactive(self, pkg, ordered, addl): - """Increments counters.interactive if the pkg is to - be merged and it's metadata has interactive set True + @param pkg_str: string + @param pkg: _emerge.Package.Package instance + @param pkg_info: dictionary + @rtype string + """ + if pkg_info.attr_display.new_slot: + pkg_str += _slot_separator + pkg_info.slot + if pkg_info.slot != pkg_info.sub_slot: + pkg_str += "/" + pkg_info.sub_slot + elif any(x.slot + "/" + x.sub_slot != "0/0" for x in pkg_info.oldbest_list + [pkg]): + pkg_str += _slot_separator + pkg_info.slot + if pkg_info.slot != pkg_info.sub_slot or \ + any(x.slot == pkg_info.slot and x.sub_slot != pkg_info.sub_slot for x in pkg_info.oldbest_list): + pkg_str += "/" + pkg_info.sub_slot + return pkg_str + + def _append_repository(self, pkg_str, pkg, pkg_info): + """Potentially appends repository to package string. + @param pkg_str: string @param pkg: _emerge.Package.Package instance - @param ordered: boolean - @param addl: already defined string to add to + @param pkg_info: dictionary + @rtype string """ - if 'interactive' in pkg.metadata.properties and \ - pkg.operation == 'merge': - addl = colorize("WARN", "I") + addl[1:] - if ordered: - self.counters.interactive += 1 - return addl - - def _set_non_root_columns(self, addl, pkg_info, pkg): + if not self.quiet_repo_display and (self.verbose_main_repo_display or + self.portdb.repositories.mainRepo() is None or + any(x.repo != self.portdb.repositories.mainRepo().name for x in pkg_info.oldbest_list + [pkg])): + pkg_str += _repo_separator + pkg.repo + return pkg_str + + def _set_non_root_columns(self, pkg, pkg_info): """sets the indent level and formats the output - @param addl: already defined string to add to - @param pkg_info: dictionary @param pkg: _emerge.Package.Package instance + @param pkg_info: dictionary @rtype string """ ver_str = pkg_info.ver - if self.conf.verbosity == 3 and not self.quiet_repo_display and (self.verbose_main_repo_display or - any(x.repo != self.portdb.repositories.mainRepo().name for x in pkg_info.oldbest_list + [pkg])): - ver_str += _repo_separator + pkg.repo + if self.conf.verbosity == 3: + ver_str = self._append_slot(ver_str, pkg, pkg_info) + ver_str = self._append_repository(ver_str, pkg, pkg_info) if self.conf.quiet: - myprint = addl + " " + self.indent + \ + myprint = _unicode(pkg_info.attr_display) + " " + self.indent + \ self.pkgprint(pkg_info.cp, pkg_info) myprint = myprint+darkblue(" "+ver_str)+" " myprint = myprint+pkg_info.oldbest @@ -434,7 +451,8 @@ class Display(object): self.indent, self.pkgprint(pkg.cp, pkg_info)) else: myprint = "[%s %s] %s%s" % \ - (self.pkgprint(pkg.type_name, pkg_info), addl, + (self.pkgprint(pkg.type_name, pkg_info), + pkg_info.attr_display, self.indent, self.pkgprint(pkg.cp, pkg_info)) if (self.newlp-nc_len(myprint)) > 0: myprint = myprint+(" "*(self.newlp-nc_len(myprint))) @@ -446,21 +464,20 @@ class Display(object): return myprint - def _set_root_columns(self, addl, pkg_info, pkg): + def _set_root_columns(self, pkg, pkg_info): """sets the indent level and formats the output - @param addl: already defined string to add to - @param pkg_info: dictionary @param pkg: _emerge.Package.Package instance + @param pkg_info: dictionary @rtype string Modifies self.verboseadd """ ver_str = pkg_info.ver - if self.conf.verbosity == 3 and not self.quiet_repo_display and (self.verbose_main_repo_display or - any(x.repo != self.portdb.repositories.mainRepo().name for x in pkg_info.oldbest_list + [pkg])): - ver_str += _repo_separator + pkg.repo + if self.conf.verbosity == 3: + ver_str = self._append_slot(ver_str, pkg, pkg_info) + ver_str = self._append_repository(ver_str, pkg, pkg_info) if self.conf.quiet: - myprint = addl + " " + self.indent + \ + myprint = _unicode(pkg_info.attr_display) + " " + self.indent + \ self.pkgprint(pkg_info.cp, pkg_info) myprint = myprint+" "+green(ver_str)+" " myprint = myprint+pkg_info.oldbest @@ -473,7 +490,8 @@ class Display(object): addl, self.indent, self.pkgprint(pkg.cp, pkg_info)) else: myprint = "[%s %s] %s%s" % \ - (self.pkgprint(pkg.type_name, pkg_info), addl, + (self.pkgprint(pkg.type_name, pkg_info), + pkg_info.attr_display, self.indent, self.pkgprint(pkg.cp, pkg_info)) if (self.newlp-nc_len(myprint)) > 0: myprint = myprint+(" "*(self.newlp-nc_len(myprint))) @@ -484,18 +502,17 @@ class Display(object): return myprint - def _set_no_columns(self, pkg, pkg_info, addl): + def _set_no_columns(self, pkg, pkg_info): """prints pkg info without column indentation. @param pkg: _emerge.Package.Package instance @param pkg_info: dictionary - @param addl: the current text to add for the next line to output @rtype the updated addl """ pkg_str = pkg.cpv - if self.conf.verbosity == 3 and not self.quiet_repo_display and (self.verbose_main_repo_display or - any(x.repo != self.portdb.repositories.mainRepo().name for x in pkg_info.oldbest_list + [pkg])): - pkg_str += _repo_separator + pkg.repo + if self.conf.verbosity == 3: + pkg_str = self._append_slot(pkg_str, pkg, pkg_info) + pkg_str = self._append_repository(pkg_str, pkg, pkg_info) if not pkg_info.merge: addl = self.empty_space_in_brackets() myprint = "[%s%s] %s%s %s" % \ @@ -506,53 +523,10 @@ class Display(object): else: myprint = "[%s %s] %s%s %s" % \ (self.pkgprint(pkg.type_name, pkg_info), - addl, self.indent, + pkg_info.attr_display, self.indent, self.pkgprint(pkg_str, pkg_info), pkg_info.oldbest) return myprint - - def _insert_slot(self, pkg, pkg_info, myinslotlist): - """Adds slot info to the message - - @return addl: formatted slot info - @return myoldbest: installed version list - Modifies self.counters.downgrades, self.counters.upgrades, - self.counters.binary - """ - addl = " " + pkg_info.fetch_symbol - if not cpvequal(pkg.cpv, - best([pkg.cpv] + [x.cpv for x in myinslotlist])): - # Downgrade in slot - addl += turquoise("U")+blue("D") - if pkg_info.ordered: - self.counters.downgrades += 1 - if pkg.type_name == "binary": - self.counters.binary += 1 - else: - # Update in slot - addl += turquoise("U") + " " - if pkg_info.ordered: - self.counters.upgrades += 1 - if pkg.type_name == "binary": - self.counters.binary += 1 - return addl - - - def _new_slot(self, pkg, pkg_info): - """New slot, mark it new. - - @return addl: formatted slot info - @return myoldbest: installed version list - Modifies self.counters.newslot, self.counters.binary - """ - addl = " " + green("NS") + pkg_info.fetch_symbol + " " - if pkg_info.ordered: - self.counters.newslot += 1 - if pkg.type_name == "binary": - self.counters.binary += 1 - return addl - - def print_messages(self, show_repos): """Performs the actual output printing of the pre-formatted messages @@ -588,9 +562,9 @@ class Display(object): """ writemsg_stdout('\n%s\n' % (self.counters,), noiselevel=-1) if show_repos: - # Use _unicode_decode() to force unicode format string so + # Use unicode_literals to force unicode format string so # that RepoDisplay.__unicode__() is called in python2. - writemsg_stdout(_unicode_decode("%s") % (self.conf.repo_display,), + writemsg_stdout("%s" % (self.conf.repo_display,), noiselevel=-1) return @@ -642,15 +616,24 @@ class Display(object): self.counters.restrict_fetch_satisfied """ pkg_info = PkgInfo() + pkg_info.cp = pkg.cp + pkg_info.ver = self.get_ver_str(pkg) + pkg_info.slot = pkg.slot + pkg_info.sub_slot = pkg.sub_slot + pkg_info.repo_name = pkg.repo pkg_info.ordered = ordered - pkg_info.fetch_symbol = " " pkg_info.operation = pkg.operation pkg_info.merge = ordered and pkg_info.operation == "merge" if not pkg_info.merge and pkg_info.operation == "merge": pkg_info.operation = "nomerge" pkg_info.built = pkg.type_name != "ebuild" pkg_info.ebuild_path = None - pkg_info.repo_name = pkg.repo + if ordered: + if pkg_info.merge: + if pkg.type_name == "binary": + self.counters.binary += 1 + elif pkg_info.operation == "uninstall": + self.counters.uninst += 1 if pkg.type_name == "ebuild": pkg_info.ebuild_path = self.portdb.findname( pkg.cpv, myrepo=pkg_info.repo_name) @@ -660,22 +643,30 @@ class Display(object): pkg_info.repo_path_real = os.path.dirname(os.path.dirname( os.path.dirname(pkg_info.ebuild_path))) else: - pkg_info.repo_path_real = \ - self.portdb.getRepositoryPath(pkg.metadata["repository"]) + pkg_info.repo_path_real = self.portdb.getRepositoryPath(pkg.repo) pkg_info.use = list(self.conf.pkg_use_enabled(pkg)) if not pkg.built and pkg.operation == 'merge' and \ - 'fetch' in pkg.metadata.restrict: + 'fetch' in pkg.restrict: if pkg_info.ordered: self.counters.restrict_fetch += 1 + pkg_info.attr_display.fetch_restrict = True if not self.portdb.getfetchsizes(pkg.cpv, useflags=pkg_info.use, myrepo=pkg.repo): - pkg_info.fetch_symbol = green("f") + pkg_info.attr_display.fetch_restrict_satisfied = True if pkg_info.ordered: self.counters.restrict_fetch_satisfied += 1 else: - pkg_info.fetch_symbol = red("F") if pkg_info.ebuild_path is not None: self.restrict_fetch_list[pkg] = pkg_info + + if self.vardb.cpv_exists(pkg.cpv): + # Do a cpv match first, in case the SLOT has changed. + pkg_info.previous_pkg = self.vardb.match_pkgs('=' + pkg.cpv)[0] + else: + slot_matches = self.vardb.match_pkgs(pkg.slot_atom) + if slot_matches: + pkg_info.previous_pkg = slot_matches[0] + return pkg_info @@ -686,15 +677,14 @@ class Display(object): @param pkg_info: dictionay Modifies self.changelogs """ - inst_matches = self.vardb.match(pkg.slot_atom) - if inst_matches: + if pkg_info.previous_pkg is not None: ebuild_path_cl = pkg_info.ebuild_path if ebuild_path_cl is None: # binary package ebuild_path_cl = self.portdb.findname(pkg.cpv, myrepo=pkg.repo) if ebuild_path_cl is not None: self.changelogs.extend(_calc_changelog( - ebuild_path_cl, inst_matches[0], pkg.cpv)) + ebuild_path_cl, pkg_info.previous_pkg, pkg.cpv)) return @@ -734,12 +724,10 @@ class Display(object): @param pkg: _emerge.Package.Package instance @rtype string """ - ver_str = list(catpkgsplit(pkg.cpv)[2:]) - if ver_str[1] == "r0": - ver_str[1] = "" - else: - ver_str[1] = "-" + ver_str[1] - return ver_str[0]+ver_str[1] + ver_str = pkg.cpv.version + if ver_str.endswith("-r0"): + ver_str = ver_str[:-3] + return ver_str def _get_installed_best(self, pkg, pkg_info): @@ -751,24 +739,21 @@ class Display(object): @param pkg: _emerge.Package.Package instance @param pkg_info: dictionay @rtype addl, myoldbest: list, myinslotlist: list - Modifies self.counters.reinst, self.counters.binary, self.counters.new + Modifies self.counters.reinst, self.counters.new """ myoldbest = [] myinslotlist = None installed_versions = self.vardb.match_pkgs(pkg.cp) if self.vardb.cpv_exists(pkg.cpv): - addl = " "+yellow("R")+pkg_info.fetch_symbol+" " - installed_version = self.vardb.match_pkgs(pkg.cpv)[0] - if not self.quiet_repo_display and installed_version.repo != pkg.repo: + pkg_info.attr_display.replace = True + installed_version = pkg_info.previous_pkg + if installed_version.slot != pkg.slot or installed_version.sub_slot != pkg.sub_slot or \ + not self.quiet_repo_display and installed_version.repo != pkg.repo: myoldbest = [installed_version] if pkg_info.ordered: if pkg_info.merge: self.counters.reinst += 1 - if pkg.type_name == "binary": - self.counters.binary += 1 - elif pkg_info.operation == "uninstall": - self.counters.uninst += 1 # filter out old-style virtual matches elif installed_versions and \ installed_versions[0].cp == pkg.cp: @@ -780,19 +765,31 @@ class Display(object): myinslotlist = None if myinslotlist: myoldbest = myinslotlist[:] - addl = self._insert_slot(pkg, pkg_info, myinslotlist) + if not cpvequal(pkg.cpv, + best([pkg.cpv] + [x.cpv for x in myinslotlist])): + # Downgrade in slot + pkg_info.attr_display.new_version = True + pkg_info.attr_display.downgrade = True + if pkg_info.ordered: + self.counters.downgrades += 1 + else: + # Update in slot + pkg_info.attr_display.new_version = True + if pkg_info.ordered: + self.counters.upgrades += 1 else: myoldbest = installed_versions - addl = self._new_slot(pkg, pkg_info) + pkg_info.attr_display.new = True + pkg_info.attr_display.new_slot = True + if pkg_info.ordered: + self.counters.newslot += 1 if self.conf.changelog: self.do_changelog(pkg, pkg_info) else: - addl = " " + green("N") + " " + pkg_info.fetch_symbol + " " + pkg_info.attr_display.new = True if pkg_info.ordered: self.counters.new += 1 - if pkg.type_name == "binary": - self.counters.binary += 1 - return addl, myoldbest, myinslotlist + return myoldbest, myinslotlist def __call__(self, depgraph, mylist, favorites=None, verbosity=None): @@ -813,7 +810,7 @@ class Display(object): # files to fetch list - avoids counting a same file twice # in size display (verbose mode) self.myfetchlist = set() - + self.quiet_repo_display = "--quiet-repo-display" in depgraph._frozen_config.myopts if self.quiet_repo_display: # Use this set to detect when all the "repoadd" strings are "[0]" @@ -831,47 +828,52 @@ class Display(object): self.indent = " " * depth if isinstance(pkg, Blocker): - if self._blockers(pkg, fetch_symbol=" "): - continue + self._blockers(pkg) else: pkg_info = self.set_pkg_info(pkg, ordered) - addl, pkg_info.oldbest_list, myinslotlist = \ + pkg_info.oldbest_list, myinslotlist = \ self._get_installed_best(pkg, pkg_info) + if ordered and pkg_info.merge and \ + not pkg_info.attr_display.new: + for arg, atom in depgraph._iter_atoms_for_pkg(pkg): + if arg.force_reinstall: + pkg_info.attr_display.force_reinstall = True + break + self.verboseadd = "" if self.quiet_repo_display: self.repoadd = None - self._display_use(pkg, pkg_info.oldbest_list, myinslotlist) - self.recheck_hidden(pkg) + self._display_use(pkg, pkg_info) if self.conf.verbosity == 3: if self.quiet_repo_display: self.verbose_size(pkg, repoadd_set, pkg_info) else: self.verbose_size(pkg, None, pkg_info) - pkg_info.cp = pkg.cp - pkg_info.ver = self.get_ver_str(pkg) - self.oldlp = self.conf.columnwidth - 30 self.newlp = self.oldlp - 30 - pkg_info.oldbest = self.convert_myoldbest(pkg, pkg_info.oldbest_list) + pkg_info.oldbest = self.convert_myoldbest(pkg, pkg_info) pkg_info.system, pkg_info.world = \ self.check_system_world(pkg) - addl = self.set_interactive(pkg, pkg_info.ordered, addl) + if 'interactive' in pkg.properties and \ + pkg.operation == 'merge': + pkg_info.attr_display.interactive = True + if ordered: + self.counters.interactive += 1 if self.include_mask_str(): - addl += self.gen_mask_str(pkg) + pkg_info.attr_display.mask = self.gen_mask_str(pkg) if pkg.root_config.settings["ROOT"] != "/": if pkg_info.oldbest: pkg_info.oldbest += " " if self.conf.columns: - myprint = self._set_non_root_columns( - addl, pkg_info, pkg) + myprint = self._set_non_root_columns(pkg, pkg_info) else: pkg_str = pkg.cpv - if self.conf.verbosity == 3 and not self.quiet_repo_display and (self.verbose_main_repo_display or - any(x.repo != self.portdb.repositories.mainRepo().name for x in pkg_info.oldbest_list + [pkg])): - pkg_str += _repo_separator + pkg.repo + if self.conf.verbosity == 3: + pkg_str = self._append_slot(pkg_str, pkg, pkg_info) + pkg_str = self._append_repository(pkg_str, pkg, pkg_info) if not pkg_info.merge: addl = self.empty_space_in_brackets() myprint = "[%s%s] " % ( @@ -880,17 +882,16 @@ class Display(object): ) else: myprint = "[%s %s] " % ( - self.pkgprint(pkg.type_name, pkg_info), addl) + self.pkgprint(pkg.type_name, pkg_info), + pkg_info.attr_display) myprint += self.indent + \ self.pkgprint(pkg_str, pkg_info) + " " + \ pkg_info.oldbest + darkgreen("to " + pkg.root) else: if self.conf.columns: - myprint = self._set_root_columns( - addl, pkg_info, pkg) + myprint = self._set_root_columns(pkg, pkg_info) else: - myprint = self._set_no_columns( - pkg, pkg_info, addl) + myprint = self._set_no_columns(pkg, pkg_info) if self.conf.columns and pkg.operation == "uninstall": continue diff --git a/portage_with_autodep/pym/_emerge/resolver/output.pyo b/portage_with_autodep/pym/_emerge/resolver/output.pyo Binary files differindex bd2ae2f..e3a3c7f 100644 --- a/portage_with_autodep/pym/_emerge/resolver/output.pyo +++ b/portage_with_autodep/pym/_emerge/resolver/output.pyo diff --git a/portage_with_autodep/pym/_emerge/resolver/output_helpers.py b/portage_with_autodep/pym/_emerge/resolver/output_helpers.py index e751dd8..cfa6910 100644 --- a/portage_with_autodep/pym/_emerge/resolver/output_helpers.py +++ b/portage_with_autodep/pym/_emerge/resolver/output_helpers.py @@ -1,9 +1,12 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 """Contains private support functions for the Display class in output.py """ + +from __future__ import unicode_literals + __all__ = ( ) @@ -15,9 +18,10 @@ from portage import os from portage import _encodings, _unicode_encode from portage._sets.base import InternalPackageSet from portage.output import (blue, bold, colorize, create_color_func, - green, red, teal, yellow) + green, red, teal, turquoise, yellow) bad = create_color_func("BAD") from portage.util import shlex_split, writemsg +from portage.util.SlotObject import SlotObject from portage.versions import catpkgsplit from _emerge.Blocker import Blocker @@ -245,10 +249,9 @@ def _format_size(mysize): mystr=mystr[:mycount]+","+mystr[mycount:] return mystr+" kB" - def _create_use_string(conf, name, cur_iuse, iuse_forced, cur_use, old_iuse, old_use, - is_new, reinst_flags): + is_new, feature_flags, reinst_flags): if not conf.print_use_string: return "" @@ -266,6 +269,7 @@ def _create_use_string(conf, name, cur_iuse, iuse_forced, cur_use, any_iuse = cur_iuse.union(old_iuse) any_iuse = list(any_iuse) any_iuse.sort() + for flag in any_iuse: flag_str = None isEnabled = False @@ -299,7 +303,9 @@ def _create_use_string(conf, name, cur_iuse, iuse_forced, cur_use, elif flag in old_use: flag_str = green("-" + flag) + "*" if flag_str: - if flag in iuse_forced: + if flag in feature_flags: + flag_str = "{" + flag_str + "}" + elif flag in iuse_forced: flag_str = "(" + flag_str + ")" if isEnabled: enabled.append(flag_str) @@ -611,9 +617,10 @@ class PkgInfo(object): information about the pkg being printed. """ - __slots__ = ("built", "cp", "ebuild_path", "fetch_symbol", "merge", - "oldbest", "oldbest_list", "operation", "ordered", - "repo_name", "repo_path_real", "system", "use", "ver", "world") + __slots__ = ("attr_display", "built", "cp", + "ebuild_path", "fetch_symbol", "merge", + "oldbest", "oldbest_list", "operation", "ordered", "previous_pkg", + "repo_name", "repo_path_real", "slot", "sub_slot", "system", "use", "ver", "world") def __init__(self): @@ -626,9 +633,74 @@ class PkgInfo(object): self.oldbest_list = [] self.operation = '' self.ordered = False + self.previous_pkg = None self.repo_path_real = '' self.repo_name = '' + self.slot = '' + self.sub_slot = '' self.system = False self.use = '' self.ver = '' self.world = False + self.attr_display = PkgAttrDisplay() + +class PkgAttrDisplay(SlotObject): + + __slots__ = ("downgrade", "fetch_restrict", "fetch_restrict_satisfied", + "force_reinstall", + "interactive", "mask", "new", "new_slot", "new_version", "replace") + + def __str__(self): + output = [] + + if self.interactive: + output.append(colorize("WARN", "I")) + else: + output.append(" ") + + if self.new or self.force_reinstall: + if self.force_reinstall: + output.append(red("r")) + else: + output.append(green("N")) + else: + output.append(" ") + + if self.new_slot or self.replace: + if self.replace: + output.append(yellow("R")) + else: + output.append(green("S")) + else: + output.append(" ") + + if self.fetch_restrict or self.fetch_restrict_satisfied: + if self.fetch_restrict_satisfied: + output.append(green("f")) + else: + output.append(red("F")) + else: + output.append(" ") + + if self.new_version: + output.append(turquoise("U")) + else: + output.append(" ") + + if self.downgrade: + output.append(blue("D")) + else: + output.append(" ") + + if self.mask is not None: + output.append(self.mask) + + return "".join(output) + + if sys.hexversion < 0x3000000: + + __unicode__ = __str__ + + def __str__(self): + return _unicode_encode(self.__unicode__(), + encoding=_encodings['content']) diff --git a/portage_with_autodep/pym/_emerge/resolver/output_helpers.pyo b/portage_with_autodep/pym/_emerge/resolver/output_helpers.pyo Binary files differindex ae39dd4..225f4bc 100644 --- a/portage_with_autodep/pym/_emerge/resolver/output_helpers.pyo +++ b/portage_with_autodep/pym/_emerge/resolver/output_helpers.pyo diff --git a/portage_with_autodep/pym/_emerge/resolver/slot_collision.py b/portage_with_autodep/pym/_emerge/resolver/slot_collision.py index a1c8714..a193baa 100644 --- a/portage_with_autodep/pym/_emerge/resolver/slot_collision.py +++ b/portage_with_autodep/pym/_emerge/resolver/slot_collision.py @@ -1,10 +1,11 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import print_function +from __future__ import print_function, unicode_literals import sys +from portage import _encodings, _unicode_encode from _emerge.AtomArg import AtomArg from _emerge.Package import Package from _emerge.PackageArg import PackageArg @@ -150,7 +151,7 @@ class slot_conflict_handler(object): if self.debug: writemsg("\nNew configuration:\n", noiselevel=-1) for pkg in config: - writemsg(" " + str(pkg) + "\n", noiselevel=-1) + writemsg(" %s\n" % (pkg,), noiselevel=-1) writemsg("\n", noiselevel=-1) new_solutions = self._check_configuration(config, all_conflict_atoms_by_slotatom, conflict_nodes) @@ -225,10 +226,14 @@ class slot_conflict_handler(object): new_change = {} for pkg in solution: for flag, state in solution[pkg].items(): + real_flag = pkg.iuse.get_real_flag(flag) + if real_flag is None: + # Triggered by use-dep defaults. + continue if state == "enabled" and flag not in _pkg_use_enabled(pkg): - new_change.setdefault(pkg, {})[flag] = True + new_change.setdefault(pkg, {})[real_flag] = True elif state == "disabled" and flag in _pkg_use_enabled(pkg): - new_change.setdefault(pkg, {})[flag] = False + new_change.setdefault(pkg, {})[real_flag] = False return new_change def _prepare_conflict_msg_and_check_for_specificity(self): @@ -236,6 +241,7 @@ class slot_conflict_handler(object): Print all slot conflicts in a human readable way. """ _pkg_use_enabled = self.depgraph._pkg_use_enabled + verboseconflicts = "--verbose-conflicts" in self.myopts msg = self.conflict_msg indent = " " msg.append("\n!!! Multiple package instances within a single " + \ @@ -245,14 +251,14 @@ class slot_conflict_handler(object): for (slot_atom, root), pkgs \ in self.slot_collision_info.items(): - msg.append(str(slot_atom)) + msg.append("%s" % (slot_atom,)) if root != self.depgraph._frozen_config._running_root.root: msg.append(" for %s" % (root,)) msg.append("\n\n") for pkg in pkgs: msg.append(indent) - msg.append(str(pkg)) + msg.append("%s" % (pkg,)) parent_atoms = self.all_parents.get(pkg) if parent_atoms: #Create a list of collision reasons and map them to sets @@ -275,19 +281,29 @@ class slot_conflict_handler(object): if not atom_without_use_set.findAtomForPackage(other_pkg, \ modified_use=_pkg_use_enabled(other_pkg)): - #The version range does not match. - sub_type = None - if atom.operator in (">=", ">"): - sub_type = "ge" - elif atom.operator in ("=", "~"): - sub_type = "eq" - elif atom.operator in ("<=", "<"): - sub_type = "le" - - atoms = collision_reasons.get(("version", sub_type), set()) - atoms.add((ppkg, atom, other_pkg)) - num_all_specific_atoms += 1 - collision_reasons[("version", sub_type)] = atoms + if atom.operator is not None: + # The version range does not match. + sub_type = None + if atom.operator in (">=", ">"): + sub_type = "ge" + elif atom.operator in ("=", "~"): + sub_type = "eq" + elif atom.operator in ("<=", "<"): + sub_type = "le" + + key = ("version", sub_type) + atoms = collision_reasons.get(key, set()) + atoms.add((ppkg, atom, other_pkg)) + num_all_specific_atoms += 1 + collision_reasons[key] = atoms + else: + # The sub_slot does not match. + key = ("sub-slot", atom.sub_slot) + atoms = collision_reasons.get(key, set()) + atoms.add((ppkg, atom, other_pkg)) + num_all_specific_atoms += 1 + collision_reasons[key] = atoms + elif not atom_set.findAtomForPackage(other_pkg, \ modified_use=_pkg_use_enabled(other_pkg)): missing_iuse = other_pkg.iuse.get_missing_iuse( @@ -302,6 +318,26 @@ class slot_conflict_handler(object): #Use conditionals not met. violated_atom = atom.violated_conditionals(_pkg_use_enabled(other_pkg), \ other_pkg.iuse.is_valid_flag) + if violated_atom.use is None: + # Something like bug #453400 caused the + # above findAtomForPackage call to + # return None unexpectedly. + msg = ("\n\n!!! BUG: Detected " + "USE dep match inconsistency:\n" + "\tppkg: %s\n" + "\tviolated_atom: %s\n" + "\tatom: %s unevaluated: %s\n" + "\tother_pkg: %s IUSE: %s USE: %s\n" % + (ppkg, + violated_atom, + atom, + atom.unevaluated_atom, + other_pkg, + sorted(other_pkg.iuse.all), + sorted(_pkg_use_enabled(other_pkg)))) + writemsg(msg, noiselevel=-2) + raise AssertionError( + 'BUG: USE dep match inconsistency') for flag in violated_atom.use.enabled.union(violated_atom.use.disabled): atoms = collision_reasons.get(("use", flag), set()) atoms.add((ppkg, atom, other_pkg)) @@ -332,7 +368,14 @@ class slot_conflict_handler(object): best_matches[atom.cp] = (ppkg, atom) else: best_matches[atom.cp] = (ppkg, atom) - selected_for_display.update(best_matches.values()) + if verboseconflicts: + selected_for_display.add((ppkg, atom)) + if not verboseconflicts: + selected_for_display.update( + best_matches.values()) + elif type == "sub-slot": + for ppkg, atom, other_pkg in parents: + selected_for_display.add((ppkg, atom)) elif type == "use": #Prefer atoms with unconditional use deps over, because it's #not possible to change them on the parent, which means there @@ -377,7 +420,7 @@ class slot_conflict_handler(object): def highlight_violations(atom, version, use=[]): """Colorize parts of an atom""" - atom_str = str(atom) + atom_str = "%s" % (atom,) if version: op = atom.operator ver = None @@ -432,24 +475,27 @@ class slot_conflict_handler(object): (PackageArg, AtomArg)): # For PackageArg and AtomArg types, it's # redundant to display the atom attribute. - msg.append(str(parent)) + msg.append("%s" % (parent,)) else: # Display the specific atom from SetArg or # Package types. version_violated = False + sub_slot_violated = False use = [] for (type, sub_type), parents in collision_reasons.items(): for x in parents: if parent == x[0] and atom == x[1]: if type == "version": version_violated = True + elif type == "sub-slot": + sub_slot_violated = True elif type == "use": use.append(sub_type) break atom_str = highlight_violations(atom.unevaluated_atom, version_violated, use) - if version_violated: + if version_violated or sub_slot_violated: self.is_a_version_conflict = True msg.append("%s required by %s" % (atom_str, parent)) @@ -547,7 +593,9 @@ class slot_conflict_handler(object): if pkg.iuse.all.symmetric_difference(other_pkg.iuse.all) \ or _pkg_use_enabled(pkg).symmetric_difference(_pkg_use_enabled(other_pkg)): if self.debug: - writemsg(str(pkg) + " has pending USE changes. Rejecting configuration.\n", noiselevel=-1) + writemsg(("%s has pending USE changes. " + "Rejecting configuration.\n") % (pkg,), + noiselevel=-1) return False #A list of dicts. Keeps one dict per slot conflict. [ { flag1: "enabled" }, { flag2: "disabled" } ] @@ -570,16 +618,18 @@ class slot_conflict_handler(object): if not i.findAtomForPackage(pkg, modified_use=_pkg_use_enabled(pkg)): #Version range does not match. if self.debug: - writemsg(str(pkg) + " does not satify all version requirements." + \ - " Rejecting configuration.\n", noiselevel=-1) + writemsg(("%s does not satify all version " + "requirements. Rejecting configuration.\n") % + (pkg,), noiselevel=-1) return False if not pkg.iuse.is_valid_flag(atom.unevaluated_atom.use.required): #Missing IUSE. #FIXME: This needs to support use dep defaults. if self.debug: - writemsg(str(pkg) + " misses needed flags from IUSE." + \ - " Rejecting configuration.\n", noiselevel=-1) + writemsg(("%s misses needed flags from IUSE." + " Rejecting configuration.\n") % (pkg,), + noiselevel=-1) return False if not isinstance(ppkg, Package) or ppkg.installed: @@ -604,8 +654,9 @@ class slot_conflict_handler(object): #We can't change USE of an installed package (only of an ebuild, but that is already #part of the conflict, isn't it? if self.debug: - writemsg(str(pkg) + ": installed package would need USE changes." + \ - " Rejecting configuration.\n", noiselevel=-1) + writemsg(("%s: installed package would need USE" + " changes. Rejecting configuration.\n") % (pkg,), + noiselevel=-1) return False #Compute the required USE changes. A flag can be forced to "enabled" or "disabled", @@ -659,7 +710,7 @@ class slot_conflict_handler(object): if self.debug: writemsg("All involved flags:\n", noiselevel=-1) for id, involved_flags in enumerate(all_involved_flags): - writemsg(" " + str(config[id]) + "\n", noiselevel=-1) + writemsg(" %s\n" % (config[id],), noiselevel=-1) for flag, state in involved_flags.items(): writemsg(" " + flag + ": " + state + "\n", noiselevel=-1) @@ -742,7 +793,7 @@ class slot_conflict_handler(object): inner_first = False else: msg += ", " - msg += flag + ": " + str(state) + msg += flag + ": %s" % (state,) msg += "}" msg += "]\n" writemsg(msg, noiselevel=-1) @@ -846,8 +897,9 @@ class slot_conflict_handler(object): #We managed to create a new problem with our changes. is_valid_solution = False if self.debug: - writemsg("new conflict introduced: " + str(pkg) + \ - " does not match " + new_atom + " from " + str(ppkg) + "\n", noiselevel=-1) + writemsg(("new conflict introduced: %s" + " does not match %s from %s\n") % + (pkg, new_atom, ppkg), noiselevel=-1) break if not is_valid_solution: @@ -855,7 +907,7 @@ class slot_conflict_handler(object): #Make sure the changes don't violate REQUIRED_USE for pkg in required_changes: - required_use = pkg.metadata.get("REQUIRED_USE") + required_use = pkg._metadata.get("REQUIRED_USE") if not required_use: continue @@ -934,8 +986,16 @@ class _solution_candidate_generator(object): else: return self.value == other.value def __str__(self): - return str(self.value) - + return "%s" % (self.value,) + + if sys.hexversion < 0x3000000: + + __unicode__ = __str__ + + def __str__(self): + return _unicode_encode(self.__unicode__(), + encoding=_encodings['content'], errors='backslashreplace') + def __init__(self, all_involved_flags): #A copy of all_involved_flags with all "cond" values #replaced by a _value_helper object. diff --git a/portage_with_autodep/pym/_emerge/resolver/slot_collision.pyo b/portage_with_autodep/pym/_emerge/resolver/slot_collision.pyo Binary files differindex 1fc3a13..2ed98c5 100644 --- a/portage_with_autodep/pym/_emerge/resolver/slot_collision.pyo +++ b/portage_with_autodep/pym/_emerge/resolver/slot_collision.pyo diff --git a/portage_with_autodep/pym/_emerge/search.py b/portage_with_autodep/pym/_emerge/search.py index 5abc8a0..bd74fb7 100644 --- a/portage_with_autodep/pym/_emerge/search.py +++ b/portage_with_autodep/pym/_emerge/search.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -69,7 +69,7 @@ class search(object): return db.aux_get(*args, **kwargs) except KeyError: pass - raise + raise KeyError(args[0]) def _findname(self, *args, **kwargs): for db in self._dbs: diff --git a/portage_with_autodep/pym/_emerge/search.pyo b/portage_with_autodep/pym/_emerge/search.pyo Binary files differindex 055a734..51eef02 100644 --- a/portage_with_autodep/pym/_emerge/search.pyo +++ b/portage_with_autodep/pym/_emerge/search.pyo diff --git a/portage_with_autodep/pym/_emerge/show_invalid_depstring_notice.pyo b/portage_with_autodep/pym/_emerge/show_invalid_depstring_notice.pyo Binary files differindex 337e135..4b3d724 100644 --- a/portage_with_autodep/pym/_emerge/show_invalid_depstring_notice.pyo +++ b/portage_with_autodep/pym/_emerge/show_invalid_depstring_notice.pyo diff --git a/portage_with_autodep/pym/_emerge/stdout_spinner.py b/portage_with_autodep/pym/_emerge/stdout_spinner.py index 5ad31f0..670686a 100644 --- a/portage_with_autodep/pym/_emerge/stdout_spinner.py +++ b/portage_with_autodep/pym/_emerge/stdout_spinner.py @@ -1,4 +1,4 @@ -# Copyright 1999-2009 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import platform @@ -53,17 +53,18 @@ class stdout_spinner(object): def update_basic(self): self.spinpos = (self.spinpos + 1) % 500 if self._return_early(): - return + return True if (self.spinpos % 100) == 0: if self.spinpos == 0: sys.stdout.write(". ") else: sys.stdout.write(".") sys.stdout.flush() + return True def update_scroll(self): if self._return_early(): - return + return True if(self.spinpos >= len(self.scroll_sequence)): sys.stdout.write(darkgreen(" \b\b\b" + self.scroll_sequence[ len(self.scroll_sequence) - 1 - (self.spinpos % len(self.scroll_sequence))])) @@ -71,13 +72,15 @@ class stdout_spinner(object): sys.stdout.write(green("\b " + self.scroll_sequence[self.spinpos])) sys.stdout.flush() self.spinpos = (self.spinpos + 1) % (2 * len(self.scroll_sequence)) + return True def update_twirl(self): self.spinpos = (self.spinpos + 1) % len(self.twirl_sequence) if self._return_early(): - return + return True sys.stdout.write("\b\b " + self.twirl_sequence[self.spinpos]) sys.stdout.flush() + return True def update_quiet(self): - return + return True diff --git a/portage_with_autodep/pym/_emerge/stdout_spinner.pyo b/portage_with_autodep/pym/_emerge/stdout_spinner.pyo Binary files differindex b091171..d7a93b1 100644 --- a/portage_with_autodep/pym/_emerge/stdout_spinner.pyo +++ b/portage_with_autodep/pym/_emerge/stdout_spinner.pyo diff --git a/portage_with_autodep/pym/_emerge/sync/__init__.pyo b/portage_with_autodep/pym/_emerge/sync/__init__.pyo Binary files differindex 7314f80..4163c1a 100644 --- a/portage_with_autodep/pym/_emerge/sync/__init__.pyo +++ b/portage_with_autodep/pym/_emerge/sync/__init__.pyo diff --git a/portage_with_autodep/pym/_emerge/sync/getaddrinfo_validate.pyo b/portage_with_autodep/pym/_emerge/sync/getaddrinfo_validate.pyo Binary files differindex 8e41377..9dba228 100644 --- a/portage_with_autodep/pym/_emerge/sync/getaddrinfo_validate.pyo +++ b/portage_with_autodep/pym/_emerge/sync/getaddrinfo_validate.pyo diff --git a/portage_with_autodep/pym/_emerge/sync/old_tree_timestamp.pyo b/portage_with_autodep/pym/_emerge/sync/old_tree_timestamp.pyo Binary files differindex 5b59c5a..e5d4588 100644 --- a/portage_with_autodep/pym/_emerge/sync/old_tree_timestamp.pyo +++ b/portage_with_autodep/pym/_emerge/sync/old_tree_timestamp.pyo diff --git a/portage_with_autodep/pym/_emerge/unmerge.pyo b/portage_with_autodep/pym/_emerge/unmerge.pyo Binary files differindex c16c9ec..c0cb01b 100644 --- a/portage_with_autodep/pym/_emerge/unmerge.pyo +++ b/portage_with_autodep/pym/_emerge/unmerge.pyo diff --git a/portage_with_autodep/pym/_emerge/userquery.pyo b/portage_with_autodep/pym/_emerge/userquery.pyo Binary files differindex 5492f90..5c2fc9e 100644 --- a/portage_with_autodep/pym/_emerge/userquery.pyo +++ b/portage_with_autodep/pym/_emerge/userquery.pyo |