diff options
Diffstat (limited to 'portage_with_autodep/pym/portage/dbapi/_MergeProcess.py')
| -rw-r--r-- | portage_with_autodep/pym/portage/dbapi/_MergeProcess.py | 214 |
1 files changed, 124 insertions, 90 deletions
diff --git a/portage_with_autodep/pym/portage/dbapi/_MergeProcess.py b/portage_with_autodep/pym/portage/dbapi/_MergeProcess.py index b5f6a0b..956dbb9 100644 --- a/portage_with_autodep/pym/portage/dbapi/_MergeProcess.py +++ b/portage_with_autodep/pym/portage/dbapi/_MergeProcess.py @@ -1,7 +1,8 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import io +import platform import signal import sys import traceback @@ -10,10 +11,11 @@ import errno import fcntl import portage from portage import os, _unicode_decode +from portage.util._ctypes import find_library import portage.elog.messages -from _emerge.SpawnProcess import SpawnProcess +from portage.util._async.ForkProcess import ForkProcess -class MergeProcess(SpawnProcess): +class MergeProcess(ForkProcess): """ Merge packages in a subprocess, so the Scheduler can run in the main thread while files are moved or copied asynchronously. @@ -40,11 +42,20 @@ class MergeProcess(SpawnProcess): settings.reset() settings.setcpv(cpv, mydb=self.mydbapi) + # This caches the libc library lookup in the current + # process, so that it's only done once rather than + # for each child process. + if platform.system() == "Linux" and \ + "merge-sync" in settings.features: + find_library("c") + # Inherit stdin by default, so that the pdb SIGUSR1 # handler is usable for the subprocess. if self.fd_pipes is None: self.fd_pipes = {} - self.fd_pipes.setdefault(0, sys.stdin.fileno()) + else: + self.fd_pipes = self.fd_pipes.copy() + self.fd_pipes.setdefault(0, portage._get_stdin().fileno()) super(MergeProcess, self)._start() @@ -90,7 +101,7 @@ class MergeProcess(SpawnProcess): reporter(msg, phase=phase, key=key, out=out) if event & self.scheduler.IO_HUP: - self.scheduler.unregister(self._elog_reg_id) + self.scheduler.source_remove(self._elog_reg_id) self._elog_reg_id = None os.close(self._elog_reader_fd) self._elog_reader_fd = None @@ -101,12 +112,24 @@ class MergeProcess(SpawnProcess): def _spawn(self, args, fd_pipes, **kwargs): """ Fork a subprocess, apply local settings, and call - dblink.merge(). + dblink.merge(). TODO: Share code with ForkProcess. """ elog_reader_fd, elog_writer_fd = os.pipe() + fcntl.fcntl(elog_reader_fd, fcntl.F_SETFL, fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK) + + # FD_CLOEXEC is enabled by default in Python >=3.4. + if sys.hexversion < 0x3040000: + try: + fcntl.FD_CLOEXEC + except AttributeError: + pass + else: + fcntl.fcntl(elog_reader_fd, fcntl.F_SETFD, + fcntl.fcntl(elog_reader_fd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC) + blockers = None if self.blockers is not None: # Query blockers in the main process, since closing @@ -116,10 +139,9 @@ class MergeProcess(SpawnProcess): blockers = self.blockers() mylink = portage.dblink(self.mycat, self.mypkg, settings=self.settings, treetype=self.treetype, vartree=self.vartree, - blockers=blockers, scheduler=self.scheduler, - pipe=elog_writer_fd) + blockers=blockers, pipe=elog_writer_fd) fd_pipes[elog_writer_fd] = elog_writer_fd - self._elog_reg_id = self.scheduler.register(elog_reader_fd, + self._elog_reg_id = self.scheduler.io_add_watch(elog_reader_fd, self._registered_events, self._elog_output_handler) # If a concurrent emerge process tries to install a package @@ -133,88 +155,100 @@ class MergeProcess(SpawnProcess): if not self.unmerge: counter = self.vartree.dbapi.counter_tick() - pid = os.fork() - if pid != 0: - if not isinstance(pid, int): - raise AssertionError( - "fork returned non-integer: %s" % (repr(pid),)) - - os.close(elog_writer_fd) - self._elog_reader_fd = elog_reader_fd - self._buf = "" - self._elog_keys = set() - - # invalidate relevant vardbapi caches - if self.vartree.dbapi._categories is not None: - self.vartree.dbapi._categories = None - self.vartree.dbapi._pkgs_changed = True - self.vartree.dbapi._clear_pkg_cache(mylink) - - portage.process.spawned_pids.append(pid) - return [pid] - - os.close(elog_reader_fd) - portage.locks._close_fds() - # Disable close_fds since we don't exec (see _setup_pipes docstring). - portage.process._setup_pipes(fd_pipes, close_fds=False) - - # Use default signal handlers since the ones inherited - # from the parent process are irrelevant here. - signal.signal(signal.SIGINT, signal.SIG_DFL) - signal.signal(signal.SIGTERM, signal.SIG_DFL) - - portage.output.havecolor = self.settings.get('NOCOLOR') \ - not in ('yes', 'true') - - # In this subprocess we want mylink._display_merge() to use - # stdout/stderr directly since they are pipes. This behavior - # is triggered when mylink._scheduler is None. - mylink._scheduler = None - - # Avoid wastful updates of the vdb cache. - self.vartree.dbapi._flush_cache_enabled = False - - # In this subprocess we don't want PORTAGE_BACKGROUND to - # suppress stdout/stderr output since they are pipes. We - # also don't want to open PORTAGE_LOG_FILE, since it will - # already be opened by the parent process, so we set the - # "subprocess" value for use in conditional logging code - # involving PORTAGE_LOG_FILE. - if not self.unmerge: - # unmerge phases have separate logs - if self.settings.get("PORTAGE_BACKGROUND") == "1": - self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "1" - else: - self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "0" - self.settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE") - self.settings["PORTAGE_BACKGROUND"] = "subprocess" - self.settings.backup_changes("PORTAGE_BACKGROUND") - - rval = 1 + parent_pid = os.getpid() + pid = None try: - if self.unmerge: - if not mylink.exists(): - rval = os.EX_OK - elif mylink.unmerge( - ldpath_mtimes=self.prev_mtimes) == os.EX_OK: - mylink.lockdb() - try: - mylink.delete() - finally: - mylink.unlockdb() - rval = os.EX_OK - else: - rval = mylink.merge(self.pkgloc, self.infloc, - myebuild=self.myebuild, mydbapi=self.mydbapi, - prev_mtimes=self.prev_mtimes, counter=counter) - except SystemExit: - raise - except: - traceback.print_exc() + pid = os.fork() + + if pid != 0: + if not isinstance(pid, int): + raise AssertionError( + "fork returned non-integer: %s" % (repr(pid),)) + + os.close(elog_writer_fd) + self._elog_reader_fd = elog_reader_fd + self._buf = "" + self._elog_keys = set() + # Discard messages which will be collected by the subprocess, + # in order to avoid duplicates (bug #446136). + portage.elog.messages.collect_messages(key=mylink.mycpv) + + # invalidate relevant vardbapi caches + if self.vartree.dbapi._categories is not None: + self.vartree.dbapi._categories = None + self.vartree.dbapi._pkgs_changed = True + self.vartree.dbapi._clear_pkg_cache(mylink) + + return [pid] + + os.close(elog_reader_fd) + + # Use default signal handlers in order to avoid problems + # killing subprocesses as reported in bug #353239. + signal.signal(signal.SIGINT, signal.SIG_DFL) + signal.signal(signal.SIGTERM, signal.SIG_DFL) + + portage.locks._close_fds() + # We don't exec, so use close_fds=False + # (see _setup_pipes docstring). + portage.process._setup_pipes(fd_pipes, close_fds=False) + + portage.output.havecolor = self.settings.get('NOCOLOR') \ + not in ('yes', 'true') + + # Avoid wastful updates of the vdb cache. + self.vartree.dbapi._flush_cache_enabled = False + + # In this subprocess we don't want PORTAGE_BACKGROUND to + # suppress stdout/stderr output since they are pipes. We + # also don't want to open PORTAGE_LOG_FILE, since it will + # already be opened by the parent process, so we set the + # "subprocess" value for use in conditional logging code + # involving PORTAGE_LOG_FILE. + if not self.unmerge: + # unmerge phases have separate logs + if self.settings.get("PORTAGE_BACKGROUND") == "1": + self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "1" + else: + self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "0" + self.settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE") + self.settings["PORTAGE_BACKGROUND"] = "subprocess" + self.settings.backup_changes("PORTAGE_BACKGROUND") + + rval = 1 + try: + if self.unmerge: + if not mylink.exists(): + rval = os.EX_OK + elif mylink.unmerge( + ldpath_mtimes=self.prev_mtimes) == os.EX_OK: + mylink.lockdb() + try: + mylink.delete() + finally: + mylink.unlockdb() + rval = os.EX_OK + else: + rval = mylink.merge(self.pkgloc, self.infloc, + myebuild=self.myebuild, mydbapi=self.mydbapi, + prev_mtimes=self.prev_mtimes, counter=counter) + except SystemExit: + raise + except: + traceback.print_exc() + # os._exit() skips stderr flush! + sys.stderr.flush() + finally: + os._exit(rval) + finally: - # Call os._exit() from finally block, in order to suppress any - # finally blocks from earlier in the call stack. See bug #345289. - os._exit(rval) + if pid == 0 or (pid is None and os.getpid() != parent_pid): + # Call os._exit() from a finally block in order + # to suppress any finally blocks from earlier + # in the call stack (see bug #345289). This + # finally block has to be setup before the fork + # in order to avoid a race condition. + os._exit(1) def _unregister(self): """ @@ -231,7 +265,7 @@ class MergeProcess(SpawnProcess): self._unlock_vdb() if self._elog_reg_id is not None: - self.scheduler.unregister(self._elog_reg_id) + self.scheduler.source_remove(self._elog_reg_id) self._elog_reg_id = None if self._elog_reader_fd is not None: os.close(self._elog_reader_fd) |