diff options
| -rw-r--r-- | pym/_emerge/AbstractEbuildProcess.py | 3 | ||||
| -rw-r--r-- | pym/_emerge/EbuildFetcher.py | 3 | ||||
| -rw-r--r-- | pym/_emerge/EbuildPhase.py | 13 | ||||
| -rw-r--r-- | pym/_emerge/EbuildProcess.py | 10 | ||||
| -rw-r--r-- | pym/_emerge/MiscFunctionsProcess.py | 8 | ||||
| -rw-r--r-- | pym/portage/__init__.py | 2199 | ||||
| -rw-r--r-- | pym/portage/dbapi/bintree.py | 3 | ||||
| -rw-r--r-- | pym/portage/dbapi/vartree.py | 5 | ||||
| -rw-r--r-- | pym/portage/package/ebuild/_pty.py | 192 | ||||
| -rw-r--r-- | pym/portage/package/ebuild/doebuild.py | 1759 | ||||
| -rw-r--r-- | pym/portage/package/ebuild/fetch.py | 9 | ||||
| -rw-r--r-- | pym/portage/package/ebuild/prepare_build_dirs.py | 304 | ||||
| -rw-r--r-- | pym/portage/tests/ebuild/test_pty_eof.py | 5 |
13 files changed, 2296 insertions, 2217 deletions
diff --git a/pym/_emerge/AbstractEbuildProcess.py b/pym/_emerge/AbstractEbuildProcess.py index db599019..0d8be6fb 100644 --- a/pym/_emerge/AbstractEbuildProcess.py +++ b/pym/_emerge/AbstractEbuildProcess.py @@ -5,6 +5,7 @@ from _emerge.SpawnProcess import SpawnProcess import portage from portage import os +from portage.package.ebuild._pty import _create_pty_or_pipe class AbstractEbuildProcess(SpawnProcess): @@ -13,7 +14,7 @@ class AbstractEbuildProcess(SpawnProcess): def _pipe(self, fd_pipes): stdout_pipe = fd_pipes.get(1) got_pty, master_fd, slave_fd = \ - portage._create_pty_or_pipe(copy_term_size=stdout_pipe) + _create_pty_or_pipe(copy_term_size=stdout_pipe) return (master_fd, slave_fd) def _can_log(self, slave_fd): diff --git a/pym/_emerge/EbuildFetcher.py b/pym/_emerge/EbuildFetcher.py index 58e4640f..eb423ff0 100644 --- a/pym/_emerge/EbuildFetcher.py +++ b/pym/_emerge/EbuildFetcher.py @@ -11,6 +11,7 @@ from portage import _encodings from portage import _unicode_encode import codecs from portage.elog.messages import eerror +from portage.package.ebuild._pty import _create_pty_or_pipe class EbuildFetcher(SpawnProcess): @@ -142,7 +143,7 @@ class EbuildFetcher(SpawnProcess): return os.pipe() stdout_pipe = fd_pipes.get(1) got_pty, master_fd, slave_fd = \ - portage._create_pty_or_pipe(copy_term_size=stdout_pipe) + _create_pty_or_pipe(copy_term_size=stdout_pipe) return (master_fd, slave_fd) def _set_returncode(self, wait_retval): diff --git a/pym/_emerge/EbuildPhase.py b/pym/_emerge/EbuildPhase.py index 89bdb078..81b0e1f8 100644 --- a/pym/_emerge/EbuildPhase.py +++ b/pym/_emerge/EbuildPhase.py @@ -5,6 +5,9 @@ from _emerge.MiscFunctionsProcess import MiscFunctionsProcess from _emerge.EbuildProcess import EbuildProcess from _emerge.CompositeTask import CompositeTask +from portage.package.ebuild.doebuild import _check_build_log, \ + _post_phase_cmds, _post_src_install_chost_fix, \ + _post_src_install_uid_fix from portage.util import writemsg, writemsg_stdout import portage from portage import os @@ -18,8 +21,6 @@ class EbuildPhase(CompositeTask): __slots__ = ("background", "pkg", "phase", "scheduler", "settings", "tree") - _post_phase_cmds = portage._post_phase_cmds - def _start(self): ebuild_process = EbuildProcess(background=self.background, @@ -39,7 +40,7 @@ class EbuildPhase(CompositeTask): encoding=_encodings['fs'], errors='strict'), mode='a', encoding=_encodings['content'], errors='replace') try: - portage._check_build_log(self.settings, out=out) + _check_build_log(self.settings, out=out) msg = _unicode_decode(out.getvalue(), encoding=_encodings['content'], errors='replace') if msg: @@ -66,12 +67,12 @@ class EbuildPhase(CompositeTask): encoding=_encodings['fs'], errors='strict'), mode='a', encoding=_encodings['content'], errors='replace') out = log_file - portage._post_src_install_chost_fix(settings) - portage._post_src_install_uid_fix(settings, out=out) + _post_src_install_chost_fix(settings) + _post_src_install_uid_fix(settings, out=out) if log_file is not None: log_file.close() - post_phase_cmds = self._post_phase_cmds.get(self.phase) + post_phase_cmds = _post_phase_cmds.get(self.phase) if post_phase_cmds is not None: post_phase = MiscFunctionsProcess(background=self.background, commands=post_phase_cmds, phase=self.phase, pkg=self.pkg, diff --git a/pym/_emerge/EbuildProcess.py b/pym/_emerge/EbuildProcess.py index 597caf4d..7cd164ee 100644 --- a/pym/_emerge/EbuildProcess.py +++ b/pym/_emerge/EbuildProcess.py @@ -3,8 +3,9 @@ # $Id$ from _emerge.AbstractEbuildProcess import AbstractEbuildProcess -import portage from portage import os +from portage.package.ebuild.doebuild import doebuild, \ + _doebuild_exit_status_check_and_log, _post_phase_userpriv_perms class EbuildProcess(AbstractEbuildProcess): @@ -26,8 +27,9 @@ class EbuildProcess(AbstractEbuildProcess): settings = self.settings ebuild_path = settings["EBUILD"] debug = settings.get("PORTAGE_DEBUG") == "1" + - rval = portage.doebuild(ebuild_path, self.phase, + rval = doebuild(ebuild_path, self.phase, root_config.root, settings, debug, mydbapi=mydbapi, tree=tree, **kwargs) @@ -37,12 +39,12 @@ class EbuildProcess(AbstractEbuildProcess): AbstractEbuildProcess._set_returncode(self, wait_retval) if self.phase not in ("clean", "cleanrm"): - self.returncode = portage._doebuild_exit_status_check_and_log( + self.returncode = _doebuild_exit_status_check_and_log( self.settings, self.phase, self.returncode) if self.phase == "test" and self.returncode != os.EX_OK and \ "test-fail-continue" in self.settings.features: self.returncode = os.EX_OK - portage._post_phase_userpriv_perms(self.settings) + _post_phase_userpriv_perms(self.settings) diff --git a/pym/_emerge/MiscFunctionsProcess.py b/pym/_emerge/MiscFunctionsProcess.py index 63d7873e..75c68cf1 100644 --- a/pym/_emerge/MiscFunctionsProcess.py +++ b/pym/_emerge/MiscFunctionsProcess.py @@ -5,6 +5,8 @@ from _emerge.AbstractEbuildProcess import AbstractEbuildProcess import portage from portage import os +from portage.package.ebuild.doebuild import spawn, \ + _doebuild_exit_status_check_and_log, _doebuild_exit_status_unlink class MiscFunctionsProcess(AbstractEbuildProcess): """ @@ -23,7 +25,7 @@ class MiscFunctionsProcess(AbstractEbuildProcess): self.args = [portage._shell_quote(misc_sh_binary)] + self.commands self.logfile = settings.get("PORTAGE_LOG_FILE") - portage._doebuild_exit_status_unlink( + _doebuild_exit_status_unlink( settings.get("EBUILD_EXIT_STATUS_FILE")) AbstractEbuildProcess._start(self) @@ -31,11 +33,11 @@ class MiscFunctionsProcess(AbstractEbuildProcess): def _spawn(self, args, **kwargs): settings = self.settings debug = settings.get("PORTAGE_DEBUG") == "1" - return portage.spawn(" ".join(args), settings, + return spawn(" ".join(args), settings, debug=debug, **kwargs) def _set_returncode(self, wait_retval): AbstractEbuildProcess._set_returncode(self, wait_retval) - self.returncode = portage._doebuild_exit_status_check_and_log( + self.returncode = _doebuild_exit_status_check_and_log( self.settings, self.phase, self.returncode) diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 39196cc9..cf2cf260 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -102,9 +102,12 @@ try: 'portage.manifest:Manifest', 'portage.output', 'portage.output:bold,colorize', + 'portage.package.ebuild.doebuild:doebuild,' + \ + 'doebuild_environment,spawn,spawnebuild', 'portage.package.ebuild.config:autouse,best_from_dict,' + \ 'check_config_instance,config', 'portage.package.ebuild.fetch:fetch', + 'portage.package.ebuild.prepare_build_dirs:prepare_build_dirs', 'portage.process', 'portage.process:atexit_register,run_exitfuncs', 'portage.update:dep_transform,fixdbentries,grab_updates,' + \ @@ -997,385 +1000,6 @@ def ExtractKernelVersion(base_dir): return (version,None) -def _can_test_pty_eof(): - """ - The _test_pty_eof() function seems to hang on most - kernels other than Linux. - This was reported for the following kernels which used to work fine - without this EOF test: Darwin, AIX, FreeBSD. They seem to hang on - the slave_file.close() call. Note that Python's implementation of - openpty on Solaris already caused random hangs without this EOF test - and hence is globally disabled. - @rtype: bool - @returns: True if _test_pty_eof() won't hang, False otherwise. - """ - return platform.system() in ("Linux",) - -def _test_pty_eof(): - """ - Returns True if this issues is fixed for the currently - running version of python: http://bugs.python.org/issue5380 - Raises an EnvironmentError from openpty() if it fails. - """ - - use_fork = False - - import array, fcntl, pty, select, termios - test_string = 2 * "blah blah blah\n" - test_string = _unicode_decode(test_string, - encoding='utf_8', errors='strict') - - # may raise EnvironmentError - master_fd, slave_fd = pty.openpty() - - # Non-blocking mode is required for Darwin kernel. - fcntl.fcntl(master_fd, fcntl.F_SETFL, - fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK) - - # Disable post-processing of output since otherwise weird - # things like \n -> \r\n transformations may occur. - mode = termios.tcgetattr(slave_fd) - mode[1] &= ~termios.OPOST - termios.tcsetattr(slave_fd, termios.TCSANOW, mode) - - # Simulate a subprocess writing some data to the - # slave end of the pipe, and then exiting. - pid = None - if use_fork: - pids = process.spawn_bash(_unicode_encode("echo -n '%s'" % test_string, - encoding='utf_8', errors='strict'), env=os.environ, - fd_pipes={0:sys.stdin.fileno(), 1:slave_fd, 2:slave_fd}, - returnpid=True) - if isinstance(pids, int): - os.close(master_fd) - os.close(slave_fd) - raise EnvironmentError('spawn failed') - pid = pids[0] - else: - os.write(slave_fd, _unicode_encode(test_string, - encoding='utf_8', errors='strict')) - os.close(slave_fd) - - # If using a fork, we must wait for the child here, - # in order to avoid a race condition that would - # lead to inconsistent results. - if pid is not None: - os.waitpid(pid, 0) - - master_file = os.fdopen(master_fd, 'rb') - eof = False - data = [] - iwtd = [master_file] - owtd = [] - ewtd = [] - - while not eof: - - events = select.select(iwtd, owtd, ewtd) - if not events[0]: - eof = True - break - - buf = array.array('B') - try: - buf.fromfile(master_file, 1024) - except EOFError: - eof = True - except IOError: - # This is where data loss occurs. - eof = True - - if not buf: - eof = True - else: - data.append(_unicode_decode(buf.tostring(), - encoding='utf_8', errors='strict')) - - master_file.close() - - return test_string == ''.join(data) - -# If _test_pty_eof() can't be used for runtime detection of -# http://bugs.python.org/issue5380, openpty can't safely be used -# unless we can guarantee that the current version of python has -# been fixed (affects all current versions of python3). When -# this issue is fixed in python3, we can add another sys.hexversion -# conditional to enable openpty support in the fixed versions. -if sys.hexversion >= 0x3000000 and not _can_test_pty_eof(): - _disable_openpty = True -else: - # Disable the use of openpty on Solaris as it seems Python's openpty - # implementation doesn't play nice on Solaris with Portage's - # behaviour causing hangs/deadlocks. - # Additional note for the future: on Interix, pipes do NOT work, so - # _disable_openpty on Interix must *never* be True - _disable_openpty = platform.system() in ("SunOS",) -_tested_pty = False - -if not _can_test_pty_eof(): - # Skip _test_pty_eof() on systems where it hangs. - _tested_pty = True - -def _create_pty_or_pipe(copy_term_size=None): - """ - Try to create a pty and if then fails then create a normal - pipe instead. - - @param copy_term_size: If a tty file descriptor is given - then the term size will be copied to the pty. - @type copy_term_size: int - @rtype: tuple - @returns: A tuple of (is_pty, master_fd, slave_fd) where - is_pty is True if a pty was successfully allocated, and - False if a normal pipe was allocated. - """ - - got_pty = False - - global _disable_openpty, _tested_pty - if not (_tested_pty or _disable_openpty): - try: - if not _test_pty_eof(): - _disable_openpty = True - except EnvironmentError as e: - _disable_openpty = True - writemsg("openpty failed: '%s'\n" % str(e), - noiselevel=-1) - del e - _tested_pty = True - - if _disable_openpty: - master_fd, slave_fd = os.pipe() - else: - from pty import openpty - try: - master_fd, slave_fd = openpty() - got_pty = True - except EnvironmentError as e: - _disable_openpty = True - writemsg("openpty failed: '%s'\n" % str(e), - noiselevel=-1) - del e - master_fd, slave_fd = os.pipe() - - if got_pty: - # Disable post-processing of output since otherwise weird - # things like \n -> \r\n transformations may occur. - import termios - mode = termios.tcgetattr(slave_fd) - mode[1] &= ~termios.OPOST - termios.tcsetattr(slave_fd, termios.TCSANOW, mode) - - if got_pty and \ - copy_term_size is not None and \ - os.isatty(copy_term_size): - from portage.output import get_term_size, set_term_size - rows, columns = get_term_size() - set_term_size(rows, columns, slave_fd) - - return (got_pty, master_fd, slave_fd) - -# XXX This would be to replace getstatusoutput completely. -# XXX Issue: cannot block execution. Deadlock condition. -def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakeroot=0, **keywords): - """ - Spawn a subprocess with extra portage-specific options. - Optiosn include: - - Sandbox: Sandbox means the spawned process will be limited in its ability t - read and write files (normally this means it is restricted to ${D}/) - SElinux Sandbox: Enables sandboxing on SElinux - Reduced Privileges: Drops privilages such that the process runs as portage:portage - instead of as root. - - Notes: os.system cannot be used because it messes with signal handling. Instead we - use the portage.process spawn* family of functions. - - This function waits for the process to terminate. - - @param mystring: Command to run - @type mystring: String - @param mysettings: Either a Dict of Key,Value pairs or an instance of portage.config - @type mysettings: Dictionary or config instance - @param debug: Ignored - @type debug: Boolean - @param free: Enable sandboxing for this process - @type free: Boolean - @param droppriv: Drop to portage:portage when running this command - @type droppriv: Boolean - @param sesandbox: Enable SELinux Sandboxing (toggles a context switch) - @type sesandbox: Boolean - @param fakeroot: Run this command with faked root privileges - @type fakeroot: Boolean - @param keywords: Extra options encoded as a dict, to be passed to spawn - @type keywords: Dictionary - @rtype: Integer - @returns: - 1. The return code of the spawned process. - """ - - if isinstance(mysettings, dict): - env=mysettings - keywords["opt_name"]="[ %s ]" % "portage" - else: - check_config_instance(mysettings) - env=mysettings.environ() - if mysettings.mycpv is not None: - keywords["opt_name"] = "[%s]" % mysettings.mycpv - else: - keywords["opt_name"] = "[%s/%s]" % \ - (mysettings.get("CATEGORY",""), mysettings.get("PF","")) - - fd_pipes = keywords.get("fd_pipes") - if fd_pipes is None: - fd_pipes = { - 0:sys.stdin.fileno(), - 1:sys.stdout.fileno(), - 2:sys.stderr.fileno(), - } - # In some cases the above print statements don't flush stdout, so - # it needs to be flushed before allowing a child process to use it - # so that output always shows in the correct order. - stdout_filenos = (sys.stdout.fileno(), sys.stderr.fileno()) - for fd in fd_pipes.values(): - if fd in stdout_filenos: - sys.stdout.flush() - sys.stderr.flush() - break - - # The default policy for the sesandbox domain only allows entry (via exec) - # from shells and from binaries that belong to portage (the number of entry - # points is minimized). The "tee" binary is not among the allowed entry - # points, so it is spawned outside of the sesandbox domain and reads from a - # pseudo-terminal that connects two domains. - logfile = keywords.get("logfile") - mypids = [] - master_fd = None - slave_fd = None - fd_pipes_orig = None - got_pty = False - if logfile: - del keywords["logfile"] - if 1 not in fd_pipes or 2 not in fd_pipes: - raise ValueError(fd_pipes) - - got_pty, master_fd, slave_fd = \ - _create_pty_or_pipe(copy_term_size=fd_pipes[1]) - - if not got_pty and 'sesandbox' in mysettings.features \ - and mysettings.selinux_enabled(): - # With sesandbox, logging works through a pty but not through a - # normal pipe. So, disable logging if ptys are broken. - # See Bug #162404. - logfile = None - os.close(master_fd) - master_fd = None - os.close(slave_fd) - slave_fd = None - - if logfile: - - fd_pipes.setdefault(0, sys.stdin.fileno()) - fd_pipes_orig = fd_pipes.copy() - - # We must set non-blocking mode before we close the slave_fd - # since otherwise the fcntl call can fail on FreeBSD (the child - # process might have already exited and closed slave_fd so we - # have to keep it open in order to avoid FreeBSD potentially - # generating an EAGAIN exception). - import fcntl - fcntl.fcntl(master_fd, fcntl.F_SETFL, - fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK) - - fd_pipes[0] = fd_pipes_orig[0] - fd_pipes[1] = slave_fd - fd_pipes[2] = slave_fd - keywords["fd_pipes"] = fd_pipes - - features = mysettings.features - # TODO: Enable fakeroot to be used together with droppriv. The - # fake ownership/permissions will have to be converted to real - # permissions in the merge phase. - fakeroot = fakeroot and uid != 0 and portage.process.fakeroot_capable - if droppriv and not uid and portage_gid and portage_uid: - keywords.update({"uid":portage_uid,"gid":portage_gid, - "groups":userpriv_groups,"umask":0o02}) - if not free: - free=((droppriv and "usersandbox" not in features) or \ - (not droppriv and "sandbox" not in features and \ - "usersandbox" not in features and not fakeroot)) - - if not free and not (fakeroot or process.sandbox_capable): - free = True - - if free or "SANDBOX_ACTIVE" in os.environ: - keywords["opt_name"] += " bash" - spawn_func = portage.process.spawn_bash - elif fakeroot: - keywords["opt_name"] += " fakeroot" - keywords["fakeroot_state"] = os.path.join(mysettings["T"], "fakeroot.state") - spawn_func = portage.process.spawn_fakeroot - else: - keywords["opt_name"] += " sandbox" - spawn_func = portage.process.spawn_sandbox - - if sesandbox: - spawn_func = selinux.spawn_wrapper(spawn_func, - mysettings["PORTAGE_SANDBOX_T"]) - - returnpid = keywords.get("returnpid") - keywords["returnpid"] = True - try: - mypids.extend(spawn_func(mystring, env=env, **keywords)) - finally: - if logfile: - os.close(slave_fd) - - if returnpid: - return mypids - - if logfile: - log_file = open(_unicode_encode(logfile), mode='ab') - apply_secpass_permissions(logfile, - uid=portage_uid, gid=portage_gid, mode=0o664) - stdout_file = os.fdopen(os.dup(fd_pipes_orig[1]), 'wb') - master_file = os.fdopen(master_fd, 'rb') - iwtd = [master_file] - owtd = [] - ewtd = [] - import array, select - buffsize = 65536 - eof = False - while not eof: - events = select.select(iwtd, owtd, ewtd) - for f in events[0]: - # Use non-blocking mode to prevent read - # calls from blocking indefinitely. - buf = array.array('B') - try: - buf.fromfile(f, buffsize) - except EOFError: - pass - if not buf: - eof = True - break - if f is master_file: - buf.tofile(stdout_file) - stdout_file.flush() - buf.tofile(log_file) - log_file.flush() - log_file.close() - stdout_file.close() - master_file.close() - pid = mypids[-1] - retval = os.waitpid(pid, 0)[1] - portage.process.spawned_pids.remove(pid) - if retval != os.EX_OK: - if retval & 0xff: - return (retval & 0xff) << 8 - return retval >> 8 - return retval - def digestgen(myarchives=None, mysettings=None, overwrite=None, manifestonly=None, myportdb=None): """ @@ -1719,498 +1343,6 @@ def digestcheck(myfiles, mysettings, strict=0, justmanifest=0): return 0 return 1 -# parse actionmap to spawn ebuild with the appropriate args -def spawnebuild(mydo, actionmap, mysettings, debug, alwaysdep=0, - logfile=None, fd_pipes=None, returnpid=False): - if not returnpid and \ - (alwaysdep or "noauto" not in mysettings.features): - # process dependency first - if "dep" in actionmap[mydo]: - retval = spawnebuild(actionmap[mydo]["dep"], actionmap, - mysettings, debug, alwaysdep=alwaysdep, logfile=logfile, - fd_pipes=fd_pipes, returnpid=returnpid) - if retval: - return retval - - eapi = mysettings["EAPI"] - - if mydo == "configure" and eapi in ("0", "1"): - return os.EX_OK - - if mydo == "prepare" and eapi in ("0", "1"): - return os.EX_OK - - if mydo == "pretend" and eapi in ("0", "1", "2", "3", "3_pre2"): - return os.EX_OK - - kwargs = actionmap[mydo]["args"] - mysettings["EBUILD_PHASE"] = mydo - _doebuild_exit_status_unlink( - mysettings.get("EBUILD_EXIT_STATUS_FILE")) - - try: - phase_retval = spawn(actionmap[mydo]["cmd"] % mydo, - mysettings, debug=debug, logfile=logfile, - fd_pipes=fd_pipes, returnpid=returnpid, **kwargs) - finally: - mysettings["EBUILD_PHASE"] = "" - - if returnpid: - return phase_retval - - msg = _doebuild_exit_status_check(mydo, mysettings) - if msg: - if phase_retval == os.EX_OK: - phase_retval = 1 - from textwrap import wrap - from portage.elog.messages import eerror - for l in wrap(msg, 72): - eerror(l, phase=mydo, key=mysettings.mycpv) - - _post_phase_userpriv_perms(mysettings) - if mydo == "install": - out = StringIO() - _check_build_log(mysettings, out=out) - msg = _unicode_decode(out.getvalue(), - encoding=_encodings['content'], errors='replace') - if msg: - writemsg_stdout(msg, noiselevel=-1) - if logfile is not None: - try: - f = codecs.open(_unicode_encode(logfile, - encoding=_encodings['fs'], errors='strict'), - mode='a', encoding=_encodings['content'], - errors='replace') - except EnvironmentError: - pass - else: - f.write(msg) - f.close() - if phase_retval == os.EX_OK: - _post_src_install_chost_fix(mysettings) - phase_retval = _post_src_install_checks(mysettings) - - if mydo == "test" and phase_retval != os.EX_OK and \ - "test-fail-continue" in mysettings.features: - phase_retval = os.EX_OK - - return phase_retval - -_post_phase_cmds = { - - "install" : [ - "install_qa_check", - "install_symlink_html_docs"], - - "preinst" : [ - "preinst_bsdflags", - "preinst_sfperms", - "preinst_selinux_labels", - "preinst_suid_scan", - "preinst_mask"], - - "postinst" : [ - "postinst_bsdflags"] -} - -def _post_phase_userpriv_perms(mysettings): - if "userpriv" in mysettings.features and secpass >= 2: - """ Privileged phases may have left files that need to be made - writable to a less privileged user.""" - apply_recursive_permissions(mysettings["T"], - uid=portage_uid, gid=portage_gid, dirmode=0o70, dirmask=0, - filemode=0o60, filemask=0) - -def _post_src_install_checks(mysettings): - _post_src_install_uid_fix(mysettings) - global _post_phase_cmds - retval = _spawn_misc_sh(mysettings, _post_phase_cmds["install"], - phase='internal_post_src_install') - if retval != os.EX_OK: - writemsg(_("!!! install_qa_check failed; exiting.\n"), - noiselevel=-1) - return retval - -def _check_build_log(mysettings, out=None): - """ - Search the content of $PORTAGE_LOG_FILE if it exists - and generate the following QA Notices when appropriate: - - * Automake "maintainer mode" - * command not found - * Unrecognized configure options - """ - logfile = mysettings.get("PORTAGE_LOG_FILE") - if logfile is None: - return - try: - f = codecs.open(_unicode_encode(logfile, - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['content'], errors='replace') - except EnvironmentError: - return - - am_maintainer_mode = [] - bash_command_not_found = [] - bash_command_not_found_re = re.compile( - r'(.*): line (\d*): (.*): command not found$') - command_not_found_exclude_re = re.compile(r'/configure: line ') - helper_missing_file = [] - helper_missing_file_re = re.compile( - r'^!!! (do|new).*: .* does not exist$') - - configure_opts_warn = [] - configure_opts_warn_re = re.compile( - r'^configure: WARNING: [Uu]nrecognized options: ') - - # Exclude output from dev-libs/yaz-3.0.47 which looks like this: - # - #Configuration: - # Automake: ${SHELL} /var/tmp/portage/dev-libs/yaz-3.0.47/work/yaz-3.0.47/config/missing --run automake-1.10 - am_maintainer_mode_re = re.compile(r'/missing --run ') - am_maintainer_mode_exclude_re = \ - re.compile(r'(/missing --run (autoheader|makeinfo)|^\s*Automake:\s)') - - make_jobserver_re = \ - re.compile(r'g?make\[\d+\]: warning: jobserver unavailable:') - make_jobserver = [] - - try: - for line in f: - if am_maintainer_mode_re.search(line) is not None and \ - am_maintainer_mode_exclude_re.search(line) is None: - am_maintainer_mode.append(line.rstrip("\n")) - - if bash_command_not_found_re.match(line) is not None and \ - command_not_found_exclude_re.search(line) is None: - bash_command_not_found.append(line.rstrip("\n")) - - if helper_missing_file_re.match(line) is not None: - helper_missing_file.append(line.rstrip("\n")) - - if configure_opts_warn_re.match(line) is not None: - configure_opts_warn.append(line.rstrip("\n")) - - if make_jobserver_re.match(line) is not None: - make_jobserver.append(line.rstrip("\n")) - - finally: - f.close() - - from portage.elog.messages import eqawarn - def _eqawarn(lines): - for line in lines: - eqawarn(line, phase="install", key=mysettings.mycpv, out=out) - from textwrap import wrap - wrap_width = 70 - - if am_maintainer_mode: - msg = [_("QA Notice: Automake \"maintainer mode\" detected:")] - msg.append("") - msg.extend("\t" + line for line in am_maintainer_mode) - msg.append("") - msg.extend(wrap(_( - "If you patch Makefile.am, " - "configure.in, or configure.ac then you " - "should use autotools.eclass and " - "eautomake or eautoreconf. Exceptions " - "are limited to system packages " - "for which it is impossible to run " - "autotools during stage building. " - "See http://www.gentoo.org/p" - "roj/en/qa/autofailure.xml for more information."), - wrap_width)) - _eqawarn(msg) - - if bash_command_not_found: - msg = [_("QA Notice: command not found:")] - msg.append("") - msg.extend("\t" + line for line in bash_command_not_found) - _eqawarn(msg) - - if helper_missing_file: - msg = [_("QA Notice: file does not exist:")] - msg.append("") - msg.extend("\t" + line[4:] for line in helper_missing_file) - _eqawarn(msg) - - if configure_opts_warn: - msg = [_("QA Notice: Unrecognized configure options:")] - msg.append("") - msg.extend("\t" + line for line in configure_opts_warn) - _eqawarn(msg) - - if make_jobserver: - msg = [_("QA Notice: make jobserver unavailable:")] - msg.append("") - msg.extend("\t" + line for line in make_jobserver) - _eqawarn(msg) - -def _post_src_install_chost_fix(settings): - """ - It's possible that the ebuild has changed the - CHOST variable, so revert it to the initial - setting. - """ - if settings.get('CATEGORY') == 'virtual': - return - - chost = settings.get('CHOST') - if chost: - write_atomic(os.path.join(settings['PORTAGE_BUILDDIR'], - 'build-info', 'CHOST'), chost + '\n') - -_vdb_use_conditional_keys = ('DEPEND', 'LICENSE', 'PDEPEND', - 'PROPERTIES', 'PROVIDE', 'RDEPEND', 'RESTRICT',) -_vdb_use_conditional_atoms = frozenset(['DEPEND', 'PDEPEND', 'RDEPEND']) - -def _post_src_install_uid_fix(mysettings, out=None): - """ - Files in $D with user and group bits that match the "portage" - user or group are automatically mapped to PORTAGE_INST_UID and - PORTAGE_INST_GID if necessary. The chown system call may clear - S_ISUID and S_ISGID bits, so those bits are restored if - necessary. - """ - - os = _os_merge - - inst_uid = int(mysettings["PORTAGE_INST_UID"]) - inst_gid = int(mysettings["PORTAGE_INST_GID"]) - - if bsd_chflags: - # Temporarily remove all of the flags in order to avoid EPERM errors. - os.system("mtree -c -p %s -k flags > %s" % \ - (_shell_quote(mysettings["D"]), - _shell_quote(os.path.join(mysettings["T"], "bsdflags.mtree")))) - os.system("chflags -R noschg,nouchg,nosappnd,nouappnd %s" % \ - (_shell_quote(mysettings["D"]),)) - os.system("chflags -R nosunlnk,nouunlnk %s 2>/dev/null" % \ - (_shell_quote(mysettings["D"]),)) - - destdir = mysettings["D"] - unicode_errors = [] - - while True: - - unicode_error = False - size = 0 - counted_inodes = set() - - for parent, dirs, files in os.walk(destdir): - try: - parent = _unicode_decode(parent, - encoding=_encodings['merge'], errors='strict') - except UnicodeDecodeError: - new_parent = _unicode_decode(parent, - encoding=_encodings['merge'], errors='replace') - new_parent = _unicode_encode(new_parent, - encoding=_encodings['merge'], errors='backslashreplace') - new_parent = _unicode_decode(new_parent, - encoding=_encodings['merge'], errors='replace') - os.rename(parent, new_parent) - unicode_error = True - unicode_errors.append(new_parent[len(destdir):]) - break - - for fname in chain(dirs, files): - try: - fname = _unicode_decode(fname, - encoding=_encodings['merge'], errors='strict') - except UnicodeDecodeError: - fpath = _os.path.join( - parent.encode(_encodings['merge']), fname) - new_fname = _unicode_decode(fname, - encoding=_encodings['merge'], errors='replace') - new_fname = _unicode_encode(new_fname, - encoding=_encodings['merge'], errors='backslashreplace') - new_fname = _unicode_decode(new_fname, - encoding=_encodings['merge'], errors='replace') - new_fpath = os.path.join(parent, new_fname) - os.rename(fpath, new_fpath) - unicode_error = True - unicode_errors.append(new_fpath[len(destdir):]) - fname = new_fname - fpath = new_fpath - else: - fpath = os.path.join(parent, fname) - - mystat = os.lstat(fpath) - if stat.S_ISREG(mystat.st_mode) and \ - mystat.st_ino not in counted_inodes: - counted_inodes.add(mystat.st_ino) - size += mystat.st_size - if mystat.st_uid != portage_uid and \ - mystat.st_gid != portage_gid: - continue - myuid = -1 - mygid = -1 - if mystat.st_uid == portage_uid: - myuid = inst_uid - if mystat.st_gid == portage_gid: - mygid = inst_gid - apply_secpass_permissions( - _unicode_encode(fpath, encoding=_encodings['merge']), - uid=myuid, gid=mygid, - mode=mystat.st_mode, stat_cached=mystat, - follow_links=False) - - if unicode_error: - break - - if not unicode_error: - break - - if unicode_errors: - from portage.elog.messages import eerror - for l in _merge_unicode_error(unicode_errors): - eerror(l, phase='install', key=mysettings.mycpv, out=out) - - build_info_dir = os.path.join(mysettings['PORTAGE_BUILDDIR'], - 'build-info') - - codecs.open(_unicode_encode(os.path.join(build_info_dir, - 'SIZE'), encoding=_encodings['fs'], errors='strict'), - 'w', encoding=_encodings['repo.content'], - errors='strict').write(str(size) + '\n') - - codecs.open(_unicode_encode(os.path.join(build_info_dir, - 'BUILD_TIME'), encoding=_encodings['fs'], errors='strict'), - 'w', encoding=_encodings['repo.content'], - errors='strict').write(str(int(time.time())) + '\n') - - use = frozenset(mysettings['PORTAGE_USE'].split()) - for k in _vdb_use_conditional_keys: - v = mysettings.configdict['pkg'].get(k) - if v is None: - continue - v = dep.paren_reduce(v) - v = dep.use_reduce(v, uselist=use) - v = dep.paren_normalize(v) - v = dep.paren_enclose(v) - if not v: - continue - if v in _vdb_use_conditional_atoms: - v_split = [] - for x in v.split(): - try: - x = dep.Atom(x) - except exception.InvalidAtom: - v_split.append(x) - else: - v_split.append(str(x.evaluate_conditionals(use))) - v = ' '.join(v_split) - codecs.open(_unicode_encode(os.path.join(build_info_dir, - k), encoding=_encodings['fs'], errors='strict'), - mode='w', encoding=_encodings['repo.content'], - errors='strict').write(v + '\n') - - if bsd_chflags: - # Restore all of the flags saved above. - os.system("mtree -e -p %s -U -k flags < %s > /dev/null" % \ - (_shell_quote(mysettings["D"]), - _shell_quote(os.path.join(mysettings["T"], "bsdflags.mtree")))) - -def _merge_unicode_error(errors): - from textwrap import wrap - lines = [] - - msg = _("This package installs one or more file names containing " - "characters that do not match your current locale " - "settings. The current setting for filesystem encoding is '%s'.") \ - % _encodings['merge'] - lines.extend(wrap(msg, 72)) - - lines.append("") - errors.sort() - lines.extend("\t" + x for x in errors) - lines.append("") - - if _encodings['merge'].lower().replace('_', '').replace('-', '') != 'utf8': - msg = _("For best results, UTF-8 encoding is recommended. See " - "the Gentoo Linux Localization Guide for instructions " - "about how to configure your locale for UTF-8 encoding:") - lines.extend(wrap(msg, 72)) - lines.append("") - lines.append("\t" + \ - "http://www.gentoo.org/doc/en/guide-localization.xml") - lines.append("") - - return lines - -def _post_pkg_preinst_cmd(mysettings): - """ - Post phase logic and tasks that have been factored out of - ebuild.sh. Call preinst_mask last so that INSTALL_MASK can - can be used to wipe out any gmon.out files created during - previous functions (in case any tools were built with -pg - in CFLAGS). - """ - - portage_bin_path = mysettings["PORTAGE_BIN_PATH"] - misc_sh_binary = os.path.join(portage_bin_path, - os.path.basename(MISC_SH_BINARY)) - - mysettings["EBUILD_PHASE"] = "" - global _post_phase_cmds - myargs = [_shell_quote(misc_sh_binary)] + _post_phase_cmds["preinst"] - - return myargs - -def _post_pkg_postinst_cmd(mysettings): - """ - Post phase logic and tasks that have been factored out of - build.sh. - """ - - portage_bin_path = mysettings["PORTAGE_BIN_PATH"] - misc_sh_binary = os.path.join(portage_bin_path, - os.path.basename(MISC_SH_BINARY)) - - mysettings["EBUILD_PHASE"] = "" - global _post_phase_cmds - myargs = [_shell_quote(misc_sh_binary)] + _post_phase_cmds["postinst"] - - return myargs - -def _spawn_misc_sh(mysettings, commands, phase=None, **kwargs): - """ - @param mysettings: the ebuild config - @type mysettings: config - @param commands: a list of function names to call in misc-functions.sh - @type commands: list - @rtype: int - @returns: the return value from the spawn() call - """ - - # Note: PORTAGE_BIN_PATH may differ from the global - # constant when portage is reinstalling itself. - portage_bin_path = mysettings["PORTAGE_BIN_PATH"] - misc_sh_binary = os.path.join(portage_bin_path, - os.path.basename(MISC_SH_BINARY)) - mycommand = " ".join([_shell_quote(misc_sh_binary)] + commands) - _doebuild_exit_status_unlink( - mysettings.get("EBUILD_EXIT_STATUS_FILE")) - debug = mysettings.get("PORTAGE_DEBUG") == "1" - logfile = mysettings.get("PORTAGE_LOG_FILE") - mysettings.pop("EBUILD_PHASE", None) - try: - rval = spawn(mycommand, mysettings, debug=debug, - logfile=logfile, **kwargs) - finally: - pass - - msg = _doebuild_exit_status_check(phase, mysettings) - if msg: - if rval == os.EX_OK: - rval = 1 - from textwrap import wrap - from portage.elog.messages import eerror - for l in wrap(msg, 72): - eerror(l, phase=mydo, key=mysettings.mycpv) - - return rval - _testing_eapis = frozenset() _deprecated_eapis = frozenset(["3_pre2", "3_pre1", "2_pre3", "2_pre2", "2_pre1"]) @@ -2276,1331 +1408,6 @@ def _split_ebuild_name_glep55(name): return (None, None) return (m.group(1), m.group(3)) -def doebuild_environment(myebuild, mydo, myroot, mysettings, debug, use_cache, mydbapi): - - ebuild_path = os.path.abspath(myebuild) - pkg_dir = os.path.dirname(ebuild_path) - - if "CATEGORY" in mysettings.configdict["pkg"]: - cat = mysettings.configdict["pkg"]["CATEGORY"] - else: - cat = os.path.basename(normalize_path(os.path.join(pkg_dir, ".."))) - - eapi = None - if 'parse-eapi-glep-55' in mysettings.features: - mypv, eapi = portage._split_ebuild_name_glep55( - os.path.basename(myebuild)) - else: - mypv = os.path.basename(ebuild_path)[:-7] - - mycpv = cat+"/"+mypv - mysplit = versions._pkgsplit(mypv) - if mysplit is None: - raise portage.exception.IncorrectParameter( - _("Invalid ebuild path: '%s'") % myebuild) - - # Make a backup of PORTAGE_TMPDIR prior to calling config.reset() - # so that the caller can override it. - tmpdir = mysettings["PORTAGE_TMPDIR"] - - if mydo == 'depend': - if mycpv != mysettings.mycpv: - # Don't pass in mydbapi here since the resulting aux_get - # call would lead to infinite 'depend' phase recursion. - mysettings.setcpv(mycpv) - else: - # If IUSE isn't in configdict['pkg'], it means that setcpv() - # hasn't been called with the mydb argument, so we have to - # call it here (portage code always calls setcpv properly, - # but api consumers might not). - if mycpv != mysettings.mycpv or \ - 'IUSE' not in mysettings.configdict['pkg']: - # Reload env.d variables and reset any previous settings. - mysettings.reload() - mysettings.reset() - mysettings.setcpv(mycpv, mydb=mydbapi) - - # config.reset() might have reverted a change made by the caller, - # so restore it to it's original value. - mysettings["PORTAGE_TMPDIR"] = tmpdir - - mysettings.pop("EBUILD_PHASE", None) # remove from backupenv - mysettings["EBUILD_PHASE"] = mydo - - mysettings["PORTAGE_MASTER_PID"] = str(os.getpid()) - - # We are disabling user-specific bashrc files. - mysettings["BASH_ENV"] = INVALID_ENV_FILE - - if debug: # Otherwise it overrides emerge's settings. - # We have no other way to set debug... debug can't be passed in - # due to how it's coded... Don't overwrite this so we can use it. - mysettings["PORTAGE_DEBUG"] = "1" - - mysettings["EBUILD"] = ebuild_path - mysettings["O"] = pkg_dir - mysettings.configdict["pkg"]["CATEGORY"] = cat - mysettings["FILESDIR"] = pkg_dir+"/files" - mysettings["PF"] = mypv - - if hasattr(mydbapi, '_repo_info'): - mytree = os.path.dirname(os.path.dirname(pkg_dir)) - repo_info = mydbapi._repo_info[mytree] - mysettings['PORTDIR'] = repo_info.portdir - mysettings['PORTDIR_OVERLAY'] = repo_info.portdir_overlay - - mysettings["PORTDIR"] = os.path.realpath(mysettings["PORTDIR"]) - mysettings["DISTDIR"] = os.path.realpath(mysettings["DISTDIR"]) - mysettings["RPMDIR"] = os.path.realpath(mysettings["RPMDIR"]) - - mysettings["ECLASSDIR"] = mysettings["PORTDIR"]+"/eclass" - mysettings["SANDBOX_LOG"] = mycpv.replace("/", "_-_") - - mysettings["PROFILE_PATHS"] = "\n".join(mysettings.profiles) - mysettings["P"] = mysplit[0]+"-"+mysplit[1] - mysettings["PN"] = mysplit[0] - mysettings["PV"] = mysplit[1] - mysettings["PR"] = mysplit[2] - - if portage.util.noiselimit < 0: - mysettings["PORTAGE_QUIET"] = "1" - - if mydo == 'depend' and \ - 'EAPI' not in mysettings.configdict['pkg']: - - if eapi is not None: - # From parse-eapi-glep-55 above. - pass - elif 'parse-eapi-ebuild-head' in mysettings.features: - eapi = _parse_eapi_ebuild_head( - codecs.open(_unicode_encode(ebuild_path, - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['content'], errors='replace')) - - if eapi is not None: - if not eapi_is_supported(eapi): - raise portage.exception.UnsupportedAPIException(mycpv, eapi) - mysettings.configdict['pkg']['EAPI'] = eapi - - if mydo != "depend": - # Metadata vars such as EAPI and RESTRICT are - # set by the above config.setcpv() call. - eapi = mysettings["EAPI"] - if not eapi_is_supported(eapi): - # can't do anything with this. - raise portage.exception.UnsupportedAPIException(mycpv, eapi) - - if mysplit[2] == "r0": - mysettings["PVR"]=mysplit[1] - else: - mysettings["PVR"]=mysplit[1]+"-"+mysplit[2] - - if "PATH" in mysettings: - mysplit=mysettings["PATH"].split(":") - else: - mysplit=[] - # Note: PORTAGE_BIN_PATH may differ from the global constant - # when portage is reinstalling itself. - portage_bin_path = mysettings["PORTAGE_BIN_PATH"] - if portage_bin_path not in mysplit: - mysettings["PATH"] = portage_bin_path + ":" + mysettings["PATH"] - - # Sandbox needs cannonical paths. - mysettings["PORTAGE_TMPDIR"] = os.path.realpath( - mysettings["PORTAGE_TMPDIR"]) - mysettings["BUILD_PREFIX"] = mysettings["PORTAGE_TMPDIR"]+"/portage" - mysettings["PKG_TMPDIR"] = mysettings["PORTAGE_TMPDIR"]+"/binpkgs" - - # Package {pre,post}inst and {pre,post}rm may overlap, so they must have separate - # locations in order to prevent interference. - if mydo in ("unmerge", "prerm", "postrm", "cleanrm"): - mysettings["PORTAGE_BUILDDIR"] = os.path.join( - mysettings["PKG_TMPDIR"], - mysettings["CATEGORY"], mysettings["PF"]) - else: - mysettings["PORTAGE_BUILDDIR"] = os.path.join( - mysettings["BUILD_PREFIX"], - mysettings["CATEGORY"], mysettings["PF"]) - - mysettings["HOME"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], "homedir") - mysettings["WORKDIR"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], "work") - mysettings["D"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], "image") + os.sep - mysettings["T"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], "temp") - - # Prefix forward compatability - mysettings["ED"] = mysettings["D"] - - mysettings["PORTAGE_BASHRC"] = os.path.join( - mysettings["PORTAGE_CONFIGROOT"], EBUILD_SH_ENV_FILE) - mysettings["EBUILD_EXIT_STATUS_FILE"] = os.path.join( - mysettings["PORTAGE_BUILDDIR"], ".exit_status") - - #set up KV variable -- DEP SPEEDUP :: Don't waste time. Keep var persistent. - if eapi not in ('0', '1', '2', '3', '3_pre2'): - # Discard KV for EAPIs that don't support it. Cache KV is restored - # from the backupenv whenever config.reset() is called. - mysettings.pop('KV', None) - elif mydo != 'depend' and 'KV' not in mysettings and \ - mydo in ('compile', 'config', 'configure', 'info', - 'install', 'nofetch', 'postinst', 'postrm', 'preinst', - 'prepare', 'prerm', 'setup', 'test', 'unpack'): - mykv,err1=ExtractKernelVersion(os.path.join(myroot, "usr/src/linux")) - if mykv: - # Regular source tree - mysettings["KV"]=mykv - else: - mysettings["KV"]="" - mysettings.backup_changes("KV") - - # Allow color.map to control colors associated with einfo, ewarn, etc... - mycolors = [] - for c in ("GOOD", "WARN", "BAD", "HILITE", "BRACKET"): - mycolors.append("%s=$'%s'" % \ - (c, portage.output.style_to_ansi_code(c))) - mysettings["PORTAGE_COLORMAP"] = "\n".join(mycolors) - -def prepare_build_dirs(myroot, mysettings, cleanup): - - clean_dirs = [mysettings["HOME"]] - - # We enable cleanup when we want to make sure old cruft (such as the old - # environment) doesn't interfere with the current phase. - if cleanup: - clean_dirs.append(mysettings["T"]) - - for clean_dir in clean_dirs: - try: - shutil.rmtree(clean_dir) - except OSError as oe: - if errno.ENOENT == oe.errno: - pass - elif errno.EPERM == oe.errno: - writemsg("%s\n" % oe, noiselevel=-1) - writemsg(_("Operation Not Permitted: rmtree('%s')\n") % \ - clean_dir, noiselevel=-1) - return 1 - else: - raise - - def makedirs(dir_path): - try: - os.makedirs(dir_path) - except OSError as oe: - if errno.EEXIST == oe.errno: - pass - elif errno.EPERM == oe.errno: - writemsg("%s\n" % oe, noiselevel=-1) - writemsg(_("Operation Not Permitted: makedirs('%s')\n") % \ - dir_path, noiselevel=-1) - return False - else: - raise - return True - - mysettings["PKG_LOGDIR"] = os.path.join(mysettings["T"], "logging") - - mydirs = [os.path.dirname(mysettings["PORTAGE_BUILDDIR"])] - mydirs.append(os.path.dirname(mydirs[-1])) - - try: - for mydir in mydirs: - portage.util.ensure_dirs(mydir) - portage.util.apply_secpass_permissions(mydir, - gid=portage_gid, uid=portage_uid, mode=0o70, mask=0) - for dir_key in ("PORTAGE_BUILDDIR", "HOME", "PKG_LOGDIR", "T"): - """These directories don't necessarily need to be group writable. - However, the setup phase is commonly run as a privileged user prior - to the other phases being run by an unprivileged user. Currently, - we use the portage group to ensure that the unprivleged user still - has write access to these directories in any case.""" - portage.util.ensure_dirs(mysettings[dir_key], mode=0o775) - portage.util.apply_secpass_permissions(mysettings[dir_key], - uid=portage_uid, gid=portage_gid) - except portage.exception.PermissionDenied as e: - writemsg(_("Permission Denied: %s\n") % str(e), noiselevel=-1) - return 1 - except portage.exception.OperationNotPermitted as e: - writemsg(_("Operation Not Permitted: %s\n") % str(e), noiselevel=-1) - return 1 - except portage.exception.FileNotFound as e: - writemsg(_("File Not Found: '%s'\n") % str(e), noiselevel=-1) - return 1 - - # Reset state for things like noauto and keepwork in FEATURES. - for x in ('.die_hooks',): - try: - os.unlink(os.path.join(mysettings['PORTAGE_BUILDDIR'], x)) - except OSError: - pass - - _prepare_workdir(mysettings) - if mysettings.get('EBUILD_PHASE') != 'fetch': - # Avoid spurious permissions adjustments when fetching with - # a temporary PORTAGE_TMPDIR setting (for fetchonly). - _prepare_features_dirs(mysettings) - -def _adjust_perms_msg(settings, msg): - - def write(msg): - writemsg(msg, noiselevel=-1) - - background = settings.get("PORTAGE_BACKGROUND") == "1" - log_path = settings.get("PORTAGE_LOG_FILE") - log_file = None - - if background and log_path is not None: - try: - log_file = codecs.open(_unicode_encode(log_path, - encoding=_encodings['fs'], errors='strict'), - mode='a', encoding=_encodings['content'], errors='replace') - except IOError: - def write(msg): - pass - else: - def write(msg): - log_file.write(_unicode_decode(msg)) - log_file.flush() - - try: - write(msg) - finally: - if log_file is not None: - log_file.close() - -def _prepare_features_dirs(mysettings): - - features_dirs = { - "ccache":{ - "path_dir": "/usr/lib/ccache/bin", - "basedir_var":"CCACHE_DIR", - "default_dir":os.path.join(mysettings["PORTAGE_TMPDIR"], "ccache"), - "always_recurse":False}, - "distcc":{ - "path_dir": "/usr/lib/distcc/bin", - "basedir_var":"DISTCC_DIR", - "default_dir":os.path.join(mysettings["BUILD_PREFIX"], ".distcc"), - "subdirs":("lock", "state"), - "always_recurse":True} - } - dirmode = 0o2070 - filemode = 0o60 - modemask = 0o2 - restrict = mysettings.get("PORTAGE_RESTRICT","").split() - from portage.data import secpass - droppriv = secpass >= 2 and \ - "userpriv" in mysettings.features and \ - "userpriv" not in restrict - for myfeature, kwargs in features_dirs.items(): - if myfeature in mysettings.features: - failure = False - basedir = mysettings.get(kwargs["basedir_var"]) - if basedir is None or not basedir.strip(): - basedir = kwargs["default_dir"] - mysettings[kwargs["basedir_var"]] = basedir - try: - path_dir = kwargs["path_dir"] - if not os.path.isdir(path_dir): - raise portage.exception.DirectoryNotFound(path_dir) - - mydirs = [mysettings[kwargs["basedir_var"]]] - if "subdirs" in kwargs: - for subdir in kwargs["subdirs"]: - mydirs.append(os.path.join(basedir, subdir)) - for mydir in mydirs: - modified = portage.util.ensure_dirs(mydir) - # Generally, we only want to apply permissions for - # initial creation. Otherwise, we don't know exactly what - # permissions the user wants, so should leave them as-is. - droppriv_fix = False - if droppriv: - st = os.stat(mydir) - if st.st_gid != portage_gid or \ - not dirmode == (stat.S_IMODE(st.st_mode) & dirmode): - droppriv_fix = True - if not droppriv_fix: - # Check permissions of files in the directory. - for filename in os.listdir(mydir): - try: - subdir_st = os.lstat( - os.path.join(mydir, filename)) - except OSError: - continue - if subdir_st.st_gid != portage_gid or \ - ((stat.S_ISDIR(subdir_st.st_mode) and \ - not dirmode == (stat.S_IMODE(subdir_st.st_mode) & dirmode))): - droppriv_fix = True - break - - if droppriv_fix: - _adjust_perms_msg(mysettings, - colorize("WARN", " * ") + \ - _("Adjusting permissions " - "for FEATURES=userpriv: '%s'\n") % mydir) - elif modified: - _adjust_perms_msg(mysettings, - colorize("WARN", " * ") + \ - _("Adjusting permissions " - "for FEATURES=%s: '%s'\n") % (myfeature, mydir)) - - if modified or kwargs["always_recurse"] or droppriv_fix: - def onerror(e): - raise # The feature is disabled if a single error - # occurs during permissions adjustment. - if not apply_recursive_permissions(mydir, - gid=portage_gid, dirmode=dirmode, dirmask=modemask, - filemode=filemode, filemask=modemask, onerror=onerror): - raise portage.exception.OperationNotPermitted( - _("Failed to apply recursive permissions for the portage group.")) - - except portage.exception.DirectoryNotFound as e: - failure = True - writemsg(_("\n!!! Directory does not exist: '%s'\n") % \ - (e,), noiselevel=-1) - writemsg(_("!!! Disabled FEATURES='%s'\n") % myfeature, - noiselevel=-1) - - except portage.exception.PortageException as e: - failure = True - writemsg("\n!!! %s\n" % str(e), noiselevel=-1) - writemsg(_("!!! Failed resetting perms on %s='%s'\n") % \ - (kwargs["basedir_var"], basedir), noiselevel=-1) - writemsg(_("!!! Disabled FEATURES='%s'\n") % myfeature, - noiselevel=-1) - - if failure: - mysettings.features.remove(myfeature) - mysettings['FEATURES'] = ' '.join(sorted(mysettings.features)) - time.sleep(5) - -def _prepare_workdir(mysettings): - workdir_mode = 0o700 - try: - mode = mysettings["PORTAGE_WORKDIR_MODE"] - if mode.isdigit(): - parsed_mode = int(mode, 8) - elif mode == "": - raise KeyError() - else: - raise ValueError() - if parsed_mode & 0o7777 != parsed_mode: - raise ValueError("Invalid file mode: %s" % mode) - else: - workdir_mode = parsed_mode - except KeyError as e: - writemsg(_("!!! PORTAGE_WORKDIR_MODE is unset, using %s.\n") % oct(workdir_mode)) - except ValueError as e: - if len(str(e)) > 0: - writemsg("%s\n" % e) - writemsg(_("!!! Unable to parse PORTAGE_WORKDIR_MODE='%s', using %s.\n") % \ - (mysettings["PORTAGE_WORKDIR_MODE"], oct(workdir_mode))) - mysettings["PORTAGE_WORKDIR_MODE"] = oct(workdir_mode).replace('o', '') - try: - apply_secpass_permissions(mysettings["WORKDIR"], - uid=portage_uid, gid=portage_gid, mode=workdir_mode) - except portage.exception.FileNotFound: - pass # ebuild.sh will create it - - if mysettings.get("PORT_LOGDIR", "") == "": - while "PORT_LOGDIR" in mysettings: - del mysettings["PORT_LOGDIR"] - if "PORT_LOGDIR" in mysettings: - try: - modified = portage.util.ensure_dirs(mysettings["PORT_LOGDIR"]) - if modified: - apply_secpass_permissions(mysettings["PORT_LOGDIR"], - uid=portage_uid, gid=portage_gid, mode=0o2770) - except portage.exception.PortageException as e: - writemsg("!!! %s\n" % str(e), noiselevel=-1) - writemsg(_("!!! Permission issues with PORT_LOGDIR='%s'\n") % \ - mysettings["PORT_LOGDIR"], noiselevel=-1) - writemsg(_("!!! Disabling logging.\n"), noiselevel=-1) - while "PORT_LOGDIR" in mysettings: - del mysettings["PORT_LOGDIR"] - if "PORT_LOGDIR" in mysettings and \ - os.access(mysettings["PORT_LOGDIR"], os.W_OK): - logid_path = os.path.join(mysettings["PORTAGE_BUILDDIR"], ".logid") - if not os.path.exists(logid_path): - open(_unicode_encode(logid_path), 'w') - logid_time = _unicode_decode(time.strftime("%Y%m%d-%H%M%S", - time.gmtime(os.stat(logid_path).st_mtime)), - encoding=_encodings['content'], errors='replace') - - if "split-log" in mysettings.features: - mysettings["PORTAGE_LOG_FILE"] = os.path.join( - mysettings["PORT_LOGDIR"], "build", "%s/%s:%s.log" % \ - (mysettings["CATEGORY"], mysettings["PF"], logid_time)) - else: - mysettings["PORTAGE_LOG_FILE"] = os.path.join( - mysettings["PORT_LOGDIR"], "%s:%s:%s.log" % \ - (mysettings["CATEGORY"], mysettings["PF"], logid_time)) - - util.ensure_dirs(os.path.dirname(mysettings["PORTAGE_LOG_FILE"])) - - else: - # NOTE: When sesandbox is enabled, the local SELinux security policies - # may not allow output to be piped out of the sesandbox domain. The - # current policy will allow it to work when a pty is available, but - # not through a normal pipe. See bug #162404. - mysettings["PORTAGE_LOG_FILE"] = os.path.join( - mysettings["T"], "build.log") - -def _doebuild_exit_status_check(mydo, settings): - """ - Returns an error string if the shell appeared - to exit unsuccessfully, None otherwise. - """ - exit_status_file = settings.get("EBUILD_EXIT_STATUS_FILE") - if not exit_status_file or \ - os.path.exists(exit_status_file): - return None - msg = _("The ebuild phase '%s' has exited " - "unexpectedly. This type of behavior " - "is known to be triggered " - "by things such as failed variable " - "assignments (bug #190128) or bad substitution " - "errors (bug #200313). Normally, before exiting, bash should " - "have displayed an error message above. If bash did not " - "produce an error message above, it's possible " - "that the ebuild has called `exit` when it " - "should have called `die` instead. This behavior may also " - "be triggered by a corrupt bash binary or a hardware " - "problem such as memory or cpu malfunction. If the problem is not " - "reproducible or it appears to occur randomly, then it is likely " - "to be triggered by a hardware problem. " - "If you suspect a hardware problem then you should " - "try some basic hardware diagnostics such as memtest. " - "Please do not report this as a bug unless it is consistently " - "reproducible and you are sure that your bash binary and hardware " - "are functioning properly.") % mydo - return msg - -def _doebuild_exit_status_check_and_log(settings, mydo, retval): - msg = _doebuild_exit_status_check(mydo, settings) - if msg: - if retval == os.EX_OK: - retval = 1 - from textwrap import wrap - from portage.elog.messages import eerror - for l in wrap(msg, 72): - eerror(l, phase=mydo, key=settings.mycpv) - return retval - -def _doebuild_exit_status_unlink(exit_status_file): - """ - Double check to make sure it really doesn't exist - and raise an OSError if it still does (it shouldn't). - OSError if necessary. - """ - if not exit_status_file: - return - try: - os.unlink(exit_status_file) - except OSError: - pass - if os.path.exists(exit_status_file): - os.unlink(exit_status_file) - -_doebuild_manifest_exempt_depend = 0 -_doebuild_manifest_cache = None -_doebuild_broken_ebuilds = set() -_doebuild_broken_manifests = set() - -def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, - fetchonly=0, cleanup=0, dbkey=None, use_cache=1, fetchall=0, tree=None, - mydbapi=None, vartree=None, prev_mtimes=None, - fd_pipes=None, returnpid=False): - - """ - Wrapper function that invokes specific ebuild phases through the spawning - of ebuild.sh - - @param myebuild: name of the ebuild to invoke the phase on (CPV) - @type myebuild: String - @param mydo: Phase to run - @type mydo: String - @param myroot: $ROOT (usually '/', see man make.conf) - @type myroot: String - @param mysettings: Portage Configuration - @type mysettings: instance of portage.config - @param debug: Turns on various debug information (eg, debug for spawn) - @type debug: Boolean - @param listonly: Used to wrap fetch(); passed such that fetch only lists files required. - @type listonly: Boolean - @param fetchonly: Used to wrap fetch(); passed such that files are only fetched (no other actions) - @type fetchonly: Boolean - @param cleanup: Passed to prepare_build_dirs (TODO: what does it do?) - @type cleanup: Boolean - @param dbkey: A dict (usually keys and values from the depend phase, such as KEYWORDS, USE, etc..) - @type dbkey: Dict or String - @param use_cache: Enables the cache - @type use_cache: Boolean - @param fetchall: Used to wrap fetch(), fetches all URIs (even ones invalid due to USE conditionals) - @type fetchall: Boolean - @param tree: Which tree to use ('vartree','porttree','bintree', etc..), defaults to 'porttree' - @type tree: String - @param mydbapi: a dbapi instance to pass to various functions; this should be a portdbapi instance. - @type mydbapi: portdbapi instance - @param vartree: A instance of vartree; used for aux_get calls, defaults to db[myroot]['vartree'] - @type vartree: vartree instance - @param prev_mtimes: A dict of { filename:mtime } keys used by merge() to do config_protection - @type prev_mtimes: dictionary - @param fd_pipes: A dict of mapping for pipes, { '0': stdin, '1': stdout } - for example. - @type fd_pipes: Dictionary - @param returnpid: Return a list of process IDs for a successful spawn, or - an integer value if spawn is unsuccessful. NOTE: This requires the - caller clean up all returned PIDs. - @type returnpid: Boolean - @rtype: Boolean - @returns: - 1. 0 for success - 2. 1 for error - - Most errors have an accompanying error message. - - listonly and fetchonly are only really necessary for operations involving 'fetch' - prev_mtimes are only necessary for merge operations. - Other variables may not be strictly required, many have defaults that are set inside of doebuild. - - """ - - if not tree: - writemsg("Warning: tree not specified to doebuild\n") - tree = "porttree" - global db - - # chunked out deps for each phase, so that ebuild binary can use it - # to collapse targets down. - actionmap_deps={ - "setup": [], - "unpack": ["setup"], - "prepare": ["unpack"], - "configure": ["prepare"], - "compile":["configure"], - "test": ["compile"], - "install":["test"], - "rpm": ["install"], - "package":["install"], - } - - if mydbapi is None: - mydbapi = db[myroot][tree].dbapi - - if vartree is None and mydo in ("merge", "qmerge", "unmerge"): - vartree = db[myroot]["vartree"] - - features = mysettings.features - noauto = "noauto" in features - from portage.data import secpass - - clean_phases = ("clean", "cleanrm") - validcommands = ["help","clean","prerm","postrm","cleanrm","preinst","postinst", - "config", "info", "setup", "depend", "pretend", - "fetch", "fetchall", "digest", - "unpack", "prepare", "configure", "compile", "test", - "install", "rpm", "qmerge", "merge", - "package","unmerge", "manifest"] - - if mydo not in validcommands: - validcommands.sort() - writemsg("!!! doebuild: '%s' is not one of the following valid commands:" % mydo, - noiselevel=-1) - for vcount in range(len(validcommands)): - if vcount%6 == 0: - writemsg("\n!!! ", noiselevel=-1) - writemsg(validcommands[vcount].ljust(11), noiselevel=-1) - writemsg("\n", noiselevel=-1) - return 1 - - if mydo == "fetchall": - fetchall = 1 - mydo = "fetch" - - parallel_fetchonly = mydo in ("fetch", "fetchall") and \ - "PORTAGE_PARALLEL_FETCHONLY" in mysettings - - if mydo not in clean_phases and not os.path.exists(myebuild): - writemsg("!!! doebuild: %s not found for %s\n" % (myebuild, mydo), - noiselevel=-1) - return 1 - - global _doebuild_manifest_exempt_depend - - if "strict" in features and \ - "digest" not in features and \ - tree == "porttree" and \ - mydo not in ("digest", "manifest", "help") and \ - not _doebuild_manifest_exempt_depend: - # Always verify the ebuild checksums before executing it. - global _doebuild_manifest_cache, _doebuild_broken_ebuilds, \ - _doebuild_broken_ebuilds - - if myebuild in _doebuild_broken_ebuilds: - return 1 - - pkgdir = os.path.dirname(myebuild) - manifest_path = os.path.join(pkgdir, "Manifest") - - # Avoid checking the same Manifest several times in a row during a - # regen with an empty cache. - if _doebuild_manifest_cache is None or \ - _doebuild_manifest_cache.getFullname() != manifest_path: - _doebuild_manifest_cache = None - if not os.path.exists(manifest_path): - out = portage.output.EOutput() - out.eerror(_("Manifest not found for '%s'") % (myebuild,)) - _doebuild_broken_ebuilds.add(myebuild) - return 1 - mf = Manifest(pkgdir, mysettings["DISTDIR"]) - - else: - mf = _doebuild_manifest_cache - - try: - mf.checkFileHashes("EBUILD", os.path.basename(myebuild)) - except KeyError: - out = portage.output.EOutput() - out.eerror(_("Missing digest for '%s'") % (myebuild,)) - _doebuild_broken_ebuilds.add(myebuild) - return 1 - except portage.exception.FileNotFound: - out = portage.output.EOutput() - out.eerror(_("A file listed in the Manifest " - "could not be found: '%s'") % (myebuild,)) - _doebuild_broken_ebuilds.add(myebuild) - return 1 - except portage.exception.DigestException as e: - out = portage.output.EOutput() - out.eerror(_("Digest verification failed:")) - out.eerror("%s" % e.value[0]) - out.eerror(_("Reason: %s") % e.value[1]) - out.eerror(_("Got: %s") % e.value[2]) - out.eerror(_("Expected: %s") % e.value[3]) - _doebuild_broken_ebuilds.add(myebuild) - return 1 - - if mf.getFullname() in _doebuild_broken_manifests: - return 1 - - if mf is not _doebuild_manifest_cache: - - # Make sure that all of the ebuilds are - # actually listed in the Manifest. - glep55 = 'parse-eapi-glep-55' in mysettings.features - for f in os.listdir(pkgdir): - pf = None - if glep55: - pf, eapi = _split_ebuild_name_glep55(f) - elif f[-7:] == '.ebuild': - pf = f[:-7] - if pf is not None and not mf.hasFile("EBUILD", f): - f = os.path.join(pkgdir, f) - if f not in _doebuild_broken_ebuilds: - out = portage.output.EOutput() - out.eerror(_("A file is not listed in the " - "Manifest: '%s'") % (f,)) - _doebuild_broken_manifests.add(manifest_path) - return 1 - - # Only cache it if the above stray files test succeeds. - _doebuild_manifest_cache = mf - - def exit_status_check(retval): - msg = _doebuild_exit_status_check(mydo, mysettings) - if msg: - if retval == os.EX_OK: - retval = 1 - from textwrap import wrap - from portage.elog.messages import eerror - for l in wrap(msg, 72): - eerror(l, phase=mydo, key=mysettings.mycpv) - return retval - - # Note: PORTAGE_BIN_PATH may differ from the global - # constant when portage is reinstalling itself. - portage_bin_path = mysettings["PORTAGE_BIN_PATH"] - ebuild_sh_binary = os.path.join(portage_bin_path, - os.path.basename(EBUILD_SH_BINARY)) - misc_sh_binary = os.path.join(portage_bin_path, - os.path.basename(MISC_SH_BINARY)) - - logfile=None - builddir_lock = None - tmpdir = None - tmpdir_orig = None - - try: - if mydo in ("digest", "manifest", "help"): - # Temporarily exempt the depend phase from manifest checks, in case - # aux_get calls trigger cache generation. - _doebuild_manifest_exempt_depend += 1 - - # If we don't need much space and we don't need a constant location, - # we can temporarily override PORTAGE_TMPDIR with a random temp dir - # so that there's no need for locking and it can be used even if the - # user isn't in the portage group. - if mydo in ("info",): - from tempfile import mkdtemp - tmpdir = mkdtemp() - tmpdir_orig = mysettings["PORTAGE_TMPDIR"] - mysettings["PORTAGE_TMPDIR"] = tmpdir - - doebuild_environment(myebuild, mydo, myroot, mysettings, debug, - use_cache, mydbapi) - - if mydo in clean_phases: - retval = spawn(_shell_quote(ebuild_sh_binary) + " clean", - mysettings, debug=debug, fd_pipes=fd_pipes, free=1, - logfile=None, returnpid=returnpid) - return retval - - restrict = set(mysettings.get('PORTAGE_RESTRICT', '').split()) - # get possible slot information from the deps file - if mydo == "depend": - writemsg("!!! DEBUG: dbkey: %s\n" % str(dbkey), 2) - droppriv = "userpriv" in mysettings.features - if returnpid: - mypids = spawn(_shell_quote(ebuild_sh_binary) + " depend", - mysettings, fd_pipes=fd_pipes, returnpid=True, - droppriv=droppriv) - return mypids - elif isinstance(dbkey, dict): - mysettings["dbkey"] = "" - pr, pw = os.pipe() - fd_pipes = { - 0:sys.stdin.fileno(), - 1:sys.stdout.fileno(), - 2:sys.stderr.fileno(), - 9:pw} - mypids = spawn(_shell_quote(ebuild_sh_binary) + " depend", - mysettings, - fd_pipes=fd_pipes, returnpid=True, droppriv=droppriv) - os.close(pw) # belongs exclusively to the child process now - f = os.fdopen(pr, 'rb') - for k, v in zip(auxdbkeys, - (_unicode_decode(line).rstrip('\n') for line in f)): - dbkey[k] = v - f.close() - retval = os.waitpid(mypids[0], 0)[1] - portage.process.spawned_pids.remove(mypids[0]) - # If it got a signal, return the signal that was sent, but - # shift in order to distinguish it from a return value. (just - # like portage.process.spawn() would do). - if retval & 0xff: - retval = (retval & 0xff) << 8 - else: - # Otherwise, return its exit code. - retval = retval >> 8 - if retval == os.EX_OK and len(dbkey) != len(auxdbkeys): - # Don't trust bash's returncode if the - # number of lines is incorrect. - retval = 1 - return retval - elif dbkey: - mysettings["dbkey"] = dbkey - else: - mysettings["dbkey"] = \ - os.path.join(mysettings.depcachedir, "aux_db_key_temp") - - return spawn(_shell_quote(ebuild_sh_binary) + " depend", - mysettings, - droppriv=droppriv) - - # Validate dependency metadata here to ensure that ebuilds with invalid - # data are never installed via the ebuild command. Don't bother when - # returnpid == True since there's no need to do this every time emerge - # executes a phase. - if not returnpid: - rval = _validate_deps(mysettings, myroot, mydo, mydbapi) - if rval != os.EX_OK: - return rval - - if "PORTAGE_TMPDIR" not in mysettings or \ - not os.path.isdir(mysettings["PORTAGE_TMPDIR"]): - writemsg(_("The directory specified in your " - "PORTAGE_TMPDIR variable, '%s',\n" - "does not exist. Please create this directory or " - "correct your PORTAGE_TMPDIR setting.\n") % mysettings.get("PORTAGE_TMPDIR", ""), noiselevel=-1) - return 1 - - # as some people use a separate PORTAGE_TMPDIR mount - # we prefer that as the checks below would otherwise be pointless - # for those people. - if os.path.exists(os.path.join(mysettings["PORTAGE_TMPDIR"], "portage")): - checkdir = os.path.join(mysettings["PORTAGE_TMPDIR"], "portage") - else: - checkdir = mysettings["PORTAGE_TMPDIR"] - - if not os.access(checkdir, os.W_OK): - writemsg(_("%s is not writable.\n" - "Likely cause is that you've mounted it as readonly.\n") % checkdir, - noiselevel=-1) - return 1 - else: - from tempfile import NamedTemporaryFile - fd = NamedTemporaryFile(prefix="exectest-", dir=checkdir) - os.chmod(fd.name, 0o755) - if not os.access(fd.name, os.X_OK): - writemsg(_("Can not execute files in %s\n" - "Likely cause is that you've mounted it with one of the\n" - "following mount options: 'noexec', 'user', 'users'\n\n" - "Please make sure that portage can execute files in this directory.\n") % checkdir, - noiselevel=-1) - fd.close() - return 1 - fd.close() - del checkdir - - if mydo == "unmerge": - return unmerge(mysettings["CATEGORY"], - mysettings["PF"], myroot, mysettings, vartree=vartree) - - # Build directory creation isn't required for any of these. - # In the fetch phase, the directory is needed only for RESTRICT=fetch - # in order to satisfy the sane $PWD requirement (from bug #239560) - # when pkg_nofetch is spawned. - have_build_dirs = False - if not parallel_fetchonly and \ - mydo not in ('digest', 'help', 'manifest') and \ - not (mydo == 'fetch' and 'fetch' not in restrict): - mystatus = prepare_build_dirs(myroot, mysettings, cleanup) - if mystatus: - return mystatus - have_build_dirs = True - - # emerge handles logging externally - if not returnpid: - # PORTAGE_LOG_FILE is set by the - # above prepare_build_dirs() call. - logfile = mysettings.get("PORTAGE_LOG_FILE") - - if have_build_dirs: - env_file = os.path.join(mysettings["T"], "environment") - env_stat = None - saved_env = None - try: - env_stat = os.stat(env_file) - except OSError as e: - if e.errno != errno.ENOENT: - raise - del e - if not env_stat: - saved_env = os.path.join( - os.path.dirname(myebuild), "environment.bz2") - if not os.path.isfile(saved_env): - saved_env = None - if saved_env: - retval = os.system( - "bzip2 -dc %s > %s" % \ - (_shell_quote(saved_env), - _shell_quote(env_file))) - try: - env_stat = os.stat(env_file) - except OSError as e: - if e.errno != errno.ENOENT: - raise - del e - if os.WIFEXITED(retval) and \ - os.WEXITSTATUS(retval) == os.EX_OK and \ - env_stat and env_stat.st_size > 0: - # This is a signal to ebuild.sh, so that it knows to filter - # out things like SANDBOX_{DENY,PREDICT,READ,WRITE} that - # would be preserved between normal phases. - open(_unicode_encode(env_file + '.raw'), 'w') - else: - writemsg(_("!!! Error extracting saved " - "environment: '%s'\n") % \ - saved_env, noiselevel=-1) - try: - os.unlink(env_file) - except OSError as e: - if e.errno != errno.ENOENT: - raise - del e - env_stat = None - if env_stat: - pass - else: - for var in ("ARCH", ): - value = mysettings.get(var) - if value and value.strip(): - continue - msg = _("%(var)s is not set... " - "Are you missing the '%(configroot)setc/make.profile' symlink? " - "Is the symlink correct? " - "Is your portage tree complete?") % \ - {"var": var, "configroot": mysettings["PORTAGE_CONFIGROOT"]} - from portage.elog.messages import eerror - from textwrap import wrap - for line in wrap(msg, 70): - eerror(line, phase="setup", key=mysettings.mycpv) - from portage.elog import elog_process - elog_process(mysettings.mycpv, mysettings) - return 1 - del env_file, env_stat, saved_env - _doebuild_exit_status_unlink( - mysettings.get("EBUILD_EXIT_STATUS_FILE")) - else: - mysettings.pop("EBUILD_EXIT_STATUS_FILE", None) - - # if any of these are being called, handle them -- running them out of - # the sandbox -- and stop now. - if mydo == "help": - return spawn(_shell_quote(ebuild_sh_binary) + " " + mydo, - mysettings, debug=debug, free=1, logfile=logfile) - elif mydo == "setup": - retval = spawn( - _shell_quote(ebuild_sh_binary) + " " + mydo, mysettings, - debug=debug, free=1, logfile=logfile, fd_pipes=fd_pipes, - returnpid=returnpid) - if returnpid: - return retval - retval = exit_status_check(retval) - if secpass >= 2: - """ Privileged phases may have left files that need to be made - writable to a less privileged user.""" - apply_recursive_permissions(mysettings["T"], - uid=portage_uid, gid=portage_gid, dirmode=0o70, dirmask=0, - filemode=0o60, filemask=0) - return retval - elif mydo == "preinst": - phase_retval = spawn( - _shell_quote(ebuild_sh_binary) + " " + mydo, - mysettings, debug=debug, free=1, logfile=logfile, - fd_pipes=fd_pipes, returnpid=returnpid) - - if returnpid: - return phase_retval - - phase_retval = exit_status_check(phase_retval) - if phase_retval == os.EX_OK: - _doebuild_exit_status_unlink( - mysettings.get("EBUILD_EXIT_STATUS_FILE")) - mysettings.pop("EBUILD_PHASE", None) - phase_retval = spawn( - " ".join(_post_pkg_preinst_cmd(mysettings)), - mysettings, debug=debug, free=1, logfile=logfile) - phase_retval = exit_status_check(phase_retval) - if phase_retval != os.EX_OK: - writemsg(_("!!! post preinst failed; exiting.\n"), - noiselevel=-1) - return phase_retval - elif mydo == "postinst": - phase_retval = spawn( - _shell_quote(ebuild_sh_binary) + " " + mydo, - mysettings, debug=debug, free=1, logfile=logfile, - fd_pipes=fd_pipes, returnpid=returnpid) - - if returnpid: - return phase_retval - - phase_retval = exit_status_check(phase_retval) - if phase_retval == os.EX_OK: - _doebuild_exit_status_unlink( - mysettings.get("EBUILD_EXIT_STATUS_FILE")) - mysettings.pop("EBUILD_PHASE", None) - phase_retval = spawn(" ".join(_post_pkg_postinst_cmd(mysettings)), - mysettings, debug=debug, free=1, logfile=logfile) - phase_retval = exit_status_check(phase_retval) - if phase_retval != os.EX_OK: - writemsg(_("!!! post postinst failed; exiting.\n"), - noiselevel=-1) - return phase_retval - elif mydo in ("prerm", "postrm", "config", "info"): - retval = spawn( - _shell_quote(ebuild_sh_binary) + " " + mydo, - mysettings, debug=debug, free=1, logfile=logfile, - fd_pipes=fd_pipes, returnpid=returnpid) - - if returnpid: - return retval - - retval = exit_status_check(retval) - return retval - - mycpv = "/".join((mysettings["CATEGORY"], mysettings["PF"])) - - emerge_skip_distfiles = returnpid - emerge_skip_digest = returnpid - # Only try and fetch the files if we are going to need them ... - # otherwise, if user has FEATURES=noauto and they run `ebuild clean - # unpack compile install`, we will try and fetch 4 times :/ - need_distfiles = not emerge_skip_distfiles and \ - (mydo in ("fetch", "unpack") or \ - mydo not in ("digest", "manifest") and "noauto" not in features) - alist = mysettings.configdict["pkg"].get("A") - aalist = mysettings.configdict["pkg"].get("AA") - if alist is None or aalist is None: - # Make sure we get the correct tree in case there are overlays. - mytree = os.path.realpath( - os.path.dirname(os.path.dirname(mysettings["O"]))) - useflags = mysettings["PORTAGE_USE"].split() - try: - alist = mydbapi.getFetchMap(mycpv, useflags=useflags, - mytree=mytree) - aalist = mydbapi.getFetchMap(mycpv, mytree=mytree) - except portage.exception.InvalidDependString as e: - writemsg("!!! %s\n" % str(e), noiselevel=-1) - writemsg(_("!!! Invalid SRC_URI for '%s'.\n") % mycpv, - noiselevel=-1) - del e - return 1 - mysettings.configdict["pkg"]["A"] = " ".join(alist) - mysettings.configdict["pkg"]["AA"] = " ".join(aalist) - else: - alist = set(alist.split()) - aalist = set(aalist.split()) - if ("mirror" in features) or fetchall: - fetchme = aalist - checkme = aalist - else: - fetchme = alist - checkme = alist - - if mydo == "fetch": - # Files are already checked inside fetch(), - # so do not check them again. - checkme = [] - - if not emerge_skip_distfiles and \ - need_distfiles and not fetch( - fetchme, mysettings, listonly=listonly, fetchonly=fetchonly): - return 1 - - if mydo == "fetch" and listonly: - return 0 - - try: - if mydo == "manifest": - return not digestgen(mysettings=mysettings, myportdb=mydbapi) - elif mydo == "digest": - return not digestgen(mysettings=mysettings, myportdb=mydbapi) - elif mydo != 'fetch' and not emerge_skip_digest and \ - "digest" in mysettings.features: - # Don't do this when called by emerge or when called just - # for fetch (especially parallel-fetch) since it's not needed - # and it can interfere with parallel tasks. - digestgen(mysettings=mysettings, myportdb=mydbapi) - except portage.exception.PermissionDenied as e: - writemsg(_("!!! Permission Denied: %s\n") % (e,), noiselevel=-1) - if mydo in ("digest", "manifest"): - return 1 - - # See above comment about fetching only when needed - if not emerge_skip_distfiles and \ - not digestcheck(checkme, mysettings, "strict" in features): - return 1 - - if mydo == "fetch": - return 0 - - # remove PORTAGE_ACTUAL_DISTDIR once cvs/svn is supported via SRC_URI - if (mydo != "setup" and "noauto" not in features) or mydo == "unpack": - orig_distdir = mysettings["DISTDIR"] - mysettings["PORTAGE_ACTUAL_DISTDIR"] = orig_distdir - edpath = mysettings["DISTDIR"] = \ - os.path.join(mysettings["PORTAGE_BUILDDIR"], "distdir") - portage.util.ensure_dirs(edpath, gid=portage_gid, mode=0o755) - - # Remove any unexpected files or directories. - for x in os.listdir(edpath): - symlink_path = os.path.join(edpath, x) - st = os.lstat(symlink_path) - if x in alist and stat.S_ISLNK(st.st_mode): - continue - if stat.S_ISDIR(st.st_mode): - shutil.rmtree(symlink_path) - else: - os.unlink(symlink_path) - - # Check for existing symlinks and recreate if necessary. - for x in alist: - symlink_path = os.path.join(edpath, x) - target = os.path.join(orig_distdir, x) - try: - link_target = os.readlink(symlink_path) - except OSError: - os.symlink(target, symlink_path) - else: - if link_target != target: - os.unlink(symlink_path) - os.symlink(target, symlink_path) - - #initial dep checks complete; time to process main commands - - restrict = mysettings["PORTAGE_RESTRICT"].split() - nosandbox = (("userpriv" in features) and \ - ("usersandbox" not in features) and \ - "userpriv" not in restrict and \ - "nouserpriv" not in restrict) - if nosandbox and ("userpriv" not in features or \ - "userpriv" in restrict or \ - "nouserpriv" in restrict): - nosandbox = ("sandbox" not in features and \ - "usersandbox" not in features) - - if not process.sandbox_capable: - nosandbox = True - - sesandbox = mysettings.selinux_enabled() and \ - "sesandbox" in mysettings.features - - droppriv = "userpriv" in mysettings.features and \ - "userpriv" not in restrict and \ - secpass >= 2 - - fakeroot = "fakeroot" in mysettings.features - - ebuild_sh = _shell_quote(ebuild_sh_binary) + " %s" - misc_sh = _shell_quote(misc_sh_binary) + " dyn_%s" - - # args are for the to spawn function - actionmap = { -"pretend": {"cmd":ebuild_sh, "args":{"droppriv":0, "free":1, "sesandbox":0, "fakeroot":0}}, -"setup": {"cmd":ebuild_sh, "args":{"droppriv":0, "free":1, "sesandbox":0, "fakeroot":0}}, -"unpack": {"cmd":ebuild_sh, "args":{"droppriv":droppriv, "free":0, "sesandbox":sesandbox, "fakeroot":0}}, -"prepare": {"cmd":ebuild_sh, "args":{"droppriv":droppriv, "free":0, "sesandbox":sesandbox, "fakeroot":0}}, -"configure":{"cmd":ebuild_sh, "args":{"droppriv":droppriv, "free":nosandbox, "sesandbox":sesandbox, "fakeroot":0}}, -"compile": {"cmd":ebuild_sh, "args":{"droppriv":droppriv, "free":nosandbox, "sesandbox":sesandbox, "fakeroot":0}}, -"test": {"cmd":ebuild_sh, "args":{"droppriv":droppriv, "free":nosandbox, "sesandbox":sesandbox, "fakeroot":0}}, -"install": {"cmd":ebuild_sh, "args":{"droppriv":0, "free":0, "sesandbox":sesandbox, "fakeroot":fakeroot}}, -"rpm": {"cmd":misc_sh, "args":{"droppriv":0, "free":0, "sesandbox":0, "fakeroot":fakeroot}}, -"package": {"cmd":misc_sh, "args":{"droppriv":0, "free":0, "sesandbox":0, "fakeroot":fakeroot}}, - } - - # merge the deps in so we have again a 'full' actionmap - # be glad when this can die. - for x in actionmap: - if len(actionmap_deps.get(x, [])): - actionmap[x]["dep"] = ' '.join(actionmap_deps[x]) - - if mydo in actionmap: - if mydo == "package": - # Make sure the package directory exists before executing - # this phase. This can raise PermissionDenied if - # the current user doesn't have write access to $PKGDIR. - parent_dir = os.path.join(mysettings["PKGDIR"], - mysettings["CATEGORY"]) - portage.util.ensure_dirs(parent_dir) - if not os.access(parent_dir, os.W_OK): - raise portage.exception.PermissionDenied( - "access('%s', os.W_OK)" % parent_dir) - retval = spawnebuild(mydo, - actionmap, mysettings, debug, logfile=logfile, - fd_pipes=fd_pipes, returnpid=returnpid) - elif mydo=="qmerge": - # check to ensure install was run. this *only* pops up when users - # forget it and are using ebuild - if not os.path.exists( - os.path.join(mysettings["PORTAGE_BUILDDIR"], ".installed")): - writemsg(_("!!! mydo=qmerge, but the install phase has not been run\n"), - noiselevel=-1) - return 1 - # qmerge is a special phase that implies noclean. - if "noclean" not in mysettings.features: - mysettings.features.add("noclean") - #qmerge is specifically not supposed to do a runtime dep check - retval = merge( - mysettings["CATEGORY"], mysettings["PF"], mysettings["D"], - os.path.join(mysettings["PORTAGE_BUILDDIR"], "build-info"), - myroot, mysettings, myebuild=mysettings["EBUILD"], mytree=tree, - mydbapi=mydbapi, vartree=vartree, prev_mtimes=prev_mtimes) - elif mydo=="merge": - retval = spawnebuild("install", actionmap, mysettings, debug, - alwaysdep=1, logfile=logfile, fd_pipes=fd_pipes, - returnpid=returnpid) - retval = exit_status_check(retval) - if retval != os.EX_OK: - # The merge phase handles this already. Callers don't know how - # far this function got, so we have to call elog_process() here - # so that it's only called once. - from portage.elog import elog_process - elog_process(mysettings.mycpv, mysettings) - if retval == os.EX_OK: - retval = merge(mysettings["CATEGORY"], mysettings["PF"], - mysettings["D"], os.path.join(mysettings["PORTAGE_BUILDDIR"], - "build-info"), myroot, mysettings, - myebuild=mysettings["EBUILD"], mytree=tree, mydbapi=mydbapi, - vartree=vartree, prev_mtimes=prev_mtimes) - else: - print(_("!!! Unknown mydo: %s") % mydo) - return 1 - - return retval - - finally: - - if tmpdir: - mysettings["PORTAGE_TMPDIR"] = tmpdir_orig - shutil.rmtree(tmpdir) - if builddir_lock: - portage.locks.unlockdir(builddir_lock) - - # Make sure that DISTDIR is restored to it's normal value before we return! - if "PORTAGE_ACTUAL_DISTDIR" in mysettings: - mysettings["DISTDIR"] = mysettings["PORTAGE_ACTUAL_DISTDIR"] - del mysettings["PORTAGE_ACTUAL_DISTDIR"] - - if logfile: - try: - if os.stat(logfile).st_size == 0: - os.unlink(logfile) - except OSError: - pass - - if mydo in ("digest", "manifest", "help"): - # If necessary, depend phase has been triggered by aux_get calls - # and the exemption is no longer needed. - _doebuild_manifest_exempt_depend -= 1 - -def _validate_deps(mysettings, myroot, mydo, mydbapi): - - invalid_dep_exempt_phases = \ - set(["clean", "cleanrm", "help", "prerm", "postrm"]) - dep_keys = ["DEPEND", "RDEPEND", "PDEPEND"] - misc_keys = ["LICENSE", "PROPERTIES", "PROVIDE", "RESTRICT", "SRC_URI"] - other_keys = ["SLOT"] - all_keys = dep_keys + misc_keys + other_keys - metadata = dict(zip(all_keys, - mydbapi.aux_get(mysettings.mycpv, all_keys))) - - class FakeTree(object): - def __init__(self, mydb): - self.dbapi = mydb - dep_check_trees = {myroot:{}} - dep_check_trees[myroot]["porttree"] = \ - FakeTree(fakedbapi(settings=mysettings)) - - msgs = [] - for dep_type in dep_keys: - mycheck = dep_check(metadata[dep_type], None, mysettings, - myuse="all", myroot=myroot, trees=dep_check_trees) - if not mycheck[0]: - msgs.append(" %s: %s\n %s\n" % ( - dep_type, metadata[dep_type], mycheck[1])) - - for k in misc_keys: - try: - portage.dep.use_reduce( - portage.dep.paren_reduce(metadata[k]), matchall=True) - except portage.exception.InvalidDependString as e: - msgs.append(" %s: %s\n %s\n" % ( - k, metadata[k], str(e))) - - if not metadata["SLOT"]: - msgs.append(_(" SLOT is undefined\n")) - - if msgs: - portage.util.writemsg_level(_("Error(s) in metadata for '%s':\n") % \ - (mysettings.mycpv,), level=logging.ERROR, noiselevel=-1) - for x in msgs: - portage.util.writemsg_level(x, - level=logging.ERROR, noiselevel=-1) - if mydo not in invalid_dep_exempt_phases: - return 1 - - return os.EX_OK - def _movefile(src, dest, **kwargs): """Calls movefile and raises a PortageException if an error occurs.""" if movefile(src, dest, **kwargs) is None: diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py index 9970ccab..24077280 100644 --- a/pym/portage/dbapi/bintree.py +++ b/pym/portage/dbapi/bintree.py @@ -10,6 +10,7 @@ import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.dep:dep_getkey,isjustname,match_from_list', 'portage.output:EOutput,colorize', + 'portage.package.ebuild.doebuild:_vdb_use_conditional_atoms', 'portage.update:update_dbentries', 'portage.util:ensure_dirs,normalize_path,writemsg,writemsg_stdout', 'portage.versions:best,catpkgsplit,catsplit', @@ -1065,7 +1066,7 @@ class binarytree(object): writemsg("%s: %s\n" % (k, str(e)), noiselevel=-1) raise - if k in portage._vdb_use_conditional_atoms: + if k in _vdb_use_conditional_atoms: v_split = [] for x in deps.split(): try: diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index d2615cc7..cb849c43 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -16,6 +16,9 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.elog:elog_process', 'portage.locks:lockdir,unlockdir', 'portage.output:bold,colorize', + 'portage.package.ebuild.doebuild:doebuild,doebuild_environment,' + \ + '_spawn_misc_sh', + 'portage.package.ebuild.prepare_build_dirs:prepare_build_dirs', 'portage.update:fixdbentries', 'portage.util:apply_secpass_permissions,ConfigProtect,ensure_dirs,' + \ 'writemsg,writemsg_level,write_atomic,atomic_ofstream,writedict,' + \ @@ -33,7 +36,7 @@ from portage.exception import CommandNotFound, \ from portage.localization import _ from portage import listdir, dep_expand, digraph, flatten, \ - doebuild_environment, doebuild, env_update, prepare_build_dirs, \ + env_update, \ abssymlink, movefile, _movefile, bsd_chflags, cpv_getkey # This is a special version of the os module, wrapped for unicode support. diff --git a/pym/portage/package/ebuild/_pty.py b/pym/portage/package/ebuild/_pty.py new file mode 100644 index 00000000..7cc1e442 --- /dev/null +++ b/pym/portage/package/ebuild/_pty.py @@ -0,0 +1,192 @@ +# Copyright 2010 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Id$ + +import array +import fcntl +import platform +import pty +import select +import sys +import termios + +from portage import os, _unicode_decode, _unicode_encode +from portage.process import spawn_bash +from portage.util import writemsg + +def _can_test_pty_eof(): + """ + The _test_pty_eof() function seems to hang on most + kernels other than Linux. + This was reported for the following kernels which used to work fine + without this EOF test: Darwin, AIX, FreeBSD. They seem to hang on + the slave_file.close() call. Note that Python's implementation of + openpty on Solaris already caused random hangs without this EOF test + and hence is globally disabled. + @rtype: bool + @returns: True if _test_pty_eof() won't hang, False otherwise. + """ + return platform.system() in ("Linux",) + +def _test_pty_eof(): + """ + Returns True if this issues is fixed for the currently + running version of python: http://bugs.python.org/issue5380 + Raises an EnvironmentError from openpty() if it fails. + """ + + use_fork = False + + test_string = 2 * "blah blah blah\n" + test_string = _unicode_decode(test_string, + encoding='utf_8', errors='strict') + + # may raise EnvironmentError + master_fd, slave_fd = pty.openpty() + + # Non-blocking mode is required for Darwin kernel. + fcntl.fcntl(master_fd, fcntl.F_SETFL, + fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK) + + # Disable post-processing of output since otherwise weird + # things like \n -> \r\n transformations may occur. + mode = termios.tcgetattr(slave_fd) + mode[1] &= ~termios.OPOST + termios.tcsetattr(slave_fd, termios.TCSANOW, mode) + + # Simulate a subprocess writing some data to the + # slave end of the pipe, and then exiting. + pid = None + if use_fork: + pids = spawn_bash(_unicode_encode("echo -n '%s'" % test_string, + encoding='utf_8', errors='strict'), env=os.environ, + fd_pipes={0:sys.stdin.fileno(), 1:slave_fd, 2:slave_fd}, + returnpid=True) + if isinstance(pids, int): + os.close(master_fd) + os.close(slave_fd) + raise EnvironmentError('spawn failed') + pid = pids[0] + else: + os.write(slave_fd, _unicode_encode(test_string, + encoding='utf_8', errors='strict')) + os.close(slave_fd) + + # If using a fork, we must wait for the child here, + # in order to avoid a race condition that would + # lead to inconsistent results. + if pid is not None: + os.waitpid(pid, 0) + + master_file = os.fdopen(master_fd, 'rb') + eof = False + data = [] + iwtd = [master_file] + owtd = [] + ewtd = [] + + while not eof: + + events = select.select(iwtd, owtd, ewtd) + if not events[0]: + eof = True + break + + buf = array.array('B') + try: + buf.fromfile(master_file, 1024) + except EOFError: + eof = True + except IOError: + # This is where data loss occurs. + eof = True + + if not buf: + eof = True + else: + data.append(_unicode_decode(buf.tostring(), + encoding='utf_8', errors='strict')) + + master_file.close() + + return test_string == ''.join(data) + +# If _test_pty_eof() can't be used for runtime detection of +# http://bugs.python.org/issue5380, openpty can't safely be used +# unless we can guarantee that the current version of python has +# been fixed (affects all current versions of python3). When +# this issue is fixed in python3, we can add another sys.hexversion +# conditional to enable openpty support in the fixed versions. +if sys.hexversion >= 0x3000000 and not _can_test_pty_eof(): + _disable_openpty = True +else: + # Disable the use of openpty on Solaris as it seems Python's openpty + # implementation doesn't play nice on Solaris with Portage's + # behaviour causing hangs/deadlocks. + # Additional note for the future: on Interix, pipes do NOT work, so + # _disable_openpty on Interix must *never* be True + _disable_openpty = platform.system() in ("SunOS",) +_tested_pty = False + +if not _can_test_pty_eof(): + # Skip _test_pty_eof() on systems where it hangs. + _tested_pty = True + +def _create_pty_or_pipe(copy_term_size=None): + """ + Try to create a pty and if then fails then create a normal + pipe instead. + + @param copy_term_size: If a tty file descriptor is given + then the term size will be copied to the pty. + @type copy_term_size: int + @rtype: tuple + @returns: A tuple of (is_pty, master_fd, slave_fd) where + is_pty is True if a pty was successfully allocated, and + False if a normal pipe was allocated. + """ + + got_pty = False + + global _disable_openpty, _tested_pty + if not (_tested_pty or _disable_openpty): + try: + if not _test_pty_eof(): + _disable_openpty = True + except EnvironmentError as e: + _disable_openpty = True + writemsg("openpty failed: '%s'\n" % str(e), + noiselevel=-1) + del e + _tested_pty = True + + if _disable_openpty: + master_fd, slave_fd = os.pipe() + else: + from pty import openpty + try: + master_fd, slave_fd = openpty() + got_pty = True + except EnvironmentError as e: + _disable_openpty = True + writemsg("openpty failed: '%s'\n" % str(e), + noiselevel=-1) + del e + master_fd, slave_fd = os.pipe() + + if got_pty: + # Disable post-processing of output since otherwise weird + # things like \n -> \r\n transformations may occur. + import termios + mode = termios.tcgetattr(slave_fd) + mode[1] &= ~termios.OPOST + termios.tcsetattr(slave_fd, termios.TCSANOW, mode) + + if got_pty and \ + copy_term_size is not None and \ + os.isatty(copy_term_size): + from portage.output import get_term_size, set_term_size + rows, columns = get_term_size() + set_term_size(rows, columns, slave_fd) + + return (got_pty, master_fd, slave_fd) diff --git a/pym/portage/package/ebuild/doebuild.py b/pym/portage/package/ebuild/doebuild.py new file mode 100644 index 00000000..d1595b1f --- /dev/null +++ b/pym/portage/package/ebuild/doebuild.py @@ -0,0 +1,1759 @@ +# Copyright 2010 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Id$ + +from __future__ import print_function + +__all__ = ['doebuild', 'doebuild_environment', 'spawn', 'spawnebuild'] + +import codecs +import errno +from itertools import chain +import logging +import os as _os +import re +import shutil +import stat +import sys +from textwrap import wrap +import time + +import portage +portage.proxy.lazyimport.lazyimport(globals(), + 'portage.package.ebuild.config:check_config_instance', +) + +from portage import auxdbkeys, bsd_chflags, dep_check, digestcheck, digestgen, eapi_is_supported, ExtractKernelVersion, merge, os, selinux, StringIO, unmerge, _encodings, _parse_eapi_ebuild_head, _os_merge, _shell_quote, _split_ebuild_name_glep55, _unicode_decode, _unicode_encode +from portage.const import EBUILD_SH_ENV_FILE, EBUILD_SH_BINARY, INVALID_ENV_FILE, MISC_SH_BINARY +from portage.data import portage_gid, portage_uid, secpass, uid, userpriv_groups +from portage.dbapi.virtual import fakedbapi +from portage.dep import Atom, paren_enclose, paren_normalize, paren_reduce, use_reduce +from portage.elog import elog_process +from portage.elog.messages import eerror, eqawarn +from portage.exception import DigestException, FileNotFound, IncorrectParameter, InvalidAtom, InvalidDependString, PermissionDenied, UnsupportedAPIException +from portage.localization import _ +from portage.manifest import Manifest +from portage.output import style_to_ansi_code +from portage.package.ebuild.fetch import fetch +from portage.package.ebuild.prepare_build_dirs import prepare_build_dirs +from portage.package.ebuild._pty import _create_pty_or_pipe +from portage.util import apply_recursive_permissions, apply_secpass_permissions, noiselimit, normalize_path, writemsg, writemsg_stdout, write_atomic +from portage.versions import _pkgsplit + +def doebuild_environment(myebuild, mydo, myroot, mysettings, + debug, use_cache, mydbapi): + + ebuild_path = os.path.abspath(myebuild) + pkg_dir = os.path.dirname(ebuild_path) + + if "CATEGORY" in mysettings.configdict["pkg"]: + cat = mysettings.configdict["pkg"]["CATEGORY"] + else: + cat = os.path.basename(normalize_path(os.path.join(pkg_dir, ".."))) + + eapi = None + if 'parse-eapi-glep-55' in mysettings.features: + mypv, eapi = _split_ebuild_name_glep55( + os.path.basename(myebuild)) + else: + mypv = os.path.basename(ebuild_path)[:-7] + + mycpv = cat+"/"+mypv + mysplit = _pkgsplit(mypv) + if mysplit is None: + raise IncorrectParameter( + _("Invalid ebuild path: '%s'") % myebuild) + + # Make a backup of PORTAGE_TMPDIR prior to calling config.reset() + # so that the caller can override it. + tmpdir = mysettings["PORTAGE_TMPDIR"] + + if mydo == 'depend': + if mycpv != mysettings.mycpv: + # Don't pass in mydbapi here since the resulting aux_get + # call would lead to infinite 'depend' phase recursion. + mysettings.setcpv(mycpv) + else: + # If IUSE isn't in configdict['pkg'], it means that setcpv() + # hasn't been called with the mydb argument, so we have to + # call it here (portage code always calls setcpv properly, + # but api consumers might not). + if mycpv != mysettings.mycpv or \ + 'IUSE' not in mysettings.configdict['pkg']: + # Reload env.d variables and reset any previous settings. + mysettings.reload() + mysettings.reset() + mysettings.setcpv(mycpv, mydb=mydbapi) + + # config.reset() might have reverted a change made by the caller, + # so restore it to it's original value. + mysettings["PORTAGE_TMPDIR"] = tmpdir + + mysettings.pop("EBUILD_PHASE", None) # remove from backupenv + mysettings["EBUILD_PHASE"] = mydo + + mysettings["PORTAGE_MASTER_PID"] = str(os.getpid()) + + # We are disabling user-specific bashrc files. + mysettings["BASH_ENV"] = INVALID_ENV_FILE + + if debug: # Otherwise it overrides emerge's settings. + # We have no other way to set debug... debug can't be passed in + # due to how it's coded... Don't overwrite this so we can use it. + mysettings["PORTAGE_DEBUG"] = "1" + + mysettings["EBUILD"] = ebuild_path + mysettings["O"] = pkg_dir + mysettings.configdict["pkg"]["CATEGORY"] = cat + mysettings["FILESDIR"] = pkg_dir+"/files" + mysettings["PF"] = mypv + + if hasattr(mydbapi, '_repo_info'): + mytree = os.path.dirname(os.path.dirname(pkg_dir)) + repo_info = mydbapi._repo_info[mytree] + mysettings['PORTDIR'] = repo_info.portdir + mysettings['PORTDIR_OVERLAY'] = repo_info.portdir_overlay + + mysettings["PORTDIR"] = os.path.realpath(mysettings["PORTDIR"]) + mysettings["DISTDIR"] = os.path.realpath(mysettings["DISTDIR"]) + mysettings["RPMDIR"] = os.path.realpath(mysettings["RPMDIR"]) + + mysettings["ECLASSDIR"] = mysettings["PORTDIR"]+"/eclass" + mysettings["SANDBOX_LOG"] = mycpv.replace("/", "_-_") + + mysettings["PROFILE_PATHS"] = "\n".join(mysettings.profiles) + mysettings["P"] = mysplit[0]+"-"+mysplit[1] + mysettings["PN"] = mysplit[0] + mysettings["PV"] = mysplit[1] + mysettings["PR"] = mysplit[2] + + if noiselimit < 0: + mysettings["PORTAGE_QUIET"] = "1" + + if mydo == 'depend' and \ + 'EAPI' not in mysettings.configdict['pkg']: + + if eapi is not None: + # From parse-eapi-glep-55 above. + pass + elif 'parse-eapi-ebuild-head' in mysettings.features: + eapi = _parse_eapi_ebuild_head( + codecs.open(_unicode_encode(ebuild_path, + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['content'], errors='replace')) + + if eapi is not None: + if not eapi_is_supported(eapi): + raise UnsupportedAPIException(mycpv, eapi) + mysettings.configdict['pkg']['EAPI'] = eapi + + if mydo != "depend": + # Metadata vars such as EAPI and RESTRICT are + # set by the above config.setcpv() call. + eapi = mysettings["EAPI"] + if not eapi_is_supported(eapi): + # can't do anything with this. + raise UnsupportedAPIException(mycpv, eapi) + + if mysplit[2] == "r0": + mysettings["PVR"]=mysplit[1] + else: + mysettings["PVR"]=mysplit[1]+"-"+mysplit[2] + + if "PATH" in mysettings: + mysplit=mysettings["PATH"].split(":") + else: + mysplit=[] + # Note: PORTAGE_BIN_PATH may differ from the global constant + # when portage is reinstalling itself. + portage_bin_path = mysettings["PORTAGE_BIN_PATH"] + if portage_bin_path not in mysplit: + mysettings["PATH"] = portage_bin_path + ":" + mysettings["PATH"] + + # Sandbox needs cannonical paths. + mysettings["PORTAGE_TMPDIR"] = os.path.realpath( + mysettings["PORTAGE_TMPDIR"]) + mysettings["BUILD_PREFIX"] = mysettings["PORTAGE_TMPDIR"]+"/portage" + mysettings["PKG_TMPDIR"] = mysettings["PORTAGE_TMPDIR"]+"/binpkgs" + + # Package {pre,post}inst and {pre,post}rm may overlap, so they must have separate + # locations in order to prevent interference. + if mydo in ("unmerge", "prerm", "postrm", "cleanrm"): + mysettings["PORTAGE_BUILDDIR"] = os.path.join( + mysettings["PKG_TMPDIR"], + mysettings["CATEGORY"], mysettings["PF"]) + else: + mysettings["PORTAGE_BUILDDIR"] = os.path.join( + mysettings["BUILD_PREFIX"], + mysettings["CATEGORY"], mysettings["PF"]) + + mysettings["HOME"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], "homedir") + mysettings["WORKDIR"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], "work") + mysettings["D"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], "image") + os.sep + mysettings["T"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], "temp") + + # Prefix forward compatability + mysettings["ED"] = mysettings["D"] + + mysettings["PORTAGE_BASHRC"] = os.path.join( + mysettings["PORTAGE_CONFIGROOT"], EBUILD_SH_ENV_FILE) + mysettings["EBUILD_EXIT_STATUS_FILE"] = os.path.join( + mysettings["PORTAGE_BUILDDIR"], ".exit_status") + + #set up KV variable -- DEP SPEEDUP :: Don't waste time. Keep var persistent. + if eapi not in ('0', '1', '2', '3', '3_pre2'): + # Discard KV for EAPIs that don't support it. Cache KV is restored + # from the backupenv whenever config.reset() is called. + mysettings.pop('KV', None) + elif mydo != 'depend' and 'KV' not in mysettings and \ + mydo in ('compile', 'config', 'configure', 'info', + 'install', 'nofetch', 'postinst', 'postrm', 'preinst', + 'prepare', 'prerm', 'setup', 'test', 'unpack'): + mykv,err1=ExtractKernelVersion(os.path.join(myroot, "usr/src/linux")) + if mykv: + # Regular source tree + mysettings["KV"]=mykv + else: + mysettings["KV"]="" + mysettings.backup_changes("KV") + + # Allow color.map to control colors associated with einfo, ewarn, etc... + mycolors = [] + for c in ("GOOD", "WARN", "BAD", "HILITE", "BRACKET"): + mycolors.append("%s=$'%s'" % \ + (c, style_to_ansi_code(c))) + mysettings["PORTAGE_COLORMAP"] = "\n".join(mycolors) + +def _doebuild_exit_status_check(mydo, settings): + """ + Returns an error string if the shell appeared + to exit unsuccessfully, None otherwise. + """ + exit_status_file = settings.get("EBUILD_EXIT_STATUS_FILE") + if not exit_status_file or \ + os.path.exists(exit_status_file): + return None + msg = _("The ebuild phase '%s' has exited " + "unexpectedly. This type of behavior " + "is known to be triggered " + "by things such as failed variable " + "assignments (bug #190128) or bad substitution " + "errors (bug #200313). Normally, before exiting, bash should " + "have displayed an error message above. If bash did not " + "produce an error message above, it's possible " + "that the ebuild has called `exit` when it " + "should have called `die` instead. This behavior may also " + "be triggered by a corrupt bash binary or a hardware " + "problem such as memory or cpu malfunction. If the problem is not " + "reproducible or it appears to occur randomly, then it is likely " + "to be triggered by a hardware problem. " + "If you suspect a hardware problem then you should " + "try some basic hardware diagnostics such as memtest. " + "Please do not report this as a bug unless it is consistently " + "reproducible and you are sure that your bash binary and hardware " + "are functioning properly.") % mydo + return msg + +def _doebuild_exit_status_check_and_log(settings, mydo, retval): + msg = _doebuild_exit_status_check(mydo, settings) + if msg: + if retval == os.EX_OK: + retval = 1 + for l in wrap(msg, 72): + eerror(l, phase=mydo, key=settings.mycpv) + return retval + +def _doebuild_exit_status_unlink(exit_status_file): + """ + Double check to make sure it really doesn't exist + and raise an OSError if it still does (it shouldn't). + OSError if necessary. + """ + if not exit_status_file: + return + try: + os.unlink(exit_status_file) + except OSError: + pass + if os.path.exists(exit_status_file): + os.unlink(exit_status_file) + +_doebuild_manifest_exempt_depend = 0 +_doebuild_manifest_cache = None +_doebuild_broken_ebuilds = set() +_doebuild_broken_manifests = set() + +def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, + fetchonly=0, cleanup=0, dbkey=None, use_cache=1, fetchall=0, tree=None, + mydbapi=None, vartree=None, prev_mtimes=None, + fd_pipes=None, returnpid=False): + + """ + Wrapper function that invokes specific ebuild phases through the spawning + of ebuild.sh + + @param myebuild: name of the ebuild to invoke the phase on (CPV) + @type myebuild: String + @param mydo: Phase to run + @type mydo: String + @param myroot: $ROOT (usually '/', see man make.conf) + @type myroot: String + @param mysettings: Portage Configuration + @type mysettings: instance of portage.config + @param debug: Turns on various debug information (eg, debug for spawn) + @type debug: Boolean + @param listonly: Used to wrap fetch(); passed such that fetch only lists files required. + @type listonly: Boolean + @param fetchonly: Used to wrap fetch(); passed such that files are only fetched (no other actions) + @type fetchonly: Boolean + @param cleanup: Passed to prepare_build_dirs (TODO: what does it do?) + @type cleanup: Boolean + @param dbkey: A dict (usually keys and values from the depend phase, such as KEYWORDS, USE, etc..) + @type dbkey: Dict or String + @param use_cache: Enables the cache + @type use_cache: Boolean + @param fetchall: Used to wrap fetch(), fetches all URIs (even ones invalid due to USE conditionals) + @type fetchall: Boolean + @param tree: Which tree to use ('vartree','porttree','bintree', etc..), defaults to 'porttree' + @type tree: String + @param mydbapi: a dbapi instance to pass to various functions; this should be a portdbapi instance. + @type mydbapi: portdbapi instance + @param vartree: A instance of vartree; used for aux_get calls, defaults to db[myroot]['vartree'] + @type vartree: vartree instance + @param prev_mtimes: A dict of { filename:mtime } keys used by merge() to do config_protection + @type prev_mtimes: dictionary + @param fd_pipes: A dict of mapping for pipes, { '0': stdin, '1': stdout } + for example. + @type fd_pipes: Dictionary + @param returnpid: Return a list of process IDs for a successful spawn, or + an integer value if spawn is unsuccessful. NOTE: This requires the + caller clean up all returned PIDs. + @type returnpid: Boolean + @rtype: Boolean + @returns: + 1. 0 for success + 2. 1 for error + + Most errors have an accompanying error message. + + listonly and fetchonly are only really necessary for operations involving 'fetch' + prev_mtimes are only necessary for merge operations. + Other variables may not be strictly required, many have defaults that are set inside of doebuild. + + """ + + if not tree: + writemsg("Warning: tree not specified to doebuild\n") + tree = "porttree" + + # chunked out deps for each phase, so that ebuild binary can use it + # to collapse targets down. + actionmap_deps={ + "setup": [], + "unpack": ["setup"], + "prepare": ["unpack"], + "configure": ["prepare"], + "compile":["configure"], + "test": ["compile"], + "install":["test"], + "rpm": ["install"], + "package":["install"], + } + + if mydbapi is None: + mydbapi = portage.db[myroot][tree].dbapi + + if vartree is None and mydo in ("merge", "qmerge", "unmerge"): + vartree = portage.db[myroot]["vartree"] + + features = mysettings.features + + clean_phases = ("clean", "cleanrm") + validcommands = ["help","clean","prerm","postrm","cleanrm","preinst","postinst", + "config", "info", "setup", "depend", "pretend", + "fetch", "fetchall", "digest", + "unpack", "prepare", "configure", "compile", "test", + "install", "rpm", "qmerge", "merge", + "package","unmerge", "manifest"] + + if mydo not in validcommands: + validcommands.sort() + writemsg("!!! doebuild: '%s' is not one of the following valid commands:" % mydo, + noiselevel=-1) + for vcount in range(len(validcommands)): + if vcount%6 == 0: + writemsg("\n!!! ", noiselevel=-1) + writemsg(validcommands[vcount].ljust(11), noiselevel=-1) + writemsg("\n", noiselevel=-1) + return 1 + + if mydo == "fetchall": + fetchall = 1 + mydo = "fetch" + + parallel_fetchonly = mydo in ("fetch", "fetchall") and \ + "PORTAGE_PARALLEL_FETCHONLY" in mysettings + + if mydo not in clean_phases and not os.path.exists(myebuild): + writemsg("!!! doebuild: %s not found for %s\n" % (myebuild, mydo), + noiselevel=-1) + return 1 + + global _doebuild_manifest_exempt_depend + + if "strict" in features and \ + "digest" not in features and \ + tree == "porttree" and \ + mydo not in ("digest", "manifest", "help") and \ + not _doebuild_manifest_exempt_depend: + # Always verify the ebuild checksums before executing it. + global _doebuild_manifest_cache, _doebuild_broken_ebuilds, \ + _doebuild_broken_ebuilds + + if myebuild in _doebuild_broken_ebuilds: + return 1 + + pkgdir = os.path.dirname(myebuild) + manifest_path = os.path.join(pkgdir, "Manifest") + + # Avoid checking the same Manifest several times in a row during a + # regen with an empty cache. + if _doebuild_manifest_cache is None or \ + _doebuild_manifest_cache.getFullname() != manifest_path: + _doebuild_manifest_cache = None + if not os.path.exists(manifest_path): + out = portage.output.EOutput() + out.eerror(_("Manifest not found for '%s'") % (myebuild,)) + _doebuild_broken_ebuilds.add(myebuild) + return 1 + mf = Manifest(pkgdir, mysettings["DISTDIR"]) + + else: + mf = _doebuild_manifest_cache + + try: + mf.checkFileHashes("EBUILD", os.path.basename(myebuild)) + except KeyError: + out = portage.output.EOutput() + out.eerror(_("Missing digest for '%s'") % (myebuild,)) + _doebuild_broken_ebuilds.add(myebuild) + return 1 + except FileNotFound: + out = portage.output.EOutput() + out.eerror(_("A file listed in the Manifest " + "could not be found: '%s'") % (myebuild,)) + _doebuild_broken_ebuilds.add(myebuild) + return 1 + except DigestException as e: + out = portage.output.EOutput() + out.eerror(_("Digest verification failed:")) + out.eerror("%s" % e.value[0]) + out.eerror(_("Reason: %s") % e.value[1]) + out.eerror(_("Got: %s") % e.value[2]) + out.eerror(_("Expected: %s") % e.value[3]) + _doebuild_broken_ebuilds.add(myebuild) + return 1 + + if mf.getFullname() in _doebuild_broken_manifests: + return 1 + + if mf is not _doebuild_manifest_cache: + + # Make sure that all of the ebuilds are + # actually listed in the Manifest. + glep55 = 'parse-eapi-glep-55' in mysettings.features + for f in os.listdir(pkgdir): + pf = None + if glep55: + pf, eapi = _split_ebuild_name_glep55(f) + elif f[-7:] == '.ebuild': + pf = f[:-7] + if pf is not None and not mf.hasFile("EBUILD", f): + f = os.path.join(pkgdir, f) + if f not in _doebuild_broken_ebuilds: + out = portage.output.EOutput() + out.eerror(_("A file is not listed in the " + "Manifest: '%s'") % (f,)) + _doebuild_broken_manifests.add(manifest_path) + return 1 + + # Only cache it if the above stray files test succeeds. + _doebuild_manifest_cache = mf + + def exit_status_check(retval): + msg = _doebuild_exit_status_check(mydo, mysettings) + if msg: + if retval == os.EX_OK: + retval = 1 + for l in wrap(msg, 72): + eerror(l, phase=mydo, key=mysettings.mycpv) + return retval + + # Note: PORTAGE_BIN_PATH may differ from the global + # constant when portage is reinstalling itself. + portage_bin_path = mysettings["PORTAGE_BIN_PATH"] + ebuild_sh_binary = os.path.join(portage_bin_path, + os.path.basename(EBUILD_SH_BINARY)) + misc_sh_binary = os.path.join(portage_bin_path, + os.path.basename(MISC_SH_BINARY)) + + logfile=None + builddir_lock = None + tmpdir = None + tmpdir_orig = None + + try: + if mydo in ("digest", "manifest", "help"): + # Temporarily exempt the depend phase from manifest checks, in case + # aux_get calls trigger cache generation. + _doebuild_manifest_exempt_depend += 1 + + # If we don't need much space and we don't need a constant location, + # we can temporarily override PORTAGE_TMPDIR with a random temp dir + # so that there's no need for locking and it can be used even if the + # user isn't in the portage group. + if mydo in ("info",): + from tempfile import mkdtemp + tmpdir = mkdtemp() + tmpdir_orig = mysettings["PORTAGE_TMPDIR"] + mysettings["PORTAGE_TMPDIR"] = tmpdir + + doebuild_environment(myebuild, mydo, myroot, mysettings, debug, + use_cache, mydbapi) + + if mydo in clean_phases: + retval = spawn(_shell_quote(ebuild_sh_binary) + " clean", + mysettings, debug=debug, fd_pipes=fd_pipes, free=1, + logfile=None, returnpid=returnpid) + return retval + + restrict = set(mysettings.get('PORTAGE_RESTRICT', '').split()) + # get possible slot information from the deps file + if mydo == "depend": + writemsg("!!! DEBUG: dbkey: %s\n" % str(dbkey), 2) + droppriv = "userpriv" in mysettings.features + if returnpid: + mypids = spawn(_shell_quote(ebuild_sh_binary) + " depend", + mysettings, fd_pipes=fd_pipes, returnpid=True, + droppriv=droppriv) + return mypids + elif isinstance(dbkey, dict): + mysettings["dbkey"] = "" + pr, pw = os.pipe() + fd_pipes = { + 0:sys.stdin.fileno(), + 1:sys.stdout.fileno(), + 2:sys.stderr.fileno(), + 9:pw} + mypids = spawn(_shell_quote(ebuild_sh_binary) + " depend", + mysettings, + fd_pipes=fd_pipes, returnpid=True, droppriv=droppriv) + os.close(pw) # belongs exclusively to the child process now + f = os.fdopen(pr, 'rb') + for k, v in zip(auxdbkeys, + (_unicode_decode(line).rstrip('\n') for line in f)): + dbkey[k] = v + f.close() + retval = os.waitpid(mypids[0], 0)[1] + portage.process.spawned_pids.remove(mypids[0]) + # If it got a signal, return the signal that was sent, but + # shift in order to distinguish it from a return value. (just + # like portage.process.spawn() would do). + if retval & 0xff: + retval = (retval & 0xff) << 8 + else: + # Otherwise, return its exit code. + retval = retval >> 8 + if retval == os.EX_OK and len(dbkey) != len(auxdbkeys): + # Don't trust bash's returncode if the + # number of lines is incorrect. + retval = 1 + return retval + elif dbkey: + mysettings["dbkey"] = dbkey + else: + mysettings["dbkey"] = \ + os.path.join(mysettings.depcachedir, "aux_db_key_temp") + + return spawn(_shell_quote(ebuild_sh_binary) + " depend", + mysettings, + droppriv=droppriv) + + # Validate dependency metadata here to ensure that ebuilds with invalid + # data are never installed via the ebuild command. Don't bother when + # returnpid == True since there's no need to do this every time emerge + # executes a phase. + if not returnpid: + rval = _validate_deps(mysettings, myroot, mydo, mydbapi) + if rval != os.EX_OK: + return rval + + if "PORTAGE_TMPDIR" not in mysettings or \ + not os.path.isdir(mysettings["PORTAGE_TMPDIR"]): + writemsg(_("The directory specified in your " + "PORTAGE_TMPDIR variable, '%s',\n" + "does not exist. Please create this directory or " + "correct your PORTAGE_TMPDIR setting.\n") % mysettings.get("PORTAGE_TMPDIR", ""), noiselevel=-1) + return 1 + + # as some people use a separate PORTAGE_TMPDIR mount + # we prefer that as the checks below would otherwise be pointless + # for those people. + if os.path.exists(os.path.join(mysettings["PORTAGE_TMPDIR"], "portage")): + checkdir = os.path.join(mysettings["PORTAGE_TMPDIR"], "portage") + else: + checkdir = mysettings["PORTAGE_TMPDIR"] + + if not os.access(checkdir, os.W_OK): + writemsg(_("%s is not writable.\n" + "Likely cause is that you've mounted it as readonly.\n") % checkdir, + noiselevel=-1) + return 1 + else: + from tempfile import NamedTemporaryFile + fd = NamedTemporaryFile(prefix="exectest-", dir=checkdir) + os.chmod(fd.name, 0o755) + if not os.access(fd.name, os.X_OK): + writemsg(_("Can not execute files in %s\n" + "Likely cause is that you've mounted it with one of the\n" + "following mount options: 'noexec', 'user', 'users'\n\n" + "Please make sure that portage can execute files in this directory.\n") % checkdir, + noiselevel=-1) + fd.close() + return 1 + fd.close() + del checkdir + + if mydo == "unmerge": + return unmerge(mysettings["CATEGORY"], + mysettings["PF"], myroot, mysettings, vartree=vartree) + + # Build directory creation isn't required for any of these. + # In the fetch phase, the directory is needed only for RESTRICT=fetch + # in order to satisfy the sane $PWD requirement (from bug #239560) + # when pkg_nofetch is spawned. + have_build_dirs = False + if not parallel_fetchonly and \ + mydo not in ('digest', 'help', 'manifest') and \ + not (mydo == 'fetch' and 'fetch' not in restrict): + mystatus = prepare_build_dirs(myroot, mysettings, cleanup) + if mystatus: + return mystatus + have_build_dirs = True + + # emerge handles logging externally + if not returnpid: + # PORTAGE_LOG_FILE is set by the + # above prepare_build_dirs() call. + logfile = mysettings.get("PORTAGE_LOG_FILE") + + if have_build_dirs: + env_file = os.path.join(mysettings["T"], "environment") + env_stat = None + saved_env = None + try: + env_stat = os.stat(env_file) + except OSError as e: + if e.errno != errno.ENOENT: + raise + del e + if not env_stat: + saved_env = os.path.join( + os.path.dirname(myebuild), "environment.bz2") + if not os.path.isfile(saved_env): + saved_env = None + if saved_env: + retval = os.system( + "bzip2 -dc %s > %s" % \ + (_shell_quote(saved_env), + _shell_quote(env_file))) + try: + env_stat = os.stat(env_file) + except OSError as e: + if e.errno != errno.ENOENT: + raise + del e + if os.WIFEXITED(retval) and \ + os.WEXITSTATUS(retval) == os.EX_OK and \ + env_stat and env_stat.st_size > 0: + # This is a signal to ebuild.sh, so that it knows to filter + # out things like SANDBOX_{DENY,PREDICT,READ,WRITE} that + # would be preserved between normal phases. + open(_unicode_encode(env_file + '.raw'), 'w') + else: + writemsg(_("!!! Error extracting saved " + "environment: '%s'\n") % \ + saved_env, noiselevel=-1) + try: + os.unlink(env_file) + except OSError as e: + if e.errno != errno.ENOENT: + raise + del e + env_stat = None + if env_stat: + pass + else: + for var in ("ARCH", ): + value = mysettings.get(var) + if value and value.strip(): + continue + msg = _("%(var)s is not set... " + "Are you missing the '%(configroot)setc/make.profile' symlink? " + "Is the symlink correct? " + "Is your portage tree complete?") % \ + {"var": var, "configroot": mysettings["PORTAGE_CONFIGROOT"]} + for line in wrap(msg, 70): + eerror(line, phase="setup", key=mysettings.mycpv) + elog_process(mysettings.mycpv, mysettings) + return 1 + del env_file, env_stat, saved_env + _doebuild_exit_status_unlink( + mysettings.get("EBUILD_EXIT_STATUS_FILE")) + else: + mysettings.pop("EBUILD_EXIT_STATUS_FILE", None) + + # if any of these are being called, handle them -- running them out of + # the sandbox -- and stop now. + if mydo == "help": + return spawn(_shell_quote(ebuild_sh_binary) + " " + mydo, + mysettings, debug=debug, free=1, logfile=logfile) + elif mydo == "setup": + retval = spawn( + _shell_quote(ebuild_sh_binary) + " " + mydo, mysettings, + debug=debug, free=1, logfile=logfile, fd_pipes=fd_pipes, + returnpid=returnpid) + if returnpid: + return retval + retval = exit_status_check(retval) + if secpass >= 2: + """ Privileged phases may have left files that need to be made + writable to a less privileged user.""" + apply_recursive_permissions(mysettings["T"], + uid=portage_uid, gid=portage_gid, dirmode=0o70, dirmask=0, + filemode=0o60, filemask=0) + return retval + elif mydo == "preinst": + phase_retval = spawn( + _shell_quote(ebuild_sh_binary) + " " + mydo, + mysettings, debug=debug, free=1, logfile=logfile, + fd_pipes=fd_pipes, returnpid=returnpid) + + if returnpid: + return phase_retval + + phase_retval = exit_status_check(phase_retval) + if phase_retval == os.EX_OK: + _doebuild_exit_status_unlink( + mysettings.get("EBUILD_EXIT_STATUS_FILE")) + mysettings.pop("EBUILD_PHASE", None) + phase_retval = spawn( + " ".join(_post_pkg_preinst_cmd(mysettings)), + mysettings, debug=debug, free=1, logfile=logfile) + phase_retval = exit_status_check(phase_retval) + if phase_retval != os.EX_OK: + writemsg(_("!!! post preinst failed; exiting.\n"), + noiselevel=-1) + return phase_retval + elif mydo == "postinst": + phase_retval = spawn( + _shell_quote(ebuild_sh_binary) + " " + mydo, + mysettings, debug=debug, free=1, logfile=logfile, + fd_pipes=fd_pipes, returnpid=returnpid) + + if returnpid: + return phase_retval + + phase_retval = exit_status_check(phase_retval) + if phase_retval == os.EX_OK: + _doebuild_exit_status_unlink( + mysettings.get("EBUILD_EXIT_STATUS_FILE")) + mysettings.pop("EBUILD_PHASE", None) + phase_retval = spawn(" ".join(_post_pkg_postinst_cmd(mysettings)), + mysettings, debug=debug, free=1, logfile=logfile) + phase_retval = exit_status_check(phase_retval) + if phase_retval != os.EX_OK: + writemsg(_("!!! post postinst failed; exiting.\n"), + noiselevel=-1) + return phase_retval + elif mydo in ("prerm", "postrm", "config", "info"): + retval = spawn( + _shell_quote(ebuild_sh_binary) + " " + mydo, + mysettings, debug=debug, free=1, logfile=logfile, + fd_pipes=fd_pipes, returnpid=returnpid) + + if returnpid: + return retval + + retval = exit_status_check(retval) + return retval + + mycpv = "/".join((mysettings["CATEGORY"], mysettings["PF"])) + + emerge_skip_distfiles = returnpid + emerge_skip_digest = returnpid + # Only try and fetch the files if we are going to need them ... + # otherwise, if user has FEATURES=noauto and they run `ebuild clean + # unpack compile install`, we will try and fetch 4 times :/ + need_distfiles = not emerge_skip_distfiles and \ + (mydo in ("fetch", "unpack") or \ + mydo not in ("digest", "manifest") and "noauto" not in features) + alist = mysettings.configdict["pkg"].get("A") + aalist = mysettings.configdict["pkg"].get("AA") + if alist is None or aalist is None: + # Make sure we get the correct tree in case there are overlays. + mytree = os.path.realpath( + os.path.dirname(os.path.dirname(mysettings["O"]))) + useflags = mysettings["PORTAGE_USE"].split() + try: + alist = mydbapi.getFetchMap(mycpv, useflags=useflags, + mytree=mytree) + aalist = mydbapi.getFetchMap(mycpv, mytree=mytree) + except InvalidDependString as e: + writemsg("!!! %s\n" % str(e), noiselevel=-1) + writemsg(_("!!! Invalid SRC_URI for '%s'.\n") % mycpv, + noiselevel=-1) + del e + return 1 + mysettings.configdict["pkg"]["A"] = " ".join(alist) + mysettings.configdict["pkg"]["AA"] = " ".join(aalist) + else: + alist = set(alist.split()) + aalist = set(aalist.split()) + if ("mirror" in features) or fetchall: + fetchme = aalist + checkme = aalist + else: + fetchme = alist + checkme = alist + + if mydo == "fetch": + # Files are already checked inside fetch(), + # so do not check them again. + checkme = [] + + if not emerge_skip_distfiles and \ + need_distfiles and not fetch( + fetchme, mysettings, listonly=listonly, fetchonly=fetchonly): + return 1 + + if mydo == "fetch" and listonly: + return 0 + + try: + if mydo == "manifest": + return not digestgen(mysettings=mysettings, myportdb=mydbapi) + elif mydo == "digest": + return not digestgen(mysettings=mysettings, myportdb=mydbapi) + elif mydo != 'fetch' and not emerge_skip_digest and \ + "digest" in mysettings.features: + # Don't do this when called by emerge or when called just + # for fetch (especially parallel-fetch) since it's not needed + # and it can interfere with parallel tasks. + digestgen(mysettings=mysettings, myportdb=mydbapi) + except PermissionDenied as e: + writemsg(_("!!! Permission Denied: %s\n") % (e,), noiselevel=-1) + if mydo in ("digest", "manifest"): + return 1 + + # See above comment about fetching only when needed + if not emerge_skip_distfiles and \ + not digestcheck(checkme, mysettings, "strict" in features): + return 1 + + if mydo == "fetch": + return 0 + + # remove PORTAGE_ACTUAL_DISTDIR once cvs/svn is supported via SRC_URI + if (mydo != "setup" and "noauto" not in features) or mydo == "unpack": + orig_distdir = mysettings["DISTDIR"] + mysettings["PORTAGE_ACTUAL_DISTDIR"] = orig_distdir + edpath = mysettings["DISTDIR"] = \ + os.path.join(mysettings["PORTAGE_BUILDDIR"], "distdir") + portage.util.ensure_dirs(edpath, gid=portage_gid, mode=0o755) + + # Remove any unexpected files or directories. + for x in os.listdir(edpath): + symlink_path = os.path.join(edpath, x) + st = os.lstat(symlink_path) + if x in alist and stat.S_ISLNK(st.st_mode): + continue + if stat.S_ISDIR(st.st_mode): + shutil.rmtree(symlink_path) + else: + os.unlink(symlink_path) + + # Check for existing symlinks and recreate if necessary. + for x in alist: + symlink_path = os.path.join(edpath, x) + target = os.path.join(orig_distdir, x) + try: + link_target = os.readlink(symlink_path) + except OSError: + os.symlink(target, symlink_path) + else: + if link_target != target: + os.unlink(symlink_path) + os.symlink(target, symlink_path) + + #initial dep checks complete; time to process main commands + + restrict = mysettings["PORTAGE_RESTRICT"].split() + nosandbox = (("userpriv" in features) and \ + ("usersandbox" not in features) and \ + "userpriv" not in restrict and \ + "nouserpriv" not in restrict) + if nosandbox and ("userpriv" not in features or \ + "userpriv" in restrict or \ + "nouserpriv" in restrict): + nosandbox = ("sandbox" not in features and \ + "usersandbox" not in features) + + if not portage.process.sandbox_capable: + nosandbox = True + + sesandbox = mysettings.selinux_enabled() and \ + "sesandbox" in mysettings.features + + droppriv = "userpriv" in mysettings.features and \ + "userpriv" not in restrict and \ + secpass >= 2 + + fakeroot = "fakeroot" in mysettings.features + + ebuild_sh = _shell_quote(ebuild_sh_binary) + " %s" + misc_sh = _shell_quote(misc_sh_binary) + " dyn_%s" + + # args are for the to spawn function + actionmap = { +"pretend": {"cmd":ebuild_sh, "args":{"droppriv":0, "free":1, "sesandbox":0, "fakeroot":0}}, +"setup": {"cmd":ebuild_sh, "args":{"droppriv":0, "free":1, "sesandbox":0, "fakeroot":0}}, +"unpack": {"cmd":ebuild_sh, "args":{"droppriv":droppriv, "free":0, "sesandbox":sesandbox, "fakeroot":0}}, +"prepare": {"cmd":ebuild_sh, "args":{"droppriv":droppriv, "free":0, "sesandbox":sesandbox, "fakeroot":0}}, +"configure":{"cmd":ebuild_sh, "args":{"droppriv":droppriv, "free":nosandbox, "sesandbox":sesandbox, "fakeroot":0}}, +"compile": {"cmd":ebuild_sh, "args":{"droppriv":droppriv, "free":nosandbox, "sesandbox":sesandbox, "fakeroot":0}}, +"test": {"cmd":ebuild_sh, "args":{"droppriv":droppriv, "free":nosandbox, "sesandbox":sesandbox, "fakeroot":0}}, +"install": {"cmd":ebuild_sh, "args":{"droppriv":0, "free":0, "sesandbox":sesandbox, "fakeroot":fakeroot}}, +"rpm": {"cmd":misc_sh, "args":{"droppriv":0, "free":0, "sesandbox":0, "fakeroot":fakeroot}}, +"package": {"cmd":misc_sh, "args":{"droppriv":0, "free":0, "sesandbox":0, "fakeroot":fakeroot}}, + } + + # merge the deps in so we have again a 'full' actionmap + # be glad when this can die. + for x in actionmap: + if len(actionmap_deps.get(x, [])): + actionmap[x]["dep"] = ' '.join(actionmap_deps[x]) + + if mydo in actionmap: + if mydo == "package": + # Make sure the package directory exists before executing + # this phase. This can raise PermissionDenied if + # the current user doesn't have write access to $PKGDIR. + parent_dir = os.path.join(mysettings["PKGDIR"], + mysettings["CATEGORY"]) + portage.util.ensure_dirs(parent_dir) + if not os.access(parent_dir, os.W_OK): + raise PermissionDenied( + "access('%s', os.W_OK)" % parent_dir) + retval = spawnebuild(mydo, + actionmap, mysettings, debug, logfile=logfile, + fd_pipes=fd_pipes, returnpid=returnpid) + elif mydo=="qmerge": + # check to ensure install was run. this *only* pops up when users + # forget it and are using ebuild + if not os.path.exists( + os.path.join(mysettings["PORTAGE_BUILDDIR"], ".installed")): + writemsg(_("!!! mydo=qmerge, but the install phase has not been run\n"), + noiselevel=-1) + return 1 + # qmerge is a special phase that implies noclean. + if "noclean" not in mysettings.features: + mysettings.features.add("noclean") + #qmerge is specifically not supposed to do a runtime dep check + retval = merge( + mysettings["CATEGORY"], mysettings["PF"], mysettings["D"], + os.path.join(mysettings["PORTAGE_BUILDDIR"], "build-info"), + myroot, mysettings, myebuild=mysettings["EBUILD"], mytree=tree, + mydbapi=mydbapi, vartree=vartree, prev_mtimes=prev_mtimes) + elif mydo=="merge": + retval = spawnebuild("install", actionmap, mysettings, debug, + alwaysdep=1, logfile=logfile, fd_pipes=fd_pipes, + returnpid=returnpid) + retval = exit_status_check(retval) + if retval != os.EX_OK: + # The merge phase handles this already. Callers don't know how + # far this function got, so we have to call elog_process() here + # so that it's only called once. + from portage.elog import elog_process + elog_process(mysettings.mycpv, mysettings) + if retval == os.EX_OK: + retval = merge(mysettings["CATEGORY"], mysettings["PF"], + mysettings["D"], os.path.join(mysettings["PORTAGE_BUILDDIR"], + "build-info"), myroot, mysettings, + myebuild=mysettings["EBUILD"], mytree=tree, mydbapi=mydbapi, + vartree=vartree, prev_mtimes=prev_mtimes) + else: + print(_("!!! Unknown mydo: %s") % mydo) + return 1 + + return retval + + finally: + + if tmpdir: + mysettings["PORTAGE_TMPDIR"] = tmpdir_orig + shutil.rmtree(tmpdir) + if builddir_lock: + portage.locks.unlockdir(builddir_lock) + + # Make sure that DISTDIR is restored to it's normal value before we return! + if "PORTAGE_ACTUAL_DISTDIR" in mysettings: + mysettings["DISTDIR"] = mysettings["PORTAGE_ACTUAL_DISTDIR"] + del mysettings["PORTAGE_ACTUAL_DISTDIR"] + + if logfile: + try: + if os.stat(logfile).st_size == 0: + os.unlink(logfile) + except OSError: + pass + + if mydo in ("digest", "manifest", "help"): + # If necessary, depend phase has been triggered by aux_get calls + # and the exemption is no longer needed. + _doebuild_manifest_exempt_depend -= 1 + +def _validate_deps(mysettings, myroot, mydo, mydbapi): + + invalid_dep_exempt_phases = \ + set(["clean", "cleanrm", "help", "prerm", "postrm"]) + dep_keys = ["DEPEND", "RDEPEND", "PDEPEND"] + misc_keys = ["LICENSE", "PROPERTIES", "PROVIDE", "RESTRICT", "SRC_URI"] + other_keys = ["SLOT"] + all_keys = dep_keys + misc_keys + other_keys + metadata = dict(zip(all_keys, + mydbapi.aux_get(mysettings.mycpv, all_keys))) + + class FakeTree(object): + def __init__(self, mydb): + self.dbapi = mydb + dep_check_trees = {myroot:{}} + dep_check_trees[myroot]["porttree"] = \ + FakeTree(fakedbapi(settings=mysettings)) + + msgs = [] + for dep_type in dep_keys: + mycheck = dep_check(metadata[dep_type], None, mysettings, + myuse="all", myroot=myroot, trees=dep_check_trees) + if not mycheck[0]: + msgs.append(" %s: %s\n %s\n" % ( + dep_type, metadata[dep_type], mycheck[1])) + + for k in misc_keys: + try: + use_reduce( + paren_reduce(metadata[k]), matchall=True) + except InvalidDependString as e: + msgs.append(" %s: %s\n %s\n" % ( + k, metadata[k], str(e))) + + if not metadata["SLOT"]: + msgs.append(_(" SLOT is undefined\n")) + + if msgs: + portage.util.writemsg_level(_("Error(s) in metadata for '%s':\n") % \ + (mysettings.mycpv,), level=logging.ERROR, noiselevel=-1) + for x in msgs: + portage.util.writemsg_level(x, + level=logging.ERROR, noiselevel=-1) + if mydo not in invalid_dep_exempt_phases: + return 1 + + return os.EX_OK + +# XXX This would be to replace getstatusoutput completely. +# XXX Issue: cannot block execution. Deadlock condition. +def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakeroot=0, **keywords): + """ + Spawn a subprocess with extra portage-specific options. + Optiosn include: + + Sandbox: Sandbox means the spawned process will be limited in its ability t + read and write files (normally this means it is restricted to ${D}/) + SElinux Sandbox: Enables sandboxing on SElinux + Reduced Privileges: Drops privilages such that the process runs as portage:portage + instead of as root. + + Notes: os.system cannot be used because it messes with signal handling. Instead we + use the portage.process spawn* family of functions. + + This function waits for the process to terminate. + + @param mystring: Command to run + @type mystring: String + @param mysettings: Either a Dict of Key,Value pairs or an instance of portage.config + @type mysettings: Dictionary or config instance + @param debug: Ignored + @type debug: Boolean + @param free: Enable sandboxing for this process + @type free: Boolean + @param droppriv: Drop to portage:portage when running this command + @type droppriv: Boolean + @param sesandbox: Enable SELinux Sandboxing (toggles a context switch) + @type sesandbox: Boolean + @param fakeroot: Run this command with faked root privileges + @type fakeroot: Boolean + @param keywords: Extra options encoded as a dict, to be passed to spawn + @type keywords: Dictionary + @rtype: Integer + @returns: + 1. The return code of the spawned process. + """ + + if isinstance(mysettings, dict): + env=mysettings + keywords["opt_name"]="[ %s ]" % "portage" + else: + check_config_instance(mysettings) + env=mysettings.environ() + if mysettings.mycpv is not None: + keywords["opt_name"] = "[%s]" % mysettings.mycpv + else: + keywords["opt_name"] = "[%s/%s]" % \ + (mysettings.get("CATEGORY",""), mysettings.get("PF","")) + + fd_pipes = keywords.get("fd_pipes") + if fd_pipes is None: + fd_pipes = { + 0:sys.stdin.fileno(), + 1:sys.stdout.fileno(), + 2:sys.stderr.fileno(), + } + # In some cases the above print statements don't flush stdout, so + # it needs to be flushed before allowing a child process to use it + # so that output always shows in the correct order. + stdout_filenos = (sys.stdout.fileno(), sys.stderr.fileno()) + for fd in fd_pipes.values(): + if fd in stdout_filenos: + sys.stdout.flush() + sys.stderr.flush() + break + + # The default policy for the sesandbox domain only allows entry (via exec) + # from shells and from binaries that belong to portage (the number of entry + # points is minimized). The "tee" binary is not among the allowed entry + # points, so it is spawned outside of the sesandbox domain and reads from a + # pseudo-terminal that connects two domains. + logfile = keywords.get("logfile") + mypids = [] + master_fd = None + slave_fd = None + fd_pipes_orig = None + got_pty = False + if logfile: + del keywords["logfile"] + if 1 not in fd_pipes or 2 not in fd_pipes: + raise ValueError(fd_pipes) + + got_pty, master_fd, slave_fd = \ + _create_pty_or_pipe(copy_term_size=fd_pipes[1]) + + if not got_pty and 'sesandbox' in mysettings.features \ + and mysettings.selinux_enabled(): + # With sesandbox, logging works through a pty but not through a + # normal pipe. So, disable logging if ptys are broken. + # See Bug #162404. + logfile = None + os.close(master_fd) + master_fd = None + os.close(slave_fd) + slave_fd = None + + if logfile: + + fd_pipes.setdefault(0, sys.stdin.fileno()) + fd_pipes_orig = fd_pipes.copy() + + # We must set non-blocking mode before we close the slave_fd + # since otherwise the fcntl call can fail on FreeBSD (the child + # process might have already exited and closed slave_fd so we + # have to keep it open in order to avoid FreeBSD potentially + # generating an EAGAIN exception). + import fcntl + fcntl.fcntl(master_fd, fcntl.F_SETFL, + fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK) + + fd_pipes[0] = fd_pipes_orig[0] + fd_pipes[1] = slave_fd + fd_pipes[2] = slave_fd + keywords["fd_pipes"] = fd_pipes + + features = mysettings.features + # TODO: Enable fakeroot to be used together with droppriv. The + # fake ownership/permissions will have to be converted to real + # permissions in the merge phase. + fakeroot = fakeroot and uid != 0 and portage.process.fakeroot_capable + if droppriv and not uid and portage_gid and portage_uid: + keywords.update({"uid":portage_uid,"gid":portage_gid, + "groups":userpriv_groups,"umask":0o02}) + if not free: + free=((droppriv and "usersandbox" not in features) or \ + (not droppriv and "sandbox" not in features and \ + "usersandbox" not in features and not fakeroot)) + + if not free and not (fakeroot or portage.process.sandbox_capable): + free = True + + if free or "SANDBOX_ACTIVE" in os.environ: + keywords["opt_name"] += " bash" + spawn_func = portage.process.spawn_bash + elif fakeroot: + keywords["opt_name"] += " fakeroot" + keywords["fakeroot_state"] = os.path.join(mysettings["T"], "fakeroot.state") + spawn_func = portage.process.spawn_fakeroot + else: + keywords["opt_name"] += " sandbox" + spawn_func = portage.process.spawn_sandbox + + if sesandbox: + spawn_func = selinux.spawn_wrapper(spawn_func, + mysettings["PORTAGE_SANDBOX_T"]) + + returnpid = keywords.get("returnpid") + keywords["returnpid"] = True + try: + mypids.extend(spawn_func(mystring, env=env, **keywords)) + finally: + if logfile: + os.close(slave_fd) + + if returnpid: + return mypids + + if logfile: + log_file = open(_unicode_encode(logfile), mode='ab') + apply_secpass_permissions(logfile, + uid=portage_uid, gid=portage_gid, mode=0o664) + stdout_file = os.fdopen(os.dup(fd_pipes_orig[1]), 'wb') + master_file = os.fdopen(master_fd, 'rb') + iwtd = [master_file] + owtd = [] + ewtd = [] + import array, select + buffsize = 65536 + eof = False + while not eof: + events = select.select(iwtd, owtd, ewtd) + for f in events[0]: + # Use non-blocking mode to prevent read + # calls from blocking indefinitely. + buf = array.array('B') + try: + buf.fromfile(f, buffsize) + except EOFError: + pass + if not buf: + eof = True + break + if f is master_file: + buf.tofile(stdout_file) + stdout_file.flush() + buf.tofile(log_file) + log_file.flush() + log_file.close() + stdout_file.close() + master_file.close() + pid = mypids[-1] + retval = os.waitpid(pid, 0)[1] + portage.process.spawned_pids.remove(pid) + if retval != os.EX_OK: + if retval & 0xff: + return (retval & 0xff) << 8 + return retval >> 8 + return retval + + +# parse actionmap to spawn ebuild with the appropriate args +def spawnebuild(mydo, actionmap, mysettings, debug, alwaysdep=0, + logfile=None, fd_pipes=None, returnpid=False): + if not returnpid and \ + (alwaysdep or "noauto" not in mysettings.features): + # process dependency first + if "dep" in actionmap[mydo]: + retval = spawnebuild(actionmap[mydo]["dep"], actionmap, + mysettings, debug, alwaysdep=alwaysdep, logfile=logfile, + fd_pipes=fd_pipes, returnpid=returnpid) + if retval: + return retval + + eapi = mysettings["EAPI"] + + if mydo == "configure" and eapi in ("0", "1"): + return os.EX_OK + + if mydo == "prepare" and eapi in ("0", "1"): + return os.EX_OK + + if mydo == "pretend" and eapi in ("0", "1", "2", "3", "3_pre2"): + return os.EX_OK + + kwargs = actionmap[mydo]["args"] + mysettings["EBUILD_PHASE"] = mydo + _doebuild_exit_status_unlink( + mysettings.get("EBUILD_EXIT_STATUS_FILE")) + + try: + phase_retval = spawn(actionmap[mydo]["cmd"] % mydo, + mysettings, debug=debug, logfile=logfile, + fd_pipes=fd_pipes, returnpid=returnpid, **kwargs) + finally: + mysettings["EBUILD_PHASE"] = "" + + if returnpid: + return phase_retval + + msg = _doebuild_exit_status_check(mydo, mysettings) + if msg: + if phase_retval == os.EX_OK: + phase_retval = 1 + for l in wrap(msg, 72): + eerror(l, phase=mydo, key=mysettings.mycpv) + + _post_phase_userpriv_perms(mysettings) + if mydo == "install": + out = StringIO() + _check_build_log(mysettings, out=out) + msg = _unicode_decode(out.getvalue(), + encoding=_encodings['content'], errors='replace') + if msg: + writemsg_stdout(msg, noiselevel=-1) + if logfile is not None: + try: + f = codecs.open(_unicode_encode(logfile, + encoding=_encodings['fs'], errors='strict'), + mode='a', encoding=_encodings['content'], + errors='replace') + except EnvironmentError: + pass + else: + f.write(msg) + f.close() + if phase_retval == os.EX_OK: + _post_src_install_chost_fix(mysettings) + phase_retval = _post_src_install_checks(mysettings) + + if mydo == "test" and phase_retval != os.EX_OK and \ + "test-fail-continue" in mysettings.features: + phase_retval = os.EX_OK + + return phase_retval + +_post_phase_cmds = { + + "install" : [ + "install_qa_check", + "install_symlink_html_docs"], + + "preinst" : [ + "preinst_bsdflags", + "preinst_sfperms", + "preinst_selinux_labels", + "preinst_suid_scan", + "preinst_mask"], + + "postinst" : [ + "postinst_bsdflags"] +} + +def _post_phase_userpriv_perms(mysettings): + if "userpriv" in mysettings.features and secpass >= 2: + """ Privileged phases may have left files that need to be made + writable to a less privileged user.""" + apply_recursive_permissions(mysettings["T"], + uid=portage_uid, gid=portage_gid, dirmode=0o70, dirmask=0, + filemode=0o60, filemask=0) + +def _post_src_install_checks(mysettings): + _post_src_install_uid_fix(mysettings) + global _post_phase_cmds + retval = _spawn_misc_sh(mysettings, _post_phase_cmds["install"], + phase='internal_post_src_install') + if retval != os.EX_OK: + writemsg(_("!!! install_qa_check failed; exiting.\n"), + noiselevel=-1) + return retval + +def _check_build_log(mysettings, out=None): + """ + Search the content of $PORTAGE_LOG_FILE if it exists + and generate the following QA Notices when appropriate: + + * Automake "maintainer mode" + * command not found + * Unrecognized configure options + """ + logfile = mysettings.get("PORTAGE_LOG_FILE") + if logfile is None: + return + try: + f = codecs.open(_unicode_encode(logfile, + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['content'], errors='replace') + except EnvironmentError: + return + + am_maintainer_mode = [] + bash_command_not_found = [] + bash_command_not_found_re = re.compile( + r'(.*): line (\d*): (.*): command not found$') + command_not_found_exclude_re = re.compile(r'/configure: line ') + helper_missing_file = [] + helper_missing_file_re = re.compile( + r'^!!! (do|new).*: .* does not exist$') + + configure_opts_warn = [] + configure_opts_warn_re = re.compile( + r'^configure: WARNING: [Uu]nrecognized options: ') + + # Exclude output from dev-libs/yaz-3.0.47 which looks like this: + # + #Configuration: + # Automake: ${SHELL} /var/tmp/portage/dev-libs/yaz-3.0.47/work/yaz-3.0.47/config/missing --run automake-1.10 + am_maintainer_mode_re = re.compile(r'/missing --run ') + am_maintainer_mode_exclude_re = \ + re.compile(r'(/missing --run (autoheader|makeinfo)|^\s*Automake:\s)') + + make_jobserver_re = \ + re.compile(r'g?make\[\d+\]: warning: jobserver unavailable:') + make_jobserver = [] + + try: + for line in f: + if am_maintainer_mode_re.search(line) is not None and \ + am_maintainer_mode_exclude_re.search(line) is None: + am_maintainer_mode.append(line.rstrip("\n")) + + if bash_command_not_found_re.match(line) is not None and \ + command_not_found_exclude_re.search(line) is None: + bash_command_not_found.append(line.rstrip("\n")) + + if helper_missing_file_re.match(line) is not None: + helper_missing_file.append(line.rstrip("\n")) + + if configure_opts_warn_re.match(line) is not None: + configure_opts_warn.append(line.rstrip("\n")) + + if make_jobserver_re.match(line) is not None: + make_jobserver.append(line.rstrip("\n")) + + finally: + f.close() + + def _eqawarn(lines): + for line in lines: + eqawarn(line, phase="install", key=mysettings.mycpv, out=out) + wrap_width = 70 + + if am_maintainer_mode: + msg = [_("QA Notice: Automake \"maintainer mode\" detected:")] + msg.append("") + msg.extend("\t" + line for line in am_maintainer_mode) + msg.append("") + msg.extend(wrap(_( + "If you patch Makefile.am, " + "configure.in, or configure.ac then you " + "should use autotools.eclass and " + "eautomake or eautoreconf. Exceptions " + "are limited to system packages " + "for which it is impossible to run " + "autotools during stage building. " + "See http://www.gentoo.org/p" + "roj/en/qa/autofailure.xml for more information."), + wrap_width)) + _eqawarn(msg) + + if bash_command_not_found: + msg = [_("QA Notice: command not found:")] + msg.append("") + msg.extend("\t" + line for line in bash_command_not_found) + _eqawarn(msg) + + if helper_missing_file: + msg = [_("QA Notice: file does not exist:")] + msg.append("") + msg.extend("\t" + line[4:] for line in helper_missing_file) + _eqawarn(msg) + + if configure_opts_warn: + msg = [_("QA Notice: Unrecognized configure options:")] + msg.append("") + msg.extend("\t" + line for line in configure_opts_warn) + _eqawarn(msg) + + if make_jobserver: + msg = [_("QA Notice: make jobserver unavailable:")] + msg.append("") + msg.extend("\t" + line for line in make_jobserver) + _eqawarn(msg) + +def _post_src_install_chost_fix(settings): + """ + It's possible that the ebuild has changed the + CHOST variable, so revert it to the initial + setting. + """ + if settings.get('CATEGORY') == 'virtual': + return + + chost = settings.get('CHOST') + if chost: + write_atomic(os.path.join(settings['PORTAGE_BUILDDIR'], + 'build-info', 'CHOST'), chost + '\n') + +_vdb_use_conditional_keys = ('DEPEND', 'LICENSE', 'PDEPEND', + 'PROPERTIES', 'PROVIDE', 'RDEPEND', 'RESTRICT',) +_vdb_use_conditional_atoms = frozenset(['DEPEND', 'PDEPEND', 'RDEPEND']) + +def _post_src_install_uid_fix(mysettings, out=None): + """ + Files in $D with user and group bits that match the "portage" + user or group are automatically mapped to PORTAGE_INST_UID and + PORTAGE_INST_GID if necessary. The chown system call may clear + S_ISUID and S_ISGID bits, so those bits are restored if + necessary. + """ + + os = _os_merge + + inst_uid = int(mysettings["PORTAGE_INST_UID"]) + inst_gid = int(mysettings["PORTAGE_INST_GID"]) + + if bsd_chflags: + # Temporarily remove all of the flags in order to avoid EPERM errors. + os.system("mtree -c -p %s -k flags > %s" % \ + (_shell_quote(mysettings["D"]), + _shell_quote(os.path.join(mysettings["T"], "bsdflags.mtree")))) + os.system("chflags -R noschg,nouchg,nosappnd,nouappnd %s" % \ + (_shell_quote(mysettings["D"]),)) + os.system("chflags -R nosunlnk,nouunlnk %s 2>/dev/null" % \ + (_shell_quote(mysettings["D"]),)) + + destdir = mysettings["D"] + unicode_errors = [] + + while True: + + unicode_error = False + size = 0 + counted_inodes = set() + + for parent, dirs, files in os.walk(destdir): + try: + parent = _unicode_decode(parent, + encoding=_encodings['merge'], errors='strict') + except UnicodeDecodeError: + new_parent = _unicode_decode(parent, + encoding=_encodings['merge'], errors='replace') + new_parent = _unicode_encode(new_parent, + encoding=_encodings['merge'], errors='backslashreplace') + new_parent = _unicode_decode(new_parent, + encoding=_encodings['merge'], errors='replace') + os.rename(parent, new_parent) + unicode_error = True + unicode_errors.append(new_parent[len(destdir):]) + break + + for fname in chain(dirs, files): + try: + fname = _unicode_decode(fname, + encoding=_encodings['merge'], errors='strict') + except UnicodeDecodeError: + fpath = _os.path.join( + parent.encode(_encodings['merge']), fname) + new_fname = _unicode_decode(fname, + encoding=_encodings['merge'], errors='replace') + new_fname = _unicode_encode(new_fname, + encoding=_encodings['merge'], errors='backslashreplace') + new_fname = _unicode_decode(new_fname, + encoding=_encodings['merge'], errors='replace') + new_fpath = os.path.join(parent, new_fname) + os.rename(fpath, new_fpath) + unicode_error = True + unicode_errors.append(new_fpath[len(destdir):]) + fname = new_fname + fpath = new_fpath + else: + fpath = os.path.join(parent, fname) + + mystat = os.lstat(fpath) + if stat.S_ISREG(mystat.st_mode) and \ + mystat.st_ino not in counted_inodes: + counted_inodes.add(mystat.st_ino) + size += mystat.st_size + if mystat.st_uid != portage_uid and \ + mystat.st_gid != portage_gid: + continue + myuid = -1 + mygid = -1 + if mystat.st_uid == portage_uid: + myuid = inst_uid + if mystat.st_gid == portage_gid: + mygid = inst_gid + apply_secpass_permissions( + _unicode_encode(fpath, encoding=_encodings['merge']), + uid=myuid, gid=mygid, + mode=mystat.st_mode, stat_cached=mystat, + follow_links=False) + + if unicode_error: + break + + if not unicode_error: + break + + if unicode_errors: + from portage.elog.messages import eerror + for l in _merge_unicode_error(unicode_errors): + eerror(l, phase='install', key=mysettings.mycpv, out=out) + + build_info_dir = os.path.join(mysettings['PORTAGE_BUILDDIR'], + 'build-info') + + codecs.open(_unicode_encode(os.path.join(build_info_dir, + 'SIZE'), encoding=_encodings['fs'], errors='strict'), + 'w', encoding=_encodings['repo.content'], + errors='strict').write(str(size) + '\n') + + codecs.open(_unicode_encode(os.path.join(build_info_dir, + 'BUILD_TIME'), encoding=_encodings['fs'], errors='strict'), + 'w', encoding=_encodings['repo.content'], + errors='strict').write(str(int(time.time())) + '\n') + + use = frozenset(mysettings['PORTAGE_USE'].split()) + for k in _vdb_use_conditional_keys: + v = mysettings.configdict['pkg'].get(k) + if v is None: + continue + v = paren_reduce(v) + v = use_reduce(v, uselist=use) + v = paren_normalize(v) + v = paren_enclose(v) + if not v: + continue + if v in _vdb_use_conditional_atoms: + v_split = [] + for x in v.split(): + try: + x = Atom(x) + except InvalidAtom: + v_split.append(x) + else: + v_split.append(str(x.evaluate_conditionals(use))) + v = ' '.join(v_split) + codecs.open(_unicode_encode(os.path.join(build_info_dir, + k), encoding=_encodings['fs'], errors='strict'), + mode='w', encoding=_encodings['repo.content'], + errors='strict').write(v + '\n') + + if bsd_chflags: + # Restore all of the flags saved above. + os.system("mtree -e -p %s -U -k flags < %s > /dev/null" % \ + (_shell_quote(mysettings["D"]), + _shell_quote(os.path.join(mysettings["T"], "bsdflags.mtree")))) + +def _merge_unicode_error(errors): + lines = [] + + msg = _("This package installs one or more file names containing " + "characters that do not match your current locale " + "settings. The current setting for filesystem encoding is '%s'.") \ + % _encodings['merge'] + lines.extend(wrap(msg, 72)) + + lines.append("") + errors.sort() + lines.extend("\t" + x for x in errors) + lines.append("") + + if _encodings['merge'].lower().replace('_', '').replace('-', '') != 'utf8': + msg = _("For best results, UTF-8 encoding is recommended. See " + "the Gentoo Linux Localization Guide for instructions " + "about how to configure your locale for UTF-8 encoding:") + lines.extend(wrap(msg, 72)) + lines.append("") + lines.append("\t" + \ + "http://www.gentoo.org/doc/en/guide-localization.xml") + lines.append("") + + return lines + +def _post_pkg_preinst_cmd(mysettings): + """ + Post phase logic and tasks that have been factored out of + ebuild.sh. Call preinst_mask last so that INSTALL_MASK can + can be used to wipe out any gmon.out files created during + previous functions (in case any tools were built with -pg + in CFLAGS). + """ + + portage_bin_path = mysettings["PORTAGE_BIN_PATH"] + misc_sh_binary = os.path.join(portage_bin_path, + os.path.basename(MISC_SH_BINARY)) + + mysettings["EBUILD_PHASE"] = "" + global _post_phase_cmds + myargs = [_shell_quote(misc_sh_binary)] + _post_phase_cmds["preinst"] + + return myargs + +def _post_pkg_postinst_cmd(mysettings): + """ + Post phase logic and tasks that have been factored out of + build.sh. + """ + + portage_bin_path = mysettings["PORTAGE_BIN_PATH"] + misc_sh_binary = os.path.join(portage_bin_path, + os.path.basename(MISC_SH_BINARY)) + + mysettings["EBUILD_PHASE"] = "" + global _post_phase_cmds + myargs = [_shell_quote(misc_sh_binary)] + _post_phase_cmds["postinst"] + + return myargs + +def _spawn_misc_sh(mysettings, commands, phase=None, **kwargs): + """ + @param mysettings: the ebuild config + @type mysettings: config + @param commands: a list of function names to call in misc-functions.sh + @type commands: list + @rtype: int + @returns: the return value from the spawn() call + """ + + # Note: PORTAGE_BIN_PATH may differ from the global + # constant when portage is reinstalling itself. + portage_bin_path = mysettings["PORTAGE_BIN_PATH"] + misc_sh_binary = os.path.join(portage_bin_path, + os.path.basename(MISC_SH_BINARY)) + mycommand = " ".join([_shell_quote(misc_sh_binary)] + commands) + _doebuild_exit_status_unlink( + mysettings.get("EBUILD_EXIT_STATUS_FILE")) + debug = mysettings.get("PORTAGE_DEBUG") == "1" + logfile = mysettings.get("PORTAGE_LOG_FILE") + mysettings.pop("EBUILD_PHASE", None) + try: + rval = spawn(mycommand, mysettings, debug=debug, + logfile=logfile, **kwargs) + finally: + pass + + msg = _doebuild_exit_status_check(phase, mysettings) + if msg: + if rval == os.EX_OK: + rval = 1 + for l in wrap(msg, 72): + eerror(l, phase=phase, key=mysettings.mycpv) + + return rval diff --git a/pym/portage/package/ebuild/fetch.py b/pym/portage/package/ebuild/fetch.py index f3862258..c9de1870 100644 --- a/pym/portage/package/ebuild/fetch.py +++ b/pym/portage/package/ebuild/fetch.py @@ -15,7 +15,13 @@ import shutil import stat import sys -from portage import check_config_instance, doebuild_environment, OrderedDict, os, prepare_build_dirs, selinux, _encodings, _shell_quote, _unicode_encode + +import portage +portage.proxy.lazyimport.lazyimport(globals(), + 'portage.package.ebuild.config:check_config_instance,config', +) + +from portage import doebuild_environment, OrderedDict, os, prepare_build_dirs, selinux, _encodings, _shell_quote, _unicode_encode from portage.checksum import perform_md5, verify_all from portage.const import BASH_BINARY, CUSTOM_MIRRORS_FILE, EBUILD_SH_BINARY, GLOBAL_CONFIG_PATH from portage.data import portage_gid, portage_uid, secpass, userpriv_groups @@ -24,7 +30,6 @@ from portage.localization import _ from portage.locks import lockfile, unlockfile from portage.manifest import Manifest from portage.output import colorize, EOutput -from portage.package.ebuild.config import config from portage.util import apply_recursive_permissions, apply_secpass_permissions, ensure_dirs, grabdict, shlex_split, varexpand, writemsg, writemsg_level, writemsg_stdout from portage.process import spawn diff --git a/pym/portage/package/ebuild/prepare_build_dirs.py b/pym/portage/package/ebuild/prepare_build_dirs.py new file mode 100644 index 00000000..83031d68 --- /dev/null +++ b/pym/portage/package/ebuild/prepare_build_dirs.py @@ -0,0 +1,304 @@ +# Copyright 2010 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Id$ + +__all__ = ['prepare_build_dirs'] + +import codecs +import errno +import shutil +import stat +import time + +from portage import os, _encodings, _unicode_encode, _unicode_decode +from portage.data import portage_gid, portage_uid, secpass +from portage.exception import DirectoryNotFound, FileNotFound, \ + OperationNotPermitted, PermissionDenied, PortageException +from portage.localization import _ +from portage.output import colorize +from portage.util import apply_recursive_permissions, \ + apply_secpass_permissions, ensure_dirs, writemsg + +def prepare_build_dirs(myroot, mysettings, cleanup): + + clean_dirs = [mysettings["HOME"]] + + # We enable cleanup when we want to make sure old cruft (such as the old + # environment) doesn't interfere with the current phase. + if cleanup: + clean_dirs.append(mysettings["T"]) + + for clean_dir in clean_dirs: + try: + shutil.rmtree(clean_dir) + except OSError as oe: + if errno.ENOENT == oe.errno: + pass + elif errno.EPERM == oe.errno: + writemsg("%s\n" % oe, noiselevel=-1) + writemsg(_("Operation Not Permitted: rmtree('%s')\n") % \ + clean_dir, noiselevel=-1) + return 1 + else: + raise + + def makedirs(dir_path): + try: + os.makedirs(dir_path) + except OSError as oe: + if errno.EEXIST == oe.errno: + pass + elif errno.EPERM == oe.errno: + writemsg("%s\n" % oe, noiselevel=-1) + writemsg(_("Operation Not Permitted: makedirs('%s')\n") % \ + dir_path, noiselevel=-1) + return False + else: + raise + return True + + mysettings["PKG_LOGDIR"] = os.path.join(mysettings["T"], "logging") + + mydirs = [os.path.dirname(mysettings["PORTAGE_BUILDDIR"])] + mydirs.append(os.path.dirname(mydirs[-1])) + + try: + for mydir in mydirs: + ensure_dirs(mydir) + apply_secpass_permissions(mydir, + gid=portage_gid, uid=portage_uid, mode=0o70, mask=0) + for dir_key in ("PORTAGE_BUILDDIR", "HOME", "PKG_LOGDIR", "T"): + """These directories don't necessarily need to be group writable. + However, the setup phase is commonly run as a privileged user prior + to the other phases being run by an unprivileged user. Currently, + we use the portage group to ensure that the unprivleged user still + has write access to these directories in any case.""" + ensure_dirs(mysettings[dir_key], mode=0o775) + apply_secpass_permissions(mysettings[dir_key], + uid=portage_uid, gid=portage_gid) + except PermissionDenied as e: + writemsg(_("Permission Denied: %s\n") % str(e), noiselevel=-1) + return 1 + except OperationNotPermitted as e: + writemsg(_("Operation Not Permitted: %s\n") % str(e), noiselevel=-1) + return 1 + except FileNotFound as e: + writemsg(_("File Not Found: '%s'\n") % str(e), noiselevel=-1) + return 1 + + # Reset state for things like noauto and keepwork in FEATURES. + for x in ('.die_hooks',): + try: + os.unlink(os.path.join(mysettings['PORTAGE_BUILDDIR'], x)) + except OSError: + pass + + _prepare_workdir(mysettings) + if mysettings.get('EBUILD_PHASE') != 'fetch': + # Avoid spurious permissions adjustments when fetching with + # a temporary PORTAGE_TMPDIR setting (for fetchonly). + _prepare_features_dirs(mysettings) + +def _adjust_perms_msg(settings, msg): + + def write(msg): + writemsg(msg, noiselevel=-1) + + background = settings.get("PORTAGE_BACKGROUND") == "1" + log_path = settings.get("PORTAGE_LOG_FILE") + log_file = None + + if background and log_path is not None: + try: + log_file = codecs.open(_unicode_encode(log_path, + encoding=_encodings['fs'], errors='strict'), + mode='a', encoding=_encodings['content'], errors='replace') + except IOError: + def write(msg): + pass + else: + def write(msg): + log_file.write(_unicode_decode(msg)) + log_file.flush() + + try: + write(msg) + finally: + if log_file is not None: + log_file.close() + +def _prepare_features_dirs(mysettings): + + features_dirs = { + "ccache":{ + "path_dir": "/usr/lib/ccache/bin", + "basedir_var":"CCACHE_DIR", + "default_dir":os.path.join(mysettings["PORTAGE_TMPDIR"], "ccache"), + "always_recurse":False}, + "distcc":{ + "path_dir": "/usr/lib/distcc/bin", + "basedir_var":"DISTCC_DIR", + "default_dir":os.path.join(mysettings["BUILD_PREFIX"], ".distcc"), + "subdirs":("lock", "state"), + "always_recurse":True} + } + dirmode = 0o2070 + filemode = 0o60 + modemask = 0o2 + restrict = mysettings.get("PORTAGE_RESTRICT","").split() + droppriv = secpass >= 2 and \ + "userpriv" in mysettings.features and \ + "userpriv" not in restrict + for myfeature, kwargs in features_dirs.items(): + if myfeature in mysettings.features: + failure = False + basedir = mysettings.get(kwargs["basedir_var"]) + if basedir is None or not basedir.strip(): + basedir = kwargs["default_dir"] + mysettings[kwargs["basedir_var"]] = basedir + try: + path_dir = kwargs["path_dir"] + if not os.path.isdir(path_dir): + raise DirectoryNotFound(path_dir) + + mydirs = [mysettings[kwargs["basedir_var"]]] + if "subdirs" in kwargs: + for subdir in kwargs["subdirs"]: + mydirs.append(os.path.join(basedir, subdir)) + for mydir in mydirs: + modified = ensure_dirs(mydir) + # Generally, we only want to apply permissions for + # initial creation. Otherwise, we don't know exactly what + # permissions the user wants, so should leave them as-is. + droppriv_fix = False + if droppriv: + st = os.stat(mydir) + if st.st_gid != portage_gid or \ + not dirmode == (stat.S_IMODE(st.st_mode) & dirmode): + droppriv_fix = True + if not droppriv_fix: + # Check permissions of files in the directory. + for filename in os.listdir(mydir): + try: + subdir_st = os.lstat( + os.path.join(mydir, filename)) + except OSError: + continue + if subdir_st.st_gid != portage_gid or \ + ((stat.S_ISDIR(subdir_st.st_mode) and \ + not dirmode == (stat.S_IMODE(subdir_st.st_mode) & dirmode))): + droppriv_fix = True + break + + if droppriv_fix: + _adjust_perms_msg(mysettings, + colorize("WARN", " * ") + \ + _("Adjusting permissions " + "for FEATURES=userpriv: '%s'\n") % mydir) + elif modified: + _adjust_perms_msg(mysettings, + colorize("WARN", " * ") + \ + _("Adjusting permissions " + "for FEATURES=%s: '%s'\n") % (myfeature, mydir)) + + if modified or kwargs["always_recurse"] or droppriv_fix: + def onerror(e): + raise # The feature is disabled if a single error + # occurs during permissions adjustment. + if not apply_recursive_permissions(mydir, + gid=portage_gid, dirmode=dirmode, dirmask=modemask, + filemode=filemode, filemask=modemask, onerror=onerror): + raise OperationNotPermitted( + _("Failed to apply recursive permissions for the portage group.")) + + except DirectoryNotFound as e: + failure = True + writemsg(_("\n!!! Directory does not exist: '%s'\n") % \ + (e,), noiselevel=-1) + writemsg(_("!!! Disabled FEATURES='%s'\n") % myfeature, + noiselevel=-1) + + except PortageException as e: + failure = True + writemsg("\n!!! %s\n" % str(e), noiselevel=-1) + writemsg(_("!!! Failed resetting perms on %s='%s'\n") % \ + (kwargs["basedir_var"], basedir), noiselevel=-1) + writemsg(_("!!! Disabled FEATURES='%s'\n") % myfeature, + noiselevel=-1) + + if failure: + mysettings.features.remove(myfeature) + mysettings['FEATURES'] = ' '.join(sorted(mysettings.features)) + time.sleep(5) + +def _prepare_workdir(mysettings): + workdir_mode = 0o700 + try: + mode = mysettings["PORTAGE_WORKDIR_MODE"] + if mode.isdigit(): + parsed_mode = int(mode, 8) + elif mode == "": + raise KeyError() + else: + raise ValueError() + if parsed_mode & 0o7777 != parsed_mode: + raise ValueError("Invalid file mode: %s" % mode) + else: + workdir_mode = parsed_mode + except KeyError as e: + writemsg(_("!!! PORTAGE_WORKDIR_MODE is unset, using %s.\n") % oct(workdir_mode)) + except ValueError as e: + if len(str(e)) > 0: + writemsg("%s\n" % e) + writemsg(_("!!! Unable to parse PORTAGE_WORKDIR_MODE='%s', using %s.\n") % \ + (mysettings["PORTAGE_WORKDIR_MODE"], oct(workdir_mode))) + mysettings["PORTAGE_WORKDIR_MODE"] = oct(workdir_mode).replace('o', '') + try: + apply_secpass_permissions(mysettings["WORKDIR"], + uid=portage_uid, gid=portage_gid, mode=workdir_mode) + except FileNotFound: + pass # ebuild.sh will create it + + if mysettings.get("PORT_LOGDIR", "") == "": + while "PORT_LOGDIR" in mysettings: + del mysettings["PORT_LOGDIR"] + if "PORT_LOGDIR" in mysettings: + try: + modified = ensure_dirs(mysettings["PORT_LOGDIR"]) + if modified: + apply_secpass_permissions(mysettings["PORT_LOGDIR"], + uid=portage_uid, gid=portage_gid, mode=0o2770) + except PortageException as e: + writemsg("!!! %s\n" % str(e), noiselevel=-1) + writemsg(_("!!! Permission issues with PORT_LOGDIR='%s'\n") % \ + mysettings["PORT_LOGDIR"], noiselevel=-1) + writemsg(_("!!! Disabling logging.\n"), noiselevel=-1) + while "PORT_LOGDIR" in mysettings: + del mysettings["PORT_LOGDIR"] + if "PORT_LOGDIR" in mysettings and \ + os.access(mysettings["PORT_LOGDIR"], os.W_OK): + logid_path = os.path.join(mysettings["PORTAGE_BUILDDIR"], ".logid") + if not os.path.exists(logid_path): + open(_unicode_encode(logid_path), 'w') + logid_time = _unicode_decode(time.strftime("%Y%m%d-%H%M%S", + time.gmtime(os.stat(logid_path).st_mtime)), + encoding=_encodings['content'], errors='replace') + + if "split-log" in mysettings.features: + mysettings["PORTAGE_LOG_FILE"] = os.path.join( + mysettings["PORT_LOGDIR"], "build", "%s/%s:%s.log" % \ + (mysettings["CATEGORY"], mysettings["PF"], logid_time)) + else: + mysettings["PORTAGE_LOG_FILE"] = os.path.join( + mysettings["PORT_LOGDIR"], "%s:%s:%s.log" % \ + (mysettings["CATEGORY"], mysettings["PF"], logid_time)) + + ensure_dirs(os.path.dirname(mysettings["PORTAGE_LOG_FILE"])) + + else: + # NOTE: When sesandbox is enabled, the local SELinux security policies + # may not allow output to be piped out of the sesandbox domain. The + # current policy will allow it to work when a pty is available, but + # not through a normal pipe. See bug #162404. + mysettings["PORTAGE_LOG_FILE"] = os.path.join( + mysettings["T"], "build.log") diff --git a/pym/portage/tests/ebuild/test_pty_eof.py b/pym/portage/tests/ebuild/test_pty_eof.py index 31a5df4a..b29c8f95 100644 --- a/pym/portage/tests/ebuild/test_pty_eof.py +++ b/pym/portage/tests/ebuild/test_pty_eof.py @@ -4,6 +4,7 @@ import portage from portage.tests import TestCase +from portage.package.ebuild._pty import _can_test_pty_eof, _test_pty_eof class PtyEofTestCase(TestCase): @@ -13,8 +14,8 @@ class PtyEofTestCase(TestCase): # Since it might not be fixed, mark as todo. self.todo = True # The result is only valid if openpty does not raise EnvironmentError. - if portage._can_test_pty_eof(): + if _can_test_pty_eof(): try: - self.assertEqual(portage._test_pty_eof(), True) + self.assertEqual(_test_pty_eof(), True) except EnvironmentError: pass |