diff options
Diffstat (limited to 'portage_with_autodep/pym/portage/package/ebuild/doebuild.py')
| -rw-r--r-- | portage_with_autodep/pym/portage/package/ebuild/doebuild.py | 592 |
1 files changed, 438 insertions, 154 deletions
diff --git a/portage_with_autodep/pym/portage/package/ebuild/doebuild.py b/portage_with_autodep/pym/portage/package/ebuild/doebuild.py index c76c1ed..610172f 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/doebuild.py +++ b/portage_with_autodep/pym/portage/package/ebuild/doebuild.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ['doebuild', 'doebuild_environment', 'spawn', 'spawnebuild'] @@ -10,7 +10,6 @@ from itertools import chain import logging import os as _os import re -import shutil import signal import stat import sys @@ -31,8 +30,8 @@ portage.proxy.lazyimport.lazyimport(globals(), ) from portage import auxdbkeys, bsd_chflags, \ - eapi_is_supported, merge, os, selinux, \ - unmerge, _encodings, _parse_eapi_ebuild_head, _os_merge, \ + eapi_is_supported, merge, os, selinux, shutil, \ + unmerge, _encodings, _os_merge, \ _shell_quote, _unicode_decode, _unicode_encode from portage.const import EBUILD_SH_ENV_FILE, EBUILD_SH_ENV_DIR, \ EBUILD_SH_BINARY, INVALID_ENV_FILE, MISC_SH_BINARY @@ -42,16 +41,16 @@ from portage.dbapi.porttree import _parse_uri_map from portage.dep import Atom, check_required_use, \ human_readable_required_use, paren_enclose, use_reduce from portage.eapi import eapi_exports_KV, eapi_exports_merge_type, \ - eapi_exports_replace_vars, eapi_has_required_use, \ - eapi_has_src_prepare_and_src_configure, eapi_has_pkg_pretend -from portage.elog import elog_process + eapi_exports_replace_vars, eapi_exports_REPOSITORY, \ + eapi_has_required_use, eapi_has_src_prepare_and_src_configure, \ + eapi_has_pkg_pretend +from portage.elog import elog_process, _preload_elog_modules from portage.elog.messages import eerror, eqawarn from portage.exception import DigestException, FileNotFound, \ IncorrectParameter, InvalidDependString, PermissionDenied, \ UnsupportedAPIException from portage.localization import _ -from portage.manifest import Manifest -from portage.output import style_to_ansi_code +from portage.output import colormap from portage.package.ebuild.prepare_build_dirs import prepare_build_dirs from portage.util import apply_recursive_permissions, \ apply_secpass_permissions, noiselimit, normalize_path, \ @@ -116,6 +115,38 @@ def _spawn_phase(phase, settings, actionmap=None, **kwargs): ebuild_phase.wait() return ebuild_phase.returncode +def _doebuild_path(settings, eapi=None): + """ + Generate the PATH variable. + """ + + # Note: PORTAGE_BIN_PATH may differ from the global constant + # when portage is reinstalling itself. + portage_bin_path = settings["PORTAGE_BIN_PATH"] + eprefix = settings["EPREFIX"] + prerootpath = [x for x in settings.get("PREROOTPATH", "").split(":") if x] + rootpath = [x for x in settings.get("ROOTPATH", "").split(":") if x] + + prefixes = [] + if eprefix: + prefixes.append(eprefix) + prefixes.append("/") + + path = [] + + if eapi not in (None, "0", "1", "2", "3"): + path.append(os.path.join(portage_bin_path, "ebuild-helpers", "4")) + + path.append(os.path.join(portage_bin_path, "ebuild-helpers")) + path.extend(prerootpath) + + for prefix in prefixes: + for x in ("usr/local/sbin", "usr/local/bin", "usr/sbin", "usr/bin", "sbin", "bin"): + path.append(os.path.join(prefix, x)) + + path.extend(rootpath) + settings["PATH"] = ":".join(path) + def doebuild_environment(myebuild, mydo, myroot=None, settings=None, debug=False, use_cache=None, db=None): """ @@ -143,20 +174,32 @@ def doebuild_environment(myebuild, mydo, myroot=None, settings=None, ebuild_path = os.path.abspath(myebuild) pkg_dir = os.path.dirname(ebuild_path) mytree = os.path.dirname(os.path.dirname(pkg_dir)) - - if "CATEGORY" in mysettings.configdict["pkg"]: - cat = mysettings.configdict["pkg"]["CATEGORY"] - else: - cat = os.path.basename(normalize_path(os.path.join(pkg_dir, ".."))) - mypv = os.path.basename(ebuild_path)[:-7] - - mycpv = cat+"/"+mypv - mysplit = _pkgsplit(mypv) + mysplit = _pkgsplit(mypv, eapi=mysettings.configdict["pkg"].get("EAPI")) if mysplit is None: raise IncorrectParameter( _("Invalid ebuild path: '%s'") % myebuild) + if mysettings.mycpv is not None and \ + mysettings.configdict["pkg"].get("PF") == mypv and \ + "CATEGORY" in mysettings.configdict["pkg"]: + # Assume that PF is enough to assume that we've got + # the correct CATEGORY, though this is not really + # a solid assumption since it's possible (though + # unlikely) that two packages in different + # categories have the same PF. Callers should call + # setcpv or create a clean clone of a locked config + # instance in order to ensure that this assumption + # does not fail like in bug #408817. + cat = mysettings.configdict["pkg"]["CATEGORY"] + mycpv = mysettings.mycpv + elif os.path.basename(pkg_dir) in (mysplit[0], mypv): + # portdbapi or vardbapi + cat = os.path.basename(os.path.dirname(pkg_dir)) + mycpv = cat + "/" + mypv + else: + raise AssertionError("unable to determine CATEGORY") + # Make a backup of PORTAGE_TMPDIR prior to calling config.reset() # so that the caller can override it. tmpdir = mysettings["PORTAGE_TMPDIR"] @@ -208,11 +251,11 @@ def doebuild_environment(myebuild, mydo, myroot=None, settings=None, mysettings["FILESDIR"] = pkg_dir+"/files" mysettings["PF"] = mypv - if hasattr(mydbapi, '_repo_info'): - repo_info = mydbapi._repo_info[mytree] - mysettings['PORTDIR'] = repo_info.portdir - mysettings['PORTDIR_OVERLAY'] = repo_info.portdir_overlay - mysettings.configdict["pkg"]["PORTAGE_REPO_NAME"] = repo_info.name + if hasattr(mydbapi, 'repositories'): + repo = mydbapi.repositories.get_repo_for_location(mytree) + mysettings['PORTDIR'] = repo.eclass_db.porttrees[0] + mysettings['PORTDIR_OVERLAY'] = ' '.join(repo.eclass_db.porttrees[1:]) + mysettings.configdict["pkg"]["PORTAGE_REPO_NAME"] = repo.name mysettings["PORTDIR"] = os.path.realpath(mysettings["PORTDIR"]) mysettings["DISTDIR"] = os.path.realpath(mysettings["DISTDIR"]) @@ -235,16 +278,6 @@ def doebuild_environment(myebuild, mydo, myroot=None, settings=None, else: mysettings["PVR"]=mysplit[1]+"-"+mysplit[2] - if "PATH" in mysettings: - mysplit=mysettings["PATH"].split(":") - else: - mysplit=[] - # Note: PORTAGE_BIN_PATH may differ from the global constant - # when portage is reinstalling itself. - portage_bin_path = mysettings["PORTAGE_BIN_PATH"] - if portage_bin_path not in mysplit: - mysettings["PATH"] = portage_bin_path + ":" + mysettings["PATH"] - # All temporary directories should be subdirectories of # $PORTAGE_TMPDIR/portage, since it's common for /tmp and /var/tmp # to be mounted with the "noexec" option (see bug #346899). @@ -268,7 +301,9 @@ def doebuild_environment(myebuild, mydo, myroot=None, settings=None, mysettings["T"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], "temp") # Prefix forward compatability - mysettings["ED"] = mysettings["D"] + eprefix_lstrip = mysettings["EPREFIX"].lstrip(os.sep) + mysettings["ED"] = os.path.join( + mysettings["D"], eprefix_lstrip).rstrip(os.sep) + os.sep mysettings["PORTAGE_BASHRC"] = os.path.join( mysettings["PORTAGE_CONFIGROOT"], EBUILD_SH_ENV_FILE) @@ -276,37 +311,41 @@ def doebuild_environment(myebuild, mydo, myroot=None, settings=None, mysettings["PORTAGE_CONFIGROOT"], EBUILD_SH_ENV_DIR) # Allow color.map to control colors associated with einfo, ewarn, etc... - mycolors = [] - for c in ("GOOD", "WARN", "BAD", "HILITE", "BRACKET"): - mycolors.append("%s=$'%s'" % \ - (c, style_to_ansi_code(c))) - mysettings["PORTAGE_COLORMAP"] = "\n".join(mycolors) - - # All EAPI dependent code comes last, so that essential variables - # like PORTAGE_BUILDDIR are still initialized even in cases when + mysettings["PORTAGE_COLORMAP"] = colormap() + + if "COLUMNS" not in mysettings: + # Set COLUMNS, in order to prevent unnecessary stty calls + # inside the set_colors function of isolated-functions.sh. + # We cache the result in os.environ, in order to avoid + # multiple stty calls in cases when get_term_size() falls + # back to stty due to a missing or broken curses module. + columns = os.environ.get("COLUMNS") + if columns is None: + rows, columns = portage.output.get_term_size() + if columns < 1: + # Force a sane value for COLUMNS, so that tools + # like ls don't complain (see bug #394091). + columns = 80 + columns = str(columns) + os.environ["COLUMNS"] = columns + mysettings["COLUMNS"] = columns + + # EAPI is always known here, even for the "depend" phase, because + # EbuildMetadataPhase gets it from _parse_eapi_ebuild_head(). + eapi = mysettings.configdict['pkg']['EAPI'] + _doebuild_path(mysettings, eapi=eapi) + + # All EAPI dependent code comes last, so that essential variables like + # PATH and PORTAGE_BUILDDIR are still initialized even in cases when # UnsupportedAPIException needs to be raised, which can be useful # when uninstalling a package that has corrupt EAPI metadata. - eapi = None - if mydo == 'depend' and 'EAPI' not in mysettings.configdict['pkg']: - if eapi is None and 'parse-eapi-ebuild-head' in mysettings.features: - eapi = _parse_eapi_ebuild_head( - io.open(_unicode_encode(ebuild_path, - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['content'], errors='replace')) - - if eapi is not None: - if not eapi_is_supported(eapi): - raise UnsupportedAPIException(mycpv, eapi) - mysettings.configdict['pkg']['EAPI'] = eapi + if not eapi_is_supported(eapi): + raise UnsupportedAPIException(mycpv, eapi) - if mydo != "depend": - # Metadata vars such as EAPI and RESTRICT are - # set by the above config.setcpv() call. - eapi = mysettings["EAPI"] - if not eapi_is_supported(eapi): - # can't do anything with this. - raise UnsupportedAPIException(mycpv, eapi) + if eapi_exports_REPOSITORY(eapi) and "PORTAGE_REPO_NAME" in mysettings.configdict["pkg"]: + mysettings.configdict["pkg"]["REPOSITORY"] = mysettings.configdict["pkg"]["PORTAGE_REPO_NAME"] + if mydo != "depend": if hasattr(mydbapi, "getFetchMap") and \ ("A" not in mysettings.configdict["pkg"] or \ "AA" not in mysettings.configdict["pkg"]): @@ -331,22 +370,41 @@ def doebuild_environment(myebuild, mydo, myroot=None, settings=None, else: mysettings.configdict["pkg"]["AA"] = " ".join(uri_map) - if not eapi_exports_KV(eapi): - # Discard KV for EAPIs that don't support it. Cache KV is restored - # from the backupenv whenever config.reset() is called. - mysettings.pop('KV', None) - elif mydo != 'depend' and 'KV' not in mysettings and \ - mydo in ('compile', 'config', 'configure', 'info', - 'install', 'nofetch', 'postinst', 'postrm', 'preinst', - 'prepare', 'prerm', 'setup', 'test', 'unpack'): - mykv, err1 = ExtractKernelVersion( - os.path.join(mysettings['EROOT'], "usr/src/linux")) - if mykv: - # Regular source tree - mysettings["KV"] = mykv - else: - mysettings["KV"] = "" - mysettings.backup_changes("KV") + ccache = "ccache" in mysettings.features + distcc = "distcc" in mysettings.features + if ccache or distcc: + # Use default ABI libdir in accordance with bug #355283. + libdir = None + default_abi = mysettings.get("DEFAULT_ABI") + if default_abi: + libdir = mysettings.get("LIBDIR_" + default_abi) + if not libdir: + libdir = "lib" + + if distcc: + mysettings["PATH"] = os.path.join(os.sep, eprefix_lstrip, + "usr", libdir, "distcc", "bin") + ":" + mysettings["PATH"] + + if ccache: + mysettings["PATH"] = os.path.join(os.sep, eprefix_lstrip, + "usr", libdir, "ccache", "bin") + ":" + mysettings["PATH"] + + if not eapi_exports_KV(eapi): + # Discard KV for EAPIs that don't support it. Cached KV is restored + # from the backupenv whenever config.reset() is called. + mysettings.pop('KV', None) + elif 'KV' not in mysettings and \ + mydo in ('compile', 'config', 'configure', 'info', + 'install', 'nofetch', 'postinst', 'postrm', 'preinst', + 'prepare', 'prerm', 'setup', 'test', 'unpack'): + mykv, err1 = ExtractKernelVersion( + os.path.join(mysettings['EROOT'], "usr/src/linux")) + if mykv: + # Regular source tree + mysettings["KV"] = mykv + else: + mysettings["KV"] = "" + mysettings.backup_changes("KV") _doebuild_manifest_cache = None _doebuild_broken_ebuilds = set() @@ -356,7 +414,7 @@ _doebuild_commands_without_builddir = ( 'fetch', 'fetchall', 'help', 'manifest' ) -def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, +def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, fetchonly=0, cleanup=0, dbkey=None, use_cache=1, fetchall=0, tree=None, mydbapi=None, vartree=None, prev_mtimes=None, fd_pipes=None, returnpid=False): @@ -368,10 +426,10 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, @type myebuild: String @param mydo: Phase to run @type mydo: String - @param myroot: $ROOT (usually '/', see man make.conf) - @type myroot: String - @param mysettings: Portage Configuration - @type mysettings: instance of portage.config + @param _unused: Deprecated (use settings["ROOT"] instead) + @type _unused: String + @param settings: Portage Configuration + @type settings: instance of portage.config @param debug: Turns on various debug information (eg, debug for spawn) @type debug: Boolean @param listonly: Used to wrap fetch(); passed such that fetch only lists files required. @@ -403,7 +461,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, caller clean up all returned PIDs. @type returnpid: Boolean @rtype: Boolean - @returns: + @return: 1. 0 for success 2. 1 for error @@ -414,7 +472,18 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, Other variables may not be strictly required, many have defaults that are set inside of doebuild. """ - + + if settings is None: + raise TypeError("settings parameter is required") + mysettings = settings + myroot = settings['EROOT'] + + if _unused is not None and _unused != mysettings['EROOT']: + warnings.warn("The third parameter of the " + "portage.doebuild() is now unused. Use " + "settings['ROOT'] instead.", + DeprecationWarning, stacklevel=2) + if not tree: writemsg("Warning: tree not specified to doebuild\n") tree = "porttree" @@ -432,6 +501,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, "install":["test"], "rpm": ["install"], "package":["install"], + "merge" :["install"], } if mydbapi is None: @@ -480,21 +550,28 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, return 1 global _doebuild_manifest_cache + pkgdir = os.path.dirname(myebuild) + manifest_path = os.path.join(pkgdir, "Manifest") + if tree == "porttree": + repo_config = mysettings.repositories.get_repo_for_location( + os.path.dirname(os.path.dirname(pkgdir))) + else: + repo_config = None + mf = None if "strict" in features and \ "digest" not in features and \ tree == "porttree" and \ + not repo_config.thin_manifest and \ mydo not in ("digest", "manifest", "help") and \ - not portage._doebuild_manifest_exempt_depend: + not portage._doebuild_manifest_exempt_depend and \ + not (repo_config.allow_missing_manifest and not os.path.exists(manifest_path)): # Always verify the ebuild checksums before executing it. global _doebuild_broken_ebuilds if myebuild in _doebuild_broken_ebuilds: return 1 - pkgdir = os.path.dirname(myebuild) - manifest_path = os.path.join(pkgdir, "Manifest") - # Avoid checking the same Manifest several times in a row during a # regen with an empty cache. if _doebuild_manifest_cache is None or \ @@ -505,7 +582,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, out.eerror(_("Manifest not found for '%s'") % (myebuild,)) _doebuild_broken_ebuilds.add(myebuild) return 1 - mf = Manifest(pkgdir, mysettings["DISTDIR"]) + mf = repo_config.load_manifest(pkgdir, mysettings["DISTDIR"]) else: mf = _doebuild_manifest_cache @@ -513,10 +590,12 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, try: mf.checkFileHashes("EBUILD", os.path.basename(myebuild)) except KeyError: - out = portage.output.EOutput() - out.eerror(_("Missing digest for '%s'") % (myebuild,)) - _doebuild_broken_ebuilds.add(myebuild) - return 1 + if not (mf.allow_missing and + os.path.basename(myebuild) not in mf.fhashdict["EBUILD"]): + out = portage.output.EOutput() + out.eerror(_("Missing digest for '%s'") % (myebuild,)) + _doebuild_broken_ebuilds.add(myebuild) + return 1 except FileNotFound: out = portage.output.EOutput() out.eerror(_("A file listed in the Manifest " @@ -536,7 +615,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, if mf.getFullname() in _doebuild_broken_manifests: return 1 - if mf is not _doebuild_manifest_cache: + if mf is not _doebuild_manifest_cache and not mf.allow_missing: # Make sure that all of the ebuilds are # actually listed in the Manifest. @@ -553,8 +632,8 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, _doebuild_broken_manifests.add(manifest_path) return 1 - # Only cache it if the above stray files test succeeds. - _doebuild_manifest_cache = mf + # We cache it only after all above checks succeed. + _doebuild_manifest_cache = mf logfile=None builddir_lock = None @@ -594,7 +673,6 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, if builddir_lock is not None: builddir_lock.unlock() - restrict = set(mysettings.get('PORTAGE_RESTRICT', '').split()) # get possible slot information from the deps file if mydo == "depend": writemsg("!!! DEBUG: dbkey: %s\n" % str(dbkey), 2) @@ -654,6 +732,13 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, if rval != os.EX_OK: return rval + else: + # FEATURES=noauto only makes sense for porttree, and we don't want + # it to trigger redundant sourcing of the ebuild for API consumers + # that are using binary packages + if "noauto" in mysettings.features: + mysettings.features.discard("noauto") + # The info phase is special because it uses mkdtemp so and # user (not necessarily in the portage group) can run it. if mydo not in ('info',) and \ @@ -666,6 +751,73 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, return unmerge(mysettings["CATEGORY"], mysettings["PF"], myroot, mysettings, vartree=vartree) + phases_to_run = set() + if "noauto" in mysettings.features or \ + mydo not in actionmap_deps: + phases_to_run.add(mydo) + else: + phase_stack = [mydo] + while phase_stack: + x = phase_stack.pop() + if x in phases_to_run: + continue + phases_to_run.add(x) + phase_stack.extend(actionmap_deps.get(x, [])) + del phase_stack + + alist = set(mysettings.configdict["pkg"].get("A", "").split()) + + unpacked = False + if tree != "porttree": + pass + elif "unpack" not in phases_to_run: + unpacked = os.path.exists(os.path.join( + mysettings["PORTAGE_BUILDDIR"], ".unpacked")) + else: + try: + workdir_st = os.stat(mysettings["WORKDIR"]) + except OSError: + pass + else: + newstuff = False + if not os.path.exists(os.path.join( + mysettings["PORTAGE_BUILDDIR"], ".unpacked")): + writemsg_stdout(_( + ">>> Not marked as unpacked; recreating WORKDIR...\n")) + newstuff = True + else: + for x in alist: + writemsg_stdout(">>> Checking %s's mtime...\n" % x) + try: + x_st = os.stat(os.path.join( + mysettings["DISTDIR"], x)) + except OSError: + # file not fetched yet + x_st = None + + if x_st is None or x_st.st_mtime > workdir_st.st_mtime: + writemsg_stdout(_(">>> Timestamp of " + "%s has changed; recreating WORKDIR...\n") % x) + newstuff = True + break + + if newstuff: + if builddir_lock is None and \ + 'PORTAGE_BUILDIR_LOCKED' not in mysettings: + builddir_lock = EbuildBuildDir( + scheduler=PollScheduler().sched_iface, + settings=mysettings) + builddir_lock.lock() + try: + _spawn_phase("clean", mysettings) + finally: + if builddir_lock is not None: + builddir_lock.unlock() + builddir_lock = None + else: + writemsg_stdout(_(">>> WORKDIR is up-to-date, keeping...\n")) + unpacked = True + # Build directory creation isn't required for any of these. # In the fetch phase, the directory is needed only for RESTRICT=fetch # in order to satisfy the sane $PWD requirement (from bug #239560) @@ -739,10 +891,9 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, # Only try and fetch the files if we are going to need them ... # otherwise, if user has FEATURES=noauto and they run `ebuild clean # unpack compile install`, we will try and fetch 4 times :/ - need_distfiles = tree == "porttree" and \ + need_distfiles = tree == "porttree" and not unpacked and \ (mydo in ("fetch", "unpack") or \ mydo not in ("digest", "manifest") and "noauto" not in features) - alist = set(mysettings.configdict["pkg"].get("A", "").split()) if need_distfiles: src_uri, = mydbapi.aux_get(mysettings.mycpv, @@ -783,10 +934,14 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, return 0 return 1 - if mydo == "fetch": + if need_distfiles: # Files are already checked inside fetch(), # so do not check them again. checkme = [] + elif unpacked: + # The unpack phase is marked as complete, so it + # would be wasteful to check distfiles again. + checkme = [] else: checkme = alist @@ -845,7 +1000,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, # this phase. This can raise PermissionDenied if # the current user doesn't have write access to $PKGDIR. if hasattr(portage, 'db'): - bintree = portage.db[mysettings["ROOT"]]["bintree"] + bintree = portage.db[mysettings['EROOT']]['bintree'] mysettings["PORTAGE_BINPKG_TMPFILE"] = \ bintree.getname(mysettings.mycpv) + \ ".%s" % (os.getpid(),) @@ -866,6 +1021,13 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, if mydo == "package" and bintree is not None: bintree.inject(mysettings.mycpv, filename=mysettings["PORTAGE_BINPKG_TMPFILE"]) + else: + if "PORTAGE_BINPKG_TMPFILE" in mysettings: + try: + os.unlink(mysettings["PORTAGE_BINPKG_TMPFILE"]) + except OSError: + pass + elif mydo=="qmerge": # check to ensure install was run. this *only* pops up when users # forget it and are using ebuild @@ -877,6 +1039,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, # qmerge is a special phase that implies noclean. if "noclean" not in mysettings.features: mysettings.features.add("noclean") + _handle_self_update(mysettings, vartree.dbapi) #qmerge is specifically not supposed to do a runtime dep check retval = merge( mysettings["CATEGORY"], mysettings["PF"], mysettings["D"], @@ -893,6 +1056,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, # so that it's only called once. elog_process(mysettings.mycpv, mysettings) if retval == os.EX_OK: + _handle_self_update(mysettings, vartree.dbapi) retval = merge(mysettings["CATEGORY"], mysettings["PF"], mysettings["D"], os.path.join(mysettings["PORTAGE_BUILDDIR"], "build-info"), myroot, mysettings, @@ -944,10 +1108,31 @@ def _check_temp_dir(settings): # as some people use a separate PORTAGE_TMPDIR mount # we prefer that as the checks below would otherwise be pointless # for those people. - if os.path.exists(os.path.join(settings["PORTAGE_TMPDIR"], "portage")): - checkdir = os.path.join(settings["PORTAGE_TMPDIR"], "portage") + tmpdir = os.path.realpath(settings["PORTAGE_TMPDIR"]) + if os.path.exists(os.path.join(tmpdir, "portage")): + checkdir = os.path.realpath(os.path.join(tmpdir, "portage")) + if ("sandbox" in settings.features or + "usersandox" in settings.features) and \ + not checkdir.startswith(tmpdir + os.sep): + msg = _("The 'portage' subdirectory of the directory " + "referenced by the PORTAGE_TMPDIR variable appears to be " + "a symlink. In order to avoid sandbox violations (see bug " + "#378379), you must adjust PORTAGE_TMPDIR instead of using " + "the symlink located at '%s'. A suitable PORTAGE_TMPDIR " + "setting would be '%s'.") % \ + (os.path.join(tmpdir, "portage"), checkdir) + lines = [] + lines.append("") + lines.append("") + lines.extend(wrap(msg, 72)) + lines.append("") + for line in lines: + if line: + line = "!!! %s" % (line,) + writemsg("%s\n" % (line,), noiselevel=-1) + return 1 else: - checkdir = settings["PORTAGE_TMPDIR"] + checkdir = tmpdir if not os.access(checkdir, os.W_OK): writemsg(_("%s is not writable.\n" @@ -955,8 +1140,7 @@ def _check_temp_dir(settings): noiselevel=-1) return 1 - else: - fd = tempfile.NamedTemporaryFile(prefix="exectest-", dir=checkdir) + with tempfile.NamedTemporaryFile(prefix="exectest-", dir=checkdir) as fd: os.chmod(fd.name, 0o755) if not os.access(fd.name, os.X_OK): writemsg(_("Can not execute files in %s\n" @@ -1085,7 +1269,8 @@ def _validate_deps(mysettings, myroot, mydo, mydbapi): all_keys.add("SRC_URI") all_keys = tuple(all_keys) metadata = dict(zip(all_keys, - mydbapi.aux_get(mysettings.mycpv, all_keys))) + mydbapi.aux_get(mysettings.mycpv, all_keys, + myrepo=mysettings.get("PORTAGE_REPO_NAME")))) class FakeTree(object): def __init__(self, mydb): @@ -1173,7 +1358,7 @@ def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakero @param keywords: Extra options encoded as a dict, to be passed to spawn @type keywords: Dictionary @rtype: Integer - @returns: + @return: 1. The return code of the spawned process. """ @@ -1201,7 +1386,8 @@ def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakero # fake ownership/permissions will have to be converted to real # permissions in the merge phase. fakeroot = fakeroot and uid != 0 and portage.process.fakeroot_capable - if droppriv and not uid and portage_gid and portage_uid: + if droppriv and uid == 0 and portage_gid and portage_uid and \ + hasattr(os, "setgroups"): keywords.update({"uid":portage_uid,"gid":portage_gid, "groups":userpriv_groups,"umask":0o02}) if not free: @@ -1277,6 +1463,17 @@ def spawnebuild(mydo, actionmap, mysettings, debug, alwaysdep=0, if mydo == "pretend" and not eapi_has_pkg_pretend(eapi): return os.EX_OK + if not (mydo == "install" and "noauto" in mysettings.features): + check_file = os.path.join( + mysettings["PORTAGE_BUILDDIR"], ".%sed" % mydo.rstrip('e')) + if os.path.exists(check_file): + writemsg_stdout(_(">>> It appears that " + "'%(action)s' has already executed for '%(pkg)s'; skipping.\n") % + {"action":mydo, "pkg":mysettings["PF"]}) + writemsg_stdout(_(">>> Remove '%(file)s' to force %(action)s.\n") % + {"file":check_file, "action":mydo}) + return os.EX_OK + return _spawn_phase(mydo, mysettings, actionmap=actionmap, logfile=logfile, fd_pipes=fd_pipes, returnpid=returnpid) @@ -1285,13 +1482,14 @@ _post_phase_cmds = { "install" : [ "install_qa_check", - "install_symlink_html_docs"], + "install_symlink_html_docs", + "install_hooks"], "preinst" : [ "preinst_sfperms", "preinst_selinux_labels", "preinst_suid_scan", - "preinst_mask"] + ] } def _post_phase_userpriv_perms(mysettings): @@ -1320,7 +1518,9 @@ def _check_build_log(mysettings, out=None): except EnvironmentError: return + f_real = None if logfile.endswith('.gz'): + f_real = f f = gzip.GzipFile(filename='', mode='rb', fileobj=f) am_maintainer_mode = [] @@ -1425,19 +1625,32 @@ def _check_build_log(mysettings, out=None): msg.extend("\t" + line for line in make_jobserver) _eqawarn(msg) + f.close() + if f_real is not None: + f_real.close() + def _post_src_install_chost_fix(settings): """ It's possible that the ebuild has changed the CHOST variable, so revert it to the initial - setting. + setting. Also, revert IUSE in case it's corrupted + due to local environment settings like in bug #386829. """ - if settings.get('CATEGORY') == 'virtual': - return - chost = settings.get('CHOST') - if chost: - write_atomic(os.path.join(settings['PORTAGE_BUILDDIR'], - 'build-info', 'CHOST'), chost + '\n') + build_info_dir = os.path.join(settings['PORTAGE_BUILDDIR'], 'build-info') + + for k in ('IUSE',): + v = settings.get(k) + if v is not None: + write_atomic(os.path.join(build_info_dir, k), v + '\n') + + # The following variables are irrelevant for virtual packages. + if settings.get('CATEGORY') != 'virtual': + + for k in ('CHOST',): + v = settings.get(k) + if v is not None: + write_atomic(os.path.join(build_info_dir, k), v + '\n') _vdb_use_conditional_keys = ('DEPEND', 'LICENSE', 'PDEPEND', 'PROPERTIES', 'PROVIDE', 'RDEPEND', 'RESTRICT',) @@ -1481,6 +1694,7 @@ def _post_src_install_uid_fix(mysettings, out): _preinst_bsdflags(mysettings) destdir = mysettings["D"] + ed_len = len(mysettings["ED"]) unicode_errors = [] while True: @@ -1499,12 +1713,12 @@ def _post_src_install_uid_fix(mysettings, out): new_parent = _unicode_decode(parent, encoding=_encodings['merge'], errors='replace') new_parent = _unicode_encode(new_parent, - encoding=_encodings['merge'], errors='backslashreplace') + encoding='ascii', errors='backslashreplace') new_parent = _unicode_decode(new_parent, encoding=_encodings['merge'], errors='replace') os.rename(parent, new_parent) unicode_error = True - unicode_errors.append(new_parent[len(destdir):]) + unicode_errors.append(new_parent[ed_len:]) break for fname in chain(dirs, files): @@ -1517,13 +1731,13 @@ def _post_src_install_uid_fix(mysettings, out): new_fname = _unicode_decode(fname, encoding=_encodings['merge'], errors='replace') new_fname = _unicode_encode(new_fname, - encoding=_encodings['merge'], errors='backslashreplace') + encoding='ascii', errors='backslashreplace') new_fname = _unicode_decode(new_fname, encoding=_encodings['merge'], errors='replace') new_fpath = os.path.join(parent, new_fname) os.rename(fpath, new_fpath) unicode_error = True - unicode_errors.append(new_fpath[len(destdir):]) + unicode_errors.append(new_fpath[ed_len:]) fname = new_fname fpath = new_fpath else: @@ -1597,20 +1811,24 @@ def _post_src_install_uid_fix(mysettings, out): if unicode_errors: for l in _merge_unicode_error(unicode_errors): - eerror(l, phase='install', key=mysettings.mycpv, out=out) + eqawarn(l, phase='install', key=mysettings.mycpv, out=out) build_info_dir = os.path.join(mysettings['PORTAGE_BUILDDIR'], 'build-info') - io.open(_unicode_encode(os.path.join(build_info_dir, + f = io.open(_unicode_encode(os.path.join(build_info_dir, 'SIZE'), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['repo.content'], - errors='strict').write(_unicode_decode(str(size) + '\n')) + errors='strict') + f.write(_unicode_decode(str(size) + '\n')) + f.close() - io.open(_unicode_encode(os.path.join(build_info_dir, + f = io.open(_unicode_encode(os.path.join(build_info_dir, 'BUILD_TIME'), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['repo.content'], - errors='strict').write(_unicode_decode("%.0f\n" % (time.time(),))) + errors='strict') + f.write(_unicode_decode("%.0f\n" % (time.time(),))) + f.close() use = frozenset(mysettings['PORTAGE_USE'].split()) for k in _vdb_use_conditional_keys: @@ -1636,10 +1854,12 @@ def _post_src_install_uid_fix(mysettings, out): except OSError: pass continue - io.open(_unicode_encode(os.path.join(build_info_dir, + f = io.open(_unicode_encode(os.path.join(build_info_dir, k), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['repo.content'], - errors='strict').write(_unicode_decode(v + '\n')) + errors='strict') + f.write(_unicode_decode(v + '\n')) + f.close() _reapply_bsdflags_to_image(mysettings) @@ -1664,15 +1884,46 @@ def _post_src_install_soname_symlinks(mysettings, out): needed_filename = os.path.join(mysettings["PORTAGE_BUILDDIR"], "build-info", "NEEDED.ELF.2") + f = None try: - lines = io.open(_unicode_encode(needed_filename, + f = io.open(_unicode_encode(needed_filename, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], - errors='replace').readlines() + errors='replace') + lines = f.readlines() except IOError as e: if e.errno not in (errno.ENOENT, errno.ESTALE): raise return + finally: + if f is not None: + f.close() + + qa_no_symlink = "" + f = None + try: + f = io.open(_unicode_encode(os.path.join( + mysettings["PORTAGE_BUILDDIR"], + "build-info", "QA_SONAME_NO_SYMLINK"), + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') + qa_no_symlink = f.read() + except IOError as e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + raise + finally: + if f is not None: + f.close() + + qa_no_symlink = qa_no_symlink.split() + if qa_no_symlink: + if len(qa_no_symlink) > 1: + qa_no_symlink = "|".join("(%s)" % x for x in qa_no_symlink) + qa_no_symlink = "^(%s)$" % qa_no_symlink + else: + qa_no_symlink = "^%s$" % qa_no_symlink[0] + qa_no_symlink = re.compile(qa_no_symlink) libpaths = set(portage.util.getlibpaths( mysettings["ROOT"], env=mysettings)) @@ -1730,6 +1981,8 @@ def _post_src_install_soname_symlinks(mysettings, out): continue if not is_libdir(os.path.dirname(obj)): continue + if qa_no_symlink and qa_no_symlink.match(obj.strip(os.sep)) is not None: + continue obj_file_path = os.path.join(image_dir, obj.lstrip(os.sep)) sym_file_path = os.path.join(os.path.dirname(obj_file_path), soname) @@ -1746,8 +1999,7 @@ def _post_src_install_soname_symlinks(mysettings, out): if not missing_symlinks: return - qa_msg = ["QA Notice: Missing soname symlink(s) " + \ - "will be automatically created:"] + qa_msg = ["QA Notice: Missing soname symlink(s):"] qa_msg.append("") qa_msg.extend("\t%s -> %s" % (os.path.join( os.path.dirname(obj).lstrip(os.sep), soname), @@ -1757,20 +2009,11 @@ def _post_src_install_soname_symlinks(mysettings, out): for line in qa_msg: eqawarn(line, key=mysettings.mycpv, out=out) - _preinst_bsdflags(mysettings) - for obj, soname in missing_symlinks: - obj_file_path = os.path.join(image_dir, obj.lstrip(os.sep)) - sym_file_path = os.path.join(os.path.dirname(obj_file_path), soname) - os.symlink(os.path.basename(obj_file_path), sym_file_path) - _reapply_bsdflags_to_image(mysettings) - def _merge_unicode_error(errors): lines = [] - msg = _("This package installs one or more file names containing " - "characters that do not match your current locale " - "settings. The current setting for filesystem encoding is '%s'.") \ - % _encodings['merge'] + msg = _("QA Notice: This package installs one or more file names " + "containing characters that are not encoded with the UTF-8 encoding.") lines.extend(wrap(msg, 72)) lines.append("") @@ -1778,14 +2021,55 @@ def _merge_unicode_error(errors): lines.extend("\t" + x for x in errors) lines.append("") - if _encodings['merge'].lower().replace('_', '').replace('-', '') != 'utf8': - msg = _("For best results, UTF-8 encoding is recommended. See " - "the Gentoo Linux Localization Guide for instructions " - "about how to configure your locale for UTF-8 encoding:") - lines.extend(wrap(msg, 72)) - lines.append("") - lines.append("\t" + \ - "http://www.gentoo.org/doc/en/guide-localization.xml") - lines.append("") - return lines + +def _prepare_self_update(settings): + """ + Call this when portage is updating itself, in order to create + temporary copies of PORTAGE_BIN_PATH and PORTAGE_PYM_PATH, since + the new versions may be incompatible. An atexit hook will + automatically clean up the temporary copies. + """ + + # sanity check: ensure that that this routine only runs once + if portage._bin_path != portage.const.PORTAGE_BIN_PATH: + return + + # Load lazily referenced portage submodules into memory, + # so imports won't fail during portage upgrade/downgrade. + _preload_elog_modules(settings) + portage.proxy.lazyimport._preload_portage_submodules() + + # Make the temp directory inside $PORTAGE_TMPDIR/portage, since + # it's common for /tmp and /var/tmp to be mounted with the + # "noexec" option (see bug #346899). + build_prefix = os.path.join(settings["PORTAGE_TMPDIR"], "portage") + portage.util.ensure_dirs(build_prefix) + base_path_tmp = tempfile.mkdtemp( + "", "._portage_reinstall_.", build_prefix) + portage.process.atexit_register(shutil.rmtree, base_path_tmp) + + orig_bin_path = portage._bin_path + portage._bin_path = os.path.join(base_path_tmp, "bin") + shutil.copytree(orig_bin_path, portage._bin_path, symlinks=True) + + orig_pym_path = portage._pym_path + portage._pym_path = os.path.join(base_path_tmp, "pym") + shutil.copytree(orig_pym_path, portage._pym_path, symlinks=True) + + for dir_path in (base_path_tmp, portage._bin_path, portage._pym_path): + os.chmod(dir_path, 0o755) + +def _handle_self_update(settings, vardb): + cpv = settings.mycpv + if settings["ROOT"] == "/" and \ + portage.dep.match_from_list( + portage.const.PORTAGE_PACKAGE_ATOM, [cpv]): + inherited = frozenset(settings.get('INHERITED', '').split()) + if not vardb.cpv_exists(cpv) or \ + '9999' in cpv or \ + 'git' in inherited or \ + 'git-2' in inherited: + _prepare_self_update(settings) + return True + return False |