diff options
author | Alexander Bersenev <bay@hackerdom.ru> | 2014-02-17 17:55:51 +0600 |
---|---|---|
committer | Alexander Bersenev <bay@hackerdom.ru> | 2014-02-17 17:55:51 +0600 |
commit | 5a3f506c9ef1cfd78940b0509f10ef94b4434e29 (patch) | |
tree | 147c35a17a8bcd8ff467bb3063adab623da51fac /portage_with_autodep/pym/portage/package/ebuild | |
parent | fixed a deadlock (diff) | |
download | autodep-5a3f506c9ef1cfd78940b0509f10ef94b4434e29.tar.gz autodep-5a3f506c9ef1cfd78940b0509f10ef94b4434e29.tar.bz2 autodep-5a3f506c9ef1cfd78940b0509f10ef94b4434e29.zip |
updated portage to 2.2.8-r1
Diffstat (limited to 'portage_with_autodep/pym/portage/package/ebuild')
51 files changed, 1631 insertions, 480 deletions
diff --git a/portage_with_autodep/pym/portage/package/ebuild/__init__.pyo b/portage_with_autodep/pym/portage/package/ebuild/__init__.pyo Binary files differnew file mode 100644 index 0000000..927b4bc --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/__init__.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.py b/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.py index cd22554..0c613ce 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ( @@ -11,7 +11,7 @@ from portage.dep import ExtendedAtomDict, _repo_separator, _slot_separator from portage.localization import _ from portage.package.ebuild._config.helper import ordered_by_atom_specificity from portage.util import grabdict_package, stack_lists, writemsg -from portage.versions import cpv_getkey +from portage.versions import cpv_getkey, _pkg_str class KeywordsManager(object): """Manager class to handle keywords processing and validation""" @@ -20,7 +20,8 @@ class KeywordsManager(object): global_accept_keywords=""): self._pkeywords_list = [] rawpkeywords = [grabdict_package( - os.path.join(x, "package.keywords"), recursive=1, + os.path.join(x.location, "package.keywords"), + recursive=x.portage1_directories, verify_eapi=True) \ for x in profiles] for pkeyworddict in rawpkeywords: @@ -35,7 +36,8 @@ class KeywordsManager(object): self._p_accept_keywords = [] raw_p_accept_keywords = [grabdict_package( - os.path.join(x, "package.accept_keywords"), recursive=1, + os.path.join(x.location, "package.accept_keywords"), + recursive=x.portage1_directories, verify_eapi=True) \ for x in profiles] for d in raw_p_accept_keywords: @@ -75,10 +77,11 @@ class KeywordsManager(object): def getKeywords(self, cpv, slot, keywords, repo): - cp = cpv_getkey(cpv) - pkg = "".join((cpv, _slot_separator, slot)) - if repo and repo != Package.UNKNOWN_REPO: - pkg = "".join((pkg, _repo_separator, repo)) + if not hasattr(cpv, 'slot'): + pkg = _pkg_str(cpv, slot=slot, repo=repo) + else: + pkg = cpv + cp = pkg.cp keywords = [[x for x in keywords.split() if x != "-*"]] for pkeywords_dict in self._pkeywords_list: cpdict = pkeywords_dict.get(cp) @@ -206,12 +209,16 @@ class KeywordsManager(object): hasstable = False hastesting = False for gp in mygroups: - if gp == "*" or (gp == "-*" and len(mygroups) == 1): - writemsg(_("--- WARNING: Package '%(cpv)s' uses" - " '%(keyword)s' keyword.\n") % {"cpv": cpv, "keyword": gp}, - noiselevel=-1) - if gp == "*": - match = True + if gp == "*": + match = True + break + elif gp == "~*": + hastesting = True + for x in pgroups: + if x[:1] == "~": + match = True + break + if match: break elif gp in pgroups: match = True @@ -254,18 +261,19 @@ class KeywordsManager(object): """ pgroups = global_accept_keywords.split() + if not hasattr(cpv, 'slot'): + cpv = _pkg_str(cpv, slot=slot, repo=repo) cp = cpv_getkey(cpv) unmaskgroups = [] if self._p_accept_keywords: - cpv_slot = "%s:%s" % (cpv, slot) accept_keywords_defaults = tuple('~' + keyword for keyword in \ pgroups if keyword[:1] not in "~-") for d in self._p_accept_keywords: cpdict = d.get(cp) if cpdict: pkg_accept_keywords = \ - ordered_by_atom_specificity(cpdict, cpv_slot) + ordered_by_atom_specificity(cpdict, cpv) if pkg_accept_keywords: for x in pkg_accept_keywords: if not x: @@ -274,9 +282,8 @@ class KeywordsManager(object): pkgdict = self.pkeywordsdict.get(cp) if pkgdict: - cpv_slot = "%s:%s" % (cpv, slot) pkg_accept_keywords = \ - ordered_by_atom_specificity(pkgdict, cpv_slot, repo=repo) + ordered_by_atom_specificity(pkgdict, cpv) if pkg_accept_keywords: for x in pkg_accept_keywords: unmaskgroups.extend(x) diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.pyo Binary files differnew file mode 100644 index 0000000..15043f0 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/LicenseManager.py b/portage_with_autodep/pym/portage/package/ebuild/_config/LicenseManager.py index effd55b..f76e7e2 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/LicenseManager.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/LicenseManager.py @@ -1,4 +1,4 @@ -# Copyright 2010 Gentoo Foundation +# Copyright 201-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ( @@ -10,7 +10,7 @@ from portage.dep import ExtendedAtomDict, use_reduce from portage.exception import InvalidDependString from portage.localization import _ from portage.util import grabdict, grabdict_package, writemsg -from portage.versions import cpv_getkey +from portage.versions import cpv_getkey, _pkg_str from portage.package.ebuild._config.helper import ordered_by_atom_specificity @@ -119,8 +119,9 @@ class LicenseManager(object): cp = cpv_getkey(cpv) cpdict = self._plicensedict.get(cp) if cpdict: - cpv_slot = "%s:%s" % (cpv, slot) - plicence_list = ordered_by_atom_specificity(cpdict, cpv_slot, repo) + if not hasattr(cpv, slot): + cpv = _pkg_str(cpv, slot=slot, repo=repo) + plicence_list = ordered_by_atom_specificity(cpdict, cpv) if plicence_list: accept_license = list(self._accept_license) for x in plicence_list: diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/LicenseManager.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/LicenseManager.pyo Binary files differnew file mode 100644 index 0000000..4a38298 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/LicenseManager.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.py b/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.py index c2b115b..f7a1177 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.py @@ -5,7 +5,11 @@ __all__ = ( 'LocationsManager', ) +import collections import io +import warnings + +import portage from portage import os, eapi_is_supported, _encodings, _unicode_encode from portage.const import CUSTOM_PROFILE_PATH, GLOBAL_CONFIG_PATH, \ PROFILE_PATH, USER_CONFIG_PATH @@ -13,7 +17,20 @@ from portage.exception import DirectoryNotFound, ParseError from portage.localization import _ from portage.util import ensure_dirs, grabfile, \ normalize_path, shlex_split, writemsg +from portage.repository.config import parse_layout_conf, \ + _portage1_profiles_allow_directories + + +_PORTAGE1_DIRECTORIES = frozenset([ + 'package.mask', 'package.provided', + 'package.use', 'package.use.mask', 'package.use.force', + 'use.mask', 'use.force']) + +_profile_node = collections.namedtuple('_profile_node', + 'location portage1_directories') +_allow_parent_colon = frozenset( + ["portage-2"]) class LocationsManager(object): @@ -25,9 +42,9 @@ class LocationsManager(object): self.config_root = config_root self.target_root = target_root self._user_config = local_config - + if self.eprefix is None: - self.eprefix = "" + self.eprefix = portage.const.EPREFIX if self.config_root is None: self.config_root = self.eprefix + os.sep @@ -37,17 +54,33 @@ class LocationsManager(object): self._check_var_directory("PORTAGE_CONFIGROOT", self.config_root) self.abs_user_config = os.path.join(self.config_root, USER_CONFIG_PATH) + self.config_profile_path = config_profile_path + + def load_profiles(self, repositories, known_repository_paths): + known_repository_paths = set(os.path.realpath(x) + for x in known_repository_paths) - if config_profile_path is None: - config_profile_path = \ + known_repos = [] + for x in known_repository_paths: + try: + layout_data = {"profile-formats": + repositories.get_repo_for_location(x).profile_formats} + except KeyError: + layout_data = parse_layout_conf(x)[0] + # force a trailing '/' for ease of doing startswith checks + known_repos.append((x + '/', layout_data)) + known_repos = tuple(known_repos) + + if self.config_profile_path is None: + self.config_profile_path = \ os.path.join(self.config_root, PROFILE_PATH) - if os.path.isdir(config_profile_path): - self.profile_path = config_profile_path + if os.path.isdir(self.config_profile_path): + self.profile_path = self.config_profile_path else: - config_profile_path = \ + self.config_profile_path = \ os.path.join(self.abs_user_config, 'make.profile') - if os.path.isdir(config_profile_path): - self.profile_path = config_profile_path + if os.path.isdir(self.config_profile_path): + self.profile_path = self.config_profile_path else: self.profile_path = None else: @@ -55,19 +88,22 @@ class LocationsManager(object): # here, in order to create an empty profile # for checking dependencies of packages with # empty KEYWORDS. - self.profile_path = config_profile_path + self.profile_path = self.config_profile_path # The symlink might not exist or might not be a symlink. self.profiles = [] + self.profiles_complex = [] if self.profile_path: try: - self._addProfile(os.path.realpath(self.profile_path)) + self._addProfile(os.path.realpath(self.profile_path), + repositories, known_repos) except ParseError as e: writemsg(_("!!! Unable to parse profile: '%s'\n") % \ self.profile_path, noiselevel=-1) writemsg("!!! ParseError: %s\n" % str(e), noiselevel=-1) self.profiles = [] + self.profiles_complex = [] if self._user_config and self.profiles: custom_prof = os.path.join( @@ -75,9 +111,11 @@ class LocationsManager(object): if os.path.exists(custom_prof): self.user_profile_dir = custom_prof self.profiles.append(custom_prof) + self.profiles_complex.append(_profile_node(custom_prof, True)) del custom_prof self.profiles = tuple(self.profiles) + self.profiles_complex = tuple(self.profiles_complex) def _check_var_directory(self, varname, var): if not os.path.isdir(var): @@ -86,14 +124,45 @@ class LocationsManager(object): noiselevel=-1) raise DirectoryNotFound(var) - def _addProfile(self, currentPath): + def _addProfile(self, currentPath, repositories, known_repos): + current_abs_path = os.path.abspath(currentPath) + allow_directories = True + allow_parent_colon = True + repo_loc = None + compat_mode = False + intersecting_repos = [x for x in known_repos if current_abs_path.startswith(x[0])] + if intersecting_repos: + # protect against nested repositories. Insane configuration, but the longest + # path will be the correct one. + repo_loc, layout_data = max(intersecting_repos, key=lambda x:len(x[0])) + allow_directories = any(x in _portage1_profiles_allow_directories + for x in layout_data['profile-formats']) + compat_mode = layout_data['profile-formats'] == ('portage-1-compat',) + allow_parent_colon = any(x in _allow_parent_colon + for x in layout_data['profile-formats']) + + if compat_mode: + offenders = _PORTAGE1_DIRECTORIES.intersection(os.listdir(currentPath)) + offenders = sorted(x for x in offenders + if os.path.isdir(os.path.join(currentPath, x))) + if offenders: + warnings.warn(_("Profile '%(profile_path)s' in repository " + "'%(repo_name)s' is implicitly using 'portage-1' profile format, but " + "the repository profiles are not marked as that format. This will break " + "in the future. Please either convert the following paths " + "to files, or add\nprofile-formats = portage-1\nto the " + "repositories layout.conf. Files: '%(files)s'\n") + % dict(profile_path=currentPath, repo_name=repo_loc, + files=', '.join(offenders))) + parentsFile = os.path.join(currentPath, "parent") eapi_file = os.path.join(currentPath, "eapi") + f = None try: - eapi = io.open(_unicode_encode(eapi_file, + f = io.open(_unicode_encode(eapi_file, encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['content'], errors='replace' - ).readline().strip() + mode='r', encoding=_encodings['content'], errors='replace') + eapi = f.readline().strip() except IOError: pass else: @@ -102,21 +171,69 @@ class LocationsManager(object): "Profile contains unsupported " "EAPI '%s': '%s'") % \ (eapi, os.path.realpath(eapi_file),)) + finally: + if f is not None: + f.close() if os.path.exists(parentsFile): parents = grabfile(parentsFile) if not parents: raise ParseError( _("Empty parent file: '%s'") % parentsFile) for parentPath in parents: + abs_parent = parentPath[:1] == os.sep + if not abs_parent and allow_parent_colon: + parentPath = self._expand_parent_colon(parentsFile, + parentPath, repo_loc, repositories) + + # NOTE: This os.path.join() call is intended to ignore + # currentPath if parentPath is already absolute. parentPath = normalize_path(os.path.join( currentPath, parentPath)) + + if abs_parent or repo_loc is None or \ + not parentPath.startswith(repo_loc): + # It seems that this parent may point outside + # of the current repo, so realpath it. + parentPath = os.path.realpath(parentPath) + if os.path.exists(parentPath): - self._addProfile(parentPath) + self._addProfile(parentPath, repositories, known_repos) else: raise ParseError( _("Parent '%s' not found: '%s'") % \ (parentPath, parentsFile)) + self.profiles.append(currentPath) + self.profiles_complex.append( + _profile_node(currentPath, allow_directories)) + + def _expand_parent_colon(self, parentsFile, parentPath, + repo_loc, repositories): + colon = parentPath.find(":") + if colon == -1: + return parentPath + + if colon == 0: + if repo_loc is None: + raise ParseError( + _("Parent '%s' not found: '%s'") % \ + (parentPath, parentsFile)) + else: + parentPath = normalize_path(os.path.join( + repo_loc, 'profiles', parentPath[colon+1:])) + else: + p_repo_name = parentPath[:colon] + try: + p_repo_loc = repositories.get_location_for_name(p_repo_name) + except KeyError: + raise ParseError( + _("Parent '%s' not found: '%s'") % \ + (parentPath, parentsFile)) + else: + parentPath = normalize_path(os.path.join( + p_repo_loc, 'profiles', parentPath[colon+1:])) + + return parentPath def set_root_override(self, root_overwrite=None): # Allow ROOT setting to come from make.conf if it's not overridden diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.pyo Binary files differnew file mode 100644 index 0000000..c64d313 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.py b/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.py index df93e10..bce1152 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.py @@ -5,9 +5,12 @@ __all__ = ( 'MaskManager', ) +import warnings + from portage import os from portage.dep import ExtendedAtomDict, match_from_list, _repo_separator, _slot_separator -from portage.util import append_repo, grabfile_package, stack_lists +from portage.localization import _ +from portage.util import append_repo, grabfile_package, stack_lists, writemsg from portage.versions import cpv_getkey from _emerge.Package import Package @@ -32,30 +35,76 @@ class MaskManager(object): # repo may be often referenced by others as the master. pmask_cache = {} - def grab_pmask(loc): + def grab_pmask(loc, repo_config): if loc not in pmask_cache: - pmask_cache[loc] = grabfile_package( - os.path.join(loc, "profiles", "package.mask"), - recursive=1, remember_source_file=True, verify_eapi=True) + path = os.path.join(loc, 'profiles', 'package.mask') + pmask_cache[loc] = grabfile_package(path, + recursive=repo_config.portage1_profiles, + remember_source_file=True, verify_eapi=True) + if repo_config.portage1_profiles_compat and os.path.isdir(path): + warnings.warn(_("Repository '%(repo_name)s' is implicitly using " + "'portage-1' profile format in its profiles/package.mask, but " + "the repository profiles are not marked as that format. This will break " + "in the future. Please either convert the following paths " + "to files, or add\nprofile-formats = portage-1\nto the " + "repositories layout.conf.\n") + % dict(repo_name=repo_config.name)) + return pmask_cache[loc] repo_pkgmasklines = [] for repo in repositories.repos_with_profiles(): lines = [] - repo_lines = grab_pmask(repo.location) + repo_lines = grab_pmask(repo.location, repo) + removals = frozenset(line[0][1:] for line in repo_lines + if line[0][:1] == "-") + matched_removals = set() for master in repo.masters: - master_lines = grab_pmask(master.location) + master_lines = grab_pmask(master.location, master) + for line in master_lines: + if line[0] in removals: + matched_removals.add(line[0]) + # Since we don't stack masters recursively, there aren't any + # atoms earlier in the stack to be matched by negative atoms in + # master_lines. Also, repo_lines may contain negative atoms + # that are intended to negate atoms from a different master + # than the one with which we are currently stacking. Therefore, + # we disable warn_for_unmatched_removal here (see bug #386569). lines.append(stack_lists([master_lines, repo_lines], incremental=1, - remember_source_file=True, warn_for_unmatched_removal=True, - strict_warn_for_unmatched_removal=strict_umatched_removal)) - if not repo.masters: + remember_source_file=True, warn_for_unmatched_removal=False)) + + # It's safe to warn for unmatched removal if masters have not + # been overridden by the user, which is guaranteed when + # user_config is false (when called by repoman). + if repo.masters: + unmatched_removals = removals.difference(matched_removals) + if unmatched_removals and not user_config: + source_file = os.path.join(repo.location, + "profiles", "package.mask") + unmatched_removals = list(unmatched_removals) + if len(unmatched_removals) > 3: + writemsg( + _("--- Unmatched removal atoms in %s: %s and %s more\n") % + (source_file, + ", ".join("-" + x for x in unmatched_removals[:3]), + len(unmatched_removals) - 3), noiselevel=-1) + else: + writemsg( + _("--- Unmatched removal atom(s) in %s: %s\n") % + (source_file, + ", ".join("-" + x for x in unmatched_removals)), + noiselevel=-1) + + else: lines.append(stack_lists([repo_lines], incremental=1, - remember_source_file=True, warn_for_unmatched_removal=True, + remember_source_file=True, warn_for_unmatched_removal=not user_config, strict_warn_for_unmatched_removal=strict_umatched_removal)) repo_pkgmasklines.extend(append_repo(stack_lists(lines), repo.name, remember_source_file=True)) repo_pkgunmasklines = [] for repo in repositories.repos_with_profiles(): + if not repo.portage1_profiles: + continue repo_lines = grabfile_package(os.path.join(repo.location, "profiles", "package.unmask"), \ recursive=1, remember_source_file=True, verify_eapi=True) lines = stack_lists([repo_lines], incremental=1, \ @@ -69,9 +118,14 @@ class MaskManager(object): profile_pkgunmasklines = [] for x in profiles: profile_pkgmasklines.append(grabfile_package( - os.path.join(x, "package.mask"), recursive=1, remember_source_file=True, verify_eapi=True)) - profile_pkgunmasklines.append(grabfile_package( - os.path.join(x, "package.unmask"), recursive=1, remember_source_file=True, verify_eapi=True)) + os.path.join(x.location, "package.mask"), + recursive=x.portage1_directories, + remember_source_file=True, verify_eapi=True)) + if x.portage1_directories: + profile_pkgunmasklines.append(grabfile_package( + os.path.join(x.location, "package.unmask"), + recursive=x.portage1_directories, + remember_source_file=True, verify_eapi=True)) profile_pkgmasklines = stack_lists(profile_pkgmasklines, incremental=1, \ remember_source_file=True, warn_for_unmatched_removal=True, strict_warn_for_unmatched_removal=strict_umatched_removal) diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.pyo Binary files differnew file mode 100644 index 0000000..f48eb47 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.py b/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.py index d7ef0f6..e1ec7f4 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ( @@ -7,10 +7,10 @@ __all__ = ( from _emerge.Package import Package from portage import os -from portage.dep import ExtendedAtomDict, remove_slot, _get_useflag_re +from portage.dep import dep_getrepo, dep_getslot, ExtendedAtomDict, remove_slot, _get_useflag_re from portage.localization import _ from portage.util import grabfile, grabdict_package, read_corresponding_eapi_file, stack_lists, writemsg -from portage.versions import cpv_getkey +from portage.versions import cpv_getkey, _pkg_str from portage.package.ebuild._config.helper import ordered_by_atom_specificity @@ -65,9 +65,9 @@ class UseManager(object): self.repositories = repositories - def _parse_file_to_tuple(self, file_name): + def _parse_file_to_tuple(self, file_name, recursive=True): ret = [] - lines = grabfile(file_name, recursive=1) + lines = grabfile(file_name, recursive=recursive) eapi = read_corresponding_eapi_file(file_name) useflag_re = _get_useflag_re(eapi) for prefixed_useflag in lines: @@ -82,10 +82,10 @@ class UseManager(object): ret.append(prefixed_useflag) return tuple(ret) - def _parse_file_to_dict(self, file_name, juststrings=False): + def _parse_file_to_dict(self, file_name, juststrings=False, recursive=True): ret = {} location_dict = {} - file_dict = grabdict_package(file_name, recursive=1, verify_eapi=True) + file_dict = grabdict_package(file_name, recursive=recursive, verify_eapi=True) eapi = read_corresponding_eapi_file(file_name) useflag_re = _get_useflag_re(eapi) for k, v in file_dict.items(): @@ -132,19 +132,29 @@ class UseManager(object): return ret def _parse_profile_files_to_tuple_of_tuples(self, file_name, locations): - return tuple(self._parse_file_to_tuple(os.path.join(profile, file_name)) for profile in locations) + return tuple(self._parse_file_to_tuple( + os.path.join(profile.location, file_name), + recursive=profile.portage1_directories) + for profile in locations) def _parse_profile_files_to_tuple_of_dicts(self, file_name, locations, juststrings=False): - return tuple(self._parse_file_to_dict(os.path.join(profile, file_name), juststrings) for profile in locations) + return tuple(self._parse_file_to_dict( + os.path.join(profile.location, file_name), juststrings, + recursive=profile.portage1_directories) + for profile in locations) def getUseMask(self, pkg=None): if pkg is None: return frozenset(stack_lists( self._usemask_list, incremental=True)) + slot = None cp = getattr(pkg, "cp", None) if cp is None: - cp = cpv_getkey(remove_slot(pkg)) + slot = dep_getslot(pkg) + repo = dep_getrepo(pkg) + pkg = _pkg_str(remove_slot(pkg), slot=slot, repo=repo) + cp = pkg.cp usemask = [] if hasattr(pkg, "repo") and pkg.repo != Package.UNKNOWN_REPO: repos = [] diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.pyo Binary files differnew file mode 100644 index 0000000..2c9a609 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/VirtualsManager.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/VirtualsManager.pyo Binary files differnew file mode 100644 index 0000000..b2ebd21 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/VirtualsManager.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/__init__.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/__init__.pyo Binary files differnew file mode 100644 index 0000000..b03cc29 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/__init__.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/env_var_validation.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/env_var_validation.pyo Binary files differnew file mode 100644 index 0000000..aeee789 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/env_var_validation.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/features_set.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/features_set.pyo Binary files differnew file mode 100644 index 0000000..9854444 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/features_set.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/helper.py b/portage_with_autodep/pym/portage/package/ebuild/_config/helper.py index 4f46781..ee0c090 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/helper.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/helper.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ( @@ -24,7 +24,7 @@ def ordered_by_atom_specificity(cpdict, pkg, repo=None): order to achieve desired results (and thus corrupting the ChangeLog like ordering of the file). """ - if repo and repo != Package.UNKNOWN_REPO: + if not hasattr(pkg, 'repo') and repo and repo != Package.UNKNOWN_REPO: pkg = pkg + _repo_separator + repo results = [] diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/helper.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/helper.pyo Binary files differnew file mode 100644 index 0000000..f2b9261 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/helper.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.py b/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.py index 6d42809..1a75de9 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ( @@ -15,14 +15,14 @@ env_blacklist = frozenset(( "A", "AA", "CATEGORY", "DEPEND", "DESCRIPTION", "EAPI", "EBUILD_FORCE_TEST", "EBUILD_PHASE", "EBUILD_SKIP_MANIFEST", "ED", "EMERGE_FROM", "EPREFIX", "EROOT", - "HOMEPAGE", "INHERITED", "IUSE", + "GREP_OPTIONS", "HOMEPAGE", "INHERITED", "IUSE", "KEYWORDS", "LICENSE", "MERGE_TYPE", "PDEPEND", "PF", "PKGUSE", "PORTAGE_BACKGROUND", "PORTAGE_BACKGROUND_UNMERGE", "PORTAGE_BUILDIR_LOCKED", "PORTAGE_BUILT_USE", "PORTAGE_CONFIGROOT", "PORTAGE_IUSE", - "PORTAGE_NONFATAL", "PORTAGE_REPO_NAME", "PORTAGE_SANDBOX_COMPAT_LEVEL", - "PORTAGE_USE", "PROPERTIES", "PROVIDE", "RDEPEND", "RESTRICT", - "ROOT", "SLOT", "SRC_URI" + "PORTAGE_NONFATAL", "PORTAGE_REPO_NAME", + "PORTAGE_USE", "PROPERTIES", "PROVIDE", "RDEPEND", "REPOSITORY", + "RESTRICT", "ROOT", "SLOT", "SRC_URI" )) environ_whitelist = [] @@ -36,7 +36,7 @@ environ_whitelist = [] # environment in order to prevent sandbox from sourcing /etc/profile # in it's bashrc (causing major leakage). environ_whitelist += [ - "ACCEPT_LICENSE", "BASH_ENV", "BUILD_PREFIX", "D", + "ACCEPT_LICENSE", "BASH_ENV", "BUILD_PREFIX", "COLUMNS", "D", "DISTDIR", "DOC_SYMLINKS_DIR", "EAPI", "EBUILD", "EBUILD_FORCE_TEST", "EBUILD_PHASE", "ECLASSDIR", "ECLASS_DEPTH", "ED", @@ -50,23 +50,23 @@ environ_whitelist += [ "PORTAGE_BINPKG_TMPFILE", "PORTAGE_BIN_PATH", "PORTAGE_BUILDDIR", "PORTAGE_BUNZIP2_COMMAND", "PORTAGE_BZIP2_COMMAND", - "PORTAGE_COLORMAP", + "PORTAGE_COLORMAP", "PORTAGE_COMPRESS_EXCLUDE_SUFFIXES", "PORTAGE_CONFIGROOT", "PORTAGE_DEBUG", "PORTAGE_DEPCACHEDIR", "PORTAGE_EBUILD_EXIT_FILE", "PORTAGE_FEATURES", "PORTAGE_GID", "PORTAGE_GRPNAME", "PORTAGE_INST_GID", "PORTAGE_INST_UID", "PORTAGE_IPC_DAEMON", "PORTAGE_IUSE", - "PORTAGE_LOG_FILE", + "PORTAGE_LOG_FILE", "PORTAGE_OVERRIDE_EPREFIX", "PORTAGE_PYM_PATH", "PORTAGE_PYTHON", "PORTAGE_QUIET", "PORTAGE_REPO_NAME", "PORTAGE_RESTRICT", - "PORTAGE_SANDBOX_COMPAT_LEVEL", "PORTAGE_SIGPIPE_STATUS", + "PORTAGE_SIGPIPE_STATUS", "PORTAGE_TMPDIR", "PORTAGE_UPDATE_ENV", "PORTAGE_USERNAME", "PORTAGE_VERBOSE", "PORTAGE_WORKDIR_MODE", "PORTDIR", "PORTDIR_OVERLAY", "PREROOTPATH", "PROFILE_PATHS", "REPLACING_VERSIONS", "REPLACED_BY_VERSION", "ROOT", "ROOTPATH", "T", "TMP", "TMPDIR", "USE_EXPAND", "USE_ORDER", "WORKDIR", - "XARGS", + "XARGS", "__PORTAGE_TEST_HARDLINK_LOCKS", ] # user config variables @@ -134,8 +134,9 @@ environ_filter += [ # portage config variables and variables set directly by portage environ_filter += [ "ACCEPT_CHOSTS", "ACCEPT_KEYWORDS", "ACCEPT_PROPERTIES", "AUTOCLEAN", - "CLEAN_DELAY", "COLLISION_IGNORE", "CONFIG_PROTECT", - "CONFIG_PROTECT_MASK", "EGENCACHE_DEFAULT_OPTS", "EMERGE_DEFAULT_OPTS", + "CLEAN_DELAY", "COLLISION_IGNORE", + "CONFIG_PROTECT", "CONFIG_PROTECT_MASK", + "EGENCACHE_DEFAULT_OPTS", "EMERGE_DEFAULT_OPTS", "EMERGE_LOG_DIR", "EMERGE_WARNING_DELAY", "FETCHCOMMAND", "FETCHCOMMAND_FTP", @@ -143,7 +144,8 @@ environ_filter += [ "FETCHCOMMAND_RSYNC", "FETCHCOMMAND_SFTP", "GENTOO_MIRRORS", "NOCONFMEM", "O", "PORTAGE_BACKGROUND", "PORTAGE_BACKGROUND_UNMERGE", - "PORTAGE_BINHOST_CHUNKSIZE", "PORTAGE_BUILDIR_LOCKED", "PORTAGE_CALLER", + "PORTAGE_BINHOST", + "PORTAGE_BINHOST_CHUNKSIZE", "PORTAGE_BUILDIR_LOCKED", "PORTAGE_ELOG_CLASSES", "PORTAGE_ELOG_MAILFROM", "PORTAGE_ELOG_MAILSUBJECT", "PORTAGE_ELOG_MAILURI", "PORTAGE_ELOG_SYSTEM", @@ -156,8 +158,8 @@ environ_filter += [ "PORTAGE_RO_DISTDIRS", "PORTAGE_RSYNC_EXTRA_OPTS", "PORTAGE_RSYNC_OPTS", "PORTAGE_RSYNC_RETRIES", "PORTAGE_SYNC_STALE", - "PORTAGE_USE", "PORT_LOGDIR", - "QUICKPKG_DEFAULT_OPTS", + "PORTAGE_USE", "PORT_LOGDIR", "PORT_LOGDIR_CLEAN", + "QUICKPKG_DEFAULT_OPTS", "REPOMAN_DEFAULT_OPTS", "RESUMECOMMAND", "RESUMECOMMAND_FTP", "RESUMECOMMAND_HTTP", "RESUMECOMMAND_HTTPS", "RESUMECOMMAND_RSYNC", "RESUMECOMMAND_SFTP", diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.pyo Binary files differnew file mode 100644 index 0000000..06ea37e --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/unpack_dependencies.py b/portage_with_autodep/pym/portage/package/ebuild/_config/unpack_dependencies.py new file mode 100644 index 0000000..1375189 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/unpack_dependencies.py @@ -0,0 +1,38 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from portage import os, _supported_eapis +from portage.dep import use_reduce +from portage.eapi import eapi_has_automatic_unpack_dependencies +from portage.exception import InvalidDependString +from portage.localization import _ +from portage.util import grabfile, writemsg + +def load_unpack_dependencies_configuration(repositories): + repo_dict = {} + for repo in repositories.repos_with_profiles(): + for eapi in _supported_eapis: + if eapi_has_automatic_unpack_dependencies(eapi): + file_name = os.path.join(repo.location, "profiles", "unpack_dependencies", eapi) + lines = grabfile(file_name, recursive=True) + for line in lines: + elements = line.split() + suffix = elements[0].lower() + if len(elements) == 1: + writemsg(_("--- Missing unpack dependencies for '%s' suffix in '%s'\n") % (suffix, file_name)) + depend = " ".join(elements[1:]) + try: + use_reduce(depend, eapi=eapi) + except InvalidDependString as e: + writemsg(_("--- Invalid unpack dependencies for '%s' suffix in '%s': '%s'\n" % (suffix, file_name, e))) + else: + repo_dict.setdefault(repo.name, {}).setdefault(eapi, {})[suffix] = depend + + ret = {} + for repo in repositories.repos_with_profiles(): + for repo_name in [x.name for x in repo.masters] + [repo.name]: + for eapi in repo_dict.get(repo_name, {}): + for suffix, depend in repo_dict.get(repo_name, {}).get(eapi, {}).items(): + ret.setdefault(repo.name, {}).setdefault(eapi, {})[suffix] = depend + + return ret diff --git a/portage_with_autodep/pym/portage/package/ebuild/_eapi_invalid.py b/portage_with_autodep/pym/portage/package/ebuild/_eapi_invalid.py new file mode 100644 index 0000000..d23677d --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_eapi_invalid.py @@ -0,0 +1,54 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import textwrap + +import portage +from portage.dep import _repo_separator +from portage.elog import elog_process +from portage.elog.messages import eerror + +def eapi_invalid(self, cpv, repo_name, settings, + eapi_var, eapi_parsed, eapi_lineno): + + msg = [] + msg.extend(textwrap.wrap(("EAPI assignment in ebuild '%s%s%s' does not" + " conform with PMS section 7.3.1 (see bug #402167):") % + (cpv, _repo_separator, repo_name), 70)) + + if not eapi_parsed: + # None means the assignment was not found, while an + # empty string indicates an (invalid) empty assingment. + msg.append( + "\tvalid EAPI assignment must" + " occur on or before line: %s" % + eapi_lineno) + else: + msg.append(("\tbash returned EAPI '%s' which does not match " + "assignment on line: %s") % + (eapi_var, eapi_lineno)) + + if 'parse-eapi-ebuild-head' in settings.features: + msg.extend(textwrap.wrap(("NOTE: This error will soon" + " become unconditionally fatal in a future version of Portage," + " but at this time, it can by made non-fatal by setting" + " FEATURES=-parse-eapi-ebuild-head in" + " make.conf."), 70)) + else: + msg.extend(textwrap.wrap(("NOTE: This error will soon" + " become unconditionally fatal in a future version of Portage." + " At the earliest opportunity, please enable" + " FEATURES=parse-eapi-ebuild-head in make.conf in order to" + " make this error fatal."), 70)) + + if portage.data.secpass >= 2: + # TODO: improve elog permission error handling (bug #416231) + for line in msg: + eerror(line, phase="other", key=cpv) + elog_process(cpv, settings, + phasefilter=("other",)) + + else: + out = portage.output.EOutput() + for line in msg: + out.eerror(line) diff --git a/portage_with_autodep/pym/portage/package/ebuild/_eapi_invalid.pyo b/portage_with_autodep/pym/portage/package/ebuild/_eapi_invalid.pyo Binary files differnew file mode 100644 index 0000000..0181c03 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_eapi_invalid.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_ipc/ExitCommand.pyo b/portage_with_autodep/pym/portage/package/ebuild/_ipc/ExitCommand.pyo Binary files differnew file mode 100644 index 0000000..315cb0f --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_ipc/ExitCommand.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_ipc/IpcCommand.pyo b/portage_with_autodep/pym/portage/package/ebuild/_ipc/IpcCommand.pyo Binary files differnew file mode 100644 index 0000000..9f75518 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_ipc/IpcCommand.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.py b/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.py index fb6e61e..7bbb0e8 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import io @@ -7,6 +7,7 @@ import portage from portage import os from portage import _unicode_decode from portage.dep import Atom +from portage.eapi import eapi_has_repo_deps from portage.elog import messages as elog_messages from portage.exception import InvalidAtom from portage.package.ebuild._ipc.IpcCommand import IpcCommand @@ -26,19 +27,21 @@ class QueryCommand(IpcCommand): def __call__(self, argv): """ - @returns: tuple of (stdout, stderr, returncode) + @return: tuple of (stdout, stderr, returncode) """ cmd, root, atom_str = argv + eapi = self.settings.get('EAPI') + allow_repo = eapi_has_repo_deps(eapi) try: - atom = Atom(atom_str) + atom = Atom(atom_str, allow_repo=allow_repo) except InvalidAtom: return ('', 'invalid atom: %s\n' % atom_str, 2) warnings = [] try: - atom = Atom(atom_str, eapi=self.settings.get('EAPI')) + atom = Atom(atom_str, allow_repo=allow_repo, eapi=eapi) except InvalidAtom as e: warnings.append(_unicode_decode("QA Notice: %s: %s") % (cmd, e)) diff --git a/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.pyo b/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.pyo Binary files differnew file mode 100644 index 0000000..0e9ee96 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_ipc/__init__.pyo b/portage_with_autodep/pym/portage/package/ebuild/_ipc/__init__.pyo Binary files differnew file mode 100644 index 0000000..d9f8d25 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_ipc/__init__.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_metadata_invalid.py b/portage_with_autodep/pym/portage/package/ebuild/_metadata_invalid.py new file mode 100644 index 0000000..bcf1f7f --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_metadata_invalid.py @@ -0,0 +1,41 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import textwrap + +import portage +from portage.dep import _repo_separator +from portage.elog import elog_process +from portage.elog.messages import eerror + +def eapi_invalid(self, cpv, repo_name, settings, + eapi_var, eapi_parsed, eapi_lineno): + + msg = [] + msg.extend(textwrap.wrap(("EAPI assignment in ebuild '%s%s%s' does not" + " conform with PMS section 7.3.1 (see bug #402167):") % + (cpv, _repo_separator, repo_name), 70)) + + if not eapi_parsed: + # None means the assignment was not found, while an + # empty string indicates an (invalid) empty assingment. + msg.append( + "\tvalid EAPI assignment must" + " occur on or before line: %s" % + eapi_lineno) + else: + msg.append(("\tbash returned EAPI '%s' which does not match " + "assignment on line: %s") % + (eapi_var, eapi_lineno)) + + if portage.data.secpass >= 2: + # TODO: improve elog permission error handling (bug #416231) + for line in msg: + eerror(line, phase="other", key=cpv) + elog_process(cpv, settings, + phasefilter=("other",)) + + else: + out = portage.output.EOutput() + for line in msg: + out.eerror(line) diff --git a/portage_with_autodep/pym/portage/package/ebuild/_parallel_manifest/ManifestProcess.py b/portage_with_autodep/pym/portage/package/ebuild/_parallel_manifest/ManifestProcess.py new file mode 100644 index 0000000..44e2576 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_parallel_manifest/ManifestProcess.py @@ -0,0 +1,43 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import portage +from portage import os +from portage.exception import (FileNotFound, + PermissionDenied, PortagePackageException) +from portage.localization import _ +from portage.util._async.ForkProcess import ForkProcess + +class ManifestProcess(ForkProcess): + + __slots__ = ("cp", "distdir", "fetchlist_dict", "repo_config") + + MODIFIED = 16 + + def _run(self): + mf = self.repo_config.load_manifest( + os.path.join(self.repo_config.location, self.cp), + self.distdir, fetchlist_dict=self.fetchlist_dict) + + try: + mf.create(assumeDistHashesAlways=True) + except FileNotFound as e: + portage.writemsg(_("!!! File %s doesn't exist, can't update " + "Manifest\n") % e, noiselevel=-1) + return 1 + + except PortagePackageException as e: + portage.writemsg(("!!! %s\n") % (e,), noiselevel=-1) + return 1 + + try: + modified = mf.write(sign=False) + except PermissionDenied as e: + portage.writemsg("!!! %s: %s\n" % (_("Permission Denied"), e,), + noiselevel=-1) + return 1 + else: + if modified: + return self.MODIFIED + else: + return os.EX_OK diff --git a/portage_with_autodep/pym/portage/package/ebuild/_parallel_manifest/ManifestScheduler.py b/portage_with_autodep/pym/portage/package/ebuild/_parallel_manifest/ManifestScheduler.py new file mode 100644 index 0000000..38ac482 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_parallel_manifest/ManifestScheduler.py @@ -0,0 +1,93 @@ +# Copyright 2012-2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import portage +from portage import os +from portage.dep import _repo_separator +from portage.exception import InvalidDependString +from portage.localization import _ +from portage.util._async.AsyncScheduler import AsyncScheduler +from .ManifestTask import ManifestTask + +class ManifestScheduler(AsyncScheduler): + + def __init__(self, portdb, cp_iter=None, + gpg_cmd=None, gpg_vars=None, force_sign_key=None, **kwargs): + + AsyncScheduler.__init__(self, **kwargs) + + self._portdb = portdb + + if cp_iter is None: + cp_iter = self._iter_every_cp() + self._cp_iter = cp_iter + self._gpg_cmd = gpg_cmd + self._gpg_vars = gpg_vars + self._force_sign_key = force_sign_key + self._task_iter = self._iter_tasks() + + def _next_task(self): + return next(self._task_iter) + + def _iter_every_cp(self): + # List categories individually, in order to start yielding quicker, + # and in order to reduce latency in case of a signal interrupt. + cp_all = self._portdb.cp_all + for category in sorted(self._portdb.categories): + for cp in cp_all(categories=(category,)): + yield cp + + def _iter_tasks(self): + portdb = self._portdb + distdir = portdb.settings["DISTDIR"] + disabled_repos = set() + + for cp in self._cp_iter: + if self._terminated.is_set(): + break + # We iterate over portdb.porttrees, since it's common to + # tweak this attribute in order to adjust repo selection. + for mytree in portdb.porttrees: + if self._terminated.is_set(): + break + repo_config = portdb.repositories.get_repo_for_location(mytree) + if not repo_config.create_manifest: + if repo_config.name not in disabled_repos: + disabled_repos.add(repo_config.name) + portage.writemsg( + _(">>> Skipping creating Manifest for %s%s%s; " + "repository is configured to not use them\n") % + (cp, _repo_separator, repo_config.name), + noiselevel=-1) + continue + cpv_list = portdb.cp_list(cp, mytree=[repo_config.location]) + if not cpv_list: + continue + fetchlist_dict = {} + try: + for cpv in cpv_list: + fetchlist_dict[cpv] = \ + list(portdb.getFetchMap(cpv, mytree=mytree)) + except InvalidDependString as e: + portage.writemsg( + _("!!! %s%s%s: SRC_URI: %s\n") % + (cp, _repo_separator, repo_config.name, e), + noiselevel=-1) + self._error_count += 1 + continue + + yield ManifestTask(cp=cp, distdir=distdir, + fetchlist_dict=fetchlist_dict, repo_config=repo_config, + gpg_cmd=self._gpg_cmd, gpg_vars=self._gpg_vars, + force_sign_key=self._force_sign_key) + + def _task_exit(self, task): + + if task.returncode != os.EX_OK: + if not self._terminated_tasks: + portage.writemsg( + "Error processing %s%s%s, continuing...\n" % + (task.cp, _repo_separator, task.repo_config.name), + noiselevel=-1) + + AsyncScheduler._task_exit(self, task) diff --git a/portage_with_autodep/pym/portage/package/ebuild/_parallel_manifest/ManifestTask.py b/portage_with_autodep/pym/portage/package/ebuild/_parallel_manifest/ManifestTask.py new file mode 100644 index 0000000..0ee2b91 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_parallel_manifest/ManifestTask.py @@ -0,0 +1,186 @@ +# Copyright 2012-2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import errno +import re +import subprocess + +from portage import os +from portage import _unicode_encode, _encodings +from portage.const import MANIFEST2_IDENTIFIERS +from portage.util import (atomic_ofstream, grablines, + shlex_split, varexpand, writemsg) +from portage.util._async.PipeLogger import PipeLogger +from portage.util._async.PopenProcess import PopenProcess +from _emerge.CompositeTask import CompositeTask +from _emerge.PipeReader import PipeReader +from .ManifestProcess import ManifestProcess + +class ManifestTask(CompositeTask): + + __slots__ = ("cp", "distdir", "fetchlist_dict", "gpg_cmd", + "gpg_vars", "repo_config", "force_sign_key", "_manifest_path") + + _PGP_HEADER = b"BEGIN PGP SIGNED MESSAGE" + _manifest_line_re = re.compile(r'^(%s) ' % "|".join(MANIFEST2_IDENTIFIERS)) + _gpg_key_id_re = re.compile(r'^[0-9A-F]*$') + _gpg_key_id_lengths = (8, 16, 24, 32, 40) + + def _start(self): + self._manifest_path = os.path.join(self.repo_config.location, + self.cp, "Manifest") + manifest_proc = ManifestProcess(cp=self.cp, distdir=self.distdir, + fetchlist_dict=self.fetchlist_dict, repo_config=self.repo_config, + scheduler=self.scheduler) + self._start_task(manifest_proc, self._manifest_proc_exit) + + def _manifest_proc_exit(self, manifest_proc): + self._assert_current(manifest_proc) + if manifest_proc.returncode not in (os.EX_OK, manifest_proc.MODIFIED): + self.returncode = manifest_proc.returncode + self._current_task = None + self.wait() + return + + modified = manifest_proc.returncode == manifest_proc.MODIFIED + sign = self.gpg_cmd is not None + + if not modified and sign: + sign = self._need_signature() + if not sign and self.force_sign_key is not None \ + and os.path.exists(self._manifest_path): + self._check_sig_key() + return + + if not sign or not os.path.exists(self._manifest_path): + self.returncode = os.EX_OK + self._current_task = None + self.wait() + return + + self._start_gpg_proc() + + def _check_sig_key(self): + null_fd = os.open('/dev/null', os.O_RDONLY) + popen_proc = PopenProcess(proc=subprocess.Popen( + ["gpg", "--verify", self._manifest_path], + stdin=null_fd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT), + pipe_reader=PipeReader()) + os.close(null_fd) + popen_proc.pipe_reader.input_files = { + "producer" : popen_proc.proc.stdout} + self._start_task(popen_proc, self._check_sig_key_exit) + + @staticmethod + def _parse_gpg_key(output): + """ + Returns the first token which appears to represent a gpg key + id, or None if there is no such token. + """ + regex = ManifestTask._gpg_key_id_re + lengths = ManifestTask._gpg_key_id_lengths + for token in output.split(): + m = regex.match(token) + if m is not None and len(m.group(0)) in lengths: + return m.group(0) + return None + + @staticmethod + def _normalize_gpg_key(key_str): + """ + Strips leading "0x" and trailing "!", and converts to uppercase + (intended to be the same format as that in gpg --verify output). + """ + key_str = key_str.upper() + if key_str.startswith("0X"): + key_str = key_str[2:] + key_str = key_str.rstrip("!") + return key_str + + def _check_sig_key_exit(self, proc): + self._assert_current(proc) + + parsed_key = self._parse_gpg_key( + proc.pipe_reader.getvalue().decode('utf_8', 'replace')) + if parsed_key is not None and \ + self._normalize_gpg_key(parsed_key) == \ + self._normalize_gpg_key(self.force_sign_key): + self.returncode = os.EX_OK + self._current_task = None + self.wait() + return + + if self._was_cancelled(): + self.wait() + return + + self._strip_sig(self._manifest_path) + self._start_gpg_proc() + + @staticmethod + def _strip_sig(manifest_path): + """ + Strip an existing signature from a Manifest file. + """ + line_re = ManifestTask._manifest_line_re + lines = grablines(manifest_path) + f = None + try: + f = atomic_ofstream(manifest_path) + for line in lines: + if line_re.match(line) is not None: + f.write(line) + f.close() + f = None + finally: + if f is not None: + f.abort() + + def _start_gpg_proc(self): + gpg_vars = self.gpg_vars + if gpg_vars is None: + gpg_vars = {} + else: + gpg_vars = gpg_vars.copy() + gpg_vars["FILE"] = self._manifest_path + gpg_cmd = varexpand(self.gpg_cmd, mydict=gpg_vars) + gpg_cmd = shlex_split(gpg_cmd) + gpg_proc = PopenProcess(proc=subprocess.Popen(gpg_cmd, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT)) + # PipeLogger echos output and efficiently monitors for process + # exit by listening for the stdout EOF event. + gpg_proc.pipe_reader = PipeLogger(background=self.background, + input_fd=gpg_proc.proc.stdout, scheduler=self.scheduler) + self._start_task(gpg_proc, self._gpg_proc_exit) + + def _gpg_proc_exit(self, gpg_proc): + if self._default_exit(gpg_proc) != os.EX_OK: + self.wait() + return + + rename_args = (self._manifest_path + ".asc", self._manifest_path) + try: + os.rename(*rename_args) + except OSError as e: + writemsg("!!! rename('%s', '%s'): %s\n" % rename_args + (e,), + noiselevel=-1) + try: + os.unlink(self._manifest_path + ".asc") + except OSError: + pass + self.returncode = 1 + else: + self.returncode = os.EX_OK + + self._current_task = None + self.wait() + + def _need_signature(self): + try: + with open(_unicode_encode(self._manifest_path, + encoding=_encodings['fs'], errors='strict'), 'rb') as f: + return self._PGP_HEADER not in f.readline() + except IOError as e: + if e.errno in (errno.ENOENT, errno.ESTALE): + return False + raise diff --git a/portage_with_autodep/pym/portage/package/ebuild/_parallel_manifest/__init__.py b/portage_with_autodep/pym/portage/package/ebuild/_parallel_manifest/__init__.py new file mode 100644 index 0000000..418ad86 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_parallel_manifest/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 diff --git a/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.py b/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.py index befdc89..94f8c79 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.py @@ -1,10 +1,10 @@ # Copyright 2010-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -import shutil import tempfile from portage import os +from portage import shutil from portage.const import EBUILD_PHASES from portage.elog import elog_process from portage.package.ebuild.config import config @@ -20,7 +20,11 @@ def spawn_nofetch(portdb, ebuild_path, settings=None): to cache metadata. It will be cloned internally, in order to prevent any changes from interfering with the calling code. If settings is None then a suitable config instance will be - acquired from the given portdbapi instance. + acquired from the given portdbapi instance. Do not use the + settings parameter unless setcpv has been called on the given + instance, since otherwise it's possible to trigger issues like + bug #408817 due to fragile assumptions involving the config + state inside doebuild_environment(). A private PORTAGE_BUILDDIR will be created and cleaned up, in order to avoid any interference with any other processes. diff --git a/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.pyo b/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.pyo Binary files differnew file mode 100644 index 0000000..ac449ea --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/config.py b/portage_with_autodep/pym/portage/package/ebuild/config.py index a8c6ad6..97cbd99 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/config.py +++ b/portage_with_autodep/pym/portage/package/ebuild/config.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = [ @@ -7,7 +7,10 @@ __all__ = [ import copy from itertools import chain +import grp import logging +import platform +import pwd import re import sys import warnings @@ -21,10 +24,9 @@ from portage import bsd_chflags, \ load_mod, os, selinux, _unicode_decode from portage.const import CACHE_PATH, \ DEPCACHE_PATH, INCREMENTALS, MAKE_CONF_FILE, \ - MODULES_FILE_PATH, PORTAGE_BIN_PATH, PORTAGE_PYM_PATH, \ + MODULES_FILE_PATH, \ PRIVATE_PATH, PROFILE_PATH, USER_CONFIG_PATH, \ USER_VIRTUALS_FILE -from portage.const import _SANDBOX_COMPAT_LEVEL from portage.dbapi import dbapi from portage.dbapi.porttree import portdbapi from portage.dbapi.vartree import vartree @@ -41,7 +43,7 @@ from portage.util import ensure_dirs, getconfig, grabdict, \ grabdict_package, grabfile, grabfile_package, LazyItemsDict, \ normalize_path, shlex_split, stack_dictlist, stack_dicts, stack_lists, \ writemsg, writemsg_level -from portage.versions import catpkgsplit, catsplit, cpv_getkey +from portage.versions import catpkgsplit, catsplit, cpv_getkey, _pkg_str from portage.package.ebuild._config import special_env_vars from portage.package.ebuild._config.env_var_validation import validate_cmd_var @@ -120,11 +122,19 @@ class config(object): virtuals ...etc you look in here. """ + _constant_keys = frozenset(['PORTAGE_BIN_PATH', 'PORTAGE_GID', + 'PORTAGE_PYM_PATH']) + _setcpv_aux_keys = ('DEFINED_PHASES', 'DEPEND', 'EAPI', 'INHERITED', 'IUSE', 'REQUIRED_USE', 'KEYWORDS', 'LICENSE', 'PDEPEND', 'PROPERTIES', 'PROVIDE', 'RDEPEND', 'SLOT', 'repository', 'RESTRICT', 'LICENSE',) + _module_aliases = { + "cache.metadata_overlay.database" : "portage.cache.flat_hash.database", + "portage.cache.metadata_overlay.database" : "portage.cache.flat_hash.database", + } + _case_insensitive_vars = special_env_vars.case_insensitive_vars _default_globals = special_env_vars.default_globals _env_blacklist = special_env_vars.env_blacklist @@ -135,7 +145,8 @@ class config(object): def __init__(self, clone=None, mycpv=None, config_profile_path=None, config_incrementals=None, config_root=None, target_root=None, - _eprefix=None, local_config=True, env=None, _unmatched_removal=False): + eprefix=None, local_config=True, env=None, + _unmatched_removal=False): """ @param clone: If provided, init will use deepcopy to copy by value the instance. @type clone: Instance of config class. @@ -151,8 +162,8 @@ class config(object): @type config_root: String @param target_root: __init__ override of $ROOT env variable. @type target_root: String - @param _eprefix: set the EPREFIX variable (private, used by internal tests) - @type _eprefix: String + @param eprefix: set the EPREFIX variable (default is portage.const.EPREFIX) + @type eprefix: String @param local_config: Enables loading of local config (/etc/portage); used most by repoman to ignore local config (keywording and unmasking) @type local_config: Boolean @@ -164,10 +175,6 @@ class config(object): @type _unmatched_removal: Boolean """ - # rename local _eprefix variable for convenience - eprefix = _eprefix - del _eprefix - # When initializing the global portage.settings instance, avoid # raising exceptions whenever possible since exceptions thrown # from 'import portage' or 'import portage.exceptions' statements @@ -208,6 +215,7 @@ class config(object): self.repositories = clone.repositories self._iuse_implicit_match = clone._iuse_implicit_match self._non_user_variables = clone._non_user_variables + self._env_d_blacklist = clone._env_d_blacklist self._repo_make_defaults = clone._repo_make_defaults self.usemask = clone.usemask self.useforce = clone.useforce @@ -272,9 +280,6 @@ class config(object): eprefix = locations_manager.eprefix config_root = locations_manager.config_root - self.profiles = locations_manager.profiles - self.profile_path = locations_manager.profile_path - self.user_profile_dir = locations_manager.user_profile_dir abs_user_config = locations_manager.abs_user_config make_conf = getconfig( @@ -293,6 +298,38 @@ class config(object): eroot = locations_manager.eroot self.global_config_path = locations_manager.global_config_path + # The expand_map is used for variable substitution + # in getconfig() calls, and the getconfig() calls + # update expand_map with the value of each variable + # assignment that occurs. Variable substitution occurs + # in the following order, which corresponds to the + # order of appearance in self.lookuplist: + # + # * env.d + # * make.globals + # * make.defaults + # * make.conf + # + # Notably absent is "env", since we want to avoid any + # interaction with the calling environment that might + # lead to unexpected results. + + env_d = getconfig(os.path.join(eroot, "etc", "profile.env"), + expand=False) or {} + expand_map = env_d.copy() + self._expand_map = expand_map + + # Allow make.globals to set default paths relative to ${EPREFIX}. + expand_map["EPREFIX"] = eprefix + + make_globals = getconfig(os.path.join( + self.global_config_path, 'make.globals'), expand=expand_map) + if make_globals is None: + make_globals = {} + + for k, v in self._default_globals.items(): + make_globals.setdefault(k, v) + if config_incrementals is None: self.incrementals = INCREMENTALS else: @@ -302,14 +339,20 @@ class config(object): self.module_priority = ("user", "default") self.modules = {} - modules_loader = KeyValuePairFileLoader( - os.path.join(config_root, MODULES_FILE_PATH), None, None) + modules_file = os.path.join(config_root, MODULES_FILE_PATH) + modules_loader = KeyValuePairFileLoader(modules_file, None, None) modules_dict, modules_errors = modules_loader.load() self.modules["user"] = modules_dict if self.modules["user"] is None: self.modules["user"] = {} + user_auxdbmodule = \ + self.modules["user"].get("portdbapi.auxdbmodule") + if user_auxdbmodule is not None and \ + user_auxdbmodule in self._module_aliases: + warnings.warn("'%s' is deprecated: %s" % + (user_auxdbmodule, modules_file)) + self.modules["default"] = { - "portdbapi.metadbmodule": "portage.cache.metadata.database", "portdbapi.auxdbmodule": "portage.cache.flat_hash.database", } @@ -328,43 +371,9 @@ class config(object): self.configlist.append({}) self.configdict["pkginternal"] = self.configlist[-1] - self.packages_list = [grabfile_package(os.path.join(x, "packages"), verify_eapi=True) for x in self.profiles] - self.packages = tuple(stack_lists(self.packages_list, incremental=1)) - del self.packages_list - #self.packages = grab_stacked("packages", self.profiles, grabfile, incremental_lines=1) - - # revmaskdict - self.prevmaskdict={} - for x in self.packages: - # Negative atoms are filtered by the above stack_lists() call. - if not isinstance(x, Atom): - x = Atom(x.lstrip('*')) - self.prevmaskdict.setdefault(x.cp, []).append(x) - - # The expand_map is used for variable substitution - # in getconfig() calls, and the getconfig() calls - # update expand_map with the value of each variable - # assignment that occurs. Variable substitution occurs - # in the following order, which corresponds to the - # order of appearance in self.lookuplist: - # - # * env.d - # * make.globals - # * make.defaults - # * make.conf - # - # Notably absent is "env", since we want to avoid any - # interaction with the calling environment that might - # lead to unexpected results. - expand_map = {} - self._expand_map = expand_map - - env_d = getconfig(os.path.join(eroot, "etc", "profile.env"), - expand=expand_map) # env_d will be None if profile.env doesn't exist. if env_d: self.configdict["env.d"].update(env_d) - expand_map.update(env_d) # backupenv is used for calculating incremental variables. if env is None: @@ -390,40 +399,72 @@ class config(object): self.configdict["env"] = LazyItemsDict(self.backupenv) - for x in (self.global_config_path,): - self.mygcfg = getconfig(os.path.join(x, "make.globals"), - expand=expand_map) - if self.mygcfg: - break + self.configlist.append(make_globals) + self.configdict["globals"]=self.configlist[-1] - if self.mygcfg is None: - self.mygcfg = {} + self.make_defaults_use = [] - for k, v in self._default_globals.items(): - self.mygcfg.setdefault(k, v) + #Loading Repositories + self["PORTAGE_CONFIGROOT"] = config_root + self["ROOT"] = target_root + self["EPREFIX"] = eprefix + self["EROOT"] = eroot + known_repos = [] + portdir = "" + portdir_overlay = "" + for confs in [make_globals, make_conf, self.configdict["env"]]: + v = confs.get("PORTDIR") + if v is not None: + portdir = v + known_repos.append(v) + v = confs.get("PORTDIR_OVERLAY") + if v is not None: + portdir_overlay = v + known_repos.extend(shlex_split(v)) + known_repos = frozenset(known_repos) + self["PORTDIR"] = portdir + self["PORTDIR_OVERLAY"] = portdir_overlay + self.lookuplist = [self.configdict["env"]] + self.repositories = load_repository_config(self) - self.configlist.append(self.mygcfg) - self.configdict["globals"]=self.configlist[-1] + locations_manager.load_profiles(self.repositories, known_repos) - self.make_defaults_use = [] - self.mygcfg = {} + profiles_complex = locations_manager.profiles_complex + self.profiles = locations_manager.profiles + self.profile_path = locations_manager.profile_path + self.user_profile_dir = locations_manager.user_profile_dir + + packages_list = [grabfile_package(os.path.join(x, "packages"), + verify_eapi=True) for x in self.profiles] + self.packages = tuple(stack_lists(packages_list, incremental=1)) + + # revmaskdict + self.prevmaskdict={} + for x in self.packages: + # Negative atoms are filtered by the above stack_lists() call. + if not isinstance(x, Atom): + x = Atom(x.lstrip('*')) + self.prevmaskdict.setdefault(x.cp, []).append(x) + + + mygcfg = {} if self.profiles: mygcfg_dlists = [getconfig(os.path.join(x, "make.defaults"), expand=expand_map) for x in self.profiles] self._make_defaults = mygcfg_dlists - self.mygcfg = stack_dicts(mygcfg_dlists, + mygcfg = stack_dicts(mygcfg_dlists, incrementals=self.incrementals) - if self.mygcfg is None: - self.mygcfg = {} - self.configlist.append(self.mygcfg) + if mygcfg is None: + mygcfg = {} + self.configlist.append(mygcfg) self.configdict["defaults"]=self.configlist[-1] - self.mygcfg = getconfig( + mygcfg = getconfig( os.path.join(config_root, MAKE_CONF_FILE), tolerant=tolerant, allow_sourcing=True, expand=expand_map) or {} - self.mygcfg.update(getconfig( + mygcfg.update(getconfig( os.path.join(abs_user_config, 'make.conf'), tolerant=tolerant, allow_sourcing=True, expand=expand_map) or {}) @@ -439,10 +480,18 @@ class config(object): non_user_variables = frozenset(non_user_variables) self._non_user_variables = non_user_variables + self._env_d_blacklist = frozenset(chain( + profile_only_variables, + self._env_blacklist, + )) + env_d = self.configdict["env.d"] + for k in self._env_d_blacklist: + env_d.pop(k, None) + for k in profile_only_variables: - self.mygcfg.pop(k, None) + mygcfg.pop(k, None) - self.configlist.append(self.mygcfg) + self.configlist.append(mygcfg) self.configdict["conf"]=self.configlist[-1] self.configlist.append(LazyItemsDict()) @@ -472,25 +521,23 @@ class config(object): self["ROOT"] = target_root self.backup_changes("ROOT") + # The PORTAGE_OVERRIDE_EPREFIX variable propagates the EPREFIX + # of this config instance to any portage commands or API + # consumers running in subprocesses. self["EPREFIX"] = eprefix self.backup_changes("EPREFIX") + self["PORTAGE_OVERRIDE_EPREFIX"] = eprefix + self.backup_changes("PORTAGE_OVERRIDE_EPREFIX") self["EROOT"] = eroot self.backup_changes("EROOT") - self["PORTAGE_SANDBOX_COMPAT_LEVEL"] = _SANDBOX_COMPAT_LEVEL - self.backup_changes("PORTAGE_SANDBOX_COMPAT_LEVEL") - self._ppropertiesdict = portage.dep.ExtendedAtomDict(dict) self._penvdict = portage.dep.ExtendedAtomDict(dict) - #Loading Repositories - self.repositories = load_repository_config(self) - #filling PORTDIR and PORTDIR_OVERLAY variable for compatibility main_repo = self.repositories.mainRepo() if main_repo is not None: - main_repo = main_repo.user_location - self["PORTDIR"] = main_repo + self["PORTDIR"] = main_repo.user_location self.backup_changes("PORTDIR") # repoman controls PORTDIR_OVERLAY via the environment, so no @@ -501,11 +548,11 @@ class config(object): new_ov = [] if portdir_overlay: - whitespace_re = re.compile(r"\s") + shell_quote_re = re.compile(r"[\s\\\"'$`]") for ov in portdir_overlay: ov = normalize_path(ov) if os.path.isdir(ov): - if whitespace_re.search(ov) is not None: + if shell_quote_re.search(ov) is not None: ov = portage._shell_quote(ov) new_ov.append(ov) else: @@ -528,11 +575,11 @@ class config(object): self._repo_make_defaults[repo.name] = d #Read package.keywords and package.accept_keywords. - self._keywords_manager = KeywordsManager(self.profiles, abs_user_config, \ + self._keywords_manager = KeywordsManager(profiles_complex, abs_user_config, \ local_config, global_accept_keywords=self.configdict["defaults"].get("ACCEPT_KEYWORDS", "")) #Read all USE related files from profiles and optionally from user config. - self._use_manager = UseManager(self.repositories, self.profiles, abs_user_config, user_config=local_config) + self._use_manager = UseManager(self.repositories, profiles_complex, abs_user_config, user_config=local_config) #Initialize all USE related variables we track ourselves. self.usemask = self._use_manager.getUseMask() self.useforce = self._use_manager.getUseForce() @@ -549,7 +596,7 @@ class config(object): self.configdict["conf"].get("ACCEPT_LICENSE", "")) #Read package.mask and package.unmask from profiles and optionally from user config - self._mask_manager = MaskManager(self.repositories, self.profiles, + self._mask_manager = MaskManager(self.repositories, profiles_complex, abs_user_config, user_config=local_config, strict_umatched_removal=_unmatched_removal) @@ -597,16 +644,21 @@ class config(object): self.categories = [grabfile(os.path.join(x, "categories")) \ for x in locations_manager.profile_and_user_locations] category_re = dbapi._category_re - self.categories = tuple(sorted( + # categories used to be a tuple, but now we use a frozenset + # for hashed category validation in pordbapi.cp_list() + self.categories = frozenset( x for x in stack_lists(self.categories, incremental=1) - if category_re.match(x) is not None)) + if category_re.match(x) is not None) archlist = [grabfile(os.path.join(x, "arch.list")) \ for x in locations_manager.profile_and_user_locations] archlist = stack_lists(archlist, incremental=1) self.configdict["conf"]["PORTAGE_ARCHLIST"] = " ".join(archlist) - pkgprovidedlines = [grabfile(os.path.join(x, "package.provided"), recursive=1) for x in self.profiles] + pkgprovidedlines = [grabfile( + os.path.join(x.location, "package.provided"), + recursive=x.portage1_directories) + for x in profiles_complex] pkgprovidedlines = stack_lists(pkgprovidedlines, incremental=1) has_invalid_data = False for x in range(len(pkgprovidedlines)-1, -1, -1): @@ -649,9 +701,6 @@ class config(object): if "USE_ORDER" not in self: self.backupenv["USE_ORDER"] = "env:pkg:conf:defaults:pkginternal:repo:env.d" - self["PORTAGE_GID"] = str(portage_gid) - self.backup_changes("PORTAGE_GID") - self.depcachedir = DEPCACHE_PATH if eprefix: # See comments about make.globals and EPREFIX @@ -678,19 +727,60 @@ class config(object): self["CBUILD"] = self["CHOST"] self.backup_changes("CBUILD") - self["PORTAGE_BIN_PATH"] = PORTAGE_BIN_PATH - self.backup_changes("PORTAGE_BIN_PATH") - self["PORTAGE_PYM_PATH"] = PORTAGE_PYM_PATH - self.backup_changes("PORTAGE_PYM_PATH") + if "USERLAND" not in self: + # Set default USERLAND so that our test cases can assume that + # it's always set. This allows isolated-functions.sh to avoid + # calling uname -s when sourced. + system = platform.system() + if system is not None and \ + (system.endswith("BSD") or system == "DragonFly"): + self["USERLAND"] = "BSD" + else: + self["USERLAND"] = "GNU" + self.backup_changes("USERLAND") - for var in ("PORTAGE_INST_UID", "PORTAGE_INST_GID"): + default_inst_ids = { + "PORTAGE_INST_GID": "0", + "PORTAGE_INST_UID": "0", + } + + if eprefix: + # For prefix environments, default to the UID and GID of + # the top-level EROOT directory. try: - self[var] = str(int(self.get(var, "0"))) + eroot_st = os.stat(eroot) + except OSError: + pass + else: + default_inst_ids["PORTAGE_INST_GID"] = str(eroot_st.st_gid) + default_inst_ids["PORTAGE_INST_UID"] = str(eroot_st.st_uid) + + if "PORTAGE_USERNAME" not in self: + try: + pwd_struct = pwd.getpwuid(eroot_st.st_uid) + except KeyError: + pass + else: + self["PORTAGE_USERNAME"] = pwd_struct.pw_name + self.backup_changes("PORTAGE_USERNAME") + + if "PORTAGE_GRPNAME" not in self: + try: + grp_struct = grp.getgrgid(eroot_st.st_gid) + except KeyError: + pass + else: + self["PORTAGE_GRPNAME"] = grp_struct.gr_name + self.backup_changes("PORTAGE_GRPNAME") + + for var, default_val in default_inst_ids.items(): + try: + self[var] = str(int(self.get(var, default_val))) except ValueError: writemsg(_("!!! %s='%s' is not a valid integer. " - "Falling back to '0'.\n") % (var, self[var]), + "Falling back to %s.\n") % (var, self[var], default_val), noiselevel=-1) - self[var] = "0" + self[var] = default_val self.backup_changes(var) # initialize self.features @@ -699,21 +789,33 @@ class config(object): if bsd_chflags: self.features.add('chflags') - if 'parse-eapi-ebuild-head' in self.features: - portage._validate_cache_for_unsupported_eapis = False - self._iuse_implicit_match = _iuse_implicit_match_cache(self) self._validate_commands() - for k in self._case_insensitive_vars: - if k in self: - self[k] = self[k].lower() - self.backup_changes(k) + for k in self._case_insensitive_vars: + if k in self: + self[k] = self[k].lower() + self.backup_changes(k) + + if main_repo is not None and not main_repo.sync: + main_repo_sync = self.get("SYNC") + if main_repo_sync: + main_repo.sync = main_repo_sync + + # The first constructed config object initializes these modules, + # and subsequent calls to the _init() functions have no effect. + portage.output._init(config_root=self['PORTAGE_CONFIGROOT']) + portage.data._init(self) if mycpv: self.setcpv(mycpv) + @property + def mygcfg(self): + warnings.warn("portage.config.mygcfg is deprecated", stacklevel=3) + return {} + def _validate_commands(self): for k in special_env_vars.validate_commands: v = self.get(k) @@ -817,11 +919,26 @@ class config(object): writemsg(_("!!! INVALID ACCEPT_KEYWORDS: %s\n") % str(group), noiselevel=-1) - abs_profile_path = os.path.join(self["PORTAGE_CONFIGROOT"], - PROFILE_PATH) - if (not self.profile_path or \ - not os.path.exists(os.path.join(self.profile_path, "parent"))) and \ - os.path.exists(os.path.join(self["PORTDIR"], "profiles")): + profile_broken = not self.profile_path or \ + not os.path.exists(os.path.join(self.profile_path, "parent")) and \ + os.path.exists(os.path.join(self["PORTDIR"], "profiles")) + + if profile_broken: + abs_profile_path = None + for x in (PROFILE_PATH, 'etc/portage/make.profile'): + x = os.path.join(self["PORTAGE_CONFIGROOT"], x) + try: + os.lstat(x) + except OSError: + pass + else: + abs_profile_path = x + break + + if abs_profile_path is None: + abs_profile_path = os.path.join(self["PORTAGE_CONFIGROOT"], + PROFILE_PATH) + writemsg(_("\n\n!!! %s is not a symlink and will probably prevent most merges.\n") % abs_profile_path, noiselevel=-1) writemsg(_("!!! It should point into a profile within %s/profiles/\n") % self["PORTDIR"]) @@ -851,13 +968,32 @@ class config(object): writemsg(_("!!! FEATURES=fakeroot is enabled, but the " "fakeroot binary is not installed.\n"), noiselevel=-1) + if os.getuid() == 0 and not hasattr(os, "setgroups"): + warning_shown = False + + if "userpriv" in self.features: + writemsg(_("!!! FEATURES=userpriv is enabled, but " + "os.setgroups is not available.\n"), noiselevel=-1) + warning_shown = True + + if "userfetch" in self.features: + writemsg(_("!!! FEATURES=userfetch is enabled, but " + "os.setgroups is not available.\n"), noiselevel=-1) + warning_shown = True + + if warning_shown and platform.python_implementation() == 'PyPy': + writemsg(_("!!! See https://bugs.pypy.org/issue833 for details.\n"), + noiselevel=-1) + def load_best_module(self,property_string): best_mod = best_from_dict(property_string,self.modules,self.module_priority) mod = None try: mod = load_mod(best_mod) except ImportError: - if not best_mod.startswith("cache."): + if best_mod in self._module_aliases: + mod = load_mod(self._module_aliases[best_mod]) + elif not best_mod.startswith("cache."): raise else: best_mod = "portage." + best_mod @@ -1099,8 +1235,11 @@ class config(object): # packages since we want to save it PORTAGE_BUILT_USE for # evaluating conditional USE deps in atoms passed via IPC to # helpers like has_version and best_version. + aux_keys = set(aux_keys) + if hasattr(mydb, '_aux_cache_keys'): + aux_keys = aux_keys.intersection(mydb._aux_cache_keys) + aux_keys.add('USE') aux_keys = list(aux_keys) - aux_keys.append('USE') for k, v in zip(aux_keys, mydb.aux_get(self.mycpv, aux_keys)): pkg_configdict[k] = v built_use = frozenset(pkg_configdict.pop('USE').split()) @@ -1115,7 +1254,7 @@ class config(object): slot = pkg_configdict["SLOT"] iuse = pkg_configdict["IUSE"] if pkg is None: - cpv_slot = "%s:%s" % (self.mycpv, slot) + cpv_slot = _pkg_str(self.mycpv, slot=slot, repo=repository) else: cpv_slot = pkg pkginternaluse = [] @@ -1462,6 +1601,9 @@ class config(object): @return: A matching profile atom string or None if one is not found. """ + warnings.warn("The config._getProfileMaskAtom() method is deprecated.", + DeprecationWarning, stacklevel=2) + cp = cpv_getkey(cpv) profile_atoms = self.prevmaskdict.get(cp) if profile_atoms: @@ -1564,11 +1706,13 @@ class config(object): @return: A list of properties that have not been accepted. """ accept_properties = self._accept_properties + if not hasattr(cpv, 'slot'): + cpv = _pkg_str(cpv, slot=metadata["SLOT"], + repo=metadata.get("repository")) cp = cpv_getkey(cpv) cpdict = self._ppropertiesdict.get(cp) if cpdict: - cpv_slot = "%s:%s" % (cpv, metadata["SLOT"]) - pproperties_list = ordered_by_atom_specificity(cpdict, cpv_slot, repo=metadata.get('repository')) + pproperties_list = ordered_by_atom_specificity(cpdict, cpv) if pproperties_list: accept_properties = list(self._accept_properties) for x in pproperties_list: @@ -1699,6 +1843,8 @@ class config(object): env_d = getconfig(env_d_filename, expand=False) if env_d: # env_d will be None if profile.env doesn't exist. + for k in self._env_d_blacklist: + env_d.pop(k, None) self.configdict["env.d"].update(env_d) def regenerate(self, useonly=0, use_cache=None): @@ -2025,25 +2171,43 @@ class config(object): self._virtuals_manager._treeVirtuals = {} def __delitem__(self,mykey): - self.modifying() - for x in self.lookuplist: - if x != None: - if mykey in x: - del x[mykey] + self.pop(mykey) + + def __getitem__(self, key): + try: + return self._getitem(key) + except KeyError: + return '' # for backward compat, don't raise KeyError + + def _getitem(self, mykey): + + if mykey in self._constant_keys: + # These two point to temporary values when + # portage plans to update itself. + if mykey == "PORTAGE_BIN_PATH": + return portage._bin_path + elif mykey == "PORTAGE_PYM_PATH": + return portage._pym_path + + elif mykey == "PORTAGE_GID": + return _unicode_decode(str(portage_gid)) - def __getitem__(self,mykey): for d in self.lookuplist: - if mykey in d: + try: return d[mykey] - return '' # for backward compat, don't raise KeyError + except KeyError: + pass + + raise KeyError(mykey) def get(self, k, x=None): - for d in self.lookuplist: - if k in d: - return d[k] - return x + try: + return self._getitem(k) + except KeyError: + return x def pop(self, key, *args): + self.modifying() if len(args) > 1: raise TypeError( "pop expected at most 2 arguments, got " + \ @@ -2059,10 +2223,12 @@ class config(object): def __contains__(self, mykey): """Called to implement membership test operators (in and not in).""" - for d in self.lookuplist: - if mykey in d: - return True - return False + try: + self._getitem(mykey) + except KeyError: + return False + else: + return True def setdefault(self, k, x=None): v = self.get(k) @@ -2077,6 +2243,7 @@ class config(object): def __iter__(self): keys = set() + keys.update(self._constant_keys) for d in self.lookuplist: keys.update(d) return iter(keys) @@ -2086,7 +2253,7 @@ class config(object): def iteritems(self): for k in self: - yield (k, self[k]) + yield (k, self._getitem(k)) def items(self): return list(self.iteritems()) @@ -2168,8 +2335,14 @@ class config(object): if not eapi_exports_merge_type(eapi): mydict.pop("MERGE_TYPE", None) - # Prefix variables are supported starting with EAPI 3. - if phase == 'depend' or eapi is None or not eapi_supports_prefix(eapi): + # Prefix variables are supported beginning with EAPI 3, or when + # force-prefix is in FEATURES, since older EAPIs would otherwise be + # useless with prefix configurations. This brings compatibility with + # the prefix branch of portage, which also supports EPREFIX for all + # EAPIs (for obvious reasons). + if phase == 'depend' or \ + ('force-prefix' not in self.features and + eapi is not None and not eapi_supports_prefix(eapi)): mydict.pop("ED", None) mydict.pop("EPREFIX", None) mydict.pop("EROOT", None) diff --git a/portage_with_autodep/pym/portage/package/ebuild/config.pyo b/portage_with_autodep/pym/portage/package/ebuild/config.pyo Binary files differnew file mode 100644 index 0000000..742ee2b --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/config.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/deprecated_profile_check.pyo b/portage_with_autodep/pym/portage/package/ebuild/deprecated_profile_check.pyo Binary files differnew file mode 100644 index 0000000..2b9362b --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/deprecated_profile_check.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/digestcheck.py b/portage_with_autodep/pym/portage/package/ebuild/digestcheck.py index 1e34b14..8705639 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/digestcheck.py +++ b/portage_with_autodep/pym/portage/package/ebuild/digestcheck.py @@ -8,7 +8,6 @@ import warnings from portage import os, _encodings, _unicode_decode from portage.exception import DigestException, FileNotFound from portage.localization import _ -from portage.manifest import Manifest from portage.output import EOutput from portage.util import writemsg @@ -16,7 +15,7 @@ def digestcheck(myfiles, mysettings, strict=False, justmanifest=None, mf=None): """ Verifies checksums. Assumes all files have been downloaded. @rtype: int - @returns: 1 on success and 0 on failure + @return: 1 on success and 0 on failure """ if justmanifest is not None: @@ -28,49 +27,33 @@ def digestcheck(myfiles, mysettings, strict=False, justmanifest=None, mf=None): if mysettings.get("EBUILD_SKIP_MANIFEST") == "1": return 1 - allow_missing = "allow-missing-manifests" in mysettings.features pkgdir = mysettings["O"] - manifest_path = os.path.join(pkgdir, "Manifest") - if not os.path.exists(manifest_path): - if allow_missing: - return 1 - writemsg(_("!!! Manifest file not found: '%s'\n") % manifest_path, - noiselevel=-1) - if strict: - return 0 - else: - return 1 if mf is None: - mf = Manifest(pkgdir, mysettings["DISTDIR"]) - manifest_empty = True - for d in mf.fhashdict.values(): - if d: - manifest_empty = False - break - if manifest_empty: - writemsg(_("!!! Manifest is empty: '%s'\n") % manifest_path, - noiselevel=-1) - if strict: - return 0 - else: - return 1 + mf = mysettings.repositories.get_repo_for_location( + os.path.dirname(os.path.dirname(pkgdir))) + mf = mf.load_manifest(pkgdir, mysettings["DISTDIR"]) eout = EOutput() eout.quiet = mysettings.get("PORTAGE_QUIET", None) == "1" try: - if strict and "PORTAGE_PARALLEL_FETCHONLY" not in mysettings: - eout.ebegin(_("checking ebuild checksums ;-)")) - mf.checkTypeHashes("EBUILD") - eout.eend(0) - eout.ebegin(_("checking auxfile checksums ;-)")) - mf.checkTypeHashes("AUX") - eout.eend(0) - eout.ebegin(_("checking miscfile checksums ;-)")) - mf.checkTypeHashes("MISC", ignoreMissingFiles=True) - eout.eend(0) + if not mf.thin and strict and "PORTAGE_PARALLEL_FETCHONLY" not in mysettings: + if mf.fhashdict.get("EBUILD"): + eout.ebegin(_("checking ebuild checksums ;-)")) + mf.checkTypeHashes("EBUILD") + eout.eend(0) + if mf.fhashdict.get("AUX"): + eout.ebegin(_("checking auxfile checksums ;-)")) + mf.checkTypeHashes("AUX") + eout.eend(0) + if mf.fhashdict.get("MISC"): + eout.ebegin(_("checking miscfile checksums ;-)")) + mf.checkTypeHashes("MISC", ignoreMissingFiles=True) + eout.eend(0) for f in myfiles: eout.ebegin(_("checking %s ;-)") % f) ftype = mf.findFile(f) if ftype is None: + if mf.allow_missing: + continue eout.eend(1) writemsg(_("\n!!! Missing digest for '%s'\n") % (f,), noiselevel=-1) @@ -90,7 +73,7 @@ def digestcheck(myfiles, mysettings, strict=False, justmanifest=None, mf=None): writemsg(_("!!! Got: %s\n") % e.value[2], noiselevel=-1) writemsg(_("!!! Expected: %s\n") % e.value[3], noiselevel=-1) return 0 - if allow_missing: + if mf.thin or mf.allow_missing: # In this case we ignore any missing digests that # would otherwise be detected below. return 1 diff --git a/portage_with_autodep/pym/portage/package/ebuild/digestcheck.pyo b/portage_with_autodep/pym/portage/package/ebuild/digestcheck.pyo Binary files differnew file mode 100644 index 0000000..66987a2 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/digestcheck.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/digestgen.py b/portage_with_autodep/pym/portage/package/ebuild/digestgen.py index eb7210e..6ad3397 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/digestgen.py +++ b/portage_with_autodep/pym/portage/package/ebuild/digestgen.py @@ -17,7 +17,6 @@ from portage.dep import use_reduce from portage.exception import InvalidDependString, FileNotFound, \ PermissionDenied, PortagePackageException from portage.localization import _ -from portage.manifest import Manifest from portage.output import colorize from portage.package.ebuild.fetch import fetch from portage.util import writemsg, writemsg_stdout @@ -34,7 +33,7 @@ def digestgen(myarchives=None, mysettings=None, myportdb=None): @param myportdb: a portdbapi instance @type myportdb: portdbapi @rtype: int - @returns: 1 on success and 0 on failure + @return: 1 on success and 0 on failure """ if mysettings is None or myportdb is None: raise TypeError("portage.digestgen(): 'mysettings' and 'myportdb' parameter are required.") @@ -52,9 +51,21 @@ def digestgen(myarchives=None, mysettings=None, myportdb=None): del e return 0 mytree = os.path.dirname(os.path.dirname(mysettings["O"])) - manifest1_compat = False - mf = Manifest(mysettings["O"], mysettings["DISTDIR"], - fetchlist_dict=fetchlist_dict, manifest1_compat=manifest1_compat) + try: + mf = mysettings.repositories.get_repo_for_location(mytree) + except KeyError: + # backward compatibility + mytree = os.path.realpath(mytree) + mf = mysettings.repositories.get_repo_for_location(mytree) + + mf = mf.load_manifest(mysettings["O"], mysettings["DISTDIR"], + fetchlist_dict=fetchlist_dict) + + if not mf.allow_create: + writemsg_stdout(_(">>> Skipping creating Manifest for %s; " + "repository is configured to not use them\n") % mysettings["O"]) + return 1 + # Don't require all hashes since that can trigger excessive # fetches when sufficient digests already exist. To ease transition # while Manifest 1 is being removed, only require hashes that will @@ -102,8 +113,6 @@ def digestgen(myarchives=None, mysettings=None, myportdb=None): continue if missing_files: - mytree = os.path.realpath(os.path.dirname( - os.path.dirname(mysettings["O"]))) for myfile in missing_files: uris = set() all_restrict = set() @@ -139,8 +148,7 @@ def digestgen(myarchives=None, mysettings=None, myportdb=None): if not fetch({myfile : uris}, mysettings): myebuild = os.path.join(mysettings["O"], catsplit(cpv)[1] + ".ebuild") - spawn_nofetch(myportdb, myebuild, - settings=mysettings) + spawn_nofetch(myportdb, myebuild) writemsg(_("!!! Fetch failed for %s, can't update " "Manifest\n") % myfile, noiselevel=-1) if myfile in dist_hashes and \ @@ -183,8 +191,6 @@ def digestgen(myarchives=None, mysettings=None, myportdb=None): os.path.join(mysettings["DISTDIR"], filename)): auto_assumed.append(filename) if auto_assumed: - mytree = os.path.realpath( - os.path.dirname(os.path.dirname(mysettings["O"]))) cp = os.path.sep.join(mysettings["O"].split(os.path.sep)[-2:]) pkgs = myportdb.cp_list(cp, mytree=mytree) pkgs.sort() diff --git a/portage_with_autodep/pym/portage/package/ebuild/digestgen.pyo b/portage_with_autodep/pym/portage/package/ebuild/digestgen.pyo Binary files differnew file mode 100644 index 0000000..66876ec --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/digestgen.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/doebuild.py b/portage_with_autodep/pym/portage/package/ebuild/doebuild.py index c76c1ed..610172f 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/doebuild.py +++ b/portage_with_autodep/pym/portage/package/ebuild/doebuild.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ['doebuild', 'doebuild_environment', 'spawn', 'spawnebuild'] @@ -10,7 +10,6 @@ from itertools import chain import logging import os as _os import re -import shutil import signal import stat import sys @@ -31,8 +30,8 @@ portage.proxy.lazyimport.lazyimport(globals(), ) from portage import auxdbkeys, bsd_chflags, \ - eapi_is_supported, merge, os, selinux, \ - unmerge, _encodings, _parse_eapi_ebuild_head, _os_merge, \ + eapi_is_supported, merge, os, selinux, shutil, \ + unmerge, _encodings, _os_merge, \ _shell_quote, _unicode_decode, _unicode_encode from portage.const import EBUILD_SH_ENV_FILE, EBUILD_SH_ENV_DIR, \ EBUILD_SH_BINARY, INVALID_ENV_FILE, MISC_SH_BINARY @@ -42,16 +41,16 @@ from portage.dbapi.porttree import _parse_uri_map from portage.dep import Atom, check_required_use, \ human_readable_required_use, paren_enclose, use_reduce from portage.eapi import eapi_exports_KV, eapi_exports_merge_type, \ - eapi_exports_replace_vars, eapi_has_required_use, \ - eapi_has_src_prepare_and_src_configure, eapi_has_pkg_pretend -from portage.elog import elog_process + eapi_exports_replace_vars, eapi_exports_REPOSITORY, \ + eapi_has_required_use, eapi_has_src_prepare_and_src_configure, \ + eapi_has_pkg_pretend +from portage.elog import elog_process, _preload_elog_modules from portage.elog.messages import eerror, eqawarn from portage.exception import DigestException, FileNotFound, \ IncorrectParameter, InvalidDependString, PermissionDenied, \ UnsupportedAPIException from portage.localization import _ -from portage.manifest import Manifest -from portage.output import style_to_ansi_code +from portage.output import colormap from portage.package.ebuild.prepare_build_dirs import prepare_build_dirs from portage.util import apply_recursive_permissions, \ apply_secpass_permissions, noiselimit, normalize_path, \ @@ -116,6 +115,38 @@ def _spawn_phase(phase, settings, actionmap=None, **kwargs): ebuild_phase.wait() return ebuild_phase.returncode +def _doebuild_path(settings, eapi=None): + """ + Generate the PATH variable. + """ + + # Note: PORTAGE_BIN_PATH may differ from the global constant + # when portage is reinstalling itself. + portage_bin_path = settings["PORTAGE_BIN_PATH"] + eprefix = settings["EPREFIX"] + prerootpath = [x for x in settings.get("PREROOTPATH", "").split(":") if x] + rootpath = [x for x in settings.get("ROOTPATH", "").split(":") if x] + + prefixes = [] + if eprefix: + prefixes.append(eprefix) + prefixes.append("/") + + path = [] + + if eapi not in (None, "0", "1", "2", "3"): + path.append(os.path.join(portage_bin_path, "ebuild-helpers", "4")) + + path.append(os.path.join(portage_bin_path, "ebuild-helpers")) + path.extend(prerootpath) + + for prefix in prefixes: + for x in ("usr/local/sbin", "usr/local/bin", "usr/sbin", "usr/bin", "sbin", "bin"): + path.append(os.path.join(prefix, x)) + + path.extend(rootpath) + settings["PATH"] = ":".join(path) + def doebuild_environment(myebuild, mydo, myroot=None, settings=None, debug=False, use_cache=None, db=None): """ @@ -143,20 +174,32 @@ def doebuild_environment(myebuild, mydo, myroot=None, settings=None, ebuild_path = os.path.abspath(myebuild) pkg_dir = os.path.dirname(ebuild_path) mytree = os.path.dirname(os.path.dirname(pkg_dir)) - - if "CATEGORY" in mysettings.configdict["pkg"]: - cat = mysettings.configdict["pkg"]["CATEGORY"] - else: - cat = os.path.basename(normalize_path(os.path.join(pkg_dir, ".."))) - mypv = os.path.basename(ebuild_path)[:-7] - - mycpv = cat+"/"+mypv - mysplit = _pkgsplit(mypv) + mysplit = _pkgsplit(mypv, eapi=mysettings.configdict["pkg"].get("EAPI")) if mysplit is None: raise IncorrectParameter( _("Invalid ebuild path: '%s'") % myebuild) + if mysettings.mycpv is not None and \ + mysettings.configdict["pkg"].get("PF") == mypv and \ + "CATEGORY" in mysettings.configdict["pkg"]: + # Assume that PF is enough to assume that we've got + # the correct CATEGORY, though this is not really + # a solid assumption since it's possible (though + # unlikely) that two packages in different + # categories have the same PF. Callers should call + # setcpv or create a clean clone of a locked config + # instance in order to ensure that this assumption + # does not fail like in bug #408817. + cat = mysettings.configdict["pkg"]["CATEGORY"] + mycpv = mysettings.mycpv + elif os.path.basename(pkg_dir) in (mysplit[0], mypv): + # portdbapi or vardbapi + cat = os.path.basename(os.path.dirname(pkg_dir)) + mycpv = cat + "/" + mypv + else: + raise AssertionError("unable to determine CATEGORY") + # Make a backup of PORTAGE_TMPDIR prior to calling config.reset() # so that the caller can override it. tmpdir = mysettings["PORTAGE_TMPDIR"] @@ -208,11 +251,11 @@ def doebuild_environment(myebuild, mydo, myroot=None, settings=None, mysettings["FILESDIR"] = pkg_dir+"/files" mysettings["PF"] = mypv - if hasattr(mydbapi, '_repo_info'): - repo_info = mydbapi._repo_info[mytree] - mysettings['PORTDIR'] = repo_info.portdir - mysettings['PORTDIR_OVERLAY'] = repo_info.portdir_overlay - mysettings.configdict["pkg"]["PORTAGE_REPO_NAME"] = repo_info.name + if hasattr(mydbapi, 'repositories'): + repo = mydbapi.repositories.get_repo_for_location(mytree) + mysettings['PORTDIR'] = repo.eclass_db.porttrees[0] + mysettings['PORTDIR_OVERLAY'] = ' '.join(repo.eclass_db.porttrees[1:]) + mysettings.configdict["pkg"]["PORTAGE_REPO_NAME"] = repo.name mysettings["PORTDIR"] = os.path.realpath(mysettings["PORTDIR"]) mysettings["DISTDIR"] = os.path.realpath(mysettings["DISTDIR"]) @@ -235,16 +278,6 @@ def doebuild_environment(myebuild, mydo, myroot=None, settings=None, else: mysettings["PVR"]=mysplit[1]+"-"+mysplit[2] - if "PATH" in mysettings: - mysplit=mysettings["PATH"].split(":") - else: - mysplit=[] - # Note: PORTAGE_BIN_PATH may differ from the global constant - # when portage is reinstalling itself. - portage_bin_path = mysettings["PORTAGE_BIN_PATH"] - if portage_bin_path not in mysplit: - mysettings["PATH"] = portage_bin_path + ":" + mysettings["PATH"] - # All temporary directories should be subdirectories of # $PORTAGE_TMPDIR/portage, since it's common for /tmp and /var/tmp # to be mounted with the "noexec" option (see bug #346899). @@ -268,7 +301,9 @@ def doebuild_environment(myebuild, mydo, myroot=None, settings=None, mysettings["T"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], "temp") # Prefix forward compatability - mysettings["ED"] = mysettings["D"] + eprefix_lstrip = mysettings["EPREFIX"].lstrip(os.sep) + mysettings["ED"] = os.path.join( + mysettings["D"], eprefix_lstrip).rstrip(os.sep) + os.sep mysettings["PORTAGE_BASHRC"] = os.path.join( mysettings["PORTAGE_CONFIGROOT"], EBUILD_SH_ENV_FILE) @@ -276,37 +311,41 @@ def doebuild_environment(myebuild, mydo, myroot=None, settings=None, mysettings["PORTAGE_CONFIGROOT"], EBUILD_SH_ENV_DIR) # Allow color.map to control colors associated with einfo, ewarn, etc... - mycolors = [] - for c in ("GOOD", "WARN", "BAD", "HILITE", "BRACKET"): - mycolors.append("%s=$'%s'" % \ - (c, style_to_ansi_code(c))) - mysettings["PORTAGE_COLORMAP"] = "\n".join(mycolors) - - # All EAPI dependent code comes last, so that essential variables - # like PORTAGE_BUILDDIR are still initialized even in cases when + mysettings["PORTAGE_COLORMAP"] = colormap() + + if "COLUMNS" not in mysettings: + # Set COLUMNS, in order to prevent unnecessary stty calls + # inside the set_colors function of isolated-functions.sh. + # We cache the result in os.environ, in order to avoid + # multiple stty calls in cases when get_term_size() falls + # back to stty due to a missing or broken curses module. + columns = os.environ.get("COLUMNS") + if columns is None: + rows, columns = portage.output.get_term_size() + if columns < 1: + # Force a sane value for COLUMNS, so that tools + # like ls don't complain (see bug #394091). + columns = 80 + columns = str(columns) + os.environ["COLUMNS"] = columns + mysettings["COLUMNS"] = columns + + # EAPI is always known here, even for the "depend" phase, because + # EbuildMetadataPhase gets it from _parse_eapi_ebuild_head(). + eapi = mysettings.configdict['pkg']['EAPI'] + _doebuild_path(mysettings, eapi=eapi) + + # All EAPI dependent code comes last, so that essential variables like + # PATH and PORTAGE_BUILDDIR are still initialized even in cases when # UnsupportedAPIException needs to be raised, which can be useful # when uninstalling a package that has corrupt EAPI metadata. - eapi = None - if mydo == 'depend' and 'EAPI' not in mysettings.configdict['pkg']: - if eapi is None and 'parse-eapi-ebuild-head' in mysettings.features: - eapi = _parse_eapi_ebuild_head( - io.open(_unicode_encode(ebuild_path, - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['content'], errors='replace')) - - if eapi is not None: - if not eapi_is_supported(eapi): - raise UnsupportedAPIException(mycpv, eapi) - mysettings.configdict['pkg']['EAPI'] = eapi + if not eapi_is_supported(eapi): + raise UnsupportedAPIException(mycpv, eapi) - if mydo != "depend": - # Metadata vars such as EAPI and RESTRICT are - # set by the above config.setcpv() call. - eapi = mysettings["EAPI"] - if not eapi_is_supported(eapi): - # can't do anything with this. - raise UnsupportedAPIException(mycpv, eapi) + if eapi_exports_REPOSITORY(eapi) and "PORTAGE_REPO_NAME" in mysettings.configdict["pkg"]: + mysettings.configdict["pkg"]["REPOSITORY"] = mysettings.configdict["pkg"]["PORTAGE_REPO_NAME"] + if mydo != "depend": if hasattr(mydbapi, "getFetchMap") and \ ("A" not in mysettings.configdict["pkg"] or \ "AA" not in mysettings.configdict["pkg"]): @@ -331,22 +370,41 @@ def doebuild_environment(myebuild, mydo, myroot=None, settings=None, else: mysettings.configdict["pkg"]["AA"] = " ".join(uri_map) - if not eapi_exports_KV(eapi): - # Discard KV for EAPIs that don't support it. Cache KV is restored - # from the backupenv whenever config.reset() is called. - mysettings.pop('KV', None) - elif mydo != 'depend' and 'KV' not in mysettings and \ - mydo in ('compile', 'config', 'configure', 'info', - 'install', 'nofetch', 'postinst', 'postrm', 'preinst', - 'prepare', 'prerm', 'setup', 'test', 'unpack'): - mykv, err1 = ExtractKernelVersion( - os.path.join(mysettings['EROOT'], "usr/src/linux")) - if mykv: - # Regular source tree - mysettings["KV"] = mykv - else: - mysettings["KV"] = "" - mysettings.backup_changes("KV") + ccache = "ccache" in mysettings.features + distcc = "distcc" in mysettings.features + if ccache or distcc: + # Use default ABI libdir in accordance with bug #355283. + libdir = None + default_abi = mysettings.get("DEFAULT_ABI") + if default_abi: + libdir = mysettings.get("LIBDIR_" + default_abi) + if not libdir: + libdir = "lib" + + if distcc: + mysettings["PATH"] = os.path.join(os.sep, eprefix_lstrip, + "usr", libdir, "distcc", "bin") + ":" + mysettings["PATH"] + + if ccache: + mysettings["PATH"] = os.path.join(os.sep, eprefix_lstrip, + "usr", libdir, "ccache", "bin") + ":" + mysettings["PATH"] + + if not eapi_exports_KV(eapi): + # Discard KV for EAPIs that don't support it. Cached KV is restored + # from the backupenv whenever config.reset() is called. + mysettings.pop('KV', None) + elif 'KV' not in mysettings and \ + mydo in ('compile', 'config', 'configure', 'info', + 'install', 'nofetch', 'postinst', 'postrm', 'preinst', + 'prepare', 'prerm', 'setup', 'test', 'unpack'): + mykv, err1 = ExtractKernelVersion( + os.path.join(mysettings['EROOT'], "usr/src/linux")) + if mykv: + # Regular source tree + mysettings["KV"] = mykv + else: + mysettings["KV"] = "" + mysettings.backup_changes("KV") _doebuild_manifest_cache = None _doebuild_broken_ebuilds = set() @@ -356,7 +414,7 @@ _doebuild_commands_without_builddir = ( 'fetch', 'fetchall', 'help', 'manifest' ) -def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, +def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, fetchonly=0, cleanup=0, dbkey=None, use_cache=1, fetchall=0, tree=None, mydbapi=None, vartree=None, prev_mtimes=None, fd_pipes=None, returnpid=False): @@ -368,10 +426,10 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, @type myebuild: String @param mydo: Phase to run @type mydo: String - @param myroot: $ROOT (usually '/', see man make.conf) - @type myroot: String - @param mysettings: Portage Configuration - @type mysettings: instance of portage.config + @param _unused: Deprecated (use settings["ROOT"] instead) + @type _unused: String + @param settings: Portage Configuration + @type settings: instance of portage.config @param debug: Turns on various debug information (eg, debug for spawn) @type debug: Boolean @param listonly: Used to wrap fetch(); passed such that fetch only lists files required. @@ -403,7 +461,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, caller clean up all returned PIDs. @type returnpid: Boolean @rtype: Boolean - @returns: + @return: 1. 0 for success 2. 1 for error @@ -414,7 +472,18 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, Other variables may not be strictly required, many have defaults that are set inside of doebuild. """ - + + if settings is None: + raise TypeError("settings parameter is required") + mysettings = settings + myroot = settings['EROOT'] + + if _unused is not None and _unused != mysettings['EROOT']: + warnings.warn("The third parameter of the " + "portage.doebuild() is now unused. Use " + "settings['ROOT'] instead.", + DeprecationWarning, stacklevel=2) + if not tree: writemsg("Warning: tree not specified to doebuild\n") tree = "porttree" @@ -432,6 +501,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, "install":["test"], "rpm": ["install"], "package":["install"], + "merge" :["install"], } if mydbapi is None: @@ -480,21 +550,28 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, return 1 global _doebuild_manifest_cache + pkgdir = os.path.dirname(myebuild) + manifest_path = os.path.join(pkgdir, "Manifest") + if tree == "porttree": + repo_config = mysettings.repositories.get_repo_for_location( + os.path.dirname(os.path.dirname(pkgdir))) + else: + repo_config = None + mf = None if "strict" in features and \ "digest" not in features and \ tree == "porttree" and \ + not repo_config.thin_manifest and \ mydo not in ("digest", "manifest", "help") and \ - not portage._doebuild_manifest_exempt_depend: + not portage._doebuild_manifest_exempt_depend and \ + not (repo_config.allow_missing_manifest and not os.path.exists(manifest_path)): # Always verify the ebuild checksums before executing it. global _doebuild_broken_ebuilds if myebuild in _doebuild_broken_ebuilds: return 1 - pkgdir = os.path.dirname(myebuild) - manifest_path = os.path.join(pkgdir, "Manifest") - # Avoid checking the same Manifest several times in a row during a # regen with an empty cache. if _doebuild_manifest_cache is None or \ @@ -505,7 +582,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, out.eerror(_("Manifest not found for '%s'") % (myebuild,)) _doebuild_broken_ebuilds.add(myebuild) return 1 - mf = Manifest(pkgdir, mysettings["DISTDIR"]) + mf = repo_config.load_manifest(pkgdir, mysettings["DISTDIR"]) else: mf = _doebuild_manifest_cache @@ -513,10 +590,12 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, try: mf.checkFileHashes("EBUILD", os.path.basename(myebuild)) except KeyError: - out = portage.output.EOutput() - out.eerror(_("Missing digest for '%s'") % (myebuild,)) - _doebuild_broken_ebuilds.add(myebuild) - return 1 + if not (mf.allow_missing and + os.path.basename(myebuild) not in mf.fhashdict["EBUILD"]): + out = portage.output.EOutput() + out.eerror(_("Missing digest for '%s'") % (myebuild,)) + _doebuild_broken_ebuilds.add(myebuild) + return 1 except FileNotFound: out = portage.output.EOutput() out.eerror(_("A file listed in the Manifest " @@ -536,7 +615,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, if mf.getFullname() in _doebuild_broken_manifests: return 1 - if mf is not _doebuild_manifest_cache: + if mf is not _doebuild_manifest_cache and not mf.allow_missing: # Make sure that all of the ebuilds are # actually listed in the Manifest. @@ -553,8 +632,8 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, _doebuild_broken_manifests.add(manifest_path) return 1 - # Only cache it if the above stray files test succeeds. - _doebuild_manifest_cache = mf + # We cache it only after all above checks succeed. + _doebuild_manifest_cache = mf logfile=None builddir_lock = None @@ -594,7 +673,6 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, if builddir_lock is not None: builddir_lock.unlock() - restrict = set(mysettings.get('PORTAGE_RESTRICT', '').split()) # get possible slot information from the deps file if mydo == "depend": writemsg("!!! DEBUG: dbkey: %s\n" % str(dbkey), 2) @@ -654,6 +732,13 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, if rval != os.EX_OK: return rval + else: + # FEATURES=noauto only makes sense for porttree, and we don't want + # it to trigger redundant sourcing of the ebuild for API consumers + # that are using binary packages + if "noauto" in mysettings.features: + mysettings.features.discard("noauto") + # The info phase is special because it uses mkdtemp so and # user (not necessarily in the portage group) can run it. if mydo not in ('info',) and \ @@ -666,6 +751,73 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, return unmerge(mysettings["CATEGORY"], mysettings["PF"], myroot, mysettings, vartree=vartree) + phases_to_run = set() + if "noauto" in mysettings.features or \ + mydo not in actionmap_deps: + phases_to_run.add(mydo) + else: + phase_stack = [mydo] + while phase_stack: + x = phase_stack.pop() + if x in phases_to_run: + continue + phases_to_run.add(x) + phase_stack.extend(actionmap_deps.get(x, [])) + del phase_stack + + alist = set(mysettings.configdict["pkg"].get("A", "").split()) + + unpacked = False + if tree != "porttree": + pass + elif "unpack" not in phases_to_run: + unpacked = os.path.exists(os.path.join( + mysettings["PORTAGE_BUILDDIR"], ".unpacked")) + else: + try: + workdir_st = os.stat(mysettings["WORKDIR"]) + except OSError: + pass + else: + newstuff = False + if not os.path.exists(os.path.join( + mysettings["PORTAGE_BUILDDIR"], ".unpacked")): + writemsg_stdout(_( + ">>> Not marked as unpacked; recreating WORKDIR...\n")) + newstuff = True + else: + for x in alist: + writemsg_stdout(">>> Checking %s's mtime...\n" % x) + try: + x_st = os.stat(os.path.join( + mysettings["DISTDIR"], x)) + except OSError: + # file not fetched yet + x_st = None + + if x_st is None or x_st.st_mtime > workdir_st.st_mtime: + writemsg_stdout(_(">>> Timestamp of " + "%s has changed; recreating WORKDIR...\n") % x) + newstuff = True + break + + if newstuff: + if builddir_lock is None and \ + 'PORTAGE_BUILDIR_LOCKED' not in mysettings: + builddir_lock = EbuildBuildDir( + scheduler=PollScheduler().sched_iface, + settings=mysettings) + builddir_lock.lock() + try: + _spawn_phase("clean", mysettings) + finally: + if builddir_lock is not None: + builddir_lock.unlock() + builddir_lock = None + else: + writemsg_stdout(_(">>> WORKDIR is up-to-date, keeping...\n")) + unpacked = True + # Build directory creation isn't required for any of these. # In the fetch phase, the directory is needed only for RESTRICT=fetch # in order to satisfy the sane $PWD requirement (from bug #239560) @@ -739,10 +891,9 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, # Only try and fetch the files if we are going to need them ... # otherwise, if user has FEATURES=noauto and they run `ebuild clean # unpack compile install`, we will try and fetch 4 times :/ - need_distfiles = tree == "porttree" and \ + need_distfiles = tree == "porttree" and not unpacked and \ (mydo in ("fetch", "unpack") or \ mydo not in ("digest", "manifest") and "noauto" not in features) - alist = set(mysettings.configdict["pkg"].get("A", "").split()) if need_distfiles: src_uri, = mydbapi.aux_get(mysettings.mycpv, @@ -783,10 +934,14 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, return 0 return 1 - if mydo == "fetch": + if need_distfiles: # Files are already checked inside fetch(), # so do not check them again. checkme = [] + elif unpacked: + # The unpack phase is marked as complete, so it + # would be wasteful to check distfiles again. + checkme = [] else: checkme = alist @@ -845,7 +1000,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, # this phase. This can raise PermissionDenied if # the current user doesn't have write access to $PKGDIR. if hasattr(portage, 'db'): - bintree = portage.db[mysettings["ROOT"]]["bintree"] + bintree = portage.db[mysettings['EROOT']]['bintree'] mysettings["PORTAGE_BINPKG_TMPFILE"] = \ bintree.getname(mysettings.mycpv) + \ ".%s" % (os.getpid(),) @@ -866,6 +1021,13 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, if mydo == "package" and bintree is not None: bintree.inject(mysettings.mycpv, filename=mysettings["PORTAGE_BINPKG_TMPFILE"]) + else: + if "PORTAGE_BINPKG_TMPFILE" in mysettings: + try: + os.unlink(mysettings["PORTAGE_BINPKG_TMPFILE"]) + except OSError: + pass + elif mydo=="qmerge": # check to ensure install was run. this *only* pops up when users # forget it and are using ebuild @@ -877,6 +1039,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, # qmerge is a special phase that implies noclean. if "noclean" not in mysettings.features: mysettings.features.add("noclean") + _handle_self_update(mysettings, vartree.dbapi) #qmerge is specifically not supposed to do a runtime dep check retval = merge( mysettings["CATEGORY"], mysettings["PF"], mysettings["D"], @@ -893,6 +1056,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, # so that it's only called once. elog_process(mysettings.mycpv, mysettings) if retval == os.EX_OK: + _handle_self_update(mysettings, vartree.dbapi) retval = merge(mysettings["CATEGORY"], mysettings["PF"], mysettings["D"], os.path.join(mysettings["PORTAGE_BUILDDIR"], "build-info"), myroot, mysettings, @@ -944,10 +1108,31 @@ def _check_temp_dir(settings): # as some people use a separate PORTAGE_TMPDIR mount # we prefer that as the checks below would otherwise be pointless # for those people. - if os.path.exists(os.path.join(settings["PORTAGE_TMPDIR"], "portage")): - checkdir = os.path.join(settings["PORTAGE_TMPDIR"], "portage") + tmpdir = os.path.realpath(settings["PORTAGE_TMPDIR"]) + if os.path.exists(os.path.join(tmpdir, "portage")): + checkdir = os.path.realpath(os.path.join(tmpdir, "portage")) + if ("sandbox" in settings.features or + "usersandox" in settings.features) and \ + not checkdir.startswith(tmpdir + os.sep): + msg = _("The 'portage' subdirectory of the directory " + "referenced by the PORTAGE_TMPDIR variable appears to be " + "a symlink. In order to avoid sandbox violations (see bug " + "#378379), you must adjust PORTAGE_TMPDIR instead of using " + "the symlink located at '%s'. A suitable PORTAGE_TMPDIR " + "setting would be '%s'.") % \ + (os.path.join(tmpdir, "portage"), checkdir) + lines = [] + lines.append("") + lines.append("") + lines.extend(wrap(msg, 72)) + lines.append("") + for line in lines: + if line: + line = "!!! %s" % (line,) + writemsg("%s\n" % (line,), noiselevel=-1) + return 1 else: - checkdir = settings["PORTAGE_TMPDIR"] + checkdir = tmpdir if not os.access(checkdir, os.W_OK): writemsg(_("%s is not writable.\n" @@ -955,8 +1140,7 @@ def _check_temp_dir(settings): noiselevel=-1) return 1 - else: - fd = tempfile.NamedTemporaryFile(prefix="exectest-", dir=checkdir) + with tempfile.NamedTemporaryFile(prefix="exectest-", dir=checkdir) as fd: os.chmod(fd.name, 0o755) if not os.access(fd.name, os.X_OK): writemsg(_("Can not execute files in %s\n" @@ -1085,7 +1269,8 @@ def _validate_deps(mysettings, myroot, mydo, mydbapi): all_keys.add("SRC_URI") all_keys = tuple(all_keys) metadata = dict(zip(all_keys, - mydbapi.aux_get(mysettings.mycpv, all_keys))) + mydbapi.aux_get(mysettings.mycpv, all_keys, + myrepo=mysettings.get("PORTAGE_REPO_NAME")))) class FakeTree(object): def __init__(self, mydb): @@ -1173,7 +1358,7 @@ def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakero @param keywords: Extra options encoded as a dict, to be passed to spawn @type keywords: Dictionary @rtype: Integer - @returns: + @return: 1. The return code of the spawned process. """ @@ -1201,7 +1386,8 @@ def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakero # fake ownership/permissions will have to be converted to real # permissions in the merge phase. fakeroot = fakeroot and uid != 0 and portage.process.fakeroot_capable - if droppriv and not uid and portage_gid and portage_uid: + if droppriv and uid == 0 and portage_gid and portage_uid and \ + hasattr(os, "setgroups"): keywords.update({"uid":portage_uid,"gid":portage_gid, "groups":userpriv_groups,"umask":0o02}) if not free: @@ -1277,6 +1463,17 @@ def spawnebuild(mydo, actionmap, mysettings, debug, alwaysdep=0, if mydo == "pretend" and not eapi_has_pkg_pretend(eapi): return os.EX_OK + if not (mydo == "install" and "noauto" in mysettings.features): + check_file = os.path.join( + mysettings["PORTAGE_BUILDDIR"], ".%sed" % mydo.rstrip('e')) + if os.path.exists(check_file): + writemsg_stdout(_(">>> It appears that " + "'%(action)s' has already executed for '%(pkg)s'; skipping.\n") % + {"action":mydo, "pkg":mysettings["PF"]}) + writemsg_stdout(_(">>> Remove '%(file)s' to force %(action)s.\n") % + {"file":check_file, "action":mydo}) + return os.EX_OK + return _spawn_phase(mydo, mysettings, actionmap=actionmap, logfile=logfile, fd_pipes=fd_pipes, returnpid=returnpid) @@ -1285,13 +1482,14 @@ _post_phase_cmds = { "install" : [ "install_qa_check", - "install_symlink_html_docs"], + "install_symlink_html_docs", + "install_hooks"], "preinst" : [ "preinst_sfperms", "preinst_selinux_labels", "preinst_suid_scan", - "preinst_mask"] + ] } def _post_phase_userpriv_perms(mysettings): @@ -1320,7 +1518,9 @@ def _check_build_log(mysettings, out=None): except EnvironmentError: return + f_real = None if logfile.endswith('.gz'): + f_real = f f = gzip.GzipFile(filename='', mode='rb', fileobj=f) am_maintainer_mode = [] @@ -1425,19 +1625,32 @@ def _check_build_log(mysettings, out=None): msg.extend("\t" + line for line in make_jobserver) _eqawarn(msg) + f.close() + if f_real is not None: + f_real.close() + def _post_src_install_chost_fix(settings): """ It's possible that the ebuild has changed the CHOST variable, so revert it to the initial - setting. + setting. Also, revert IUSE in case it's corrupted + due to local environment settings like in bug #386829. """ - if settings.get('CATEGORY') == 'virtual': - return - chost = settings.get('CHOST') - if chost: - write_atomic(os.path.join(settings['PORTAGE_BUILDDIR'], - 'build-info', 'CHOST'), chost + '\n') + build_info_dir = os.path.join(settings['PORTAGE_BUILDDIR'], 'build-info') + + for k in ('IUSE',): + v = settings.get(k) + if v is not None: + write_atomic(os.path.join(build_info_dir, k), v + '\n') + + # The following variables are irrelevant for virtual packages. + if settings.get('CATEGORY') != 'virtual': + + for k in ('CHOST',): + v = settings.get(k) + if v is not None: + write_atomic(os.path.join(build_info_dir, k), v + '\n') _vdb_use_conditional_keys = ('DEPEND', 'LICENSE', 'PDEPEND', 'PROPERTIES', 'PROVIDE', 'RDEPEND', 'RESTRICT',) @@ -1481,6 +1694,7 @@ def _post_src_install_uid_fix(mysettings, out): _preinst_bsdflags(mysettings) destdir = mysettings["D"] + ed_len = len(mysettings["ED"]) unicode_errors = [] while True: @@ -1499,12 +1713,12 @@ def _post_src_install_uid_fix(mysettings, out): new_parent = _unicode_decode(parent, encoding=_encodings['merge'], errors='replace') new_parent = _unicode_encode(new_parent, - encoding=_encodings['merge'], errors='backslashreplace') + encoding='ascii', errors='backslashreplace') new_parent = _unicode_decode(new_parent, encoding=_encodings['merge'], errors='replace') os.rename(parent, new_parent) unicode_error = True - unicode_errors.append(new_parent[len(destdir):]) + unicode_errors.append(new_parent[ed_len:]) break for fname in chain(dirs, files): @@ -1517,13 +1731,13 @@ def _post_src_install_uid_fix(mysettings, out): new_fname = _unicode_decode(fname, encoding=_encodings['merge'], errors='replace') new_fname = _unicode_encode(new_fname, - encoding=_encodings['merge'], errors='backslashreplace') + encoding='ascii', errors='backslashreplace') new_fname = _unicode_decode(new_fname, encoding=_encodings['merge'], errors='replace') new_fpath = os.path.join(parent, new_fname) os.rename(fpath, new_fpath) unicode_error = True - unicode_errors.append(new_fpath[len(destdir):]) + unicode_errors.append(new_fpath[ed_len:]) fname = new_fname fpath = new_fpath else: @@ -1597,20 +1811,24 @@ def _post_src_install_uid_fix(mysettings, out): if unicode_errors: for l in _merge_unicode_error(unicode_errors): - eerror(l, phase='install', key=mysettings.mycpv, out=out) + eqawarn(l, phase='install', key=mysettings.mycpv, out=out) build_info_dir = os.path.join(mysettings['PORTAGE_BUILDDIR'], 'build-info') - io.open(_unicode_encode(os.path.join(build_info_dir, + f = io.open(_unicode_encode(os.path.join(build_info_dir, 'SIZE'), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['repo.content'], - errors='strict').write(_unicode_decode(str(size) + '\n')) + errors='strict') + f.write(_unicode_decode(str(size) + '\n')) + f.close() - io.open(_unicode_encode(os.path.join(build_info_dir, + f = io.open(_unicode_encode(os.path.join(build_info_dir, 'BUILD_TIME'), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['repo.content'], - errors='strict').write(_unicode_decode("%.0f\n" % (time.time(),))) + errors='strict') + f.write(_unicode_decode("%.0f\n" % (time.time(),))) + f.close() use = frozenset(mysettings['PORTAGE_USE'].split()) for k in _vdb_use_conditional_keys: @@ -1636,10 +1854,12 @@ def _post_src_install_uid_fix(mysettings, out): except OSError: pass continue - io.open(_unicode_encode(os.path.join(build_info_dir, + f = io.open(_unicode_encode(os.path.join(build_info_dir, k), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['repo.content'], - errors='strict').write(_unicode_decode(v + '\n')) + errors='strict') + f.write(_unicode_decode(v + '\n')) + f.close() _reapply_bsdflags_to_image(mysettings) @@ -1664,15 +1884,46 @@ def _post_src_install_soname_symlinks(mysettings, out): needed_filename = os.path.join(mysettings["PORTAGE_BUILDDIR"], "build-info", "NEEDED.ELF.2") + f = None try: - lines = io.open(_unicode_encode(needed_filename, + f = io.open(_unicode_encode(needed_filename, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], - errors='replace').readlines() + errors='replace') + lines = f.readlines() except IOError as e: if e.errno not in (errno.ENOENT, errno.ESTALE): raise return + finally: + if f is not None: + f.close() + + qa_no_symlink = "" + f = None + try: + f = io.open(_unicode_encode(os.path.join( + mysettings["PORTAGE_BUILDDIR"], + "build-info", "QA_SONAME_NO_SYMLINK"), + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') + qa_no_symlink = f.read() + except IOError as e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + raise + finally: + if f is not None: + f.close() + + qa_no_symlink = qa_no_symlink.split() + if qa_no_symlink: + if len(qa_no_symlink) > 1: + qa_no_symlink = "|".join("(%s)" % x for x in qa_no_symlink) + qa_no_symlink = "^(%s)$" % qa_no_symlink + else: + qa_no_symlink = "^%s$" % qa_no_symlink[0] + qa_no_symlink = re.compile(qa_no_symlink) libpaths = set(portage.util.getlibpaths( mysettings["ROOT"], env=mysettings)) @@ -1730,6 +1981,8 @@ def _post_src_install_soname_symlinks(mysettings, out): continue if not is_libdir(os.path.dirname(obj)): continue + if qa_no_symlink and qa_no_symlink.match(obj.strip(os.sep)) is not None: + continue obj_file_path = os.path.join(image_dir, obj.lstrip(os.sep)) sym_file_path = os.path.join(os.path.dirname(obj_file_path), soname) @@ -1746,8 +1999,7 @@ def _post_src_install_soname_symlinks(mysettings, out): if not missing_symlinks: return - qa_msg = ["QA Notice: Missing soname symlink(s) " + \ - "will be automatically created:"] + qa_msg = ["QA Notice: Missing soname symlink(s):"] qa_msg.append("") qa_msg.extend("\t%s -> %s" % (os.path.join( os.path.dirname(obj).lstrip(os.sep), soname), @@ -1757,20 +2009,11 @@ def _post_src_install_soname_symlinks(mysettings, out): for line in qa_msg: eqawarn(line, key=mysettings.mycpv, out=out) - _preinst_bsdflags(mysettings) - for obj, soname in missing_symlinks: - obj_file_path = os.path.join(image_dir, obj.lstrip(os.sep)) - sym_file_path = os.path.join(os.path.dirname(obj_file_path), soname) - os.symlink(os.path.basename(obj_file_path), sym_file_path) - _reapply_bsdflags_to_image(mysettings) - def _merge_unicode_error(errors): lines = [] - msg = _("This package installs one or more file names containing " - "characters that do not match your current locale " - "settings. The current setting for filesystem encoding is '%s'.") \ - % _encodings['merge'] + msg = _("QA Notice: This package installs one or more file names " + "containing characters that are not encoded with the UTF-8 encoding.") lines.extend(wrap(msg, 72)) lines.append("") @@ -1778,14 +2021,55 @@ def _merge_unicode_error(errors): lines.extend("\t" + x for x in errors) lines.append("") - if _encodings['merge'].lower().replace('_', '').replace('-', '') != 'utf8': - msg = _("For best results, UTF-8 encoding is recommended. See " - "the Gentoo Linux Localization Guide for instructions " - "about how to configure your locale for UTF-8 encoding:") - lines.extend(wrap(msg, 72)) - lines.append("") - lines.append("\t" + \ - "http://www.gentoo.org/doc/en/guide-localization.xml") - lines.append("") - return lines + +def _prepare_self_update(settings): + """ + Call this when portage is updating itself, in order to create + temporary copies of PORTAGE_BIN_PATH and PORTAGE_PYM_PATH, since + the new versions may be incompatible. An atexit hook will + automatically clean up the temporary copies. + """ + + # sanity check: ensure that that this routine only runs once + if portage._bin_path != portage.const.PORTAGE_BIN_PATH: + return + + # Load lazily referenced portage submodules into memory, + # so imports won't fail during portage upgrade/downgrade. + _preload_elog_modules(settings) + portage.proxy.lazyimport._preload_portage_submodules() + + # Make the temp directory inside $PORTAGE_TMPDIR/portage, since + # it's common for /tmp and /var/tmp to be mounted with the + # "noexec" option (see bug #346899). + build_prefix = os.path.join(settings["PORTAGE_TMPDIR"], "portage") + portage.util.ensure_dirs(build_prefix) + base_path_tmp = tempfile.mkdtemp( + "", "._portage_reinstall_.", build_prefix) + portage.process.atexit_register(shutil.rmtree, base_path_tmp) + + orig_bin_path = portage._bin_path + portage._bin_path = os.path.join(base_path_tmp, "bin") + shutil.copytree(orig_bin_path, portage._bin_path, symlinks=True) + + orig_pym_path = portage._pym_path + portage._pym_path = os.path.join(base_path_tmp, "pym") + shutil.copytree(orig_pym_path, portage._pym_path, symlinks=True) + + for dir_path in (base_path_tmp, portage._bin_path, portage._pym_path): + os.chmod(dir_path, 0o755) + +def _handle_self_update(settings, vardb): + cpv = settings.mycpv + if settings["ROOT"] == "/" and \ + portage.dep.match_from_list( + portage.const.PORTAGE_PACKAGE_ATOM, [cpv]): + inherited = frozenset(settings.get('INHERITED', '').split()) + if not vardb.cpv_exists(cpv) or \ + '9999' in cpv or \ + 'git' in inherited or \ + 'git-2' in inherited: + _prepare_self_update(settings) + return True + return False diff --git a/portage_with_autodep/pym/portage/package/ebuild/doebuild.pyo b/portage_with_autodep/pym/portage/package/ebuild/doebuild.pyo Binary files differnew file mode 100644 index 0000000..a6ebb1d --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/doebuild.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/fetch.py b/portage_with_autodep/pym/portage/package/ebuild/fetch.py index 5cbbf87..b795b28 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/fetch.py +++ b/portage_with_autodep/pym/portage/package/ebuild/fetch.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -10,7 +10,6 @@ import io import logging import random import re -import shutil import stat import sys import tempfile @@ -24,7 +23,7 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.package.ebuild.prepare_build_dirs:prepare_build_dirs', ) -from portage import OrderedDict, os, selinux, _encodings, \ +from portage import OrderedDict, os, selinux, shutil, _encodings, \ _shell_quote, _unicode_encode from portage.checksum import hashfunc_map, perform_md5, verify_all from portage.const import BASH_BINARY, CUSTOM_MIRRORS_FILE, \ @@ -34,7 +33,6 @@ from portage.exception import FileNotFound, OperationNotPermitted, \ PortageException, TryAgain from portage.localization import _ from portage.locks import lockfile, unlockfile -from portage.manifest import Manifest from portage.output import colorize, EOutput from portage.util import apply_recursive_permissions, \ apply_secpass_permissions, ensure_dirs, grabdict, shlex_split, \ @@ -48,6 +46,9 @@ _userpriv_spawn_kwargs = ( ("umask", 0o02), ) +def _hide_url_passwd(url): + return re.sub(r'//(.+):.+@(.+)', r'//\1:*password*@\2', url) + def _spawn_fetch(settings, args, **kwargs): """ Spawn a process with appropriate settings for fetching, including @@ -68,7 +69,8 @@ def _spawn_fetch(settings, args, **kwargs): } if "userfetch" in settings.features and \ - os.getuid() == 0 and portage_gid and portage_uid: + os.getuid() == 0 and portage_gid and portage_uid and \ + hasattr(os, "setgroups"): kwargs.update(_userpriv_spawn_kwargs) spawn_func = spawn @@ -356,7 +358,8 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, allow_missing_digests = True pkgdir = mysettings.get("O") if digests is None and not (pkgdir is None or skip_manifest): - mydigests = Manifest( + mydigests = mysettings.repositories.get_repo_for_location( + os.path.dirname(os.path.dirname(pkgdir))).load_manifest( pkgdir, mysettings["DISTDIR"]).getTypeDigests("DIST") elif digests is None or skip_manifest: # no digests because fetch was not called for a specific package @@ -612,18 +615,6 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, elif userfetch: has_space = False - if not has_space: - writemsg(_("!!! Insufficient space to store %s in %s\n") % \ - (myfile, mysettings["DISTDIR"]), noiselevel=-1) - - if has_space_superuser: - writemsg(_("!!! Insufficient privileges to use " - "remaining space.\n"), noiselevel=-1) - if userfetch: - writemsg(_("!!! You may set FEATURES=\"-userfetch\"" - " in /etc/make.conf in order to fetch with\n" - "!!! superuser privileges.\n"), noiselevel=-1) - if distdir_writable and use_locks: lock_kwargs = {} @@ -646,7 +637,10 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, match, mystat = _check_distfile( myfile_path, pruned_digests, eout) if match: - if distdir_writable: + # Skip permission adjustment for symlinks, since we don't + # want to modify anything outside of the primary DISTDIR, + # and symlinks typically point to PORTAGE_RO_DISTDIRS. + if distdir_writable and not os.path.islink(myfile_path): try: apply_secpass_permissions(myfile_path, gid=portage_gid, mode=0o664, mask=0o2, @@ -727,6 +721,20 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, os.symlink(readonly_file, myfile_path) continue + # this message is shown only after we know that + # the file is not already fetched + if not has_space: + writemsg(_("!!! Insufficient space to store %s in %s\n") % \ + (myfile, mysettings["DISTDIR"]), noiselevel=-1) + + if has_space_superuser: + writemsg(_("!!! Insufficient privileges to use " + "remaining space.\n"), noiselevel=-1) + if userfetch: + writemsg(_("!!! You may set FEATURES=\"-userfetch\"" + " in /etc/make.conf in order to fetch with\n" + "!!! superuser privileges.\n"), noiselevel=-1) + if fsmirrors and not os.path.exists(myfile_path) and has_space: for mydir in fsmirrors: mirror_file = os.path.join(mydir, myfile) @@ -746,14 +754,18 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, raise del e else: - try: - apply_secpass_permissions( - myfile_path, gid=portage_gid, mode=0o664, mask=0o2, - stat_cached=mystat) - except PortageException as e: - if not os.access(myfile_path, os.R_OK): - writemsg(_("!!! Failed to adjust permissions:" - " %s\n") % str(e), noiselevel=-1) + # Skip permission adjustment for symlinks, since we don't + # want to modify anything outside of the primary DISTDIR, + # and symlinks typically point to PORTAGE_RO_DISTDIRS. + if not os.path.islink(myfile_path): + try: + apply_secpass_permissions(myfile_path, + gid=portage_gid, mode=0o664, mask=0o2, + stat_cached=mystat) + except PortageException as e: + if not os.access(myfile_path, os.R_OK): + writemsg(_("!!! Failed to adjust permissions:" + " %s\n") % (e,), noiselevel=-1) # If the file is empty then it's obviously invalid. Remove # the empty file and try to download if possible. @@ -940,7 +952,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locfetch=fetchcommand command_var = fetchcommand_var writemsg_stdout(_(">>> Downloading '%s'\n") % \ - re.sub(r'//(.+):.+@(.+)/',r'//\1:*password*@\2/', loc)) + _hide_url_passwd(loc)) variables = { "DISTDIR": mysettings["DISTDIR"], "URI": loc, @@ -1019,18 +1031,19 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, # Fetch failed... Try the next one... Kill 404 files though. if (mystat[stat.ST_SIZE]<100000) and (len(myfile)>4) and not ((myfile[-5:]==".html") or (myfile[-4:]==".htm")): html404=re.compile("<title>.*(not found|404).*</title>",re.I|re.M) - if html404.search(io.open( + with io.open( _unicode_encode(myfile_path, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['content'], errors='replace' - ).read()): - try: - os.unlink(mysettings["DISTDIR"]+"/"+myfile) - writemsg(_(">>> Deleting invalid distfile. (Improper 404 redirect from server.)\n")) - fetched = 0 - continue - except (IOError, OSError): - pass + ) as f: + if html404.search(f.read()): + try: + os.unlink(mysettings["DISTDIR"]+"/"+myfile) + writemsg(_(">>> Deleting invalid distfile. (Improper 404 redirect from server.)\n")) + fetched = 0 + continue + except (IOError, OSError): + pass fetched = 1 continue if True: @@ -1040,7 +1053,6 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, # from another mirror... verified_ok,reason = verify_all(mysettings["DISTDIR"]+"/"+myfile, mydigests[myfile]) if not verified_ok: - print(reason) writemsg(_("!!! Fetched file: %s VERIFY FAILED!\n") % myfile, noiselevel=-1) writemsg(_("!!! Reason: %s\n") % reason[0], diff --git a/portage_with_autodep/pym/portage/package/ebuild/fetch.pyo b/portage_with_autodep/pym/portage/package/ebuild/fetch.pyo Binary files differnew file mode 100644 index 0000000..3bd81df --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/fetch.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.py b/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.py index f2af638..8a88c2f 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.py +++ b/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.py @@ -83,7 +83,13 @@ def getmaskingreason(mycpv, metadata=None, settings=None, pmasklists = [] for profile in locations: pmask_filename = os.path.join(profile, "package.mask") - pmasklists.append((pmask_filename, grablines(pmask_filename, recursive=1))) + node = None + for l, recursive_filename in grablines(pmask_filename, + recursive=1, remember_source_file=True): + if node is None or node[0] != recursive_filename: + node = (recursive_filename, []) + pmasklists.append(node) + node[1].append(l) pmaskdict = settings._mask_manager._pmaskdict if mycp in pmaskdict: diff --git a/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.pyo b/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.pyo Binary files differnew file mode 100644 index 0000000..1614244 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.py b/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.py index 4c65fcc..9bf605d 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.py +++ b/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ['getmaskingstatus'] @@ -7,11 +7,9 @@ import sys import portage from portage import eapi_is_supported, _eapi_is_deprecated -from portage.dep import match_from_list, _slot_separator, _repo_separator from portage.localization import _ from portage.package.ebuild.config import config -from portage.versions import catpkgsplit, cpv_getkey -from _emerge.Package import Package +from portage.versions import catpkgsplit, _pkg_str if sys.hexversion >= 0x3000000: basestring = str @@ -53,9 +51,6 @@ def _getmaskingstatus(mycpv, settings, portdb, myrepo=None): metadata = pkg.metadata installed = pkg.installed - mysplit = catpkgsplit(mycpv) - if not mysplit: - raise ValueError(_("invalid CPV: %s") % mycpv) if metadata is None: db_keys = list(portdb._aux_cache_keys) try: @@ -70,11 +65,14 @@ def _getmaskingstatus(mycpv, settings, portdb, myrepo=None): else: metadata["USE"] = "" - rValue = [] + if not hasattr(mycpv, 'slot'): + try: + mycpv = _pkg_str(mycpv, slot=metadata['SLOT'], + repo=metadata.get('repository')) + except portage.exception.InvalidData: + raise ValueError(_("invalid CPV: %s") % mycpv) - # profile checking - if settings._getProfileMaskAtom(mycpv, metadata): - rValue.append(_MaskReason("profile", "profile")) + rValue = [] # package.mask checking if settings._getMaskAtom(mycpv, metadata): @@ -85,8 +83,6 @@ def _getmaskingstatus(mycpv, settings, portdb, myrepo=None): mygroups = settings._getKeywords(mycpv, metadata) licenses = metadata["LICENSE"] properties = metadata["PROPERTIES"] - if eapi.startswith("-"): - eapi = eapi[1:] if not eapi_is_supported(eapi): return [_MaskReason("EAPI", "EAPI %s" % eapi)] elif _eapi_is_deprecated(eapi) and not installed: diff --git a/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.pyo b/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.pyo Binary files differnew file mode 100644 index 0000000..9cf1d9d --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.py b/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.py index 616dc2e..b8fbdc5 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.py +++ b/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.py @@ -5,11 +5,11 @@ __all__ = ['prepare_build_dirs'] import errno import gzip -import shutil import stat import time -from portage import os, _encodings, _unicode_encode, _unicode_decode +import portage +from portage import os, shutil, _encodings, _unicode_encode, _unicode_decode from portage.data import portage_gid, portage_uid, secpass from portage.exception import DirectoryNotFound, FileNotFound, \ OperationNotPermitted, PermissionDenied, PortageException @@ -118,11 +118,13 @@ def _adjust_perms_msg(settings, msg): background = settings.get("PORTAGE_BACKGROUND") == "1" log_path = settings.get("PORTAGE_LOG_FILE") log_file = None + log_file_real = None if background and log_path is not None: try: log_file = open(_unicode_encode(log_path, encoding=_encodings['fs'], errors='strict'), mode='ab') + log_file_real = log_file except IOError: def write(msg): pass @@ -139,6 +141,8 @@ def _adjust_perms_msg(settings, msg): finally: if log_file is not None: log_file.close() + if log_file_real is not log_file: + log_file_real.close() def _prepare_features_dirs(mysettings): @@ -311,7 +315,7 @@ def _prepare_workdir(mysettings): logdir = normalize_path(mysettings["PORT_LOGDIR"]) logid_path = os.path.join(mysettings["PORTAGE_BUILDDIR"], ".logid") if not os.path.exists(logid_path): - open(_unicode_encode(logid_path), 'w') + open(_unicode_encode(logid_path), 'w').close() logid_time = _unicode_decode(time.strftime("%Y%m%d-%H%M%S", time.gmtime(os.stat(logid_path).st_mtime)), encoding=_encodings['content'], errors='replace') @@ -342,13 +346,31 @@ def _prepare_workdir(mysettings): writemsg(_unicode_decode("!!! %s: %s\n") % (_("Permission Denied"), log_subdir), noiselevel=-1) + tmpdir_log_path = os.path.join( + mysettings["T"], "build.log%s" % compress_log_ext) if not logdir_subdir_ok: # NOTE: When sesandbox is enabled, the local SELinux security policies # may not allow output to be piped out of the sesandbox domain. The # current policy will allow it to work when a pty is available, but # not through a normal pipe. See bug #162404. - mysettings["PORTAGE_LOG_FILE"] = os.path.join( - mysettings["T"], "build.log%s" % compress_log_ext) + mysettings["PORTAGE_LOG_FILE"] = tmpdir_log_path + else: + # Create a symlink from tmpdir_log_path to PORTAGE_LOG_FILE, as + # requested in bug #412865. + make_new_symlink = False + try: + target = os.readlink(tmpdir_log_path) + except OSError: + make_new_symlink = True + else: + if target != mysettings["PORTAGE_LOG_FILE"]: + make_new_symlink = True + if make_new_symlink: + try: + os.unlink(tmpdir_log_path) + except OSError: + pass + os.symlink(mysettings["PORTAGE_LOG_FILE"], tmpdir_log_path) def _ensure_log_subdirs(logdir, subdir): """ @@ -358,13 +380,27 @@ def _ensure_log_subdirs(logdir, subdir): and subdir are assumed to be normalized absolute paths. """ st = os.stat(logdir) + uid = -1 gid = st.st_gid grp_mode = 0o2070 & st.st_mode + # If logdir is writable by the portage group but its uid + # is not portage_uid, then set the uid to portage_uid if + # we have privileges to do so, for compatibility with our + # default logrotate config (see bug 378451). With the + # "su portage portage" directive and logrotate-3.8.0, + # logrotate's chown call during the compression phase will + # only succeed if the log file's uid is portage_uid. + if grp_mode and gid == portage_gid and \ + portage.data.secpass >= 2: + uid = portage_uid + if st.st_uid != portage_uid: + ensure_dirs(logdir, uid=uid) + logdir_split_len = len(logdir.split(os.sep)) subdir_split = subdir.split(os.sep)[logdir_split_len:] subdir_split.reverse() current = logdir while subdir_split: current = os.path.join(current, subdir_split.pop()) - ensure_dirs(current, gid=gid, mode=grp_mode, mask=0) + ensure_dirs(current, uid=uid, gid=gid, mode=grp_mode, mask=0) diff --git a/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.pyo b/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.pyo Binary files differnew file mode 100644 index 0000000..2dcfaea --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.pyo |