diff options
| author |   Alexander Bersenev <bay@hackerdom.ru> | 2014-02-17 17:57:05 +0600 |
|---|---|---|
| committer | Alexander Bersenev <bay@hackerdom.ru> | 2014-02-17 17:57:05 +0600 |
| commit | 6563293d18daed502ccdb663f3c72b4bae5fe23a (patch) | |
| tree | d0a7d53a7c137feb4073c963408829f88ea75c92 /portage_with_autodep/pym/portage | |
| parent | updated portage to 2.2.8-r1 (diff) | |
| download | autodep-master.tar.gz autodep-master.tar.bz2 autodep-master.zip | |
Diffstat (limited to 'portage_with_autodep/pym/portage')
224 files changed, 6945 insertions, 3073 deletions
diff --git a/portage_with_autodep/pym/portage/__init__.py b/portage_with_autodep/pym/portage/__init__.py index 431dc26..2bce5d7 100644 --- a/portage_with_autodep/pym/portage/__init__.py +++ b/portage_with_autodep/pym/portage/__init__.py @@ -1,8 +1,10 @@ # portage.py -- core Portage functionality -# Copyright 1998-2011 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -VERSION="2.2.0_alpha108" +from __future__ import unicode_literals + +VERSION="2.2.8-r1" # =========================================================================== # START OF IMPORTS -- START OF IMPORTS -- START OF IMPORTS -- START OF IMPORT @@ -16,14 +18,6 @@ try: errno.ESTALE = -1 import re import types - - # Try the commands module first, since this allows us to eliminate - # the subprocess module from the baseline imports under python2. - try: - from commands import getstatusoutput as subprocess_getstatusoutput - except ImportError: - from subprocess import getstatusoutput as subprocess_getstatusoutput - import platform # Temporarily delete these imports, to ensure that only the @@ -70,6 +64,7 @@ try: 'match_from_list,match_to_list', 'portage.dep.dep_check:dep_check,dep_eval,dep_wordreduce,dep_zapdeps', 'portage.eclass_cache', + 'portage.elog', 'portage.exception', 'portage.getbinpkg', 'portage.locks', @@ -114,6 +109,7 @@ try: 'cpv_getkey@getCPFromCPV,endversion_keys,' + \ 'suffix_value@endversion,pkgcmp,pkgsplit,vercmp,ververify', 'portage.xpak', + 'subprocess', 'time', ) @@ -178,6 +174,15 @@ _encodings = { } if sys.hexversion >= 0x3000000: + + def _decode_argv(argv): + # With Python 3, the surrogateescape encoding error handler makes it + # possible to access the original argv bytes, which can be useful + # if their actual encoding does no match the filesystem encoding. + fs_encoding = sys.getfilesystemencoding() + return [_unicode_decode(x.encode(fs_encoding, 'surrogateescape')) + for x in argv] + def _unicode_encode(s, encoding=_encodings['content'], errors='backslashreplace'): if isinstance(s, str): s = s.encode(encoding, errors) @@ -187,7 +192,13 @@ if sys.hexversion >= 0x3000000: if isinstance(s, bytes): s = str(s, encoding=encoding, errors=errors) return s + + _native_string = _unicode_decode else: + + def _decode_argv(argv): + return [_unicode_decode(x) for x in argv] + def _unicode_encode(s, encoding=_encodings['content'], errors='backslashreplace'): if isinstance(s, unicode): s = s.encode(encoding, errors) @@ -198,6 +209,17 @@ else: s = unicode(s, encoding=encoding, errors=errors) return s + _native_string = _unicode_encode + +if sys.hexversion >= 0x20605f0: + def _native_kwargs(kwargs): + return kwargs +else: + # Avoid "TypeError: keywords must be strings" issue triggered + # by unicode_literals: http://bugs.python.org/issue4978 + def _native_kwargs(kwargs): + return dict((_native_string(k), v) for k, v in kwargs.iteritems()) + class _unicode_func_wrapper(object): """ Wraps a function, converts arguments from unicode to bytes, @@ -215,7 +237,7 @@ class _unicode_func_wrapper(object): self._func = func self._encoding = encoding - def __call__(self, *args, **kwargs): + def _process_args(self, args, kwargs): encoding = self._encoding wrapped_args = [_unicode_encode(x, encoding=encoding, errors='strict') @@ -227,6 +249,13 @@ class _unicode_func_wrapper(object): else: wrapped_kwargs = {} + return (wrapped_args, wrapped_kwargs) + + def __call__(self, *args, **kwargs): + + encoding = self._encoding + wrapped_args, wrapped_kwargs = self._process_args(args, kwargs) + rval = self._func(*wrapped_args, **wrapped_kwargs) # Don't use isinstance() since we don't want to convert subclasses @@ -294,12 +323,17 @@ class _unicode_module_wrapper(object): import os as _os _os_overrides = { id(_os.fdopen) : _os.fdopen, - id(_os.mkfifo) : _os.mkfifo, id(_os.popen) : _os.popen, id(_os.read) : _os.read, id(_os.system) : _os.system, } + +try: + _os_overrides[id(_os.mkfifo)] = _os.mkfifo +except AttributeError: + pass # Jython + if hasattr(_os, 'statvfs'): _os_overrides[id(_os.statvfs)] = _os.statvfs @@ -334,16 +368,25 @@ except (ImportError, OSError) as e: _python_interpreter = os.path.realpath(sys.executable) _bin_path = PORTAGE_BIN_PATH _pym_path = PORTAGE_PYM_PATH +_working_copy = VERSION == "HEAD" + +# Api consumers included in portage should set this to True. +_internal_caller = False -if sys.hexversion >= 0x3030000: - # Workaround for http://bugs.python.org/issue14007 - def _test_xml_etree_ElementTree_TreeBuilder_type(): - import subprocess - p = subprocess.Popen([_python_interpreter, "-c", - "import sys, xml.etree.ElementTree; sys.exit(not isinstance(xml.etree.ElementTree.TreeBuilder, type))"]) - if p.wait() != 0: - sys.modules["_elementtree"] = None - _test_xml_etree_ElementTree_TreeBuilder_type() +_sync_disabled_warnings = False + +def _get_stdin(): + """ + Buggy code in python's multiprocessing/process.py closes sys.stdin + and reassigns it to open(os.devnull), but fails to update the + corresponding __stdin__ reference. So, detect that case and handle + it appropriately. + """ + if not sys.__stdin__.closed: + return sys.__stdin__ + return sys.stdin + +_shell_quote_re = re.compile(r"[\s><=*\\\"'$`]") def _shell_quote(s): """ @@ -351,6 +394,8 @@ def _shell_quote(s): escape any backslashes, double-quotes, dollar signs, or backquotes in the string. """ + if _shell_quote_re.search(s) is None: + return s for letter in "\\\"$`": if letter in s: s = s.replace(letter, "\\" + letter) @@ -364,8 +409,27 @@ if platform.system() in ('FreeBSD',): @classmethod def chflags(cls, path, flags, opts=""): - cmd = 'chflags %s %o %s' % (opts, flags, _shell_quote(path)) - status, output = subprocess_getstatusoutput(cmd) + cmd = ['chflags'] + if opts: + cmd.append(opts) + cmd.append('%o' % (flags,)) + cmd.append(path) + + if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000: + # Python 3.1 _execvp throws TypeError for non-absolute executable + # path passed as bytes (see http://bugs.python.org/issue8513). + fullname = process.find_binary(cmd[0]) + if fullname is None: + raise exception.CommandNotFound(cmd[0]) + cmd[0] = fullname + + encoding = _encodings['fs'] + cmd = [_unicode_encode(x, encoding=encoding, errors='strict') + for x in cmd] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + output = proc.communicate()[0] + status = proc.wait() if os.WIFEXITED(status) and os.WEXITSTATUS(status) == os.EX_OK: return # Try to generate an ENOENT error if appropriate. @@ -378,6 +442,7 @@ if platform.system() in ('FreeBSD',): raise portage.exception.CommandNotFound('chflags') # Now we're not sure exactly why it failed or what # the real errno was, so just report EPERM. + output = _unicode_decode(output, encoding=encoding) e = OSError(errno.EPERM, output) e.errno = errno.EPERM e.filename = path @@ -406,7 +471,15 @@ def getcwd(): getcwd() def abssymlink(symlink, target=None): - "This reads symlinks, resolving the relative symlinks, and returning the absolute." + """ + This reads symlinks, resolving the relative symlinks, + and returning the absolute. + @param symlink: path of symlink (must be absolute) + @param target: the target of the symlink (as returned + by readlink) + @rtype: str + @return: the absolute path of the symlink target + """ if target is not None: mylink = target else: @@ -418,8 +491,9 @@ def abssymlink(symlink, target=None): _doebuild_manifest_exempt_depend = 0 -_testing_eapis = frozenset(["4-python"]) -_deprecated_eapis = frozenset(["4_pre1", "3_pre2", "3_pre1"]) +_testing_eapis = frozenset(["4-python", "4-slot-abi", "5-progress", "5-hdepend"]) +_deprecated_eapis = frozenset(["4_pre1", "3_pre2", "3_pre1", "5_pre1", "5_pre2"]) +_supported_eapis = frozenset([str(x) for x in range(portage.const.EAPI)] + list(_testing_eapis) + list(_deprecated_eapis)) def _eapi_is_deprecated(eapi): return eapi in _deprecated_eapis @@ -476,14 +550,13 @@ auxdbkeys = ( 'RESTRICT', 'HOMEPAGE', 'LICENSE', 'DESCRIPTION', 'KEYWORDS', 'INHERITED', 'IUSE', 'REQUIRED_USE', 'PDEPEND', 'PROVIDE', 'EAPI', - 'PROPERTIES', 'DEFINED_PHASES', 'UNUSED_05', 'UNUSED_04', + 'PROPERTIES', 'DEFINED_PHASES', 'HDEPEND', 'UNUSED_04', 'UNUSED_03', 'UNUSED_02', 'UNUSED_01', ) auxdbkeylen=len(auxdbkeys) def portageexit(): - if data.secpass > 1 and os.environ.get("SANDBOX_ON") != "1": - close_portdbapi_caches() + pass class _trees_dict(dict): __slots__ = ('_running_eroot', '_target_eroot',) @@ -494,13 +567,6 @@ class _trees_dict(dict): def create_trees(config_root=None, target_root=None, trees=None, env=None, eprefix=None): - if trees is not None: - # clean up any existing portdbapi instances - for myroot in trees: - portdb = trees[myroot]["porttree"].dbapi - portdb.close_caches() - portdbapi.portdbapi_instances.remove(portdb) - del trees[myroot]["porttree"], myroot, portdb if trees is None: trees = _trees_dict() @@ -518,7 +584,7 @@ def create_trees(config_root=None, target_root=None, trees=None, env=None, trees._target_eroot = settings['EROOT'] myroots = [(settings['EROOT'], settings)] - if settings["ROOT"] == "/": + if settings["ROOT"] == "/" and settings["EPREFIX"] == const.EPREFIX: trees._running_eroot = trees._target_eroot else: @@ -526,15 +592,15 @@ def create_trees(config_root=None, target_root=None, trees=None, env=None, # environment to apply to the config that's associated # with ROOT != "/", so pass a nearly empty dict for the env parameter. clean_env = {} - for k in ('PATH', 'PORTAGE_GRPNAME', 'PORTAGE_USERNAME', - 'SSH_AGENT_PID', 'SSH_AUTH_SOCK', 'TERM', + for k in ('PATH', 'PORTAGE_GRPNAME', 'PORTAGE_REPOSITORIES', 'PORTAGE_USERNAME', + 'PYTHONPATH', 'SSH_AGENT_PID', 'SSH_AUTH_SOCK', 'TERM', 'ftp_proxy', 'http_proxy', 'no_proxy', '__PORTAGE_TEST_HARDLINK_LOCKS'): v = settings.get(k) if v is not None: clean_env[k] = v settings = config(config_root=None, target_root="/", - env=clean_env, eprefix=eprefix) + env=clean_env, eprefix=None) settings.lock() trees._running_eroot = settings['EROOT'] myroots.append((settings['EROOT'], settings)) @@ -558,11 +624,17 @@ if VERSION == 'HEAD': if VERSION is not self: return VERSION if os.path.isdir(os.path.join(PORTAGE_BASE_PATH, '.git')): - status, output = subprocess_getstatusoutput(( - "cd %s ; git describe --tags || exit $? ; " + \ + encoding = _encodings['fs'] + cmd = [BASH_BINARY, "-c", ("cd %s ; git describe --tags || exit $? ; " + \ "if [ -n \"`git diff-index --name-only --diff-filter=M HEAD`\" ] ; " + \ "then echo modified ; git rev-list --format=%%ct -n 1 HEAD ; fi ; " + \ - "exit 0") % _shell_quote(PORTAGE_BASE_PATH)) + "exit 0") % _shell_quote(PORTAGE_BASE_PATH)] + cmd = [_unicode_encode(x, encoding=encoding, errors='strict') + for x in cmd] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + output = _unicode_decode(proc.communicate()[0], encoding=encoding) + status = proc.wait() if os.WIFEXITED(status) and os.WEXITSTATUS(status) == os.EX_OK: output_lines = output.splitlines() if output_lines: @@ -591,34 +663,17 @@ if VERSION == 'HEAD': return VERSION VERSION = _LazyVersion() -if "_legacy_globals_constructed" in globals(): - # The module has been reloaded, so perform any relevant cleanup - # and prevent memory leaks. - if "db" in _legacy_globals_constructed: - try: - db - except NameError: - pass - else: - if isinstance(db, dict) and db: - for _x in db.values(): - try: - if "porttree" in _x.lazy_items: - continue - except (AttributeError, TypeError): - continue - try: - _x = _x["porttree"].dbapi - except (AttributeError, KeyError): - continue - if not isinstance(_x, portdbapi): - continue - _x.close_caches() - try: - portdbapi.portdbapi_instances.remove(_x) - except ValueError: - pass - del _x +_legacy_global_var_names = ("archlist", "db", "features", + "groups", "mtimedb", "mtimedbfile", "pkglines", + "portdb", "profiledir", "root", "selinux_enabled", + "settings", "thirdpartymirrors") + +def _reset_legacy_globals(): + + global _legacy_globals_constructed + _legacy_globals_constructed = set() + for k in _legacy_global_var_names: + globals()[k] = _LegacyGlobalProxy(k) class _LegacyGlobalProxy(proxy.objectproxy.ObjectProxy): @@ -633,16 +688,7 @@ class _LegacyGlobalProxy(proxy.objectproxy.ObjectProxy): from portage._legacy_globals import _get_legacy_global return _get_legacy_global(name) -_legacy_global_var_names = ("archlist", "db", "features", - "groups", "mtimedb", "mtimedbfile", "pkglines", - "portdb", "profiledir", "root", "selinux_enabled", - "settings", "thirdpartymirrors") - -for k in _legacy_global_var_names: - globals()[k] = _LegacyGlobalProxy(k) -del k - -_legacy_globals_constructed = set() +_reset_legacy_globals() def _disable_legacy_globals(): """ diff --git a/portage_with_autodep/pym/portage/__init__.pyo b/portage_with_autodep/pym/portage/__init__.pyo Binary files differindex 9fc449e..ef60b98 100644 --- a/portage_with_autodep/pym/portage/__init__.pyo +++ b/portage_with_autodep/pym/portage/__init__.pyo diff --git a/portage_with_autodep/pym/portage/_global_updates.py b/portage_with_autodep/pym/portage/_global_updates.py index 5175043..9ae734b 100644 --- a/portage_with_autodep/pym/portage/_global_updates.py +++ b/portage_with_autodep/pym/portage/_global_updates.py @@ -1,4 +1,4 @@ -# Copyright 2010 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -32,22 +32,20 @@ def _global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True): """ # only do this if we're root and not running repoman/ebuild digest - retupd = False if secpass < 2 or \ "SANDBOX_ACTIVE" in os.environ or \ len(trees) != 1: - return retupd + return False + + return _do_global_updates(trees, prev_mtimes, + quiet=quiet, if_mtime_changed=if_mtime_changed) + +def _do_global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True): root = trees._running_eroot mysettings = trees[root]["vartree"].settings portdb = trees[root]["porttree"].dbapi vardb = trees[root]["vartree"].dbapi bindb = trees[root]["bintree"].dbapi - if not os.access(bindb.bintree.pkgdir, os.W_OK): - bindb = None - else: - # Call binarytree.populate(), since we want to make sure it's - # only populated with local packages here (getbinpkgs=0). - bindb.bintree.populate() world_file = os.path.join(mysettings['EROOT'], WORLD_FILE) world_list = grabfile(world_file) @@ -61,6 +59,7 @@ def _global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True): repo_map = {} timestamps = {} + retupd = False update_notice_printed = False for repo_name in portdb.getRepositories(): repo = portdb.getRepositoryPath(repo_name) @@ -115,6 +114,14 @@ def _global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True): if myupd: retupd = True + if retupd: + if os.access(bindb.bintree.pkgdir, os.W_OK): + # Call binarytree.populate(), since we want to make sure it's + # only populated with local packages here (getbinpkgs=0). + bindb.bintree.populate() + else: + bindb = None + master_repo = portdb.getRepositoryName(portdb.porttree_root) if master_repo in repo_map: repo_map['DEFAULT'] = repo_map[master_repo] @@ -237,8 +244,7 @@ def _global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True): # Update progress above is indicated by characters written to stdout so # we print a couple new lines here to separate the progress output from # what follows. - print() - print() + writemsg_stdout("\n\n") if do_upgrade_packagesmessage and bindb and \ bindb.cpv_all(): diff --git a/portage_with_autodep/pym/portage/_global_updates.pyo b/portage_with_autodep/pym/portage/_global_updates.pyo Binary files differindex 3e2e8de..fe5684e 100644 --- a/portage_with_autodep/pym/portage/_global_updates.pyo +++ b/portage_with_autodep/pym/portage/_global_updates.pyo diff --git a/portage_with_autodep/pym/portage/_legacy_globals.py b/portage_with_autodep/pym/portage/_legacy_globals.py index abffa0e..bb9691a 100644 --- a/portage_with_autodep/pym/portage/_legacy_globals.py +++ b/portage_with_autodep/pym/portage/_legacy_globals.py @@ -27,7 +27,8 @@ def _get_legacy_global(name): os.umask(0o22) kwargs = {} - for k, envvar in (("config_root", "PORTAGE_CONFIGROOT"), ("target_root", "ROOT")): + for k, envvar in (("config_root", "PORTAGE_CONFIGROOT"), + ("target_root", "ROOT"), ("eprefix", "EPREFIX")): kwargs[k] = os.environ.get(envvar) portage._initializing_globals = True diff --git a/portage_with_autodep/pym/portage/_legacy_globals.pyo b/portage_with_autodep/pym/portage/_legacy_globals.pyo Binary files differindex 2e50cbe..2a5d08a 100644 --- a/portage_with_autodep/pym/portage/_legacy_globals.pyo +++ b/portage_with_autodep/pym/portage/_legacy_globals.pyo diff --git a/portage_with_autodep/pym/portage/_selinux.py b/portage_with_autodep/pym/portage/_selinux.py index 9470978..e4621b1 100644 --- a/portage_with_autodep/pym/portage/_selinux.py +++ b/portage_with_autodep/pym/portage/_selinux.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # Don't use the unicode-wrapped os and shutil modules here since @@ -8,18 +8,18 @@ import shutil import portage from portage import _encodings -from portage import _unicode_decode -from portage import _unicode_encode +from portage import _native_string, _unicode_decode from portage.localization import _ portage.proxy.lazyimport.lazyimport(globals(), 'selinux') def copyfile(src, dest): - src = _unicode_encode(src, encoding=_encodings['fs'], errors='strict') - dest = _unicode_encode(dest, encoding=_encodings['fs'], errors='strict') + src = _native_string(src, encoding=_encodings['fs'], errors='strict') + dest = _native_string(dest, encoding=_encodings['fs'], errors='strict') (rc, ctx) = selinux.lgetfilecon(src) if rc < 0: - src = _unicode_decode(src, encoding=_encodings['fs'], errors='replace') + if sys.hexversion < 0x3000000: + src = _unicode_decode(src, encoding=_encodings['fs'], errors='replace') raise OSError(_("copyfile: Failed getting context of \"%s\".") % src) setfscreate(ctx) @@ -39,12 +39,12 @@ def is_selinux_enabled(): return selinux.is_selinux_enabled() def mkdir(target, refdir): - target = _unicode_encode(target, encoding=_encodings['fs'], errors='strict') - refdir = _unicode_encode(refdir, encoding=_encodings['fs'], errors='strict') + target = _native_string(target, encoding=_encodings['fs'], errors='strict') + refdir = _native_string(refdir, encoding=_encodings['fs'], errors='strict') (rc, ctx) = selinux.getfilecon(refdir) if rc < 0: - refdir = _unicode_decode(refdir, encoding=_encodings['fs'], - errors='replace') + if sys.hexversion < 0x3000000: + refdir = _unicode_decode(refdir, encoding=_encodings['fs'], errors='replace') raise OSError( _("mkdir: Failed getting context of reference directory \"%s\".") \ % refdir) @@ -56,11 +56,12 @@ def mkdir(target, refdir): setfscreate() def rename(src, dest): - src = _unicode_encode(src, encoding=_encodings['fs'], errors='strict') - dest = _unicode_encode(dest, encoding=_encodings['fs'], errors='strict') + src = _native_string(src, encoding=_encodings['fs'], errors='strict') + dest = _native_string(dest, encoding=_encodings['fs'], errors='strict') (rc, ctx) = selinux.lgetfilecon(src) if rc < 0: - src = _unicode_decode(src, encoding=_encodings['fs'], errors='replace') + if sys.hexversion < 0x3000000: + src = _unicode_decode(src, encoding=_encodings['fs'], errors='replace') raise OSError(_("rename: Failed getting context of \"%s\".") % src) setfscreate(ctx) @@ -75,10 +76,10 @@ def settype(newtype): return ":".join(ret) def setexec(ctx="\n"): - ctx = _unicode_encode(ctx, encoding=_encodings['content'], errors='strict') + ctx = _native_string(ctx, encoding=_encodings['content'], errors='strict') if selinux.setexeccon(ctx) < 0: - ctx = _unicode_decode(ctx, encoding=_encodings['content'], - errors='replace') + if sys.hexversion < 0x3000000: + ctx = _unicode_decode(ctx, encoding=_encodings['content'], errors='replace') if selinux.security_getenforce() == 1: raise OSError(_("Failed setting exec() context \"%s\".") % ctx) else: @@ -87,37 +88,47 @@ def setexec(ctx="\n"): noiselevel=-1) def setfscreate(ctx="\n"): - ctx = _unicode_encode(ctx, - encoding=_encodings['content'], errors='strict') + ctx = _native_string(ctx, encoding=_encodings['content'], errors='strict') if selinux.setfscreatecon(ctx) < 0: - ctx = _unicode_decode(ctx, - encoding=_encodings['content'], errors='replace') + if sys.hexversion < 0x3000000: + ctx = _unicode_decode(ctx, encoding=_encodings['content'], errors='replace') raise OSError( _("setfscreate: Failed setting fs create context \"%s\".") % ctx) -def spawn_wrapper(spawn_func, selinux_type): +class spawn_wrapper(object): + """ + Create a wrapper function for the given spawn function. When the wrapper + is called, it will adjust the arguments such that setexec() to be called + *after* the fork (thereby avoiding any interference with concurrent + threads in the calling process). + """ + __slots__ = ("_con", "_spawn_func") - selinux_type = _unicode_encode(selinux_type, - encoding=_encodings['content'], errors='strict') + def __init__(self, spawn_func, selinux_type): + self._spawn_func = spawn_func + selinux_type = _native_string(selinux_type, encoding=_encodings['content'], errors='strict') + self._con = settype(selinux_type) - def wrapper_func(*args, **kwargs): - con = settype(selinux_type) - setexec(con) - try: - return spawn_func(*args, **kwargs) - finally: - setexec() + def __call__(self, *args, **kwargs): - return wrapper_func + pre_exec = kwargs.get("pre_exec") + + def _pre_exec(): + if pre_exec is not None: + pre_exec() + setexec(self._con) + + kwargs["pre_exec"] = _pre_exec + return self._spawn_func(*args, **kwargs) def symlink(target, link, reflnk): - target = _unicode_encode(target, encoding=_encodings['fs'], errors='strict') - link = _unicode_encode(link, encoding=_encodings['fs'], errors='strict') - reflnk = _unicode_encode(reflnk, encoding=_encodings['fs'], errors='strict') + target = _native_string(target, encoding=_encodings['fs'], errors='strict') + link = _native_string(link, encoding=_encodings['fs'], errors='strict') + reflnk = _native_string(reflnk, encoding=_encodings['fs'], errors='strict') (rc, ctx) = selinux.lgetfilecon(reflnk) if rc < 0: - reflnk = _unicode_decode(reflnk, encoding=_encodings['fs'], - errors='replace') + if sys.hexversion < 0x3000000: + reflnk = _unicode_decode(reflnk, encoding=_encodings['fs'], errors='replace') raise OSError( _("symlink: Failed getting context of reference symlink \"%s\".") \ % reflnk) diff --git a/portage_with_autodep/pym/portage/_selinux.pyo b/portage_with_autodep/pym/portage/_selinux.pyo Binary files differindex 7a413e0..080613d 100644 --- a/portage_with_autodep/pym/portage/_selinux.pyo +++ b/portage_with_autodep/pym/portage/_selinux.pyo diff --git a/portage_with_autodep/pym/portage/_sets/__init__.py b/portage_with_autodep/pym/portage/_sets/__init__.py index 88a4b3b..75d1df7 100644 --- a/portage_with_autodep/pym/portage/_sets/__init__.py +++ b/portage_with_autodep/pym/portage/_sets/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2007-2011 Gentoo Foundation +# Copyright 2007-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -17,6 +17,7 @@ try: from configparser import SafeConfigParser except ImportError: from ConfigParser import SafeConfigParser, NoOptionError, ParsingError +import portage from portage import os from portage import load_mod from portage import _unicode_decode @@ -111,16 +112,51 @@ class SetConfig(object): """ parser = self._parser + parser.remove_section("world") parser.add_section("world") parser.set("world", "class", "portage.sets.base.DummyPackageSet") parser.set("world", "packages", "@selected @system") + parser.remove_section("selected") parser.add_section("selected") parser.set("selected", "class", "portage.sets.files.WorldSelectedSet") + parser.remove_section("system") parser.add_section("system") parser.set("system", "class", "portage.sets.profiles.PackagesSystemSet") + parser.remove_section("security") + parser.add_section("security") + parser.set("security", "class", "portage.sets.security.NewAffectedSet") + + parser.remove_section("usersets") + parser.add_section("usersets") + parser.set("usersets", "class", "portage.sets.files.StaticFileSet") + parser.set("usersets", "multiset", "true") + parser.set("usersets", "directory", "%(PORTAGE_CONFIGROOT)setc/portage/sets") + parser.set("usersets", "world-candidate", "true") + + parser.remove_section("live-rebuild") + parser.add_section("live-rebuild") + parser.set("live-rebuild", "class", "portage.sets.dbapi.VariableSet") + parser.set("live-rebuild", "variable", "INHERITED") + parser.set("live-rebuild", "includes", " ".join(sorted(portage.const.LIVE_ECLASSES))) + + parser.remove_section("module-rebuild") + parser.add_section("module-rebuild") + parser.set("module-rebuild", "class", "portage.sets.dbapi.OwnerSet") + parser.set("module-rebuild", "files", "/lib/modules") + + parser.remove_section("preserved-rebuild") + parser.add_section("preserved-rebuild") + parser.set("preserved-rebuild", "class", "portage.sets.libs.PreservedLibraryConsumerSet") + + parser.remove_section("x11-module-rebuild") + parser.add_section("x11-module-rebuild") + parser.set("x11-module-rebuild", "class", "portage.sets.dbapi.OwnerSet") + parser.set("x11-module-rebuild", "files", "/usr/lib/xorg/modules") + parser.set("x11-module-rebuild", "exclude-files", "/usr/bin/Xorg") + def update(self, setname, options): parser = self._parser self.errors = [] @@ -260,8 +296,8 @@ def load_default_config(settings, trees): return SetConfig(None, settings, trees) global_config_path = GLOBAL_CONFIG_PATH - if settings['EPREFIX']: - global_config_path = os.path.join(settings['EPREFIX'], + if portage.const.EPREFIX: + global_config_path = os.path.join(portage.const.EPREFIX, GLOBAL_CONFIG_PATH.lstrip(os.sep)) def _getfiles(): for path, dirs, files in os.walk(os.path.join(global_config_path, "sets")): diff --git a/portage_with_autodep/pym/portage/_sets/__init__.pyo b/portage_with_autodep/pym/portage/_sets/__init__.pyo Binary files differindex 5318dbe..500a5d3 100644 --- a/portage_with_autodep/pym/portage/_sets/__init__.pyo +++ b/portage_with_autodep/pym/portage/_sets/__init__.pyo diff --git a/portage_with_autodep/pym/portage/_sets/base.py b/portage_with_autodep/pym/portage/_sets/base.py index c8d3ae4..d368e00 100644 --- a/portage_with_autodep/pym/portage/_sets/base.py +++ b/portage_with_autodep/pym/portage/_sets/base.py @@ -126,7 +126,7 @@ class PackageSet(object): if modified_use is not None and modified_use is not pkg.use.enabled: pkg = pkg.copy() - pkg.metadata["USE"] = " ".join(modified_use) + pkg._metadata["USE"] = " ".join(modified_use) # Atoms matched via PROVIDE must be temporarily transformed since # match_from_list() only works correctly when atom.cp == pkg.cp. @@ -156,7 +156,7 @@ class PackageSet(object): for atom in atoms: if match_from_list(atom, cpv_slot_list): yield atom - provides = pkg.metadata['PROVIDE'] + provides = pkg._metadata['PROVIDE'] if not provides: return provides = provides.split() diff --git a/portage_with_autodep/pym/portage/_sets/base.pyo b/portage_with_autodep/pym/portage/_sets/base.pyo Binary files differindex 89e53be..09ade2f 100644 --- a/portage_with_autodep/pym/portage/_sets/base.pyo +++ b/portage_with_autodep/pym/portage/_sets/base.pyo diff --git a/portage_with_autodep/pym/portage/_sets/dbapi.py b/portage_with_autodep/pym/portage/_sets/dbapi.py index 4982a92..384fb3a 100644 --- a/portage_with_autodep/pym/portage/_sets/dbapi.py +++ b/portage_with_autodep/pym/portage/_sets/dbapi.py @@ -26,8 +26,7 @@ class EverythingSet(PackageSet): def load(self): myatoms = [] - db_keys = ["SLOT"] - aux_get = self._db.aux_get + pkg_str = self._db._pkg_str cp_list = self._db.cp_list for cp in self._db.cp_all(): @@ -35,8 +34,8 @@ class EverythingSet(PackageSet): # NOTE: Create SLOT atoms even when there is only one # SLOT installed, in order to avoid the possibility # of unwanted upgrades as reported in bug #338959. - slot, = aux_get(cpv, db_keys) - atom = Atom("%s:%s" % (cp, slot)) + pkg = pkg_str(cpv, None) + atom = Atom("%s:%s" % (pkg.cp, pkg.slot)) if self._filter: if self._filter(atom): myatoms.append(atom) @@ -68,20 +67,19 @@ class OwnerSet(PackageSet): """ rValue = set() vardb = self._db - aux_get = vardb.aux_get - aux_keys = ["SLOT"] + pkg_str = vardb._pkg_str if exclude_paths is None: for link, p in vardb._owners.iter_owners(paths): - slot, = aux_get(link.mycpv, aux_keys) - rValue.add("%s:%s" % (link.mycpv.cp, slot)) + pkg = pkg_str(link.mycpv, None) + rValue.add("%s:%s" % (pkg.cp, pkg.slot)) else: all_paths = set() all_paths.update(paths) all_paths.update(exclude_paths) exclude_atoms = set() for link, p in vardb._owners.iter_owners(all_paths): - slot, = aux_get(link.mycpv, aux_keys) - atom = "%s:%s" % (link.mycpv.cp, slot) + pkg = pkg_str(link.mycpv, None) + atom = "%s:%s" % (pkg.cp, pkg.slot) rValue.add(atom) if p in exclude_paths: exclude_atoms.add(atom) @@ -173,12 +171,11 @@ class DowngradeSet(PackageSet): xmatch = self._portdb.xmatch xmatch_level = "bestmatch-visible" cp_list = self._vardb.cp_list - aux_get = self._vardb.aux_get - aux_keys = ["SLOT"] + pkg_str = self._vardb._pkg_str for cp in self._vardb.cp_all(): for cpv in cp_list(cp): - slot, = aux_get(cpv, aux_keys) - slot_atom = "%s:%s" % (cp, slot) + pkg = pkg_str(cpv, None) + slot_atom = "%s:%s" % (pkg.cp, pkg.slot) ebuild = xmatch(xmatch_level, slot_atom) if not ebuild: continue @@ -326,6 +323,7 @@ class CategorySet(PackageSet): class AgeSet(EverythingSet): _operations = ["merge", "unmerge"] + _aux_keys = ('BUILD_TIME',) def __init__(self, vardb, mode="older", age=7): super(AgeSet, self).__init__(vardb) @@ -335,8 +333,12 @@ class AgeSet(EverythingSet): def _filter(self, atom): cpv = self._db.match(atom)[0] - path = self._db.getpath(cpv, filename="COUNTER") - age = (time.time() - os.stat(path).st_mtime) / (3600 * 24) + try: + date, = self._db.aux_get(cpv, self._aux_keys) + date = int(date) + except (KeyError, ValueError): + return bool(self._mode == "older") + age = (time.time() - date) / (3600 * 24) if ((self._mode == "older" and age <= self._age) \ or (self._mode == "newer" and age >= self._age)): return False @@ -355,6 +357,83 @@ class AgeSet(EverythingSet): singleBuilder = classmethod(singleBuilder) +class DateSet(EverythingSet): + _operations = ["merge", "unmerge"] + _aux_keys = ('BUILD_TIME',) + + def __init__(self, vardb, date, mode="older"): + super(DateSet, self).__init__(vardb) + self._mode = mode + self._date = date + + def _filter(self, atom): + + cpv = self._db.match(atom)[0] + try: + date, = self._db.aux_get(cpv, self._aux_keys) + date = int(date) + except (KeyError, ValueError): + return bool(self._mode == "older") + # Make sure inequality is _strict_ to exclude tested package + if ((self._mode == "older" and date < self._date) \ + or (self._mode == "newer" and date > self._date)): + return True + else: + return False + + def singleBuilder(cls, options, settings, trees): + vardbapi = trees["vartree"].dbapi + mode = options.get("mode", "older") + if str(mode).lower() not in ["newer", "older"]: + raise SetConfigError(_("invalid 'mode' value %s (use either 'newer' or 'older')") % mode) + + formats = [] + if options.get("package") is not None: + formats.append("package") + if options.get("filestamp") is not None: + formats.append("filestamp") + if options.get("seconds") is not None: + formats.append("seconds") + if options.get("date") is not None: + formats.append("date") + + if not formats: + raise SetConfigError(_("none of these options specified: 'package', 'filestamp', 'seconds', 'date'")) + elif len(formats) > 1: + raise SetConfigError(_("no more than one of these options is allowed: 'package', 'filestamp', 'seconds', 'date'")) + + format = formats[0] + + if (format == "package"): + package = options.get("package") + try: + cpv = vardbapi.match(package)[0] + date, = vardbapi.aux_get(cpv, ('BUILD_TIME',)) + date = int(date) + except (KeyError, ValueError): + raise SetConfigError(_("cannot determine installation date of package %s") % package) + elif (format == "filestamp"): + filestamp = options.get("filestamp") + try: + date = int(os.stat(filestamp).st_mtime) + except (OSError, ValueError): + raise SetConfigError(_("cannot determine 'filestamp' of '%s'") % filestamp) + elif (format == "seconds"): + try: + date = int(options.get("seconds")) + except ValueError: + raise SetConfigError(_("option 'seconds' must be an integer")) + else: + dateopt = options.get("date") + try: + dateformat = options.get("dateformat", "%x %X") + date = int(time.mktime(time.strptime(dateopt, dateformat))) + except ValueError: + raise SetConfigError(_("'date=%s' does not match 'dateformat=%s'") % (dateopt, dateformat)) + return DateSet(vardb=vardbapi, date=date, mode=mode) + + singleBuilder = classmethod(singleBuilder) + class RebuiltBinaries(EverythingSet): _operations = ('merge',) _aux_keys = ('BUILD_TIME',) diff --git a/portage_with_autodep/pym/portage/_sets/dbapi.pyo b/portage_with_autodep/pym/portage/_sets/dbapi.pyo Binary files differindex 20bf848..584cce4 100644 --- a/portage_with_autodep/pym/portage/_sets/dbapi.pyo +++ b/portage_with_autodep/pym/portage/_sets/dbapi.pyo diff --git a/portage_with_autodep/pym/portage/_sets/files.py b/portage_with_autodep/pym/portage/_sets/files.py index f19ecf6..2fb64de 100644 --- a/portage_with_autodep/pym/portage/_sets/files.py +++ b/portage_with_autodep/pym/portage/_sets/files.py @@ -1,4 +1,4 @@ -# Copyright 2007-2011 Gentoo Foundation +# Copyright 2007-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import errno @@ -11,7 +11,6 @@ from portage import _unicode_decode from portage import _unicode_encode from portage.util import grabfile, write_atomic, ensure_dirs, normalize_path from portage.const import USER_CONFIG_PATH, WORLD_FILE, WORLD_SETS_FILE -from portage.const import _ENABLE_SET_CONFIG from portage.localization import _ from portage.locks import lockfile, unlockfile from portage import portage_gid @@ -87,8 +86,8 @@ class StaticFileSet(EditablePackageSet): for a in data: matches = self.dbapi.match(a) for cpv in matches: - atoms.append("%s:%s" % (cpv_getkey(cpv), - self.dbapi.aux_get(cpv, ["SLOT"])[0])) + pkg = self.dbapi._pkg_str(cpv, None) + atoms.append("%s:%s" % (pkg.cp, pkg.slot)) # In addition to any installed slots, also try to pull # in the latest new slot that may be available. atoms.append(a) @@ -231,9 +230,8 @@ class WorldSelectedSet(EditablePackageSet): write_atomic(self._filename, "".join(sorted("%s\n" % x for x in self._atoms))) - if _ENABLE_SET_CONFIG: - write_atomic(self._filename2, - "".join(sorted("%s\n" % x for x in self._nonatoms))) + write_atomic(self._filename2, + "".join(sorted("%s\n" % x for x in self._nonatoms))) def load(self): atoms = [] @@ -263,9 +261,8 @@ class WorldSelectedSet(EditablePackageSet): else: atoms.extend(self._atoms) - if _ENABLE_SET_CONFIG: - changed2, nonatoms = self._load2() - atoms_changed |= changed2 + changed2, nonatoms = self._load2() + atoms_changed |= changed2 if atoms_changed: self._setAtoms(atoms+nonatoms) @@ -299,10 +296,14 @@ class WorldSelectedSet(EditablePackageSet): ensure_dirs(os.path.dirname(self._filename), gid=portage_gid, mode=0o2750, mask=0o2) def lock(self): + if self._lock is not None: + raise AssertionError("already locked") self._ensure_dirs() self._lock = lockfile(self._filename, wantnewlockfile=1) def unlock(self): + if self._lock is None: + raise AssertionError("not locked") unlockfile(self._lock) self._lock = None diff --git a/portage_with_autodep/pym/portage/_sets/files.pyo b/portage_with_autodep/pym/portage/_sets/files.pyo Binary files differindex eb03c00..f79345b 100644 --- a/portage_with_autodep/pym/portage/_sets/files.pyo +++ b/portage_with_autodep/pym/portage/_sets/files.pyo diff --git a/portage_with_autodep/pym/portage/_sets/libs.py b/portage_with_autodep/pym/portage/_sets/libs.py index 6c5babc..022e076 100644 --- a/portage_with_autodep/pym/portage/_sets/libs.py +++ b/portage_with_autodep/pym/portage/_sets/libs.py @@ -1,12 +1,12 @@ -# Copyright 2007-2011 Gentoo Foundation +# Copyright 2007-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function +from portage.exception import InvalidData from portage.localization import _ from portage._sets.base import PackageSet from portage._sets import get_boolean, SetConfigError -from portage.versions import cpv_getkey import portage class LibraryConsumerSet(PackageSet): @@ -22,14 +22,14 @@ class LibraryConsumerSet(PackageSet): for p in paths: for cpv in self.dbapi._linkmap.getOwners(p): try: - slot, = self.dbapi.aux_get(cpv, ["SLOT"]) - except KeyError: + pkg = self.dbapi._pkg_str(cpv, None) + except (KeyError, InvalidData): # This is expected for preserved libraries # of packages that have been uninstalled # without replacement. pass else: - rValue.add("%s:%s" % (cpv_getkey(cpv), slot)) + rValue.add("%s:%s" % (pkg.cp, pkg.slot)) return rValue class LibraryFileConsumerSet(LibraryConsumerSet): @@ -49,7 +49,8 @@ class LibraryFileConsumerSet(LibraryConsumerSet): def load(self): consumers = set() for lib in self.files: - consumers.update(self.dbapi._linkmap.findConsumers(lib)) + consumers.update( + self.dbapi._linkmap.findConsumers(lib, greedy=False)) if not consumers: return @@ -77,10 +78,10 @@ class PreservedLibraryConsumerSet(LibraryConsumerSet): for lib in libs: if self.debug: print(lib) - for x in sorted(self.dbapi._linkmap.findConsumers(lib)): + for x in sorted(self.dbapi._linkmap.findConsumers(lib, greedy=False)): print(" ", x) print("-"*40) - consumers.update(self.dbapi._linkmap.findConsumers(lib)) + consumers.update(self.dbapi._linkmap.findConsumers(lib, greedy=False)) # Don't rebuild packages just because they contain preserved # libs that happen to be consumers of other preserved libs. for libs in plib_dict.values(): diff --git a/portage_with_autodep/pym/portage/_sets/libs.pyo b/portage_with_autodep/pym/portage/_sets/libs.pyo Binary files differindex 72fc1bb..7a9f6d1 100644 --- a/portage_with_autodep/pym/portage/_sets/libs.pyo +++ b/portage_with_autodep/pym/portage/_sets/libs.pyo diff --git a/portage_with_autodep/pym/portage/_sets/profiles.pyo b/portage_with_autodep/pym/portage/_sets/profiles.pyo Binary files differindex 9502044..9ce72f1 100644 --- a/portage_with_autodep/pym/portage/_sets/profiles.pyo +++ b/portage_with_autodep/pym/portage/_sets/profiles.pyo diff --git a/portage_with_autodep/pym/portage/_sets/security.py b/portage_with_autodep/pym/portage/_sets/security.py index 7e856bc..f8dbef2 100644 --- a/portage_with_autodep/pym/portage/_sets/security.py +++ b/portage_with_autodep/pym/portage/_sets/security.py @@ -44,8 +44,8 @@ class SecuritySet(PackageSet): mydict = {} for atom in atomlist[:]: cpv = self._portdbapi.xmatch("match-all", atom)[0] - slot = self._portdbapi.aux_get(cpv, ["SLOT"])[0] - cps = "%s:%s" % (cpv.cp, slot) + pkg = self._portdbapi._pkg_str(cpv, None) + cps = "%s:%s" % (pkg.cp, pkg.slot) if not cps in mydict: mydict[cps] = (atom, cpv) else: diff --git a/portage_with_autodep/pym/portage/_sets/security.pyo b/portage_with_autodep/pym/portage/_sets/security.pyo Binary files differindex ea67514..9603bc8 100644 --- a/portage_with_autodep/pym/portage/_sets/security.pyo +++ b/portage_with_autodep/pym/portage/_sets/security.pyo diff --git a/portage_with_autodep/pym/portage/_sets/shell.pyo b/portage_with_autodep/pym/portage/_sets/shell.pyo Binary files differindex e5e4561..f153c51 100644 --- a/portage_with_autodep/pym/portage/_sets/shell.pyo +++ b/portage_with_autodep/pym/portage/_sets/shell.pyo diff --git a/portage_with_autodep/pym/portage/cache/__init__.pyo b/portage_with_autodep/pym/portage/cache/__init__.pyo Binary files differindex eb5a90e..735ab0b 100644 --- a/portage_with_autodep/pym/portage/cache/__init__.pyo +++ b/portage_with_autodep/pym/portage/cache/__init__.pyo diff --git a/portage_with_autodep/pym/portage/cache/anydbm.pyo b/portage_with_autodep/pym/portage/cache/anydbm.pyo Binary files differindex 5946da9..8422437 100644 --- a/portage_with_autodep/pym/portage/cache/anydbm.pyo +++ b/portage_with_autodep/pym/portage/cache/anydbm.pyo diff --git a/portage_with_autodep/pym/portage/cache/cache_errors.pyo b/portage_with_autodep/pym/portage/cache/cache_errors.pyo Binary files differindex 866088e..e323de7 100644 --- a/portage_with_autodep/pym/portage/cache/cache_errors.pyo +++ b/portage_with_autodep/pym/portage/cache/cache_errors.pyo diff --git a/portage_with_autodep/pym/portage/cache/ebuild_xattr.py b/portage_with_autodep/pym/portage/cache/ebuild_xattr.py index 0086e40..db6e177 100644 --- a/portage_with_autodep/pym/portage/cache/ebuild_xattr.py +++ b/portage_with_autodep/pym/portage/cache/ebuild_xattr.py @@ -1,4 +1,4 @@ -# -*- coding: UTF8 -*- +# -*- coding: utf-8 -*- # Copyright: 2009-2011 Gentoo Foundation # Author(s): Petteri Räty (betelgeuse@gentoo.org) # License: GPL2 diff --git a/portage_with_autodep/pym/portage/cache/ebuild_xattr.pyo b/portage_with_autodep/pym/portage/cache/ebuild_xattr.pyo Binary files differindex fe32dcc..405f372 100644 --- a/portage_with_autodep/pym/portage/cache/ebuild_xattr.pyo +++ b/portage_with_autodep/pym/portage/cache/ebuild_xattr.pyo diff --git a/portage_with_autodep/pym/portage/cache/flat_hash.py b/portage_with_autodep/pym/portage/cache/flat_hash.py index 2eae9f6..08dcbe8 100644 --- a/portage_with_autodep/pym/portage/cache/flat_hash.py +++ b/portage_with_autodep/pym/portage/cache/flat_hash.py @@ -1,7 +1,9 @@ -# Copyright: 2005-2011 Gentoo Foundation +# Copyright: 2005-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # Author(s): Brian Harring (ferringb@gentoo.org) +from __future__ import unicode_literals + from portage.cache import fs_template from portage.cache import cache_errors import errno @@ -11,16 +13,13 @@ import sys import os as _os from portage import os from portage import _encodings -from portage import _unicode_decode from portage import _unicode_encode +from portage.exception import InvalidData +from portage.versions import _pkg_str if sys.hexversion >= 0x3000000: long = int -# Coerce to unicode, in order to prevent TypeError when writing -# raw bytes to TextIOWrapper with python2. -_setitem_fmt = _unicode_decode("%s=%s\n") - class database(fs_template.FsBased): autocommits = True @@ -40,11 +39,10 @@ class database(fs_template.FsBased): # Don't use os.path.join, for better performance. fp = self.location + _os.sep + cpv try: - myf = io.open(_unicode_encode(fp, + with io.open(_unicode_encode(fp, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], - errors='replace') - try: + errors='replace') as myf: lines = myf.read().split("\n") if not lines[-1]: lines.pop() @@ -54,8 +52,6 @@ class database(fs_template.FsBased): # that uses mtime mangling. d['_mtime_'] = _os.fstat(myf.fileno())[stat.ST_MTIME] return d - finally: - myf.close() except (IOError, OSError) as e: if e.errno != errno.ENOENT: raise cache_errors.CacheCorruption(cpv, e) @@ -94,7 +90,10 @@ class database(fs_template.FsBased): v = values.get(k) if not v: continue - myf.write(_setitem_fmt % (k, v)) + # NOTE: This format string requires unicode_literals, so that + # k and v are coerced to unicode, in order to prevent TypeError + # when writing raw bytes to TextIOWrapper with Python 2. + myf.write("%s=%s\n" % (k, v)) finally: myf.close() self._ensure_access(fp) @@ -135,8 +134,6 @@ class database(fs_template.FsBased): del e continue for l in dir_list: - if l.endswith(".cpickle"): - continue p = os.path.join(dir_path, l) try: st = os.lstat(p) @@ -151,7 +148,11 @@ class database(fs_template.FsBased): if depth < 1: dirs.append((depth+1, p)) continue - yield p[len_base+1:] + + try: + yield _pkg_str(p[len_base+1:]) + except InvalidData: + continue class md5_database(database): diff --git a/portage_with_autodep/pym/portage/cache/flat_hash.pyo b/portage_with_autodep/pym/portage/cache/flat_hash.pyo Binary files differindex 4f568a8..90a1b36 100644 --- a/portage_with_autodep/pym/portage/cache/flat_hash.pyo +++ b/portage_with_autodep/pym/portage/cache/flat_hash.pyo diff --git a/portage_with_autodep/pym/portage/cache/flat_list.py b/portage_with_autodep/pym/portage/cache/flat_list.py deleted file mode 100644 index 7288307..0000000 --- a/portage_with_autodep/pym/portage/cache/flat_list.py +++ /dev/null @@ -1,134 +0,0 @@ -# Copyright 2005-2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.cache import fs_template -from portage.cache import cache_errors -from portage import os -from portage import _encodings -from portage import _unicode_decode -from portage import _unicode_encode -import errno -import io -import stat -import sys - -if sys.hexversion >= 0x3000000: - long = int - -# Coerce to unicode, in order to prevent TypeError when writing -# raw bytes to TextIOWrapper with python2. -_setitem_fmt = _unicode_decode("%s\n") - -# store the current key order *here*. -class database(fs_template.FsBased): - - autocommits = True - - # do not screw with this ordering. _eclasses_ needs to be last - auxdbkey_order=('DEPEND', 'RDEPEND', 'SLOT', 'SRC_URI', - 'RESTRICT', 'HOMEPAGE', 'LICENSE', 'DESCRIPTION', - 'KEYWORDS', 'IUSE', 'REQUIRED_USE', - 'PDEPEND', 'PROVIDE', 'EAPI', 'PROPERTIES', 'DEFINED_PHASES') - - def __init__(self, *args, **config): - super(database,self).__init__(*args, **config) - self.location = os.path.join(self.location, - self.label.lstrip(os.path.sep).rstrip(os.path.sep)) - - if len(self._known_keys) > len(self.auxdbkey_order) + 2: - raise Exception("less ordered keys then auxdbkeys") - if not os.path.exists(self.location): - self._ensure_dirs() - - - def _getitem(self, cpv): - d = {} - try: - myf = io.open(_unicode_encode(os.path.join(self.location, cpv), - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content'], - errors='replace') - for k,v in zip(self.auxdbkey_order, myf): - d[k] = v.rstrip("\n") - except (OSError, IOError) as e: - if errno.ENOENT == e.errno: - raise KeyError(cpv) - raise cache_errors.CacheCorruption(cpv, e) - - try: - d["_mtime_"] = os.fstat(myf.fileno())[stat.ST_MTIME] - except OSError as e: - myf.close() - raise cache_errors.CacheCorruption(cpv, e) - myf.close() - return d - - - def _setitem(self, cpv, values): - s = cpv.rfind("/") - fp=os.path.join(self.location,cpv[:s],".update.%i.%s" % (os.getpid(), cpv[s+1:])) - try: - myf = io.open(_unicode_encode(fp, - encoding=_encodings['fs'], errors='strict'), - mode='w', encoding=_encodings['repo.content'], - errors='backslashreplace') - except (OSError, IOError) as e: - if errno.ENOENT == e.errno: - try: - self._ensure_dirs(cpv) - myf = io.open(_unicode_encode(fp, - encoding=_encodings['fs'], errors='strict'), - mode='w', encoding=_encodings['repo.content'], - errors='backslashreplace') - except (OSError, IOError) as e: - raise cache_errors.CacheCorruption(cpv, e) - else: - raise cache_errors.CacheCorruption(cpv, e) - - - for x in self.auxdbkey_order: - myf.write(_setitem_fmt % (values.get(x, ""),)) - - myf.close() - self._ensure_access(fp, mtime=values["_mtime_"]) - #update written. now we move it. - new_fp = os.path.join(self.location,cpv) - try: - os.rename(fp, new_fp) - except (OSError, IOError) as e: - os.remove(fp) - raise cache_errors.CacheCorruption(cpv, e) - - - def _delitem(self, cpv): - try: - os.remove(os.path.join(self.location,cpv)) - except OSError as e: - if errno.ENOENT == e.errno: - raise KeyError(cpv) - else: - raise cache_errors.CacheCorruption(cpv, e) - - - def __contains__(self, cpv): - return os.path.exists(os.path.join(self.location, cpv)) - - - def __iter__(self): - """generator for walking the dir struct""" - dirs = [self.location] - len_base = len(self.location) - while len(dirs): - for l in os.listdir(dirs[0]): - if l.endswith(".cpickle"): - continue - p = os.path.join(dirs[0],l) - st = os.lstat(p) - if stat.S_ISDIR(st.st_mode): - dirs.append(p) - continue - yield p[len_base+1:] - dirs.pop(0) - - - def commit(self): pass diff --git a/portage_with_autodep/pym/portage/cache/flat_list.pyo b/portage_with_autodep/pym/portage/cache/flat_list.pyo Binary files differdeleted file mode 100644 index ab7dc82..0000000 --- a/portage_with_autodep/pym/portage/cache/flat_list.pyo +++ /dev/null diff --git a/portage_with_autodep/pym/portage/cache/fs_template.py b/portage_with_autodep/pym/portage/cache/fs_template.py index a82e862..0567c72 100644 --- a/portage_with_autodep/pym/portage/cache/fs_template.py +++ b/portage_with_autodep/pym/portage/cache/fs_template.py @@ -1,14 +1,14 @@ -# Copyright: 2005 Gentoo Foundation +# Copyright 2005-2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 # Author(s): Brian Harring (ferringb@gentoo.org) -# License: GPL2 +import os as _os import sys from portage.cache import template from portage import os from portage.proxy.lazyimport import lazyimport lazyimport(globals(), - 'portage.data:portage_gid', 'portage.exception:PortageException', 'portage.util:apply_permissions', ) @@ -22,14 +22,11 @@ class FsBased(template.database): attempt to ensure files have the specified owners/perms""" def __init__(self, *args, **config): - """throws InitializationError if needs args aren't specified - gid and perms aren't listed do to an oddity python currying mechanism - gid=portage_gid - perms=0665""" for x, y in (("gid", -1), ("perms", -1)): if x in config: - setattr(self, "_"+x, config[x]) + # Since Python 3.4, chown requires int type (no proxies). + setattr(self, "_" + x, int(config[x])) del config[x] else: setattr(self, "_"+x, y) @@ -78,7 +75,17 @@ class FsBased(template.database): if self._perms != -1: os.umask(um) - + def _prune_empty_dirs(self): + all_dirs = [] + for parent, dirs, files in os.walk(self.location): + for x in dirs: + all_dirs.append(_os.path.join(parent, x)) + while all_dirs: + try: + _os.rmdir(all_dirs.pop()) + except OSError: + pass + def gen_label(base, label): """if supplied label is a path, generate a unique label based upon label, and supplied base path""" if label.find(os.path.sep) == -1: diff --git a/portage_with_autodep/pym/portage/cache/fs_template.pyo b/portage_with_autodep/pym/portage/cache/fs_template.pyo Binary files differindex 6cbbc2f..de71ee6 100644 --- a/portage_with_autodep/pym/portage/cache/fs_template.pyo +++ b/portage_with_autodep/pym/portage/cache/fs_template.pyo diff --git a/portage_with_autodep/pym/portage/cache/mappings.py b/portage_with_autodep/pym/portage/cache/mappings.py index bc8ce9a..cd39a6e 100644 --- a/portage_with_autodep/pym/portage/cache/mappings.py +++ b/portage_with_autodep/pym/portage/cache/mappings.py @@ -199,10 +199,10 @@ class OrderedDict(UserDict): return iter(self._order) def __setitem__(self, key, item): - if key in self: - self._order.remove(key) + new_key = key not in self UserDict.__setitem__(self, key, item) - self._order.append(key) + if new_key: + self._order.append(key) def __delitem__(self, key): UserDict.__delitem__(self, key) diff --git a/portage_with_autodep/pym/portage/cache/mappings.pyo b/portage_with_autodep/pym/portage/cache/mappings.pyo Binary files differindex 1eb3f4f..c183246 100644 --- a/portage_with_autodep/pym/portage/cache/mappings.pyo +++ b/portage_with_autodep/pym/portage/cache/mappings.pyo diff --git a/portage_with_autodep/pym/portage/cache/metadata.py b/portage_with_autodep/pym/portage/cache/metadata.py index 9d2c3a5..6612e73 100644 --- a/portage_with_autodep/pym/portage/cache/metadata.py +++ b/portage_with_autodep/pym/portage/cache/metadata.py @@ -28,7 +28,8 @@ class database(flat_hash.database): auxdbkey_order=('DEPEND', 'RDEPEND', 'SLOT', 'SRC_URI', 'RESTRICT', 'HOMEPAGE', 'LICENSE', 'DESCRIPTION', 'KEYWORDS', 'INHERITED', 'IUSE', 'REQUIRED_USE', - 'PDEPEND', 'PROVIDE', 'EAPI', 'PROPERTIES', 'DEFINED_PHASES') + 'PDEPEND', 'PROVIDE', 'EAPI', 'PROPERTIES', + 'DEFINED_PHASES', 'HDEPEND') autocommits = True serialize_eclasses = False diff --git a/portage_with_autodep/pym/portage/cache/metadata.pyo b/portage_with_autodep/pym/portage/cache/metadata.pyo Binary files differindex c98445b..649ec5b 100644 --- a/portage_with_autodep/pym/portage/cache/metadata.pyo +++ b/portage_with_autodep/pym/portage/cache/metadata.pyo diff --git a/portage_with_autodep/pym/portage/cache/sql_template.pyo b/portage_with_autodep/pym/portage/cache/sql_template.pyo Binary files differindex e2c5974..54636ed 100644 --- a/portage_with_autodep/pym/portage/cache/sql_template.pyo +++ b/portage_with_autodep/pym/portage/cache/sql_template.pyo diff --git a/portage_with_autodep/pym/portage/cache/sqlite.py b/portage_with_autodep/pym/portage/cache/sqlite.py index fcc62ff..40db070 100644 --- a/portage_with_autodep/pym/portage/cache/sqlite.py +++ b/portage_with_autodep/pym/portage/cache/sqlite.py @@ -1,6 +1,9 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + +import re import sys from portage.cache import fs_template from portage.cache import cache_errors @@ -20,7 +23,6 @@ class database(fs_template.FsBased): # to calculate the number of pages requested, according to the following # equation: cache_bytes = page_bytes * page_count cache_bytes = 1024 * 1024 * 10 - _db_table = None def __init__(self, *args, **config): super(database, self).__init__(*args, **config) @@ -28,6 +30,7 @@ class database(fs_template.FsBased): self._allowed_keys = ["_mtime_", "_eclasses_"] self._allowed_keys.extend(self._known_keys) self._allowed_keys.sort() + self._allowed_keys_set = frozenset(self._allowed_keys) self.location = os.path.join(self.location, self.label.lstrip(os.path.sep).rstrip(os.path.sep)) @@ -37,8 +40,8 @@ class database(fs_template.FsBased): config.setdefault("autocommit", self.autocommits) config.setdefault("cache_bytes", self.cache_bytes) config.setdefault("synchronous", self.synchronous) - # Timeout for throwing a "database is locked" exception (pysqlite - # default is 5.0 seconds). + # Set longer timeout for throwing a "database is locked" exception. + # Default timeout in sqlite3 module is 5.0 seconds. config.setdefault("timeout", 15) self._db_init_connection(config) self._db_init_structures() @@ -47,11 +50,8 @@ class database(fs_template.FsBased): # sqlite3 is optional with >=python-2.5 try: import sqlite3 as db_module - except ImportError: - try: - from pysqlite2 import dbapi2 as db_module - except ImportError as e: - raise cache_errors.InitializationError(self.__class__, e) + except ImportError as e: + raise cache_errors.InitializationError(self.__class__, e) self._db_module = db_module self._db_error = db_module.Error @@ -62,7 +62,6 @@ class database(fs_template.FsBased): # Avoid potential UnicodeEncodeError in python-2.x by # only calling str() when it's absolutely necessary. s = str(s) - # This is equivalent to the _quote function from pysqlite 1.1. return "'%s'" % s.replace("'", "''") def _db_init_connection(self, config): @@ -92,9 +91,6 @@ class database(fs_template.FsBased): self._db_table["packages"]["table_name"] = mytable self._db_table["packages"]["package_id"] = "internal_db_package_id" self._db_table["packages"]["package_key"] = "portage_package_key" - self._db_table["packages"]["internal_columns"] = \ - [self._db_table["packages"]["package_id"], - self._db_table["packages"]["package_key"]] create_statement = [] create_statement.append("CREATE TABLE") create_statement.append(mytable) @@ -109,15 +105,18 @@ class database(fs_template.FsBased): create_statement.append(")") self._db_table["packages"]["create"] = " ".join(create_statement) - self._db_table["packages"]["columns"] = \ - self._db_table["packages"]["internal_columns"] + \ - self._allowed_keys cursor = self._db_cursor for k, v in self._db_table.items(): if self._db_table_exists(v["table_name"]): create_statement = self._db_table_get_create(v["table_name"]) - if create_statement != v["create"]: + table_ok, missing_keys = self._db_validate_create_statement(create_statement) + if table_ok: + if missing_keys: + for k in sorted(missing_keys): + cursor.execute("ALTER TABLE %s ADD COLUMN %s TEXT" % + (self._db_table["packages"]["table_name"], k)) + else: writemsg(_("sqlite: dropping old table: %s\n") % v["table_name"]) cursor.execute("DROP TABLE %s" % v["table_name"]) cursor.execute(v["create"]) @@ -138,6 +137,37 @@ class database(fs_template.FsBased): self._db_escape_string(table_name)) return cursor.fetchall()[0][0] + def _db_validate_create_statement(self, statement): + missing_keys = None + if statement == self._db_table["packages"]["create"]: + return True, missing_keys + + m = re.match(r'^\s*CREATE\s*TABLE\s*%s\s*\(\s*%s\s*INTEGER\s*PRIMARY\s*KEY\s*AUTOINCREMENT\s*,(.*)\)\s*$' % + (self._db_table["packages"]["table_name"], + self._db_table["packages"]["package_id"]), + statement) + if m is None: + return False, missing_keys + + unique_constraints = set([self._db_table["packages"]["package_key"]]) + missing_keys = set(self._allowed_keys) + unique_re = re.compile(r'^\s*UNIQUE\s*\(\s*(\w*)\s*\)\s*$') + column_re = re.compile(r'^\s*(\w*)\s*TEXT\s*$') + for x in m.group(1).split(","): + m = column_re.match(x) + if m is not None: + missing_keys.discard(m.group(1)) + continue + m = unique_re.match(x) + if m is not None: + unique_constraints.discard(m.group(1)) + continue + + if unique_constraints: + return False, missing_keys + + return True, missing_keys + def _db_init_cache_size(self, cache_bytes): cursor = self._db_cursor cursor.execute("PRAGMA page_size") @@ -173,13 +203,17 @@ class database(fs_template.FsBased): raise KeyError(cpv) else: raise cache_errors.CacheCorruption(cpv, "key is not unique") + result = result[0] d = {} - internal_columns = self._db_table["packages"]["internal_columns"] - column_index = -1 - for k in self._db_table["packages"]["columns"]: - column_index +=1 - if k not in internal_columns: - d[k] = result[0][column_index] + allowed_keys_set = self._allowed_keys_set + for column_index, column_info in enumerate(cursor.description): + k = column_info[0] + if k in allowed_keys_set: + v = result[column_index] + if v is None: + # This happens after a new empty column has been added. + v = "" + d[k] = v return d diff --git a/portage_with_autodep/pym/portage/cache/sqlite.pyo b/portage_with_autodep/pym/portage/cache/sqlite.pyo Binary files differindex a82d25f..aba2f6c 100644 --- a/portage_with_autodep/pym/portage/cache/sqlite.pyo +++ b/portage_with_autodep/pym/portage/cache/sqlite.pyo diff --git a/portage_with_autodep/pym/portage/cache/template.py b/portage_with_autodep/pym/portage/cache/template.py index cf1e8ae..9b8a4d3 100644 --- a/portage_with_autodep/pym/portage/cache/template.py +++ b/portage_with_autodep/pym/portage/cache/template.py @@ -1,6 +1,6 @@ -# Copyright: 2005-2012 Gentoo Foundation +# Copyright 2005-2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 # Author(s): Brian Harring (ferringb@gentoo.org) -# License: GPL2 from portage.cache import cache_errors from portage.cache.cache_errors import InvalidRestriction @@ -164,7 +164,14 @@ class database(object): def commit(self): if not self.autocommits: - raise NotImplementedError + raise NotImplementedError(self) + + def __del__(self): + # This used to be handled by an atexit hook that called + # close_portdbapi_caches() for all portdbapi instances, but that was + # prone to memory leaks for API consumers that needed to create/destroy + # many portdbapi instances. So, instead we rely on __del__. + self.sync() def __contains__(self, cpv): """This method should always be overridden. It is provided only for diff --git a/portage_with_autodep/pym/portage/cache/template.pyo b/portage_with_autodep/pym/portage/cache/template.pyo Binary files differindex 45da015..3315771 100644 --- a/portage_with_autodep/pym/portage/cache/template.pyo +++ b/portage_with_autodep/pym/portage/cache/template.pyo diff --git a/portage_with_autodep/pym/portage/cache/volatile.pyo b/portage_with_autodep/pym/portage/cache/volatile.pyo Binary files differindex fac5d55..9ebb5b8 100644 --- a/portage_with_autodep/pym/portage/cache/volatile.pyo +++ b/portage_with_autodep/pym/portage/cache/volatile.pyo diff --git a/portage_with_autodep/pym/portage/checksum.py b/portage_with_autodep/pym/portage/checksum.py index bd416ac..cd1572e 100644 --- a/portage_with_autodep/pym/portage/checksum.py +++ b/portage_with_autodep/pym/portage/checksum.py @@ -1,5 +1,5 @@ # checksum.py -- core Portage functionality -# Copyright 1998-2012 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import portage @@ -10,6 +10,8 @@ from portage import _encodings from portage import _unicode_encode import errno import stat +import sys +import subprocess import tempfile #dict of all available hash functions @@ -48,16 +50,15 @@ class _generate_hash_function(object): @type filename: String @return: The hash and size of the data """ - f = _open_file(filename) - blocksize = HASHING_BLOCKSIZE - data = f.read(blocksize) - size = 0 - checksum = self._hashobject() - while data: - checksum.update(data) - size = size + len(data) + with _open_file(filename) as f: + blocksize = HASHING_BLOCKSIZE + size = 0 + checksum = self._hashobject() data = f.read(blocksize) - f.close() + while data: + checksum.update(data) + size = size + len(data) + data = f.read(blocksize) return (checksum.hexdigest(), size) @@ -137,8 +138,10 @@ try: except ImportError: pass +_whirlpool_unaccelerated = False if "WHIRLPOOL" not in hashfunc_map: # Bundled WHIRLPOOL implementation + _whirlpool_unaccelerated = True from portage.util.whirlpool import new as _new_whirlpool whirlpoolhash = _generate_hash_function("WHIRLPOOL", _new_whirlpool, origin="bundled") @@ -161,11 +164,16 @@ hashfunc_map["size"] = getsize prelink_capable = False if os.path.exists(PRELINK_BINARY): - results = portage.subprocess_getstatusoutput( - "%s --version > /dev/null 2>&1" % (PRELINK_BINARY,)) - if (results[0] >> 8) == 0: + cmd = [PRELINK_BINARY, "--version"] + cmd = [_unicode_encode(x, encoding=_encodings['fs'], errors='strict') + for x in cmd] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + proc.communicate() + status = proc.wait() + if os.WIFEXITED(status) and os.WEXITSTATUS(status) == os.EX_OK: prelink_capable=1 - del results + del cmd, proc, status def is_prelinkable_elf(filename): f = _open_file(filename) @@ -197,6 +205,82 @@ def get_hash_origin(hashtype): raise KeyError(hashtype) return hashorigin_map.get(hashtype, "unknown") +def _filter_unaccelarated_hashes(digests): + """ + If multiple digests are available and some are unaccelerated, + then return a new dict that omits the unaccelerated ones. This + allows extreme performance problems like bug #425046 to be + avoided whenever practical, especially for cases like stage + builds where acceleration may not be available for some hashes + due to minimization of dependencies. + """ + if _whirlpool_unaccelerated and "WHIRLPOOL" in digests: + verifiable_hash_types = set(digests).intersection(hashfunc_map) + verifiable_hash_types.discard("size") + if len(verifiable_hash_types) > 1: + digests = dict(digests) + digests.pop("WHIRLPOOL") + + return digests + +class _hash_filter(object): + """ + Implements filtering for PORTAGE_CHECKSUM_FILTER. + """ + + __slots__ = ('transparent', '_tokens',) + + def __init__(self, filter_str): + tokens = filter_str.upper().split() + if not tokens or tokens[-1] == "*": + del tokens[:] + self.transparent = not tokens + tokens.reverse() + self._tokens = tuple(tokens) + + def __call__(self, hash_name): + if self.transparent: + return True + matches = ("*", hash_name) + for token in self._tokens: + if token in matches: + return True + elif token[:1] == "-": + if token[1:] in matches: + return False + return False + +def _apply_hash_filter(digests, hash_filter): + """ + Return a new dict containing the filtered digests, or the same + dict if no changes are necessary. This will always preserve at + at least one digest, in order to ensure that they are not all + discarded. + @param digests: dictionary of digests + @type digests: dict + @param hash_filter: A callable that takes a single hash name + argument, and returns True if the hash is to be used or + False otherwise + @type hash_filter: callable + """ + + verifiable_hash_types = set(digests).intersection(hashfunc_map) + verifiable_hash_types.discard("size") + modified = False + if len(verifiable_hash_types) > 1: + for k in list(verifiable_hash_types): + if not hash_filter(k): + modified = True + verifiable_hash_types.remove(k) + if len(verifiable_hash_types) == 1: + break + + if modified: + digests = dict((k, v) for (k, v) in digests.items() + if k == "size" or k in verifiable_hash_types) + + return digests + def verify_all(filename, mydict, calc_prelink=0, strict=0): """ Verify all checksums against a file. diff --git a/portage_with_autodep/pym/portage/checksum.pyo b/portage_with_autodep/pym/portage/checksum.pyo Binary files differindex 00231af..46441a3 100644 --- a/portage_with_autodep/pym/portage/checksum.pyo +++ b/portage_with_autodep/pym/portage/checksum.pyo diff --git a/portage_with_autodep/pym/portage/const.py b/portage_with_autodep/pym/portage/const.py index 614dcdb..0ac503f 100644 --- a/portage_with_autodep/pym/portage/const.py +++ b/portage_with_autodep/pym/portage/const.py @@ -1,7 +1,9 @@ # portage: Constants -# Copyright 1998-2011 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + import os # =========================================================================== @@ -27,8 +29,8 @@ import os # The variables in this file are grouped by config_root, target_root. # variables used with config_root (these need to be relative) -MAKE_CONF_FILE = "etc/make.conf" USER_CONFIG_PATH = "etc/portage" +MAKE_CONF_FILE = USER_CONFIG_PATH + "/make.conf" MODULES_FILE_PATH = USER_CONFIG_PATH + "/modules" CUSTOM_PROFILE_PATH = USER_CONFIG_PATH + "/profile" USER_VIRTUALS_FILE = USER_CONFIG_PATH + "/virtuals" @@ -36,7 +38,7 @@ EBUILD_SH_ENV_FILE = USER_CONFIG_PATH + "/bashrc" EBUILD_SH_ENV_DIR = USER_CONFIG_PATH + "/env" CUSTOM_MIRRORS_FILE = USER_CONFIG_PATH + "/mirrors" COLOR_MAP_FILE = USER_CONFIG_PATH + "/color.map" -PROFILE_PATH = "etc/make.profile" +PROFILE_PATH = USER_CONFIG_PATH + "/make.profile" MAKE_DEFAULTS_FILE = PROFILE_PATH + "/make.defaults" # FIXME: not used DEPRECATED_PROFILE_FILE = PROFILE_PATH + "/deprecated" @@ -56,7 +58,10 @@ DEPCACHE_PATH = "/var/cache/edb/dep" GLOBAL_CONFIG_PATH = "/usr/share/portage/config" # these variables are not used with target_root or config_root -PORTAGE_BASE_PATH = os.path.join(os.sep, os.sep.join(__file__.split(os.sep)[:-3])) +# NOTE: Use realpath(__file__) so that python module symlinks in site-packages +# are followed back to the real location of the whole portage installation. +PORTAGE_BASE_PATH = os.path.join(os.sep, os.sep.join(os.path.realpath( + __file__.rstrip("co")).split(os.sep)[:-3])) PORTAGE_BIN_PATH = PORTAGE_BASE_PATH + "/bin" PORTAGE_PYM_PATH = PORTAGE_BASE_PATH + "/pym" LOCALE_DATA_PATH = PORTAGE_BASE_PATH + "/locale" # FIXME: not used @@ -69,7 +74,6 @@ MOVE_BINARY = "/bin/mv" PRELINK_BINARY = "/usr/sbin/prelink" AUTODEP_LIBRARY = "/usr/lib/file_hook.so" - INVALID_ENV_FILE = "/etc/spork/is/not/valid/profile.env" REPO_NAME_FILE = "repo_name" REPO_NAME_LOC = "profiles" + "/" + REPO_NAME_FILE @@ -77,50 +81,139 @@ REPO_NAME_LOC = "profiles" + "/" + REPO_NAME_FILE PORTAGE_PACKAGE_ATOM = "sys-apps/portage" LIBC_PACKAGE_ATOM = "virtual/libc" OS_HEADERS_PACKAGE_ATOM = "virtual/os-headers" +CVS_PACKAGE_ATOM = "dev-vcs/cvs" +GIT_PACKAGE_ATOM = "dev-vcs/git" +RSYNC_PACKAGE_ATOM = "net-misc/rsync" -INCREMENTALS = ("USE", "USE_EXPAND", "USE_EXPAND_HIDDEN", - "FEATURES", "ACCEPT_KEYWORDS", - "CONFIG_PROTECT_MASK", "CONFIG_PROTECT", - "PRELINK_PATH", "PRELINK_PATH_MASK", - "PROFILE_ONLY_VARIABLES") -EBUILD_PHASES = ("pretend", "setup", "unpack", "prepare", "configure", - "compile", "test", "install", - "package", "preinst", "postinst","prerm", "postrm", - "nofetch", "config", "info", "other") +INCREMENTALS = ( + "ACCEPT_KEYWORDS", + "CONFIG_PROTECT", + "CONFIG_PROTECT_MASK", + "FEATURES", + "IUSE_IMPLICIT", + "PRELINK_PATH", + "PRELINK_PATH_MASK", + "PROFILE_ONLY_VARIABLES", + "USE", + "USE_EXPAND", + "USE_EXPAND_HIDDEN", + "USE_EXPAND_IMPLICIT", + "USE_EXPAND_UNPREFIXED", +) +EBUILD_PHASES = ( + "pretend", + "setup", + "unpack", + "prepare", + "configure", + "compile", + "test", + "install", + "package", + "preinst", + "postinst", + "prerm", + "postrm", + "nofetch", + "config", + "info", + "other", +) SUPPORTED_FEATURES = frozenset([ - "assume-digests", "binpkg-logs", "buildpkg", "buildsyspkg", "candy", - "ccache", "chflags", "clean-logs", - "collision-protect", "compress-build-logs", "compressdebug", - "config-protect-if-modified", "depcheck", "depcheckstrict", - "digest", "distcc", "distcc-pump", "distlocks", "ebuild-locks", "fakeroot", - "fail-clean", "force-mirror", "force-prefix", "getbinpkg", - "installsources", "keeptemp", "keepwork", "fixlafiles", "lmirror", - "metadata-transfer", "mirror", "multilib-strict", "news", - "noauto", "noclean", "nodoc", "noinfo", "noman", - "nostrip", "notitles", "parallel-fetch", "parallel-install", - "parse-eapi-ebuild-head", - "prelink-checksums", "preserve-libs", - "protect-owned", "python-trace", "sandbox", - "selinux", "sesandbox", "sfperms", - "sign", "skiprocheck", "split-elog", "split-log", "splitdebug", - "strict", "stricter", "suidctl", "test", "test-fail-continue", - "unknown-features-filter", "unknown-features-warn", - "unmerge-logs", "unmerge-orphans", "userfetch", "userpriv", - "usersandbox", "usersync", "webrsync-gpg", "xattr"]) - -EAPI = 4 + "assume-digests", + "binpkg-logs", + "buildpkg", + "buildsyspkg", + "candy", + "ccache", + "cgroup", + "chflags", + "clean-logs", + "collision-protect", + "compress-build-logs", + "compressdebug", + "compress-index", + "config-protect-if-modified", + "depcheck", + "depcheckstrict", + "digest", + "distcc", + "distcc-pump", + "distlocks", + "downgrade-backup", + "ebuild-locks", + "fail-clean", + "fakeroot", + "fixlafiles", + "force-mirror", + "force-prefix", + "getbinpkg", + "installsources", + "ipc-sandbox", + "keeptemp", + "keepwork", + "lmirror", + "merge-sync", + "metadata-transfer", + "mirror", + "multilib-strict", + "network-sandbox", + "news", + "noauto", + "noclean", + "nodoc", + "noinfo", + "noman", + "nostrip", + "notitles", + "parallel-fetch", + "parallel-install", + "prelink-checksums", + "preserve-libs", + "protect-owned", + "python-trace", + "sandbox", + "selinux", + "sesandbox", + "sfperms", + "sign", + "skiprocheck", + "splitdebug", + "split-elog", + "split-log", + "strict", + "stricter", + "suidctl", + "test", + "test-fail-continue", + "unknown-features-filter", + "unknown-features-warn", + "unmerge-backup", + "unmerge-logs", + "unmerge-orphans", + "userfetch", + "userpriv", + "usersandbox", + "usersync", + "webrsync-gpg", + "xattr", +]) + +EAPI = 5 HASHING_BLOCKSIZE = 32768 MANIFEST1_HASH_FUNCTIONS = ("MD5", "SHA256", "RMD160") MANIFEST1_REQUIRED_HASH = "MD5" -# Future events: +# Past events: # -# After WHIRLPOOL is supported in stable portage: -# - Add SHA256 and WHIRLPOOL to MANIFEST2_HASH_DEFAULTS. -# - Remove SHA1 and RMD160 from MANIFEST2_HASH_*. +# 20120704 - After WHIRLPOOL is supported in stable portage: # - Set manifest-hashes in gentoo-x86/metadata/layout.conf as follows: # manifest-hashes = SHA256 SHA512 WHIRLPOOL +# - Add SHA512 and WHIRLPOOL to MANIFEST2_HASH_DEFAULTS. +# - Remove SHA1 and RMD160 from MANIFEST2_HASH_*. +# +# Future events: # # After WHIRLPOOL is supported in stable portage for at least 1 year: # - Change MANIFEST2_REQUIRED_HASH to WHIRLPOOL. @@ -138,8 +231,8 @@ MANIFEST1_REQUIRED_HASH = "MD5" # After layout.conf settings correspond to defaults in stable portage: # - Remove redundant settings from gentoo-x86/metadata/layout.conf. -MANIFEST2_HASH_FUNCTIONS = ("RMD160", "SHA1", "SHA256", "SHA512", "WHIRLPOOL") -MANIFEST2_HASH_DEFAULTS = frozenset(["SHA1", "SHA256", "RMD160"]) +MANIFEST2_HASH_FUNCTIONS = ("SHA256", "SHA512", "WHIRLPOOL") +MANIFEST2_HASH_DEFAULTS = frozenset(["SHA256", "SHA512", "WHIRLPOOL"]) MANIFEST2_REQUIRED_HASH = "SHA256" MANIFEST2_IDENTIFIERS = ("AUX", "MISC", "DIST", "EBUILD") @@ -149,13 +242,35 @@ MANIFEST2_IDENTIFIERS = ("AUX", "MISC", "DIST", "EBUILD") # a config instance (since it's possible to contruct a config instance with # a different EPREFIX). Therefore, the EPREFIX constant should *NOT* be used # in the definition of any other constants within this file. -EPREFIX="" +EPREFIX = "" # pick up EPREFIX from the environment if set if "PORTAGE_OVERRIDE_EPREFIX" in os.environ: EPREFIX = os.environ["PORTAGE_OVERRIDE_EPREFIX"] if EPREFIX: EPREFIX = os.path.normpath(EPREFIX) + if EPREFIX == os.sep: + EPREFIX = "" + +VCS_DIRS = ("CVS", "RCS", "SCCS", ".bzr", ".git", ".hg", ".svn") + +# List of known live eclasses. Keep it in sync with cnf/sets/portage.conf +LIVE_ECLASSES = frozenset([ + "bzr", + "cvs", + "darcs", + "git", + "git-2", + "git-r3", + "mercurial", + "subversion", + "tla", +]) + +SUPPORTED_BINPKG_FORMATS = ("tar", "rpm") + +# Time formats used in various places like metadata.chk. +TIMESTAMP_FORMAT = "%a, %d %b %Y %H:%M:%S +0000" # to be used with time.gmtime() # =========================================================================== # END OF CONSTANTS -- END OF CONSTANTS -- END OF CONSTANTS -- END OF CONSTANT @@ -163,19 +278,5 @@ if "PORTAGE_OVERRIDE_EPREFIX" in os.environ: # Private constants for use in conditional code in order to minimize the diff # between branches. -_ENABLE_DYN_LINK_MAP = True -_ENABLE_PRESERVE_LIBS = True -_ENABLE_REPO_NAME_WARN = True +_DEPCLEAN_LIB_CHECK_DEFAULT = True _ENABLE_SET_CONFIG = True - - -# The definitions above will differ between branches, so it's useful to have -# common lines of diff context here in order to avoid merge conflicts. - -if not _ENABLE_PRESERVE_LIBS: - SUPPORTED_FEATURES = set(SUPPORTED_FEATURES) - SUPPORTED_FEATURES.remove("preserve-libs") - SUPPORTED_FEATURES = frozenset(SUPPORTED_FEATURES) - -if not _ENABLE_SET_CONFIG: - WORLD_SETS_FILE = '/dev/null' diff --git a/portage_with_autodep/pym/portage/const.py.rej b/portage_with_autodep/pym/portage/const.py.rej deleted file mode 100644 index 9fe70f8..0000000 --- a/portage_with_autodep/pym/portage/const.py.rej +++ /dev/null @@ -1,12 +0,0 @@ ---- pym/portage/const.py -+++ pym/portage/const.py -@@ -90,7 +92,8 @@ - SUPPORTED_FEATURES = frozenset([ - "allow-missing-manifests", - "assume-digests", "binpkg-logs", "buildpkg", "buildsyspkg", "candy", -- "ccache", "chflags", "collision-protect", "compress-build-logs", -+ "ccache", "chflags", "collision-protect", "compress-build-logs", -+ "depcheck", "depcheckstrict", - "digest", "distcc", "distcc-pump", "distlocks", "ebuild-locks", "fakeroot", - "fail-clean", "fixpackages", "force-mirror", "getbinpkg", - "installsources", "keeptemp", "keepwork", "fixlafiles", "lmirror", diff --git a/portage_with_autodep/pym/portage/const.pyo b/portage_with_autodep/pym/portage/const.pyo Binary files differindex 804420f..3b9812b 100644 --- a/portage_with_autodep/pym/portage/const.pyo +++ b/portage_with_autodep/pym/portage/const.pyo diff --git a/portage_with_autodep/pym/portage/cvstree.pyo b/portage_with_autodep/pym/portage/cvstree.pyo Binary files differindex 4719daf..1c18d23 100644 --- a/portage_with_autodep/pym/portage/cvstree.pyo +++ b/portage_with_autodep/pym/portage/cvstree.pyo diff --git a/portage_with_autodep/pym/portage/data.py b/portage_with_autodep/pym/portage/data.py index c4d967a..44104c2 100644 --- a/portage_with_autodep/pym/portage/data.py +++ b/portage_with_autodep/pym/portage/data.py @@ -1,13 +1,14 @@ # data.py -- Calculated/Discovered Data Values -# Copyright 1998-2011 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -import os, pwd, grp, platform +import os, pwd, grp, platform, sys import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.output:colorize', 'portage.util:writemsg', + 'subprocess' ) from portage.localization import _ @@ -85,19 +86,27 @@ def _get_global(k): elif portage.const.EPREFIX: secpass = 2 #Discover the uid and gid of the portage user/group + keyerror = False try: portage_uid = pwd.getpwnam(_get_global('_portage_username')).pw_uid - _portage_grpname = _get_global('_portage_grpname') - if platform.python_implementation() == 'PyPy': - # Somehow this prevents "TypeError: expected string" errors - # from grp.getgrnam() with PyPy 1.7 - _portage_grpname = str(_portage_grpname) - portage_gid = grp.getgrnam(_portage_grpname).gr_gid - if secpass < 1 and portage_gid in os.getgroups(): - secpass = 1 except KeyError: + keyerror = True portage_uid = 0 + + try: + portage_gid = grp.getgrnam(_get_global('_portage_grpname')).gr_gid + except KeyError: + keyerror = True portage_gid = 0 + + if secpass < 1 and portage_gid in os.getgroups(): + secpass = 1 + + # Suppress this error message if both PORTAGE_GRPNAME and + # PORTAGE_USERNAME are set to "root", for things like + # Android (see bug #454060). + if keyerror and not (_get_global('_portage_username') == "root" and + _get_global('_portage_grpname') == "root"): writemsg(colorize("BAD", _("portage: 'portage' user or group missing.")) + "\n", noiselevel=-1) writemsg(_( @@ -129,10 +138,28 @@ def _get_global(k): # Get a list of group IDs for the portage user. Do not use # grp.getgrall() since it is known to trigger spurious # SIGPIPE problems with nss_ldap. - mystatus, myoutput = \ - portage.subprocess_getstatusoutput("id -G %s" % _portage_username) - if mystatus == os.EX_OK: - for x in myoutput.split(): + cmd = ["id", "-G", _portage_username] + + if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000: + # Python 3.1 _execvp throws TypeError for non-absolute executable + # path passed as bytes (see http://bugs.python.org/issue8513). + fullname = portage.process.find_binary(cmd[0]) + if fullname is None: + globals()[k] = v + _initialized_globals.add(k) + return v + cmd[0] = fullname + + encoding = portage._encodings['content'] + cmd = [portage._unicode_encode(x, + encoding=encoding, errors='strict') for x in cmd] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + myoutput = proc.communicate()[0] + status = proc.wait() + if os.WIFEXITED(status) and os.WEXITSTATUS(status) == os.EX_OK: + for x in portage._unicode_decode(myoutput, + encoding=encoding, errors='strict').split(): try: v.append(int(x)) except ValueError: @@ -213,10 +240,18 @@ def _init(settings): if '_portage_grpname' not in _initialized_globals and \ '_portage_username' not in _initialized_globals: + # Prevents "TypeError: expected string" errors + # from grp.getgrnam() with PyPy + native_string = platform.python_implementation() == 'PyPy' + v = settings.get('PORTAGE_GRPNAME', 'portage') + if native_string: + v = portage._native_string(v) globals()['_portage_grpname'] = v _initialized_globals.add('_portage_grpname') v = settings.get('PORTAGE_USERNAME', 'portage') + if native_string: + v = portage._native_string(v) globals()['_portage_username'] = v _initialized_globals.add('_portage_username') diff --git a/portage_with_autodep/pym/portage/data.pyo b/portage_with_autodep/pym/portage/data.pyo Binary files differindex 7f749e0..8115c7a 100644 --- a/portage_with_autodep/pym/portage/data.pyo +++ b/portage_with_autodep/pym/portage/data.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/_MergeProcess.py b/portage_with_autodep/pym/portage/dbapi/_MergeProcess.py index b5f6a0b..956dbb9 100644 --- a/portage_with_autodep/pym/portage/dbapi/_MergeProcess.py +++ b/portage_with_autodep/pym/portage/dbapi/_MergeProcess.py @@ -1,7 +1,8 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import io +import platform import signal import sys import traceback @@ -10,10 +11,11 @@ import errno import fcntl import portage from portage import os, _unicode_decode +from portage.util._ctypes import find_library import portage.elog.messages -from _emerge.SpawnProcess import SpawnProcess +from portage.util._async.ForkProcess import ForkProcess -class MergeProcess(SpawnProcess): +class MergeProcess(ForkProcess): """ Merge packages in a subprocess, so the Scheduler can run in the main thread while files are moved or copied asynchronously. @@ -40,11 +42,20 @@ class MergeProcess(SpawnProcess): settings.reset() settings.setcpv(cpv, mydb=self.mydbapi) + # This caches the libc library lookup in the current + # process, so that it's only done once rather than + # for each child process. + if platform.system() == "Linux" and \ + "merge-sync" in settings.features: + find_library("c") + # Inherit stdin by default, so that the pdb SIGUSR1 # handler is usable for the subprocess. if self.fd_pipes is None: self.fd_pipes = {} - self.fd_pipes.setdefault(0, sys.stdin.fileno()) + else: + self.fd_pipes = self.fd_pipes.copy() + self.fd_pipes.setdefault(0, portage._get_stdin().fileno()) super(MergeProcess, self)._start() @@ -90,7 +101,7 @@ class MergeProcess(SpawnProcess): reporter(msg, phase=phase, key=key, out=out) if event & self.scheduler.IO_HUP: - self.scheduler.unregister(self._elog_reg_id) + self.scheduler.source_remove(self._elog_reg_id) self._elog_reg_id = None os.close(self._elog_reader_fd) self._elog_reader_fd = None @@ -101,12 +112,24 @@ class MergeProcess(SpawnProcess): def _spawn(self, args, fd_pipes, **kwargs): """ Fork a subprocess, apply local settings, and call - dblink.merge(). + dblink.merge(). TODO: Share code with ForkProcess. """ elog_reader_fd, elog_writer_fd = os.pipe() + fcntl.fcntl(elog_reader_fd, fcntl.F_SETFL, fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK) + + # FD_CLOEXEC is enabled by default in Python >=3.4. + if sys.hexversion < 0x3040000: + try: + fcntl.FD_CLOEXEC + except AttributeError: + pass + else: + fcntl.fcntl(elog_reader_fd, fcntl.F_SETFD, + fcntl.fcntl(elog_reader_fd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC) + blockers = None if self.blockers is not None: # Query blockers in the main process, since closing @@ -116,10 +139,9 @@ class MergeProcess(SpawnProcess): blockers = self.blockers() mylink = portage.dblink(self.mycat, self.mypkg, settings=self.settings, treetype=self.treetype, vartree=self.vartree, - blockers=blockers, scheduler=self.scheduler, - pipe=elog_writer_fd) + blockers=blockers, pipe=elog_writer_fd) fd_pipes[elog_writer_fd] = elog_writer_fd - self._elog_reg_id = self.scheduler.register(elog_reader_fd, + self._elog_reg_id = self.scheduler.io_add_watch(elog_reader_fd, self._registered_events, self._elog_output_handler) # If a concurrent emerge process tries to install a package @@ -133,88 +155,100 @@ class MergeProcess(SpawnProcess): if not self.unmerge: counter = self.vartree.dbapi.counter_tick() - pid = os.fork() - if pid != 0: - if not isinstance(pid, int): - raise AssertionError( - "fork returned non-integer: %s" % (repr(pid),)) - - os.close(elog_writer_fd) - self._elog_reader_fd = elog_reader_fd - self._buf = "" - self._elog_keys = set() - - # invalidate relevant vardbapi caches - if self.vartree.dbapi._categories is not None: - self.vartree.dbapi._categories = None - self.vartree.dbapi._pkgs_changed = True - self.vartree.dbapi._clear_pkg_cache(mylink) - - portage.process.spawned_pids.append(pid) - return [pid] - - os.close(elog_reader_fd) - portage.locks._close_fds() - # Disable close_fds since we don't exec (see _setup_pipes docstring). - portage.process._setup_pipes(fd_pipes, close_fds=False) - - # Use default signal handlers since the ones inherited - # from the parent process are irrelevant here. - signal.signal(signal.SIGINT, signal.SIG_DFL) - signal.signal(signal.SIGTERM, signal.SIG_DFL) - - portage.output.havecolor = self.settings.get('NOCOLOR') \ - not in ('yes', 'true') - - # In this subprocess we want mylink._display_merge() to use - # stdout/stderr directly since they are pipes. This behavior - # is triggered when mylink._scheduler is None. - mylink._scheduler = None - - # Avoid wastful updates of the vdb cache. - self.vartree.dbapi._flush_cache_enabled = False - - # In this subprocess we don't want PORTAGE_BACKGROUND to - # suppress stdout/stderr output since they are pipes. We - # also don't want to open PORTAGE_LOG_FILE, since it will - # already be opened by the parent process, so we set the - # "subprocess" value for use in conditional logging code - # involving PORTAGE_LOG_FILE. - if not self.unmerge: - # unmerge phases have separate logs - if self.settings.get("PORTAGE_BACKGROUND") == "1": - self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "1" - else: - self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "0" - self.settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE") - self.settings["PORTAGE_BACKGROUND"] = "subprocess" - self.settings.backup_changes("PORTAGE_BACKGROUND") - - rval = 1 + parent_pid = os.getpid() + pid = None try: - if self.unmerge: - if not mylink.exists(): - rval = os.EX_OK - elif mylink.unmerge( - ldpath_mtimes=self.prev_mtimes) == os.EX_OK: - mylink.lockdb() - try: - mylink.delete() - finally: - mylink.unlockdb() - rval = os.EX_OK - else: - rval = mylink.merge(self.pkgloc, self.infloc, - myebuild=self.myebuild, mydbapi=self.mydbapi, - prev_mtimes=self.prev_mtimes, counter=counter) - except SystemExit: - raise - except: - traceback.print_exc() + pid = os.fork() + + if pid != 0: + if not isinstance(pid, int): + raise AssertionError( + "fork returned non-integer: %s" % (repr(pid),)) + + os.close(elog_writer_fd) + self._elog_reader_fd = elog_reader_fd + self._buf = "" + self._elog_keys = set() + # Discard messages which will be collected by the subprocess, + # in order to avoid duplicates (bug #446136). + portage.elog.messages.collect_messages(key=mylink.mycpv) + + # invalidate relevant vardbapi caches + if self.vartree.dbapi._categories is not None: + self.vartree.dbapi._categories = None + self.vartree.dbapi._pkgs_changed = True + self.vartree.dbapi._clear_pkg_cache(mylink) + + return [pid] + + os.close(elog_reader_fd) + + # Use default signal handlers in order to avoid problems + # killing subprocesses as reported in bug #353239. + signal.signal(signal.SIGINT, signal.SIG_DFL) + signal.signal(signal.SIGTERM, signal.SIG_DFL) + + portage.locks._close_fds() + # We don't exec, so use close_fds=False + # (see _setup_pipes docstring). + portage.process._setup_pipes(fd_pipes, close_fds=False) + + portage.output.havecolor = self.settings.get('NOCOLOR') \ + not in ('yes', 'true') + + # Avoid wastful updates of the vdb cache. + self.vartree.dbapi._flush_cache_enabled = False + + # In this subprocess we don't want PORTAGE_BACKGROUND to + # suppress stdout/stderr output since they are pipes. We + # also don't want to open PORTAGE_LOG_FILE, since it will + # already be opened by the parent process, so we set the + # "subprocess" value for use in conditional logging code + # involving PORTAGE_LOG_FILE. + if not self.unmerge: + # unmerge phases have separate logs + if self.settings.get("PORTAGE_BACKGROUND") == "1": + self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "1" + else: + self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "0" + self.settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE") + self.settings["PORTAGE_BACKGROUND"] = "subprocess" + self.settings.backup_changes("PORTAGE_BACKGROUND") + + rval = 1 + try: + if self.unmerge: + if not mylink.exists(): + rval = os.EX_OK + elif mylink.unmerge( + ldpath_mtimes=self.prev_mtimes) == os.EX_OK: + mylink.lockdb() + try: + mylink.delete() + finally: + mylink.unlockdb() + rval = os.EX_OK + else: + rval = mylink.merge(self.pkgloc, self.infloc, + myebuild=self.myebuild, mydbapi=self.mydbapi, + prev_mtimes=self.prev_mtimes, counter=counter) + except SystemExit: + raise + except: + traceback.print_exc() + # os._exit() skips stderr flush! + sys.stderr.flush() + finally: + os._exit(rval) + finally: - # Call os._exit() from finally block, in order to suppress any - # finally blocks from earlier in the call stack. See bug #345289. - os._exit(rval) + if pid == 0 or (pid is None and os.getpid() != parent_pid): + # Call os._exit() from a finally block in order + # to suppress any finally blocks from earlier + # in the call stack (see bug #345289). This + # finally block has to be setup before the fork + # in order to avoid a race condition. + os._exit(1) def _unregister(self): """ @@ -231,7 +265,7 @@ class MergeProcess(SpawnProcess): self._unlock_vdb() if self._elog_reg_id is not None: - self.scheduler.unregister(self._elog_reg_id) + self.scheduler.source_remove(self._elog_reg_id) self._elog_reg_id = None if self._elog_reader_fd is not None: os.close(self._elog_reader_fd) diff --git a/portage_with_autodep/pym/portage/dbapi/_MergeProcess.pyo b/portage_with_autodep/pym/portage/dbapi/_MergeProcess.pyo Binary files differindex 5839ad8..abee4be 100644 --- a/portage_with_autodep/pym/portage/dbapi/_MergeProcess.pyo +++ b/portage_with_autodep/pym/portage/dbapi/_MergeProcess.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/__init__.py b/portage_with_autodep/pym/portage/dbapi/__init__.py index a1c5c56..a20a1e8 100644 --- a/portage_with_autodep/pym/portage/dbapi/__init__.py +++ b/portage_with_autodep/pym/portage/dbapi/__init__.py @@ -1,6 +1,8 @@ -# Copyright 1998-2012 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ["dbapi"] import re @@ -8,7 +10,7 @@ import re import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.dbapi.dep_expand:dep_expand@_dep_expand', - 'portage.dep:match_from_list', + 'portage.dep:Atom,match_from_list,_match_slot', 'portage.output:colorize', 'portage.util:cmp_sort_key,writemsg', 'portage.versions:catsplit,catpkgsplit,vercmp,_pkg_str', @@ -16,14 +18,19 @@ portage.proxy.lazyimport.lazyimport(globals(), from portage import os from portage import auxdbkeys +from portage.eapi import _get_eapi_attrs +from portage.exception import InvalidData from portage.localization import _ +from _emerge.Package import Package class dbapi(object): - _category_re = re.compile(r'^\w[-.+\w]*$') + _category_re = re.compile(r'^\w[-.+\w]*$', re.UNICODE) _categories = None _use_mutable = False _known_keys = frozenset(x for x in auxdbkeys if not x.startswith("UNUSED_0")) + _pkg_str_aux_keys = ("EAPI", "KEYWORDS", "SLOT", "repository") + def __init__(self): pass @@ -125,29 +132,52 @@ class dbapi(object): def _iter_match(self, atom, cpv_iter): cpv_iter = iter(match_from_list(atom, cpv_iter)) + if atom.repo: + cpv_iter = self._iter_match_repo(atom, cpv_iter) if atom.slot: cpv_iter = self._iter_match_slot(atom, cpv_iter) if atom.unevaluated_atom.use: cpv_iter = self._iter_match_use(atom, cpv_iter) - if atom.repo: - cpv_iter = self._iter_match_repo(atom, cpv_iter) return cpv_iter + def _pkg_str(self, cpv, repo): + """ + This is used to contruct _pkg_str instances on-demand during + matching. If cpv is a _pkg_str instance with slot attribute, + then simply return it. Otherwise, fetch metadata and construct + a _pkg_str instance. This may raise KeyError or InvalidData. + """ + try: + cpv.slot + except AttributeError: + pass + else: + return cpv + + metadata = dict(zip(self._pkg_str_aux_keys, + self.aux_get(cpv, self._pkg_str_aux_keys, myrepo=repo))) + + return _pkg_str(cpv, metadata=metadata, settings=self.settings) + def _iter_match_repo(self, atom, cpv_iter): for cpv in cpv_iter: try: - if self.aux_get(cpv, ["repository"], myrepo=atom.repo)[0] == atom.repo: - yield cpv - except KeyError: - continue + pkg_str = self._pkg_str(cpv, atom.repo) + except (KeyError, InvalidData): + pass + else: + if pkg_str.repo == atom.repo: + yield pkg_str def _iter_match_slot(self, atom, cpv_iter): for cpv in cpv_iter: try: - if self.aux_get(cpv, ["SLOT"], myrepo=atom.repo)[0] == atom.slot: - yield cpv - except KeyError: - continue + pkg_str = self._pkg_str(cpv, atom.repo) + except (KeyError, InvalidData): + pass + else: + if _match_slot(atom, pkg_str): + yield pkg_str def _iter_match_use(self, atom, cpv_iter): """ @@ -155,7 +185,7 @@ class dbapi(object): 2) Check enabled/disabled flag states. """ - aux_keys = ["IUSE", "SLOT", "USE", "repository"] + aux_keys = ["EAPI", "IUSE", "KEYWORDS", "SLOT", "USE", "repository"] for cpv in cpv_iter: try: metadata = dict(zip(aux_keys, @@ -163,17 +193,31 @@ class dbapi(object): except KeyError: continue + try: + cpv.slot + except AttributeError: + try: + cpv = _pkg_str(cpv, metadata=metadata, + settings=self.settings) + except InvalidData: + continue + if not self._match_use(atom, cpv, metadata): continue yield cpv - def _match_use(self, atom, cpv, metadata): - iuse_implicit_match = self.settings._iuse_implicit_match - iuse = frozenset(x.lstrip('+-') for x in metadata["IUSE"].split()) + def _match_use(self, atom, pkg, metadata): + eapi_attrs = _get_eapi_attrs(metadata["EAPI"]) + if eapi_attrs.iuse_effective: + iuse_implicit_match = self.settings._iuse_effective_match + else: + iuse_implicit_match = self.settings._iuse_implicit_match + usealiases = self.settings._use_manager.getUseAliases(pkg) + iuse = Package._iuse(None, metadata["IUSE"].split(), iuse_implicit_match, usealiases, metadata["EAPI"]) for x in atom.unevaluated_atom.use.required: - if x not in iuse and not iuse_implicit_match(x): + if iuse.get_real_flag(x) is None: return False if atom.use is None: @@ -183,45 +227,54 @@ class dbapi(object): # Use IUSE to validate USE settings for built packages, # in case the package manager that built this package # failed to do that for some reason (or in case of - # data corruption). - use = frozenset(x for x in metadata["USE"].split() - if x in iuse or iuse_implicit_match(x)) - missing_enabled = atom.use.missing_enabled.difference(iuse) - missing_disabled = atom.use.missing_disabled.difference(iuse) - - if atom.use.enabled: - if atom.use.enabled.intersection(missing_disabled): + # data corruption). The enabled flags must be consistent + # with implicit IUSE, in order to avoid potential + # inconsistencies in USE dep matching (see bug #453400). + use = frozenset(x for x in metadata["USE"].split() if iuse.get_real_flag(x) is not None) + missing_enabled = frozenset(x for x in atom.use.missing_enabled if iuse.get_real_flag(x) is None) + missing_disabled = frozenset(x for x in atom.use.missing_disabled if iuse.get_real_flag(x) is None) + enabled = frozenset((iuse.get_real_flag(x) or x) for x in atom.use.enabled) + disabled = frozenset((iuse.get_real_flag(x) or x) for x in atom.use.disabled) + + if enabled: + if any(x in enabled for x in missing_disabled): return False - need_enabled = atom.use.enabled.difference(use) + need_enabled = enabled.difference(use) if need_enabled: - need_enabled = need_enabled.difference(missing_enabled) - if need_enabled: + if any(x not in missing_enabled for x in need_enabled): return False - if atom.use.disabled: - if atom.use.disabled.intersection(missing_enabled): + if disabled: + if any(x in disabled for x in missing_enabled): return False - need_disabled = atom.use.disabled.intersection(use) + need_disabled = disabled.intersection(use) if need_disabled: - need_disabled = need_disabled.difference(missing_disabled) - if need_disabled: + if any(x not in missing_disabled for x in need_disabled): return False elif not self.settings.local_config: # Check masked and forced flags for repoman. - if hasattr(cpv, 'slot'): - pkg = cpv - else: - pkg = _pkg_str(cpv, slot=metadata["SLOT"], - repo=metadata.get("repository")) - usemask = self.settings._getUseMask(pkg) - if usemask.intersection(atom.use.enabled): + usemask = self.settings._getUseMask(pkg, + stable=self.settings._parent_stable) + if any(x in usemask for x in atom.use.enabled): return False - useforce = self.settings._getUseForce(pkg).difference(usemask) - if useforce.intersection(atom.use.disabled): + useforce = self.settings._getUseForce(pkg, + stable=self.settings._parent_stable) + if any(x in useforce and x not in usemask + for x in atom.use.disabled): return False + # Check unsatisfied use-default deps + if atom.use.enabled: + missing_disabled = frozenset(x for x in atom.use.missing_disabled if iuse.get_real_flag(x) is None) + if any(x in atom.use.enabled for x in missing_disabled): + return False + if atom.use.disabled: + missing_enabled = frozenset(x for x in atom.use.missing_enabled if iuse.get_real_flag(x) is None) + if any(x in atom.use.disabled for x in missing_enabled): + return False + return True def invalidentry(self, mypath): @@ -249,23 +302,30 @@ class dbapi(object): maxval = len(cpv_all) aux_get = self.aux_get aux_update = self.aux_update - meta_keys = ["DEPEND", "RDEPEND", "PDEPEND", "PROVIDE", 'repository'] + update_keys = Package._dep_keys + ("PROVIDE",) + meta_keys = update_keys + self._pkg_str_aux_keys repo_dict = None if isinstance(updates, dict): repo_dict = updates - from portage.update import update_dbentries if onUpdate: onUpdate(maxval, 0) if onProgress: onProgress(maxval, 0) for i, cpv in enumerate(cpv_all): - metadata = dict(zip(meta_keys, aux_get(cpv, meta_keys))) - repo = metadata.pop('repository') + try: + metadata = dict(zip(meta_keys, aux_get(cpv, meta_keys))) + except KeyError: + continue + try: + pkg = _pkg_str(cpv, metadata=metadata, settings=self.settings) + except InvalidData: + continue + metadata = dict((k, metadata[k]) for k in update_keys) if repo_dict is None: updates_list = updates else: try: - updates_list = repo_dict[repo] + updates_list = repo_dict[pkg.repo] except KeyError: try: updates_list = repo_dict['DEFAULT'] @@ -275,7 +335,8 @@ class dbapi(object): if not updates_list: continue - metadata_updates = update_dbentries(updates_list, metadata) + metadata_updates = \ + portage.update_dbentries(updates_list, metadata, parent=pkg) if metadata_updates: aux_update(cpv, metadata_updates) if onUpdate: @@ -286,27 +347,39 @@ class dbapi(object): def move_slot_ent(self, mylist, repo_match=None): """This function takes a sequence: Args: - mylist: a sequence of (package, originalslot, newslot) + mylist: a sequence of (atom, originalslot, newslot) repo_match: callable that takes single repo_name argument and returns True if the update should be applied Returns: The number of slotmoves this function did """ - pkg = mylist[1] + atom = mylist[1] origslot = mylist[2] newslot = mylist[3] - origmatches = self.match(pkg) + + try: + atom.with_slot + except AttributeError: + atom = Atom(atom).with_slot(origslot) + else: + atom = atom.with_slot(origslot) + + origmatches = self.match(atom) moves = 0 if not origmatches: return moves for mycpv in origmatches: - slot = self.aux_get(mycpv, ["SLOT"])[0] - if slot != origslot: + try: + mycpv = self._pkg_str(mycpv, atom.repo) + except (KeyError, InvalidData): continue - if repo_match is not None \ - and not repo_match(self.aux_get(mycpv, ['repository'])[0]): + if repo_match is not None and not repo_match(mycpv.repo): continue moves += 1 + if "/" not in newslot and \ + mycpv.sub_slot and \ + mycpv.sub_slot not in (mycpv.slot, newslot): + newslot = "%s/%s" % (newslot, mycpv.sub_slot) mydata = {"SLOT": newslot+"\n"} self.aux_update(mycpv, mydata) return moves diff --git a/portage_with_autodep/pym/portage/dbapi/__init__.pyo b/portage_with_autodep/pym/portage/dbapi/__init__.pyo Binary files differindex e7b494d..d4d47b2 100644 --- a/portage_with_autodep/pym/portage/dbapi/__init__.pyo +++ b/portage_with_autodep/pym/portage/dbapi/__init__.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/_expand_new_virt.py b/portage_with_autodep/pym/portage/dbapi/_expand_new_virt.py index d379b4c..9aa603d 100644 --- a/portage_with_autodep/pym/portage/dbapi/_expand_new_virt.py +++ b/portage_with_autodep/pym/portage/dbapi/_expand_new_virt.py @@ -1,8 +1,11 @@ -# Copyright 2011 Gentoo Foundation +# Copyright 2011-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + import portage from portage.dep import Atom, _get_useflag_re +from portage.eapi import _get_eapi_attrs def expand_new_virt(vardb, atom): """ @@ -44,6 +47,7 @@ def expand_new_virt(vardb, atom): yield atom continue + eapi_attrs = _get_eapi_attrs(eapi) # Validate IUSE and IUSE, for early detection of vardb corruption. useflag_re = _get_useflag_re(eapi) valid_iuse = [] @@ -54,7 +58,11 @@ def expand_new_virt(vardb, atom): valid_iuse.append(x) valid_iuse = frozenset(valid_iuse) - iuse_implicit_match = vardb.settings._iuse_implicit_match + if eapi_attrs.iuse_effective: + iuse_implicit_match = vardb.settings._iuse_effective_match + else: + iuse_implicit_match = vardb.settings._iuse_implicit_match + valid_use = [] for x in use.split(): if x in valid_iuse or iuse_implicit_match(x): diff --git a/portage_with_autodep/pym/portage/dbapi/_expand_new_virt.pyo b/portage_with_autodep/pym/portage/dbapi/_expand_new_virt.pyo Binary files differindex 6c23a7e..7884393 100644 --- a/portage_with_autodep/pym/portage/dbapi/_expand_new_virt.pyo +++ b/portage_with_autodep/pym/portage/dbapi/_expand_new_virt.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/bintree.py b/portage_with_autodep/pym/portage/dbapi/bintree.py index a8027ee..b1f67ae 100644 --- a/portage_with_autodep/pym/portage/dbapi/bintree.py +++ b/portage_with_autodep/pym/portage/dbapi/bintree.py @@ -1,16 +1,18 @@ -# Copyright 1998-2012 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ["bindbapi", "binarytree"] import portage portage.proxy.lazyimport.lazyimport(globals(), - 'portage.checksum:hashfunc_map,perform_multiple_checksums,verify_all', + 'portage.checksum:hashfunc_map,perform_multiple_checksums,' + \ + 'verify_all,_apply_hash_filter,_hash_filter', 'portage.dbapi.dep_expand:dep_expand', - 'portage.dep:dep_getkey,isjustname,match_from_list', + 'portage.dep:dep_getkey,isjustname,isvalidatom,match_from_list', 'portage.output:EOutput,colorize', 'portage.locks:lockfile,unlockfile', - 'portage.package.ebuild.doebuild:_vdb_use_conditional_atoms', 'portage.package.ebuild.fetch:_check_distfile,_hide_url_passwd', 'portage.update:update_dbentries', 'portage.util:atomic_ofstream,ensure_dirs,normalize_path,' + \ @@ -41,7 +43,9 @@ import subprocess import sys import tempfile import textwrap +import traceback import warnings +from gzip import GzipFile from itertools import chain try: from urllib.parse import urlparse @@ -49,8 +53,16 @@ except ImportError: from urlparse import urlparse if sys.hexversion >= 0x3000000: + _unicode = str basestring = str long = int +else: + _unicode = unicode + +class UseCachedCopyOfRemoteIndex(Exception): + # If the local copy is recent enough + # then fetching the remote index can be skipped. + pass class bindbapi(fakedbapi): _known_keys = frozenset(list(fakedbapi._known_keys) + \ @@ -63,9 +75,10 @@ class bindbapi(fakedbapi): self.cpdict={} # Selectively cache metadata in order to optimize dep matching. self._aux_cache_keys = set( - ["BUILD_TIME", "CHOST", "DEPEND", "EAPI", "IUSE", "KEYWORDS", + ["BUILD_TIME", "CHOST", "DEPEND", "EAPI", + "HDEPEND", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROPERTIES", "PROVIDE", - "RDEPEND", "repository", "RESTRICT", "SLOT", "USE", "DEFINED_PHASES", + "RDEPEND", "repository", "RESTRICT", "SLOT", "USE", "DEFINED_PHASES" ]) self._aux_cache_slot_dict = slot_dict_class(self._aux_cache_keys) self._aux_cache = {} @@ -128,15 +141,15 @@ class bindbapi(fakedbapi): if myval: mydata[x] = " ".join(myval.split()) - if not mydata.setdefault('EAPI', _unicode_decode('0')): - mydata['EAPI'] = _unicode_decode('0') + if not mydata.setdefault('EAPI', '0'): + mydata['EAPI'] = '0' if cache_me: aux_cache = self._aux_cache_slot_dict() for x in self._aux_cache_keys: - aux_cache[x] = mydata.get(x, _unicode_decode('')) + aux_cache[x] = mydata.get(x, '') self._aux_cache[mycpv] = aux_cache - return [mydata.get(x, _unicode_decode('')) for x in wants] + return [mydata.get(x, '') for x in wants] def aux_update(self, cpv, values): if not self.bintree.populated: @@ -248,7 +261,7 @@ def _pkgindex_cpv_map_latest_build(pkgindex): class binarytree(object): "this tree scans for a list of all packages available in PKGDIR" - def __init__(self, _unused=None, pkgdir=None, + def __init__(self, _unused=DeprecationWarning, pkgdir=None, virtual=DeprecationWarning, settings=None): if pkgdir is None: @@ -257,11 +270,11 @@ class binarytree(object): if settings is None: raise TypeError("settings parameter is required") - if _unused is not None and _unused != settings['ROOT']: - warnings.warn("The root parameter of the " + if _unused is not DeprecationWarning: + warnings.warn("The first parameter of the " "portage.dbapi.bintree.binarytree" - " constructor is now unused. Use " - "settings['ROOT'] instead.", + " constructor is now unused. Instead " + "settings['ROOT'] is used.", DeprecationWarning, stacklevel=2) if virtual is not DeprecationWarning: @@ -293,22 +306,26 @@ class binarytree(object): self._pkgindex_keys.update(["CPV", "MTIME", "SIZE"]) self._pkgindex_aux_keys = \ ["BUILD_TIME", "CHOST", "DEPEND", "DESCRIPTION", "EAPI", - "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROPERTIES", - "PROVIDE", "RDEPEND", "repository", "SLOT", "USE", "DEFINED_PHASES", + "HDEPEND", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROPERTIES", + "PROVIDE", "RESTRICT", "RDEPEND", "repository", "SLOT", "USE", "DEFINED_PHASES", "BASE_URI"] self._pkgindex_aux_keys = list(self._pkgindex_aux_keys) self._pkgindex_use_evaluated_keys = \ - ("LICENSE", "RDEPEND", "DEPEND", - "PDEPEND", "PROPERTIES", "PROVIDE") + ("DEPEND", "HDEPEND", "LICENSE", "RDEPEND", + "PDEPEND", "PROPERTIES", "PROVIDE", "RESTRICT") self._pkgindex_header_keys = set([ "ACCEPT_KEYWORDS", "ACCEPT_LICENSE", - "ACCEPT_PROPERTIES", "CBUILD", + "ACCEPT_PROPERTIES", "ACCEPT_RESTRICT", "CBUILD", "CONFIG_PROTECT", "CONFIG_PROTECT_MASK", "FEATURES", - "GENTOO_MIRRORS", "INSTALL_MASK", "SYNC", "USE"]) + "GENTOO_MIRRORS", "INSTALL_MASK", "IUSE_IMPLICIT", "USE", + "USE_EXPAND", "USE_EXPAND_HIDDEN", "USE_EXPAND_IMPLICIT", + "USE_EXPAND_UNPREFIXED"]) self._pkgindex_default_pkg_data = { "BUILD_TIME" : "", + "DEFINED_PHASES" : "", "DEPEND" : "", "EAPI" : "0", + "HDEPEND" : "", "IUSE" : "", "KEYWORDS": "", "LICENSE" : "", @@ -320,7 +337,6 @@ class binarytree(object): "RESTRICT": "", "SLOT" : "0", "USE" : "", - "DEFINED_PHASES" : "", } self._pkgindex_inherited_keys = ["CHOST", "repository"] @@ -378,15 +394,24 @@ class binarytree(object): if not origmatches: return moves for mycpv in origmatches: + try: + mycpv = self.dbapi._pkg_str(mycpv, None) + except (KeyError, InvalidData): + continue mycpv_cp = portage.cpv_getkey(mycpv) if mycpv_cp != origcp: # Ignore PROVIDE virtual match. continue if repo_match is not None \ - and not repo_match(self.dbapi.aux_get(mycpv, - ['repository'])[0]): + and not repo_match(mycpv.repo): + continue + + # Use isvalidatom() to check if this move is valid for the + # EAPI (characters allowed in package names may vary). + if not isvalidatom(newcp, eapi=mycpv.eapi): continue - mynewcpv = mycpv.replace(mycpv_cp, str(newcp), 1) + + mynewcpv = mycpv.replace(mycpv_cp, _unicode(newcp), 1) myoldpkg = catsplit(mycpv)[1] mynewpkg = catsplit(mynewcpv)[1] @@ -405,7 +430,7 @@ class binarytree(object): moves += 1 mytbz2 = portage.xpak.tbz2(tbz2path) mydata = mytbz2.get_data() - updated_items = update_dbentries([mylist], mydata) + updated_items = update_dbentries([mylist], mydata, parent=mycpv) mydata.update(updated_items) mydata[b'PF'] = \ _unicode_encode(mynewpkg + "\n", @@ -541,6 +566,20 @@ class binarytree(object): if not os.path.isdir(path): raise + def _file_permissions(self, path): + try: + pkgdir_st = os.stat(self.pkgdir) + except OSError: + pass + else: + pkgdir_gid = pkgdir_st.st_gid + pkgdir_grp_mode = 0o0060 & pkgdir_st.st_mode + try: + portage.util.apply_permissions(path, gid=pkgdir_gid, + mode=pkgdir_grp_mode, mask=0) + except PortageException: + pass + def _move_to_all(self, cpv): """If the file exists, move it. Whether or not it exists, update state for future getname() calls.""" @@ -796,9 +835,7 @@ class binarytree(object): del pkgindex.packages[:] pkgindex.packages.extend(iter(metadata.values())) self._update_pkgindex_header(pkgindex.header) - f = atomic_ofstream(self._pkgindex_file) - pkgindex.write(f) - f.close() + self._pkgindex_write(pkgindex) if getbinpkgs and not self.settings["PORTAGE_BINHOST"]: writemsg(_("!!! PORTAGE_BINHOST unset, but use is requested.\n"), @@ -841,6 +878,7 @@ class binarytree(object): if e.errno != errno.ENOENT: raise local_timestamp = pkgindex.header.get("TIMESTAMP", None) + remote_timestamp = None rmt_idx = self._new_pkgindex() proc = None tmp_filename = None @@ -849,41 +887,76 @@ class binarytree(object): # protocols and requires the base url to have a trailing # slash, so join manually... url = base_url.rstrip("/") + "/Packages" - try: - f = _urlopen(url) - except IOError: - path = parsed_url.path.rstrip("/") + "/Packages" + f = None + + # Don't use urlopen for https, since it doesn't support + # certificate/hostname verification (bug #469888). + if parsed_url.scheme not in ('https',): + try: + f = _urlopen(url, if_modified_since=local_timestamp) + if hasattr(f, 'headers') and f.headers.get('timestamp', ''): + remote_timestamp = f.headers.get('timestamp') + except IOError as err: + if hasattr(err, 'code') and err.code == 304: # not modified (since local_timestamp) + raise UseCachedCopyOfRemoteIndex() + + if parsed_url.scheme in ('ftp', 'http', 'https'): + # This protocol is supposedly supported by urlopen, + # so apparently there's a problem with the url + # or a bug in urlopen. + if self.settings.get("PORTAGE_DEBUG", "0") != "0": + traceback.print_exc() - if parsed_url.scheme == 'sftp': - # The sftp command complains about 'Illegal seek' if - # we try to make it write to /dev/stdout, so use a - # temp file instead. - fd, tmp_filename = tempfile.mkstemp() - os.close(fd) - if port is not None: - port_args = ['-P', "%s" % (port,)] - proc = subprocess.Popen(['sftp'] + port_args + \ - [user_passwd + host + ":" + path, tmp_filename]) - if proc.wait() != os.EX_OK: raise - f = open(tmp_filename, 'rb') - elif parsed_url.scheme == 'ssh': + + if f is None: + + path = parsed_url.path.rstrip("/") + "/Packages" + + if parsed_url.scheme == 'ssh': + # Use a pipe so that we can terminate the download + # early if we detect that the TIMESTAMP header + # matches that of the cached Packages file. + ssh_args = ['ssh'] if port is not None: - port_args = ['-p', "%s" % (port,)] - proc = subprocess.Popen(['ssh'] + port_args + \ - [user_passwd + host, '--', 'cat', path], + ssh_args.append("-p%s" % (port,)) + # NOTE: shlex evaluates embedded quotes + ssh_args.extend(portage.util.shlex_split( + self.settings.get("PORTAGE_SSH_OPTS", ""))) + ssh_args.append(user_passwd + host) + ssh_args.append('--') + ssh_args.append('cat') + ssh_args.append(path) + + proc = subprocess.Popen(ssh_args, stdout=subprocess.PIPE) f = proc.stdout else: setting = 'FETCHCOMMAND_' + parsed_url.scheme.upper() fcmd = self.settings.get(setting) if not fcmd: - raise + fcmd = self.settings.get('FETCHCOMMAND') + if not fcmd: + raise EnvironmentError("FETCHCOMMAND is unset") + fd, tmp_filename = tempfile.mkstemp() tmp_dirname, tmp_basename = os.path.split(tmp_filename) os.close(fd) - success = portage.getbinpkg.file_get(url, - tmp_dirname, fcmd=fcmd, filename=tmp_basename) + + fcmd_vars = { + "DISTDIR": tmp_dirname, + "FILE": tmp_basename, + "URI": url + } + + for k in ("PORTAGE_SSH_OPTS",): + try: + fcmd_vars[k] = self.settings[k] + except KeyError: + pass + + success = portage.getbinpkg.file_get( + fcmd=fcmd, fcmd_vars=fcmd_vars) if not success: raise EnvironmentError("%s failed" % (setting,)) f = open(tmp_filename, 'rb') @@ -892,7 +965,8 @@ class binarytree(object): _encodings['repo.content'], errors='replace') try: rmt_idx.readHeader(f_dec) - remote_timestamp = rmt_idx.header.get("TIMESTAMP", None) + if not remote_timestamp: # in case it had not been read from HTTP header + remote_timestamp = rmt_idx.header.get("TIMESTAMP", None) if not remote_timestamp: # no timestamp in the header, something's wrong pkgindex = None @@ -920,6 +994,12 @@ class binarytree(object): writemsg("\n\n!!! %s\n" % \ _("Timed out while closing connection to binhost"), noiselevel=-1) + except UseCachedCopyOfRemoteIndex: + writemsg_stdout("\n") + writemsg_stdout( + colorize("GOOD", _("Local copy of remote index is up-to-date and will be used.")) + \ + "\n") + rmt_idx = pkgindex except EnvironmentError as e: writemsg(_("\n\n!!! Error fetching binhost package" \ " info from '%s'\n") % _hide_url_passwd(base_url)) @@ -988,75 +1068,7 @@ class binarytree(object): # Local package instances override remote instances. for cpv in metadata: self._remotepkgs.pop(cpv, None) - continue - try: - chunk_size = long(self.settings["PORTAGE_BINHOST_CHUNKSIZE"]) - if chunk_size < 8: - chunk_size = 8 - except (ValueError, KeyError): - chunk_size = 3000 - writemsg_stdout("\n") - writemsg_stdout( - colorize("GOOD", _("Fetching bininfo from ")) + \ - _hide_url_passwd(base_url) + "\n") - remotepkgs = portage.getbinpkg.dir_get_metadata( - base_url, chunk_size=chunk_size) - - for mypkg, remote_metadata in remotepkgs.items(): - mycat = remote_metadata.get("CATEGORY") - if mycat is None: - #old-style or corrupt package - writemsg(_("!!! Invalid remote binary package: %s\n") % mypkg, - noiselevel=-1) - continue - mycat = mycat.strip() - try: - fullpkg = _pkg_str(mycat+"/"+mypkg[:-5]) - except InvalidData: - writemsg(_("!!! Invalid remote binary package: %s\n") % mypkg, - noiselevel=-1) - continue - - if fullpkg in metadata: - # When using this old protocol, comparison with the remote - # package isn't supported, so the local package is always - # preferred even if getbinpkgsonly is enabled. - continue - - if not self.dbapi._category_re.match(mycat): - writemsg(_("!!! Remote binary package has an " \ - "unrecognized category: '%s'\n") % fullpkg, - noiselevel=-1) - writemsg(_("!!! '%s' has a category that is not" \ - " listed in %setc/portage/categories\n") % \ - (fullpkg, self.settings["PORTAGE_CONFIGROOT"]), - noiselevel=-1) - continue - mykey = portage.cpv_getkey(fullpkg) - try: - # invalid tbz2's can hurt things. - self.dbapi.cpv_inject(fullpkg) - for k, v in remote_metadata.items(): - remote_metadata[k] = v.strip() - remote_metadata["BASE_URI"] = base_url - - # Eliminate metadata values with names that digestCheck - # uses, since they are not valid when using the old - # protocol. Typically this is needed for SIZE metadata - # which corresponds to the size of the unpacked files - # rather than the binpkg file size, triggering digest - # verification failures as reported in bug #303211. - remote_metadata.pop('SIZE', None) - for k in portage.checksum.hashfunc_map: - remote_metadata.pop(k, None) - - self._remotepkgs[fullpkg] = remote_metadata - except SystemExit as e: - raise - except: - writemsg(_("!!! Failed to inject remote binary package: %s\n") % fullpkg, - noiselevel=-1) - continue + self.populated=1 def inject(self, cpv, filename=None): @@ -1110,6 +1122,10 @@ class binarytree(object): if not samefile: self._ensure_dir(os.path.dirname(new_filename)) _movefile(filename, new_filename, mysettings=self.settings) + full_path = new_filename + + self._file_permissions(full_path) + if self._all_directory and \ self.getname(cpv).split(os.path.sep)[-2] == "All": self._create_symlink(cpv) @@ -1157,13 +1173,35 @@ class binarytree(object): pkgindex.packages.append(d) self._update_pkgindex_header(pkgindex.header) - f = atomic_ofstream(os.path.join(self.pkgdir, "Packages")) - pkgindex.write(f) - f.close() + self._pkgindex_write(pkgindex) + finally: if pkgindex_lock: unlockfile(pkgindex_lock) + def _pkgindex_write(self, pkgindex): + contents = codecs.getwriter(_encodings['repo.content'])(io.BytesIO()) + pkgindex.write(contents) + contents = contents.getvalue() + atime = mtime = long(pkgindex.header["TIMESTAMP"]) + output_files = [(atomic_ofstream(self._pkgindex_file, mode="wb"), + self._pkgindex_file, None)] + + if "compress-index" in self.settings.features: + gz_fname = self._pkgindex_file + ".gz" + fileobj = atomic_ofstream(gz_fname, mode="wb") + output_files.append((GzipFile(filename='', mode="wb", + fileobj=fileobj, mtime=mtime), gz_fname, fileobj)) + + for f, fname, f_close in output_files: + f.write(contents) + f.close() + if f_close is not None: + f_close.close() + self._file_permissions(fname) + # some seconds might have elapsed since TIMESTAMP + os.utime(fname, (atime, mtime)) + def _pkgindex_entry(self, cpv): """ Performs checksums and evaluates USE flag conditionals. @@ -1223,6 +1261,16 @@ class binarytree(object): else: header.pop(k, None) + # These values may be useful for using a binhost without + # having a local copy of the profile (bug #470006). + for k in self.settings.get("USE_EXPAND_IMPLICIT", "").split(): + k = "USE_EXPAND_VALUES_" + k + v = self.settings.get(k) + if v: + header[k] = v + else: + header.pop(k, None) + def _pkgindex_version_supported(self, pkgindex): version = pkgindex.header.get("VERSION") if version: @@ -1235,11 +1283,6 @@ class binarytree(object): def _eval_use_flags(self, cpv, metadata): use = frozenset(metadata["USE"].split()) - raw_use = use - iuse = set(f.lstrip("-+") for f in metadata["IUSE"].split()) - use = [f for f in use if f in iuse] - use.sort() - metadata["USE"] = " ".join(use) for k in self._pkgindex_use_evaluated_keys: if k.endswith('DEPEND'): token_class = Atom @@ -1248,7 +1291,7 @@ class binarytree(object): try: deps = metadata[k] - deps = use_reduce(deps, uselist=raw_use, token_class=token_class) + deps = use_reduce(deps, uselist=use, token_class=token_class) deps = paren_enclose(deps) except portage.exception.InvalidDependString as e: writemsg("%s: %s\n" % (k, str(e)), @@ -1313,6 +1356,8 @@ class binarytree(object): """Returns the URI to the Packages file for a given package.""" return self._pkgindex_uri.get(pkgname) + + def gettbz2(self, pkgname): """Fetches the package from a remote site, if necessary. Attempts to resume if the file appears to be partially downloaded.""" @@ -1320,7 +1365,7 @@ class binarytree(object): tbz2name = os.path.basename(tbz2_path) resume = False if os.path.exists(tbz2_path): - if (tbz2name not in self.invalids): + if tbz2name[:-5] not in self.invalids: return else: resume = True @@ -1370,19 +1415,14 @@ class binarytree(object): f.close() return pkgindex - def digestCheck(self, pkg): - """ - Verify digests for the given package and raise DigestException - if verification fails. - @rtype: bool - @return: True if digests could be located, False otherwise. - """ - cpv = pkg - if not isinstance(cpv, basestring): + def _get_digests(self, pkg): + + try: cpv = pkg.cpv - pkg = None + except AttributeError: + cpv = pkg - pkg_path = self.getname(cpv) + digests = {} metadata = None if self._remotepkgs is None or cpv not in self._remotepkgs: for d in self._load_pkgindex().packages: @@ -1392,9 +1432,8 @@ class binarytree(object): else: metadata = self._remotepkgs[cpv] if metadata is None: - return False + return digests - digests = {} for k in hashfunc_map: v = metadata.get(k) if not v: @@ -1408,9 +1447,31 @@ class binarytree(object): writemsg(_("!!! Malformed SIZE attribute in remote " \ "metadata for '%s'\n") % cpv) + return digests + + def digestCheck(self, pkg): + """ + Verify digests for the given package and raise DigestException + if verification fails. + @rtype: bool + @return: True if digests could be located, False otherwise. + """ + + digests = self._get_digests(pkg) + if not digests: return False + try: + cpv = pkg.cpv + except AttributeError: + cpv = pkg + + pkg_path = self.getname(cpv) + hash_filter = _hash_filter( + self.settings.get("PORTAGE_CHECKSUM_FILTER", "")) + if not hash_filter.transparent: + digests = _apply_hash_filter(digests, hash_filter) eout = EOutput() eout.quiet = self.settings.get("PORTAGE_QUIET") == "1" ok, st = _check_distfile(pkg_path, digests, eout, show_errors=0) @@ -1426,9 +1487,7 @@ class binarytree(object): "Get a slot for a catpkg; assume it exists." myslot = "" try: - myslot = self.dbapi.aux_get(mycatpkg,["SLOT"])[0] - except SystemExit as e: - raise - except Exception as e: + myslot = self.dbapi._pkg_str(mycatpkg, None).slot + except KeyError: pass return myslot diff --git a/portage_with_autodep/pym/portage/dbapi/bintree.pyo b/portage_with_autodep/pym/portage/dbapi/bintree.pyo Binary files differindex f99f377..90d0c6a 100644 --- a/portage_with_autodep/pym/portage/dbapi/bintree.pyo +++ b/portage_with_autodep/pym/portage/dbapi/bintree.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/cpv_expand.py b/portage_with_autodep/pym/portage/dbapi/cpv_expand.py index 947194c..70ee782 100644 --- a/portage_with_autodep/pym/portage/dbapi/cpv_expand.py +++ b/portage_with_autodep/pym/portage/dbapi/cpv_expand.py @@ -1,6 +1,8 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ["cpv_expand"] import portage diff --git a/portage_with_autodep/pym/portage/dbapi/cpv_expand.pyo b/portage_with_autodep/pym/portage/dbapi/cpv_expand.pyo Binary files differindex cf1a428..7c38720 100644 --- a/portage_with_autodep/pym/portage/dbapi/cpv_expand.pyo +++ b/portage_with_autodep/pym/portage/dbapi/cpv_expand.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/dep_expand.py b/portage_with_autodep/pym/portage/dbapi/dep_expand.py index ac8ccf4..3de5d8f 100644 --- a/portage_with_autodep/pym/portage/dbapi/dep_expand.py +++ b/portage_with_autodep/pym/portage/dbapi/dep_expand.py @@ -1,6 +1,8 @@ -# Copyright 2010 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ["dep_expand"] import re @@ -23,7 +25,7 @@ def dep_expand(mydep, mydb=None, use_cache=1, settings=None): if mydep[0] == "*": mydep = mydep[1:] orig_dep = mydep - has_cat = '/' in orig_dep + has_cat = '/' in orig_dep.split(':')[0] if not has_cat: alphanum = re.search(r'\w', orig_dep) if alphanum: diff --git a/portage_with_autodep/pym/portage/dbapi/dep_expand.pyo b/portage_with_autodep/pym/portage/dbapi/dep_expand.pyo Binary files differindex b323f5b..bcaf8e3 100644 --- a/portage_with_autodep/pym/portage/dbapi/dep_expand.pyo +++ b/portage_with_autodep/pym/portage/dbapi/dep_expand.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/porttree.py b/portage_with_autodep/pym/portage/dbapi/porttree.py index c5ee770..fc3fc03 100644 --- a/portage_with_autodep/pym/portage/dbapi/porttree.py +++ b/portage_with_autodep/pym/portage/dbapi/porttree.py @@ -1,6 +1,8 @@ -# Copyright 1998-2012 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = [ "close_portdbapi_caches", "FetchlistDict", "portagetree", "portdbapi" ] @@ -10,7 +12,7 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.checksum', 'portage.data:portage_gid,secpass', 'portage.dbapi.dep_expand:dep_expand', - 'portage.dep:Atom,dep_getkey,match_from_list,use_reduce', + 'portage.dep:Atom,dep_getkey,match_from_list,use_reduce,_match_slot', 'portage.package.ebuild.doebuild:doebuild', 'portage.util:ensure_dirs,shlex_split,writemsg,writemsg_level', 'portage.util.listdir:listdir', @@ -22,7 +24,8 @@ from portage.cache.cache_errors import CacheError from portage.cache.mappings import Mapping from portage.dbapi import dbapi from portage.exception import PortageException, \ - FileNotFound, InvalidAtom, InvalidDependString, InvalidPackageName + FileNotFound, InvalidAtom, InvalidData, \ + InvalidDependString, InvalidPackageName from portage.localization import _ from portage import eclass_cache, \ @@ -32,21 +35,74 @@ from portage import os from portage import _encodings from portage import _unicode_encode from portage import OrderedDict +from portage.util._eventloop.EventLoop import EventLoop +from portage.util._eventloop.global_event_loop import global_event_loop from _emerge.EbuildMetadataPhase import EbuildMetadataPhase -from _emerge.PollScheduler import PollScheduler import os as _os import sys import traceback import warnings +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse + if sys.hexversion >= 0x3000000: basestring = str long = int +def close_portdbapi_caches(): + # The python interpreter does _not_ guarantee that destructors are + # called for objects that remain when the interpreter exits, so we + # use an atexit hook to call destructors for any global portdbapi + # instances that may have been constructed. + try: + portage._legacy_globals_constructed + except AttributeError: + pass + else: + if "db" in portage._legacy_globals_constructed: + try: + db = portage.db + except AttributeError: + pass + else: + if isinstance(db, dict): + for x in db.values(): + try: + if "porttree" in x.lazy_items: + continue + except (AttributeError, TypeError): + continue + try: + x = x.pop("porttree").dbapi + except (AttributeError, KeyError): + continue + if not isinstance(x, portdbapi): + continue + x.close_caches() + +portage.process.atexit_register(close_portdbapi_caches) + +# It used to be necessary for API consumers to remove portdbapi instances +# from portdbapi_instances, in order to avoid having accumulated instances +# consume memory. Now, portdbapi_instances is just an empty dummy list, so +# for backward compatibility, ignore ValueError for removal on non-existent +# items. +class _dummy_list(list): + def remove(self, item): + # TODO: Trigger a DeprecationWarning here, after stable portage + # has dummy portdbapi_instances. + try: + list.remove(self, item) + except ValueError: + pass + class portdbapi(dbapi): """this tree will scan a portage directory located at root (passed to init)""" - portdbapi_instances = [] + portdbapi_instances = _dummy_list() _use_mutable = True @property @@ -64,14 +120,13 @@ class portdbapi(dbapi): return None return main_repo.eclass_db - def __init__(self, _unused_param=None, mysettings=None): + def __init__(self, _unused_param=DeprecationWarning, mysettings=None): """ @param _unused_param: deprecated, use mysettings['PORTDIR'] instead @type _unused_param: None @param mysettings: an immutable config instance @type mysettings: portage.config """ - portdbapi.portdbapi_instances.append(self) from portage import config if mysettings: @@ -80,7 +135,7 @@ class portdbapi(dbapi): from portage import settings self.settings = config(clone=settings) - if _unused_param is not None: + if _unused_param is not DeprecationWarning: warnings.warn("The first parameter of the " + \ "portage.dbapi.porttree.portdbapi" + \ " constructor is unused since portage-2.1.8. " + \ @@ -95,7 +150,6 @@ class portdbapi(dbapi): # this purpose because doebuild makes many changes to the config # instance that is passed in. self.doebuild_settings = config(clone=self.settings) - self._scheduler = PollScheduler().sched_iface self.depcachedir = os.path.realpath(self.settings.depcachedir) if os.environ.get("SANDBOX_ON") == "1": @@ -152,10 +206,10 @@ class portdbapi(dbapi): # portage group. depcachedir_unshared = True else: - cache_kwargs.update({ + cache_kwargs.update(portage._native_kwargs({ 'gid' : portage_gid, 'perms' : 0o664 - }) + })) # If secpass < 1, we don't want to write to the cache # since then we won't be able to apply group permissions @@ -186,13 +240,25 @@ class portdbapi(dbapi): self._pregen_auxdb[x] = cache # Selectively cache metadata in order to optimize dep matching. self._aux_cache_keys = set( - ["DEPEND", "EAPI", "INHERITED", "IUSE", "KEYWORDS", "LICENSE", + ["DEPEND", "EAPI", "HDEPEND", + "INHERITED", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROPERTIES", "PROVIDE", "RDEPEND", "repository", "RESTRICT", "SLOT", "DEFINED_PHASES", "REQUIRED_USE"]) self._aux_cache = {} self._broken_ebuilds = set() + @property + def _event_loop(self): + if portage._internal_caller: + # For internal portage usage, the global_event_loop is safe. + return global_event_loop() + else: + # For external API consumers, use a local EventLoop, since + # we don't want to assume that it's safe to override the + # global SIGCHLD handler. + return EventLoop(main=False) + def _create_pregen_cache(self, tree): conf = self.repositories.get_repo_for_location(tree) cache = conf.get_pregenerated_cache( @@ -202,6 +268,13 @@ class portdbapi(dbapi): cache.ec = self.repositories.get_repo_for_location(tree).eclass_db except AttributeError: pass + + if not cache.complete_eclass_entries: + warnings.warn( + ("Repository '%s' used deprecated 'pms' cache format. " + "Please migrate to 'md5-dict' format.") % (conf.name,), + DeprecationWarning) + return cache def _init_cache_dirs(self): @@ -446,7 +519,7 @@ class portdbapi(dbapi): proc = EbuildMetadataPhase(cpv=mycpv, ebuild_hash=ebuild_hash, portdb=self, - repo_path=mylocation, scheduler=self._scheduler, + repo_path=mylocation, scheduler=self._event_loop, settings=self.doebuild_settings) proc.start() @@ -626,13 +699,14 @@ class portdbapi(dbapi): else: return 0 - def cp_all(self, categories=None, trees=None): + def cp_all(self, categories=None, trees=None, reverse=False): """ This returns a list of all keys in our tree or trees @param categories: optional list of categories to search or defaults to self.settings.categories @param trees: optional list of trees to search the categories in or defaults to self.porttrees + @param reverse: reverse sort order (default is False) @rtype list of [cat/pkg,...] """ d = {} @@ -651,7 +725,7 @@ class portdbapi(dbapi): continue d[atom.cp] = None l = list(d) - l.sort() + l.sort(reverse=reverse) return l def cp_list(self, mycp, use_cache=1, mytree=None): @@ -825,18 +899,24 @@ class portdbapi(dbapi): # ebuild not in this repo, or masked by corruption continue - if visibility_filter and not self._visible(cpv, metadata): + try: + pkg_str = _pkg_str(cpv, metadata=metadata, + settings=self.settings) + except InvalidData: + continue + + if visibility_filter and not self._visible(pkg_str, metadata): continue if mydep.slot is not None and \ - mydep.slot != metadata["SLOT"]: + not _match_slot(mydep, pkg_str): continue if mydep.unevaluated_atom.use is not None and \ - not self._match_use(mydep, cpv, metadata): + not self._match_use(mydep, pkg_str, metadata): continue - myval.append(cpv) + myval.append(pkg_str) # only yield a given cpv once break @@ -959,19 +1039,16 @@ class portdbapi(dbapi): return False if settings._getMissingProperties(cpv, metadata): return False + if settings._getMissingRestrict(cpv, metadata): + return False except InvalidDependString: return False return True -def close_portdbapi_caches(): - for i in portdbapi.portdbapi_instances: - i.close_caches() - -portage.process.atexit_register(portage.portageexit) - class portagetree(object): - def __init__(self, root=None, virtual=DeprecationWarning, settings=None): + def __init__(self, root=DeprecationWarning, virtual=DeprecationWarning, + settings=None): """ Constructor for a PortageTree @@ -987,7 +1064,7 @@ class portagetree(object): settings = portage.settings self.settings = settings - if root is not None and root != settings['ROOT']: + if root is not DeprecationWarning: warnings.warn("The root parameter of the " + \ "portage.dbapi.porttree.portagetree" + \ " constructor is now unused. Use " + \ @@ -1055,10 +1132,8 @@ class portagetree(object): "Get a slot for a catpkg; assume it exists." myslot = "" try: - myslot = self.dbapi.aux_get(mycatpkg, ["SLOT"])[0] - except SystemExit: - raise - except Exception: + myslot = self.dbapi._pkg_str(mycatpkg, None).slot + except KeyError: pass return myslot @@ -1130,9 +1205,18 @@ def _parse_uri_map(cpv, metadata, use=None): uri_set = uri_map.get(distfile) if uri_set is None: - uri_set = set() + # Use OrderedDict to preserve order from SRC_URI + # while ensuring uniqueness. + uri_set = OrderedDict() uri_map[distfile] = uri_set - uri_set.add(uri) - uri = None + + # SRC_URI may contain a file name with no scheme, and in + # this case it does not belong in uri_set. + if urlparse(uri).scheme: + uri_set[uri] = True + + # Convert OrderedDicts to tuples. + for k, v in uri_map.items(): + uri_map[k] = tuple(v) return uri_map diff --git a/portage_with_autodep/pym/portage/dbapi/porttree.pyo b/portage_with_autodep/pym/portage/dbapi/porttree.pyo Binary files differindex fb57919..43ce4a8 100644 --- a/portage_with_autodep/pym/portage/dbapi/porttree.pyo +++ b/portage_with_autodep/pym/portage/dbapi/porttree.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/vartree.py b/portage_with_autodep/pym/portage/dbapi/vartree.py index 517c873..ed62323 100644 --- a/portage_with_autodep/pym/portage/dbapi/vartree.py +++ b/portage_with_autodep/pym/portage/dbapi/vartree.py @@ -1,6 +1,8 @@ -# Copyright 1998-2012 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = [ "vardbapi", "vartree", "dblink"] + \ ["write_contents", "tar_contents"] @@ -11,8 +13,10 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.data:portage_gid,portage_uid,secpass', 'portage.dbapi.dep_expand:dep_expand', 'portage.dbapi._MergeProcess:MergeProcess', - 'portage.dep:dep_getkey,isjustname,match_from_list,' + \ - 'use_reduce,_slot_re', + 'portage.dbapi._SyncfsProcess:SyncfsProcess', + 'portage.dep:dep_getkey,isjustname,isvalidatom,match_from_list,' + \ + 'use_reduce,_slot_separator,_repo_separator', + 'portage.eapi:_get_eapi_attrs', 'portage.elog:collect_ebuild_messages,collect_messages,' + \ 'elog_process,_merge_logentries', 'portage.locks:lockdir,unlockdir,lockfile,unlockfile', @@ -20,7 +24,7 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.package.ebuild.doebuild:doebuild_environment,' + \ '_merge_unicode_error', '_spawn_phase', 'portage.package.ebuild.prepare_build_dirs:prepare_build_dirs', - 'portage.update:fixdbentries', + 'portage.package.ebuild._ipc.QueryCommand:QueryCommand', 'portage.util:apply_secpass_permissions,ConfigProtect,ensure_dirs,' + \ 'writemsg,writemsg_level,write_atomic,atomic_ofstream,writedict,' + \ 'grabdict,normalize_path,new_protect_filename', @@ -30,15 +34,17 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.util.movefile:movefile', 'portage.util._dyn_libs.PreservedLibsRegistry:PreservedLibsRegistry', 'portage.util._dyn_libs.LinkageMapELF:LinkageMapELF@LinkageMap', + 'portage.util._async.SchedulerInterface:SchedulerInterface', + 'portage.util._eventloop.EventLoop:EventLoop', + 'portage.util._eventloop.global_event_loop:global_event_loop', 'portage.versions:best,catpkgsplit,catsplit,cpv_getkey,vercmp,' + \ - '_pkgsplit@pkgsplit,_pkg_str', + '_get_slot_re,_pkgsplit@pkgsplit,_pkg_str,_unknown_repo', 'subprocess', 'tarfile', ) from portage.const import CACHE_PATH, CONFIG_MEMORY_FILE, \ PORTAGE_PACKAGE_ATOM, PRIVATE_PATH, VDB_PATH -from portage.const import _ENABLE_DYN_LINK_MAP, _ENABLE_PRESERVE_LIBS from portage.dbapi import dbapi from portage.exception import CommandNotFound, \ InvalidData, InvalidLocation, InvalidPackageName, \ @@ -59,8 +65,8 @@ from portage import _unicode_encode from _emerge.EbuildBuildDir import EbuildBuildDir from _emerge.EbuildPhase import EbuildPhase from _emerge.emergelog import emergelog -from _emerge.PollScheduler import PollScheduler from _emerge.MiscFunctionsProcess import MiscFunctionsProcess +from _emerge.SpawnProcess import SpawnProcess import errno import fnmatch @@ -70,6 +76,7 @@ import io from itertools import chain import logging import os as _os +import platform import pwd import re import stat @@ -108,7 +115,8 @@ class vardbapi(dbapi): _aux_cache_keys_re = re.compile(r'^NEEDED\..*$') _aux_multi_line_re = re.compile(r'^(CONTENTS|NEEDED\..*)$') - def __init__(self, _unused_param=None, categories=None, settings=None, vartree=None): + def __init__(self, _unused_param=DeprecationWarning, + categories=None, settings=None, vartree=None): """ The categories parameter is unused since the dbapi class now has a categories property that is generated from the @@ -138,11 +146,11 @@ class vardbapi(dbapi): settings = portage.settings self.settings = settings - if _unused_param is not None and _unused_param != settings['ROOT']: + if _unused_param is not DeprecationWarning: warnings.warn("The first parameter of the " "portage.dbapi.vartree.vardbapi" - " constructor is now unused. Use " - "settings['ROOT'] instead.", + " constructor is now unused. Instead " + "settings['ROOT'] is used.", DeprecationWarning, stacklevel=2) self._eroot = settings['EROOT'] @@ -159,7 +167,7 @@ class vardbapi(dbapi): self.vartree = vartree self._aux_cache_keys = set( ["BUILD_TIME", "CHOST", "COUNTER", "DEPEND", "DESCRIPTION", - "EAPI", "HOMEPAGE", "IUSE", "KEYWORDS", + "EAPI", "HDEPEND", "HOMEPAGE", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROPERTIES", "PROVIDE", "RDEPEND", "repository", "RESTRICT" , "SLOT", "USE", "DEFINED_PHASES", ]) @@ -169,15 +177,9 @@ class vardbapi(dbapi): self._counter_path = os.path.join(self._eroot, CACHE_PATH, "counter") - self._plib_registry = None - if _ENABLE_PRESERVE_LIBS: - self._plib_registry = PreservedLibsRegistry(settings["ROOT"], - os.path.join(self._eroot, PRIVATE_PATH, - "preserved_libs_registry")) - - self._linkmap = None - if _ENABLE_DYN_LINK_MAP: - self._linkmap = LinkageMap(self) + self._plib_registry = PreservedLibsRegistry(settings["ROOT"], + os.path.join(self._eroot, PRIVATE_PATH, "preserved_libs_registry")) + self._linkmap = LinkageMap(self) self._owners = self._owners_db(self) self._cached_counter = None @@ -318,14 +320,24 @@ class vardbapi(dbapi): if not origmatches: return moves for mycpv in origmatches: + try: + mycpv = self._pkg_str(mycpv, None) + except (KeyError, InvalidData): + continue mycpv_cp = cpv_getkey(mycpv) if mycpv_cp != origcp: # Ignore PROVIDE virtual match. continue if repo_match is not None \ - and not repo_match(self.aux_get(mycpv, ['repository'])[0]): + and not repo_match(mycpv.repo): continue - mynewcpv = mycpv.replace(mycpv_cp, str(newcp), 1) + + # Use isvalidatom() to check if this move is valid for the + # EAPI (characters allowed in package names may vary). + if not isvalidatom(newcp, eapi=mycpv.eapi): + continue + + mynewcpv = mycpv.replace(mycpv_cp, _unicode(newcp), 1) mynewcat = catsplit(newcp)[0] origpath = self.getpath(mycpv) if not os.path.exists(origpath): @@ -355,7 +367,7 @@ class vardbapi(dbapi): del e write_atomic(os.path.join(newpath, "PF"), new_pf+"\n") write_atomic(os.path.join(newpath, "CATEGORY"), mynewcat+"\n") - fixdbentries([mylist], newpath) + return moves def cp_list(self, mycp, use_cache=1): @@ -363,7 +375,10 @@ class vardbapi(dbapi): if mysplit[0] == '*': mysplit[0] = mysplit[0][1:] try: - mystat = os.stat(self.getpath(mysplit[0])).st_mtime + if sys.hexversion >= 0x3030000: + mystat = os.stat(self.getpath(mysplit[0])).st_mtime_ns + else: + mystat = os.stat(self.getpath(mysplit[0])).st_mtime except OSError: mystat = 0 if use_cache and mycp in self.cpcache: @@ -498,7 +513,10 @@ class vardbapi(dbapi): return list(self._iter_match(mydep, self.cp_list(mydep.cp, use_cache=use_cache))) try: - curmtime = os.stat(os.path.join(self._eroot, VDB_PATH, mycat)).st_mtime + if sys.hexversion >= 0x3030000: + curmtime = os.stat(os.path.join(self._eroot, VDB_PATH, mycat)).st_mtime_ns + else: + curmtime = os.stat(os.path.join(self._eroot, VDB_PATH, mycat)).st_mtime except (IOError, OSError): curmtime=0 @@ -553,31 +571,32 @@ class vardbapi(dbapi): def _aux_cache_init(self): aux_cache = None open_kwargs = {} - if sys.hexversion >= 0x3000000: + if sys.hexversion >= 0x3000000 and sys.hexversion < 0x3020000: # Buffered io triggers extreme performance issues in # Unpickler.load() (problem observed with python-3.0.1). # Unfortunately, performance is still poor relative to - # python-2.x, but buffering makes it much worse. + # python-2.x, but buffering makes it much worse (problem + # appears to be solved in Python >=3.2 at least). open_kwargs["buffering"] = 0 try: - f = open(_unicode_encode(self._aux_cache_filename, + with open(_unicode_encode(self._aux_cache_filename, encoding=_encodings['fs'], errors='strict'), - mode='rb', **open_kwargs) - mypickle = pickle.Unpickler(f) - try: - mypickle.find_global = None - except AttributeError: - # TODO: If py3k, override Unpickler.find_class(). - pass - aux_cache = mypickle.load() - f.close() - del f - except (AttributeError, EOFError, EnvironmentError, ValueError, pickle.UnpicklingError) as e: + mode='rb', **open_kwargs) as f: + mypickle = pickle.Unpickler(f) + try: + mypickle.find_global = None + except AttributeError: + # TODO: If py3k, override Unpickler.find_class(). + pass + aux_cache = mypickle.load() + except (SystemExit, KeyboardInterrupt): + raise + except Exception as e: if isinstance(e, EnvironmentError) and \ getattr(e, 'errno', None) in (errno.ENOENT, errno.EACCES): pass else: - writemsg(_unicode_decode(_("!!! Error loading '%s': %s\n")) % \ + writemsg(_("!!! Error loading '%s': %s\n") % \ (self._aux_cache_filename, e), noiselevel=-1) del e @@ -644,7 +663,8 @@ class vardbapi(dbapi): if e.errno != errno.ENOENT: raise raise KeyError(mycpv) - mydir_mtime = mydir_stat[stat.ST_MTIME] + # Use float mtime when available. + mydir_mtime = mydir_stat.st_mtime pkg_data = self._aux_cache["packages"].get(mycpv) pull_me = cache_these.union(wants) mydata = {"_mtime_" : mydir_mtime} @@ -657,13 +677,18 @@ class vardbapi(dbapi): pkg_data = None else: cache_mtime, metadata = pkg_data - if not isinstance(cache_mtime, (long, int)) or \ + if not isinstance(cache_mtime, (float, long, int)) or \ not isinstance(metadata, dict): pkg_data = None if pkg_data: cache_mtime, metadata = pkg_data - cache_valid = cache_mtime == mydir_mtime + if isinstance(cache_mtime, float): + cache_valid = cache_mtime == mydir_stat.st_mtime + else: + # Cache may contain integer mtime. + cache_valid = cache_mtime == mydir_stat[stat.ST_MTIME] + if cache_valid: # Migrate old metadata to unicode. for k, v in metadata.items(): @@ -687,10 +712,11 @@ class vardbapi(dbapi): (mydir_mtime, cache_data) self._aux_cache["modified"].add(mycpv) - if _slot_re.match(mydata['SLOT']) is None: + eapi_attrs = _get_eapi_attrs(mydata['EAPI']) + if _get_slot_re(eapi_attrs).match(mydata['SLOT']) is None: # Empty or invalid slot triggers InvalidAtom exceptions when # generating slot atoms for packages, so translate it to '0' here. - mydata['SLOT'] = _unicode_decode('0') + mydata['SLOT'] = '0' return [mydata[x] for x in wants] @@ -715,21 +741,18 @@ class vardbapi(dbapi): results[x] = st[stat.ST_MTIME] continue try: - myf = io.open( + with io.open( _unicode_encode(os.path.join(mydir, x), encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], - errors='replace') - try: - myd = myf.read() - finally: - myf.close() + errors='replace') as f: + myd = f.read() except IOError: if x not in self._aux_cache_keys and \ self._aux_cache_keys_re.match(x) is None: env_keys.append(x) continue - myd = _unicode_decode('') + myd = '' # Preserve \n for metadata that is known to # contain multiple lines. @@ -743,13 +766,13 @@ class vardbapi(dbapi): for k in env_keys: v = env_results.get(k) if v is None: - v = _unicode_decode('') + v = '' if self._aux_multi_line_re.match(k) is None: v = " ".join(v.split()) results[k] = v if results.get("EAPI") == "": - results[_unicode_decode("EAPI")] = _unicode_decode('0') + results["EAPI"] = '0' return results @@ -869,11 +892,17 @@ class vardbapi(dbapi): del myroot counter = -1 try: - cfile = io.open( + with io.open( _unicode_encode(self._counter_path, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], - errors='replace') + errors='replace') as f: + try: + counter = long(f.readline().strip()) + except (OverflowError, ValueError) as e: + writemsg(_("!!! COUNTER file is corrupt: '%s'\n") % + self._counter_path, noiselevel=-1) + writemsg("!!! %s\n" % (e,), noiselevel=-1) except EnvironmentError as e: # Silently allow ENOENT since files under # /var/cache/ are allowed to disappear. @@ -882,17 +911,6 @@ class vardbapi(dbapi): self._counter_path, noiselevel=-1) writemsg("!!! %s\n" % str(e), noiselevel=-1) del e - else: - try: - try: - counter = long(cfile.readline().strip()) - finally: - cfile.close() - except (OverflowError, ValueError) as e: - writemsg(_("!!! COUNTER file is corrupt: '%s'\n") % \ - self._counter_path, noiselevel=-1) - writemsg("!!! %s\n" % str(e), noiselevel=-1) - del e if self._cached_counter == counter: max_counter = counter @@ -984,16 +1002,31 @@ class vardbapi(dbapi): relative_filename = filename[root_len:] contents_key = pkg._match_contents(relative_filename) if contents_key: - del new_contents[contents_key] + # It's possible for two different paths to refer to the same + # contents_key, due to directory symlinks. Therefore, pass a + # default value to pop, in order to avoid a KeyError which + # could otherwise be triggered (see bug #454400). + new_contents.pop(contents_key, None) removed += 1 if removed: - self._bump_mtime(pkg.mycpv) - f = atomic_ofstream(os.path.join(pkg.dbdir, "CONTENTS")) - write_contents(new_contents, root, f) - f.close() - self._bump_mtime(pkg.mycpv) - pkg._clear_contents_cache() + self.writeContentsToContentsFile(pkg, new_contents) + + def writeContentsToContentsFile(self, pkg, new_contents): + """ + @param pkg: package to write contents file for + @type pkg: dblink + @param new_contents: contents to write to CONTENTS file + @type new_contents: contents dictionary of the form + {u'/path/to/file' : (contents_attribute 1, ...), ...} + """ + root = self.settings['ROOT'] + self._bump_mtime(pkg.mycpv) + f = atomic_ofstream(os.path.join(pkg.dbdir, "CONTENTS")) + write_contents(new_contents, root, f) + f.close() + self._bump_mtime(pkg.mycpv) + pkg._clear_contents_cache() class _owners_cache(object): """ @@ -1238,18 +1271,35 @@ class vardbapi(dbapi): name = os.path.basename(path.rstrip(os.path.sep)) path_info_list.append((path, name, is_basename)) + # Do work via the global event loop, so that it can be used + # for indication of progress during the search (bug #461412). + event_loop = (portage._internal_caller and + global_event_loop() or EventLoop(main=False)) root = self._vardb._eroot - for cpv in self._vardb.cpv_all(): - dblnk = self._vardb._dblink(cpv) + def search_pkg(cpv): + dblnk = self._vardb._dblink(cpv) for path, name, is_basename in path_info_list: if is_basename: for p in dblnk.getcontents(): if os.path.basename(p) == name: - yield dblnk, p[len(root):] + search_pkg.results.append((dblnk, p[len(root):])) else: if dblnk.isowner(path): - yield dblnk, path + search_pkg.results.append((dblnk, path)) + search_pkg.complete = True + return False + + search_pkg.results = [] + + for cpv in self._vardb.cpv_all(): + del search_pkg.results[:] + search_pkg.complete = False + event_loop.idle_add(search_pkg, cpv) + while not search_pkg.complete: + event_loop.iteration() + for result in search_pkg.results: + yield result class vartree(object): "this tree will scan a var/db/pkg database located at root (passed to init)" @@ -1370,7 +1420,7 @@ class vartree(object): def getslot(self, mycatpkg): "Get a slot for a catpkg; assume it exists." try: - return self.dbapi.aux_get(mycatpkg, ["SLOT"])[0] + return self.dbapi._pkg_str(mycatpkg, None).slot except KeyError: return "" @@ -1463,11 +1513,16 @@ class dblink(object): self._contents_inodes = None self._contents_basenames = None self._linkmap_broken = False + self._device_path_map = {} self._hardlink_merge_map = {} self._hash_key = (self._eroot, self.mycpv) self._protect_obj = None self._pipe = pipe + # When necessary, this attribute is modified for + # compliance with RESTRICT=preserve-libs. + self._preserve_libs = "preserve-libs" in mysettings.features + def __hash__(self): return hash(self._hash_key) @@ -1510,7 +1565,11 @@ class dblink(object): """ Remove this entry from the database """ - if not os.path.exists(self.dbdir): + try: + os.lstat(self.dbdir) + except OSError as e: + if e.errno not in (errno.ENOENT, errno.ENOTDIR, errno.ESTALE): + raise return # Check validity of self.dbdir before attempting to remove it. @@ -1527,6 +1586,14 @@ class dblink(object): pass self.vartree.dbapi._remove(self) + # Use self.dbroot since we need an existing path for syncfs. + try: + self._merged_path(self.dbroot, os.lstat(self.dbroot)) + except OSError: + pass + + self._post_merge_sync() + def clearcontents(self): """ For a given db entry (self), erase the CONTENTS values. @@ -1552,18 +1619,18 @@ class dblink(object): return self.contentscache pkgfiles = {} try: - myc = io.open(_unicode_encode(contents_file, + with io.open(_unicode_encode(contents_file, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], - errors='replace') + errors='replace') as f: + mylines = f.readlines() except EnvironmentError as e: if e.errno != errno.ENOENT: raise del e self.contentscache = pkgfiles return pkgfiles - mylines = myc.readlines() - myc.close() + null_byte = "\0" normalize_needed = self._normalize_needed contents_re = self._contents_re @@ -1578,7 +1645,7 @@ class dblink(object): if myroot == os.path.sep: myroot = None # used to generate parent dir entries - dir_entry = (_unicode_decode("dir"),) + dir_entry = ("dir",) eroot_split_len = len(self.settings["EROOT"].split(os.sep)) - 1 pos = 0 errors = [] @@ -1678,8 +1745,11 @@ class dblink(object): unmerge_preserve = \ self._find_libs_to_preserve(unmerge=True) counter = self.vartree.dbapi.cpv_counter(self.mycpv) - plib_registry.unregister(self.mycpv, - self.settings["SLOT"], counter) + try: + slot = self.mycpv.slot + except AttributeError: + slot = _pkg_str(self.mycpv, slot=self.settings["SLOT"]).slot + plib_registry.unregister(self.mycpv, slot, counter) if unmerge_preserve: for path in sorted(unmerge_preserve): contents_key = self._match_contents(path) @@ -1689,7 +1759,7 @@ class dblink(object): self._display_merge(_(">>> needed %s %s\n") % \ (obj_type, contents_key), noiselevel=-1) plib_registry.register(self.mycpv, - self.settings["SLOT"], counter, unmerge_preserve) + slot, counter, unmerge_preserve) # Remove the preserved files from our contents # so that they won't be unmerged. self.vartree.dbapi.removeFromContents(self, @@ -1759,7 +1829,8 @@ class dblink(object): if self._scheduler is None: # We create a scheduler instance and use it to # log unmerge output separately from merge output. - self._scheduler = PollScheduler().sched_iface + self._scheduler = SchedulerInterface(portage._internal_caller and + global_event_loop() or EventLoop(main=False)) if self.settings.get("PORTAGE_BACKGROUND") == "subprocess": if self.settings.get("PORTAGE_BACKGROUND_UNMERGE") == "1": self.settings["PORTAGE_BACKGROUND"] = "1" @@ -1775,11 +1846,16 @@ class dblink(object): showMessage = self._display_merge if self.vartree.dbapi._categories is not None: self.vartree.dbapi._categories = None + + # When others_in_slot is not None, the backup has already been + # handled by the caller. + caller_handles_backup = others_in_slot is not None + # When others_in_slot is supplied, the security check has already been # done for this slot, so it shouldn't be repeated until the next # replacement or unmerge operation. if others_in_slot is None: - slot = self.vartree.dbapi.aux_get(self.mycpv, ["SLOT"])[0] + slot = self.vartree.dbapi._pkg_str(self.mycpv, None).slot slot_matches = self.vartree.dbapi.match( "%s:%s" % (portage.cpv_getkey(self.mycpv), slot)) others_in_slot = [] @@ -1823,8 +1899,9 @@ class dblink(object): except UnsupportedAPIException as e: eapi_unsupported = e - self._prune_plib_registry(unmerge=True, needed=needed, - preserve_paths=preserve_paths) + if self._preserve_libs and "preserve-libs" in \ + self.settings["PORTAGE_RESTRICT"].split(): + self._preserve_libs = False builddir_lock = None scheduler = self._scheduler @@ -1832,7 +1909,7 @@ class dblink(object): try: # Only create builddir_lock if the caller # has not already acquired the lock. - if "PORTAGE_BUILDIR_LOCKED" not in self.settings: + if "PORTAGE_BUILDDIR_LOCKED" not in self.settings: builddir_lock = EbuildBuildDir( scheduler=scheduler, settings=self.settings) @@ -1840,6 +1917,19 @@ class dblink(object): prepare_build_dirs(settings=self.settings, cleanup=True) log_path = self.settings.get("PORTAGE_LOG_FILE") + # Do this before the following _prune_plib_registry call, since + # that removes preserved libraries from our CONTENTS, and we + # may want to backup those libraries first. + if not caller_handles_backup: + retval = self._pre_unmerge_backup(background) + if retval != os.EX_OK: + showMessage(_("!!! FAILED prerm: quickpkg: %s\n") % retval, + level=logging.ERROR, noiselevel=-1) + return retval + + self._prune_plib_registry(unmerge=True, needed=needed, + preserve_paths=preserve_paths) + # Log the error after PORTAGE_LOG_FILE is initialized # by prepare_build_dirs above. if eapi_unsupported: @@ -1848,7 +1938,7 @@ class dblink(object): showMessage(_("!!! FAILED prerm: %s\n") % \ os.path.join(self.dbdir, "EAPI"), level=logging.ERROR, noiselevel=-1) - showMessage(_unicode_decode("%s\n") % (eapi_unsupported,), + showMessage("%s\n" % (eapi_unsupported,), level=logging.ERROR, noiselevel=-1) elif os.path.isfile(myebuildpath): phase = EbuildPhase(background=background, @@ -2037,7 +2127,7 @@ class dblink(object): if others_in_slot is None: others_in_slot = [] - slot = self.vartree.dbapi.aux_get(self.mycpv, ["SLOT"])[0] + slot = self.vartree.dbapi._pkg_str(self.mycpv, None).slot slot_matches = self.vartree.dbapi.match( "%s:%s" % (portage.cpv_getkey(self.mycpv), slot)) for cur_cpv in slot_matches: @@ -2062,7 +2152,9 @@ class dblink(object): #process symlinks second-to-last, directories last. mydirs = set() - modprotect = os.path.join(self._eroot, "lib/modules/") + + uninstall_ignore = portage.util.shlex_split( + self.settings.get("UNINSTALL_IGNORE", "")) def unlink(file_name, lstatobj): if bsd_chflags: @@ -2092,6 +2184,14 @@ class dblink(object): self._eerror("postrm", ["Could not chmod or unlink '%s': %s" % \ (file_name, ose)]) + else: + + # Even though the file no longer exists, we log it + # here so that _unmerge_dirs can see that we've + # removed a file from this device, and will record + # the parent directory for a syncfs call. + self._merged_path(file_name, lstatobj, exists=False) + finally: if bsd_chflags and pflags != 0: # Restore the parent flags we saved before unlinking @@ -2169,6 +2269,24 @@ class dblink(object): if lstatobj is None: show_unmerge("---", unmerge_desc["!found"], file_type, obj) continue + + f_match = obj[len(eroot)-1:] + ignore = False + for pattern in uninstall_ignore: + if fnmatch.fnmatch(f_match, pattern): + ignore = True + break + + if not ignore: + if islink and f_match in \ + ("/lib", "/usr/lib", "/usr/local/lib"): + # Ignore libdir symlinks for bug #423127. + ignore = True + + if ignore: + show_unmerge("---", unmerge_desc["cfgpro"], file_type, obj) + continue + # don't use EROOT, CONTENTS entries already contain EPREFIX if obj.startswith(real_root): relative_path = obj[real_root_len:] @@ -2178,8 +2296,9 @@ class dblink(object): is_owned = True break - if file_type == "sym" and is_owned and \ - (islink and statobj and stat.S_ISDIR(statobj.st_mode)): + if is_owned and islink and \ + file_type in ("sym", "dir") and \ + statobj and stat.S_ISDIR(statobj.st_mode): # A new instance of this package claims the file, so # don't unmerge it. If the file is symlink to a # directory and the unmerging package installed it as @@ -2211,18 +2330,6 @@ class dblink(object): continue elif relative_path in cfgfiledict: stale_confmem.append(relative_path) - # next line includes a tweak to protect modules from being unmerged, - # but we don't protect modules from being overwritten if they are - # upgraded. We effectively only want one half of the config protection - # functionality for /lib/modules. For portage-ng both capabilities - # should be able to be independently specified. - # TODO: For rebuilds, re-parent previous modules to the new - # installed instance (so they are not orphans). For normal - # uninstall (not rebuild/reinstall), remove the modules along - # with all other files (leave no orphans). - if obj.startswith(modprotect): - show_unmerge("---", unmerge_desc["cfgpro"], file_type, obj) - continue # Don't unlink symlinks to directories here since that can # remove /lib and /usr/lib symlinks. @@ -2244,12 +2351,12 @@ class dblink(object): show_unmerge("---", unmerge_desc["!mtime"], file_type, obj) continue - if pkgfiles[objkey][0] == "dir": + if file_type == "dir" and not islink: if lstatobj is None or not stat.S_ISDIR(lstatobj.st_mode): show_unmerge("---", unmerge_desc["!dir"], file_type, obj) continue mydirs.add((obj, (lstatobj.st_dev, lstatobj.st_ino))) - elif pkgfiles[objkey][0] == "sym": + elif file_type == "sym" or (file_type == "dir" and islink): if not islink: show_unmerge("---", unmerge_desc["!sym"], file_type, obj) continue @@ -2359,7 +2466,11 @@ class dblink(object): if protected_symlinks: msg = "One or more symlinks to directories have been " + \ "preserved in order to ensure that files installed " + \ - "via these symlinks remain accessible:" + "via these symlinks remain accessible. " + \ + "This indicates that the mentioned symlink(s) may " + \ + "be obsolete remnants of an old install, and it " + \ + "may be appropriate to replace a given symlink " + \ + "with the directory that it points to." lines = textwrap.wrap(msg, 72) lines.append("") flat_list = set() @@ -2369,7 +2480,7 @@ class dblink(object): lines.append("\t%s" % (os.path.join(real_root, f.lstrip(os.sep)))) lines.append("") - self._elog("eerror", "postrm", lines) + self._elog("elog", "postrm", lines) # Remove stale entries from config memory. if stale_confmem: @@ -2501,15 +2612,19 @@ class dblink(object): raise del e show_unmerge("!!!", "", "obj", child) + try: + parent_name = os.path.dirname(obj) + parent_stat = os.stat(parent_name) + if bsd_chflags: lstatobj = os.lstat(obj) if lstatobj.st_flags != 0: bsd_chflags.lchflags(obj, 0) - parent_name = os.path.dirname(obj) + # Use normal stat/chflags for the parent since we want to # follow any symlinks to the real parent directory. - pflags = os.stat(parent_name).st_flags + pflags = parent_stat.st_flags if pflags != 0: bsd_chflags.chflags(parent_name, 0) try: @@ -2518,13 +2633,34 @@ class dblink(object): if bsd_chflags and pflags != 0: # Restore the parent flags we saved before unlinking bsd_chflags.chflags(parent_name, pflags) + + # Record the parent directory for use in syncfs calls. + # Note that we use a realpath and a regular stat here, since + # we want to follow any symlinks back to the real device where + # the real parent directory resides. + self._merged_path(os.path.realpath(parent_name), parent_stat) + show_unmerge("<<<", "", "dir", obj) except EnvironmentError as e: if e.errno not in ignored_rmdir_errnos: raise if e.errno != errno.ENOENT: show_unmerge("---", unmerge_desc["!empty"], "dir", obj) - del e + + # Since we didn't remove this directory, record the directory + # itself for use in syncfs calls, if we have removed another + # file from the same device. + # Note that we use a realpath and a regular stat here, since + # we want to follow any symlinks back to the real device where + # the real directory resides. + try: + dir_stat = os.stat(obj) + except OSError: + pass + else: + if dir_stat.st_dev in self._device_path_map: + self._merged_path(os.path.realpath(obj), dir_stat) + else: # When a directory is successfully removed, there's # no need to protect symlinks that point to it. @@ -2751,7 +2887,7 @@ class dblink(object): self.vartree.dbapi._linkmap is None or \ self.vartree.dbapi._plib_registry is None or \ (not unmerge and self._installed_instance is None) or \ - "preserve-libs" not in self.settings.features: + not self._preserve_libs: return set() os = _os_merge @@ -3335,7 +3471,10 @@ class dblink(object): else: logdir = os.path.join(self.settings["T"], "logging") ebuild_logentries = collect_ebuild_messages(logdir) - py_logentries = collect_messages(key=cpv).get(cpv, {}) + # phasefilter is irrelevant for the above collect_ebuild_messages + # call, since this package instance has a private logdir. However, + # it may be relevant for the following collect_messages call. + py_logentries = collect_messages(key=cpv, phasefilter=phasefilter).get(cpv, {}) logentries = _merge_logentries(py_logentries, ebuild_logentries) funcnames = { "INFO": "einfo", @@ -3356,7 +3495,9 @@ class dblink(object): str_buffer.append(' '.join(fields)) str_buffer.append('\n') if str_buffer: - os.write(self._pipe, _unicode_encode(''.join(str_buffer))) + str_buffer = _unicode_encode(''.join(str_buffer)) + while str_buffer: + str_buffer = str_buffer[os.write(self._pipe, str_buffer):] def _emerge_log(self, msg): emergelog(False, msg) @@ -3414,6 +3555,7 @@ class dblink(object): level=logging.ERROR, noiselevel=-1) return 1 + is_binpkg = self.settings.get("EMERGE_FROM") == "binary" slot = '' for var_name in ('CHOST', 'SLOT'): if var_name == 'CHOST' and self.cat == 'virtual': @@ -3423,22 +3565,18 @@ class dblink(object): pass continue - f = None try: - f = io.open(_unicode_encode( + with io.open(_unicode_encode( os.path.join(inforoot, var_name), encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], - errors='replace') - val = f.readline().strip() + errors='replace') as f: + val = f.readline().strip() except EnvironmentError as e: if e.errno != errno.ENOENT: raise del e val = '' - finally: - if f is not None: - f.close() if var_name == 'SLOT': slot = val @@ -3451,7 +3589,9 @@ class dblink(object): return 1 write_atomic(os.path.join(inforoot, var_name), slot + '\n') - if val != self.settings.get(var_name, ''): + # This check only applies when built from source, since + # inforoot values are written just after src_install. + if not is_binpkg and val != self.settings.get(var_name, ''): self._eqawarn('preinst', [_("QA Notice: Expected %(var_name)s='%(expected_value)s', got '%(actual_value)s'\n") % \ {"var_name":var_name, "expected_value":self.settings.get(var_name, ''), "actual_value":val}]) @@ -3462,30 +3602,47 @@ class dblink(object): if not os.path.exists(self.dbcatdir): ensure_dirs(self.dbcatdir) + # NOTE: We use SLOT obtained from the inforoot + # directory, in order to support USE=multislot. + # Use _pkg_str discard the sub-slot part if necessary. + slot = _pkg_str(self.mycpv, slot=slot).slot cp = self.mysplit[0] slot_atom = "%s:%s" % (cp, slot) - # filter any old-style virtual matches - slot_matches = [cpv for cpv in self.vartree.dbapi.match(slot_atom) \ - if cpv_getkey(cpv) == cp] - - if self.mycpv not in slot_matches and \ - self.vartree.dbapi.cpv_exists(self.mycpv): - # handle multislot or unapplied slotmove - slot_matches.append(self.mycpv) - - others_in_slot = [] - from portage import config - for cur_cpv in slot_matches: - # Clone the config in case one of these has to be unmerged since - # we need it to have private ${T} etc... for things like elog. - settings_clone = config(clone=self.settings) - settings_clone.pop("PORTAGE_BUILDIR_LOCKED", None) - settings_clone.reset() - others_in_slot.append(dblink(self.cat, catsplit(cur_cpv)[1], - settings=settings_clone, - vartree=self.vartree, treetype="vartree", - scheduler=self._scheduler, pipe=self._pipe)) + self.lockdb() + try: + # filter any old-style virtual matches + slot_matches = [cpv for cpv in self.vartree.dbapi.match(slot_atom) + if cpv_getkey(cpv) == cp] + + if self.mycpv not in slot_matches and \ + self.vartree.dbapi.cpv_exists(self.mycpv): + # handle multislot or unapplied slotmove + slot_matches.append(self.mycpv) + + others_in_slot = [] + for cur_cpv in slot_matches: + # Clone the config in case one of these has to be unmerged, + # since we need it to have private ${T} etc... for things + # like elog. + settings_clone = portage.config(clone=self.settings) + settings_clone.pop("PORTAGE_BUILDDIR_LOCKED", None) + settings_clone.setcpv(cur_cpv, mydb=self.vartree.dbapi) + if self._preserve_libs and "preserve-libs" in \ + settings_clone["PORTAGE_RESTRICT"].split(): + self._preserve_libs = False + others_in_slot.append(dblink(self.cat, catsplit(cur_cpv)[1], + settings=settings_clone, + vartree=self.vartree, treetype="vartree", + scheduler=self._scheduler, pipe=self._pipe)) + finally: + self.unlockdb() + + # If any instance has RESTRICT=preserve-libs, then + # restrict it for all instances. + if not self._preserve_libs: + for dblnk in others_in_slot: + dblnk._preserve_libs = False retval = self._security_check(others_in_slot) if retval: @@ -3596,6 +3753,13 @@ class dblink(object): # to an infinite recursion loop. mylinklist.append(relative_path) + myto = _unicode_decode( + _os.readlink(_unicode_encode(fpath, + encoding=_encodings['merge'], errors='strict')), + encoding=_encodings['merge'], errors='replace') + if line_ending_re.search(myto) is not None: + paths_with_newlines.append(relative_path) + if unicode_error: break @@ -3647,7 +3811,7 @@ class dblink(object): _("Manually run `emerge --unmerge =%s` if you " "really want to remove the above files. Set " "PORTAGE_PACKAGE_EMPTY_ABORT=\"0\" in " - "/etc/make.conf if you do not want to " + "/etc/portage/make.conf if you do not want to " "abort in cases like this.") % other_dblink.mycpv, wrap_width)) eerror(msg) @@ -3713,7 +3877,9 @@ class dblink(object): " enough information to determine if a real problem" " exists. Please do NOT file a bug report at" " http://bugs.gentoo.org unless you report exactly which" - " two packages install the same file(s). Once again," + " two packages install the same file(s). See" + " http://wiki.gentoo.org/wiki/Knowledge_Base:Blockers" + " for tips on how to solve the problem. And once again," " please do NOT file a bug report unless you have" " completely understood the above message.") @@ -3748,17 +3914,28 @@ class dblink(object): # get_owners is slow for large numbers of files, so # don't look them all up. collisions = collisions[:20] + + pkg_info_strs = {} self.lockdb() try: owners = self.vartree.dbapi._owners.get_owners(collisions) self.vartree.dbapi.flush_cache() + + for pkg in owners: + pkg = self.vartree.dbapi._pkg_str(pkg.mycpv, None) + pkg_info_str = "%s%s%s" % (pkg, + _slot_separator, pkg.slot) + if pkg.repo != _unknown_repo: + pkg_info_str += "%s%s" % (_repo_separator, + pkg.repo) + pkg_info_strs[pkg] = pkg_info_str + finally: self.unlockdb() for pkg, owned_files in owners.items(): - cpv = pkg.mycpv msg = [] - msg.append("%s" % cpv) + msg.append(pkg_info_strs[pkg.mycpv]) for f in sorted(owned_files): msg.append("\t%s" % os.path.join(destroot, f.lstrip(os.path.sep))) @@ -3814,6 +3991,20 @@ class dblink(object): self.delete() ensure_dirs(self.dbtmpdir) + downgrade = False + if self._installed_instance is not None and \ + vercmp(self.mycpv.version, + self._installed_instance.mycpv.version) < 0: + downgrade = True + + if self._installed_instance is not None: + rval = self._pre_merge_backup(self._installed_instance, downgrade) + if rval != os.EX_OK: + showMessage(_("!!! FAILED preinst: ") + + "quickpkg: %s\n" % rval, + level=logging.ERROR, noiselevel=-1) + return rval + # run preinst script showMessage(_(">>> Merging %(cpv)s to %(destroot)s\n") % \ {"cpv":self.mycpv, "destroot":destroot}) @@ -3835,32 +4026,26 @@ class dblink(object): # write local package counter for recording if counter is None: counter = self.vartree.dbapi.counter_tick(mycpv=self.mycpv) - f = io.open(_unicode_encode(os.path.join(self.dbtmpdir, 'COUNTER'), + with io.open(_unicode_encode(os.path.join(self.dbtmpdir, 'COUNTER'), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['repo.content'], - errors='backslashreplace') - f.write(_unicode_decode(str(counter))) - f.close() + errors='backslashreplace') as f: + f.write("%s" % counter) self.updateprotect() #if we have a file containing previously-merged config file md5sums, grab it. self.vartree.dbapi._fs_lock() try: + # Always behave like --noconfmem is enabled for downgrades + # so that people who don't know about this option are less + # likely to get confused when doing upgrade/downgrade cycles. cfgfiledict = grabdict(self.vartree.dbapi._conf_mem_file) - if "NOCONFMEM" in self.settings: + if "NOCONFMEM" in self.settings or downgrade: cfgfiledict["IGNORE"]=1 else: cfgfiledict["IGNORE"]=0 - # Always behave like --noconfmem is enabled for downgrades - # so that people who don't know about this option are less - # likely to get confused when doing upgrade/downgrade cycles. - for other in others_in_slot: - if vercmp(self.mycpv.version, other.mycpv.version) < 0: - cfgfiledict["IGNORE"] = 1 - break - rval = self._merge_contents(srcroot, destroot, cfgfiledict) if rval != os.EX_OK: return rval @@ -3970,6 +4155,7 @@ class dblink(object): try: self.delete() _movefile(self.dbtmpdir, self.dbpkgdir, mysettings=self.settings) + self._merged_path(self.dbpkgdir, os.lstat(self.dbpkgdir)) finally: self.unlockdb() @@ -4014,9 +4200,9 @@ class dblink(object): self.vartree.dbapi.lock() try: try: - slot, counter = self.vartree.dbapi.aux_get( - cpv, ["SLOT", "COUNTER"]) - except KeyError: + slot = self.vartree.dbapi._pkg_str(cpv, None).slot + counter = self.vartree.dbapi.cpv_counter(cpv) + except (KeyError, InvalidData): pass else: has_vdb_entry = True @@ -4085,6 +4271,7 @@ class dblink(object): # For gcc upgrades, preserved libs have to be removed after the # the library path has been updated. self._prune_plib_registry() + self._post_merge_sync() return os.EX_OK @@ -4100,7 +4287,7 @@ class dblink(object): x = -1 while True: x += 1 - backup_p = p + '.backup.' + str(x).rjust(4, '0') + backup_p = '%s.backup.%04d' % (p, x) try: os.lstat(backup_p) except OSError: @@ -4201,8 +4388,9 @@ class dblink(object): @type stufftomerge: String or List @param cfgfiledict: { File:mtime } mapping for config_protected files @type cfgfiledict: Dictionary - @param thismtime: The current time (typically long(time.time()) - @type thismtime: Long + @param thismtime: None or new mtime for merged files (expressed in seconds + in Python <3.3 and nanoseconds in Python >=3.3) + @type thismtime: None or Int @rtype: None or Boolean @return: 1. True on failure @@ -4227,18 +4415,18 @@ class dblink(object): # this is supposed to merge a list of files. There will be 2 forms of argument passing. if isinstance(stufftomerge, basestring): #A directory is specified. Figure out protection paths, listdir() it and process it. - mergelist = os.listdir(join(srcroot, stufftomerge)) - offset = stufftomerge + mergelist = [join(stufftomerge, child) for child in \ + os.listdir(join(srcroot, stufftomerge))] else: - mergelist = stufftomerge - offset = "" + mergelist = stufftomerge[:] - for i, x in enumerate(mergelist): + while mergelist: - mysrc = join(srcroot, offset, x) - mydest = join(destroot, offset, x) + relative_path = mergelist.pop() + mysrc = join(srcroot, relative_path) + mydest = join(destroot, relative_path) # myrealdest is mydest without the $ROOT prefix (makes a difference if ROOT!="/") - myrealdest = join(sep, offset, x) + myrealdest = join(sep, relative_path) # stat file once, test using S_* macros many times (faster that way) mystat = os.lstat(mysrc) mymode = mystat[stat.ST_MODE] @@ -4333,9 +4521,26 @@ class dblink(object): mymtime = movefile(mysrc, mydest, newmtime=thismtime, sstat=mystat, mysettings=self.settings, encoding=_encodings['merge']) + + try: + self._merged_path(mydest, os.lstat(mydest)) + except OSError: + pass + if mymtime != None: + # Use lexists, since if the target happens to be a broken + # symlink then that should trigger an independent warning. + if not (os.path.lexists(myrealto) or + os.path.lexists(join(srcroot, myabsto))): + self._eqawarn('preinst', + [_("QA Notice: Symbolic link /%s points to /%s which does not exist.") + % (relative_path, myabsto)]) + showMessage(">>> %s -> %s\n" % (mydest, myto)) - outfile.write("sym "+myrealdest+" -> "+myto+" "+str(mymtime)+"\n") + if sys.hexversion >= 0x3030000: + outfile.write("sym "+myrealdest+" -> "+myto+" "+str(mymtime // 1000000000)+"\n") + else: + outfile.write("sym "+myrealdest+" -> "+myto+" "+str(mymtime)+"\n") else: showMessage(_("!!! Failed to move file.\n"), level=logging.ERROR, noiselevel=-1) @@ -4429,11 +4634,17 @@ class dblink(object): os.chmod(mydest, mystat[0]) os.chown(mydest, mystat[4], mystat[5]) showMessage(">>> %s/\n" % mydest) + + try: + self._merged_path(mydest, os.lstat(mydest)) + except OSError: + pass + outfile.write("dir "+myrealdest+"\n") # recurse and merge this directory - if self.mergeme(srcroot, destroot, outfile, secondhand, - join(offset, x), cfgfiledict, thismtime): - return 1 + mergelist.extend(join(relative_path, child) for child in + os.listdir(join(srcroot, relative_path))) + elif stat.S_ISREG(mymode): # we are merging a regular file mymd5 = perform_md5(mysrc, calc_prelink=calc_prelink) @@ -4489,7 +4700,10 @@ class dblink(object): cfgprot = cfgfiledict["IGNORE"] if not moveme: zing = "---" - mymtime = mystat[stat.ST_MTIME] + if sys.hexversion >= 0x3030000: + mymtime = mystat.st_mtime_ns + else: + mymtime = mystat[stat.ST_MTIME] else: moveme = 1 cfgprot = 1 @@ -4525,8 +4739,16 @@ class dblink(object): hardlink_candidates.append(mydest) zing = ">>>" + try: + self._merged_path(mydest, os.lstat(mydest)) + except OSError: + pass + if mymtime != None: - outfile.write("obj "+myrealdest+" "+mymd5+" "+str(mymtime)+"\n") + if sys.hexversion >= 0x3030000: + outfile.write("obj "+myrealdest+" "+mymd5+" "+str(mymtime // 1000000000)+"\n") + else: + outfile.write("obj "+myrealdest+" "+mymd5+" "+str(mymtime)+"\n") showMessage("%s %s\n" % (zing,mydest)) else: # we are merging a fifo or device node @@ -4537,6 +4759,12 @@ class dblink(object): sstat=mystat, mysettings=self.settings, encoding=_encodings['merge']) is not None: zing = ">>>" + + try: + self._merged_path(mydest, os.lstat(mydest)) + except OSError: + pass + else: return 1 if stat.S_ISFIFO(mymode): @@ -4545,6 +4773,52 @@ class dblink(object): outfile.write("dev %s\n" % myrealdest) showMessage(zing + " " + mydest + "\n") + def _merged_path(self, path, lstatobj, exists=True): + previous_path = self._device_path_map.get(lstatobj.st_dev) + if previous_path is None or previous_path is False or \ + (exists and len(path) < len(previous_path)): + if exists: + self._device_path_map[lstatobj.st_dev] = path + else: + # This entry is used to indicate that we've unmerged + # a file from this device, and later, this entry is + # replaced by a parent directory. + self._device_path_map[lstatobj.st_dev] = False + + def _post_merge_sync(self): + """ + Call this after merge or unmerge, in order to sync relevant files to + disk and avoid data-loss in the event of a power failure. This method + does nothing if FEATURES=merge-sync is disabled. + """ + if not self._device_path_map or \ + "merge-sync" not in self.settings.features: + return + + returncode = None + if platform.system() == "Linux": + + paths = [] + for path in self._device_path_map.values(): + if path is not False: + paths.append(path) + paths = tuple(paths) + + proc = SyncfsProcess(paths=paths, + scheduler=(self._scheduler or + portage._internal_caller and global_event_loop() or + EventLoop(main=False))) + proc.start() + returncode = proc.wait() + + if returncode is None or returncode != os.EX_OK: + try: + proc = subprocess.Popen(["sync"]) + except EnvironmentError: + pass + else: + proc.wait() + def merge(self, mergeroot, inforoot, myroot=None, myebuild=None, cleanup=0, mydbapi=None, prev_mtimes=None, counter=None): """ @@ -4557,7 +4831,8 @@ class dblink(object): self.lockdb() self.vartree.dbapi._bump_mtime(self.mycpv) if self._scheduler is None: - self._scheduler = PollScheduler().sched_iface + self._scheduler = SchedulerInterface(portage._internal_caller and + global_event_loop() or EventLoop(main=False)) try: retval = self.treewalk(mergeroot, myroot, inforoot, myebuild, cleanup=cleanup, mydbapi=mydbapi, prev_mtimes=prev_mtimes, @@ -4608,11 +4883,12 @@ class dblink(object): "returns contents of a file with whitespace converted to spaces" if not os.path.exists(self.dbdir+"/"+name): return "" - mydata = io.open( + with io.open( _unicode_encode(os.path.join(self.dbdir, name), encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace' - ).read().split() + ) as f: + mydata = f.read().split() return " ".join(mydata) def copyfile(self,fname): @@ -4621,10 +4897,11 @@ class dblink(object): def getfile(self,fname): if not os.path.exists(self.dbdir+"/"+fname): return "" - return io.open(_unicode_encode(os.path.join(self.dbdir, fname), + with io.open(_unicode_encode(os.path.join(self.dbdir, fname), encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace' - ).read() + ) as f: + return f.read() def setfile(self,fname,data): kwargs = {} @@ -4633,16 +4910,18 @@ class dblink(object): else: kwargs['mode'] = 'w' kwargs['encoding'] = _encodings['repo.content'] - write_atomic(os.path.join(self.dbdir, fname), data, **kwargs) + write_atomic(os.path.join(self.dbdir, fname), data, + **portage._native_kwargs(kwargs)) def getelements(self,ename): if not os.path.exists(self.dbdir+"/"+ename): return [] - mylines = io.open(_unicode_encode( + with io.open(_unicode_encode( os.path.join(self.dbdir, ename), encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace' - ).readlines() + ) as f: + mylines = f.readlines() myreturn = [] for x in mylines: for y in x[:-1].split(): @@ -4650,23 +4929,82 @@ class dblink(object): return myreturn def setelements(self,mylist,ename): - myelement = io.open(_unicode_encode( + with io.open(_unicode_encode( os.path.join(self.dbdir, ename), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['repo.content'], - errors='backslashreplace') - for x in mylist: - myelement.write(_unicode_decode(x+"\n")) - myelement.close() + errors='backslashreplace') as f: + for x in mylist: + f.write("%s\n" % x) def isregular(self): "Is this a regular package (does it have a CATEGORY file? A dblink can be virtual *and* regular)" return os.path.exists(os.path.join(self.dbdir, "CATEGORY")) + def _pre_merge_backup(self, backup_dblink, downgrade): + + if ("unmerge-backup" in self.settings.features or + (downgrade and "downgrade-backup" in self.settings.features)): + return self._quickpkg_dblink(backup_dblink, False, None) + + return os.EX_OK + + def _pre_unmerge_backup(self, background): + + if "unmerge-backup" in self.settings.features : + logfile = None + if self.settings.get("PORTAGE_BACKGROUND") != "subprocess": + logfile = self.settings.get("PORTAGE_LOG_FILE") + return self._quickpkg_dblink(self, background, logfile) + + return os.EX_OK + + def _quickpkg_dblink(self, backup_dblink, background, logfile): + + trees = QueryCommand.get_db()[self.settings["EROOT"]] + bintree = trees["bintree"] + binpkg_path = bintree.getname(backup_dblink.mycpv) + if os.path.exists(binpkg_path) and \ + catsplit(backup_dblink.mycpv)[1] not in bintree.invalids: + return os.EX_OK + + self.lockdb() + try: + + if not backup_dblink.exists(): + # It got unmerged by a concurrent process. + return os.EX_OK + + # Call quickpkg for support of QUICKPKG_DEFAULT_OPTS and stuff. + quickpkg_binary = os.path.join(self.settings["PORTAGE_BIN_PATH"], + "quickpkg") + + # Let quickpkg inherit the global vartree config's env. + env = dict(self.vartree.settings.items()) + env["__PORTAGE_INHERIT_VARDB_LOCK"] = "1" + + pythonpath = [x for x in env.get('PYTHONPATH', '').split(":") if x] + if not pythonpath or \ + not os.path.samefile(pythonpath[0], portage._pym_path): + pythonpath.insert(0, portage._pym_path) + env['PYTHONPATH'] = ":".join(pythonpath) + + quickpkg_proc = SpawnProcess( + args=[portage._python_interpreter, quickpkg_binary, + "=%s" % (backup_dblink.mycpv,)], + background=background, env=env, + scheduler=self._scheduler, logfile=logfile) + quickpkg_proc.start() + + return quickpkg_proc.wait() + + finally: + self.unlockdb() + def merge(mycat, mypkg, pkgloc, infloc, myroot=None, settings=None, myebuild=None, mytree=None, mydbapi=None, vartree=None, prev_mtimes=None, blockers=None, - scheduler=None): + scheduler=None, fd_pipes=None): """ @param myroot: ignored, settings['EROOT'] is used instead """ @@ -4681,10 +5019,12 @@ def merge(mycat, mypkg, pkgloc, infloc, merge_task = MergeProcess( mycat=mycat, mypkg=mypkg, settings=settings, treetype=mytree, vartree=vartree, - scheduler=(scheduler or PollScheduler().sched_iface), + scheduler=(scheduler or portage._internal_caller and + global_event_loop() or EventLoop(main=False)), background=background, blockers=blockers, pkgloc=pkgloc, infloc=infloc, myebuild=myebuild, mydbapi=mydbapi, - prev_mtimes=prev_mtimes, logfile=settings.get('PORTAGE_LOG_FILE')) + prev_mtimes=prev_mtimes, logfile=settings.get('PORTAGE_LOG_FILE'), + fd_pipes=fd_pipes) merge_task.start() retcode = merge_task.wait() return retcode @@ -4864,13 +5204,11 @@ def tar_contents(contents, root, tar, protect=None, onProgress=None): tar.addfile(tarinfo, f) f.close() else: - f = open(_unicode_encode(path, + with open(_unicode_encode(path, encoding=encoding, - errors='strict'), 'rb') - try: + errors='strict'), 'rb') as f: tar.addfile(tarinfo, f) - finally: - f.close() + else: tar.addfile(tarinfo) if onProgress: diff --git a/portage_with_autodep/pym/portage/dbapi/vartree.pyo b/portage_with_autodep/pym/portage/dbapi/vartree.pyo Binary files differindex 7c186cf..745d15f 100644 --- a/portage_with_autodep/pym/portage/dbapi/vartree.pyo +++ b/portage_with_autodep/pym/portage/dbapi/vartree.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/virtual.py b/portage_with_autodep/pym/portage/dbapi/virtual.py index da15983..ba9745c 100644 --- a/portage_with_autodep/pym/portage/dbapi/virtual.py +++ b/portage_with_autodep/pym/portage/dbapi/virtual.py @@ -1,6 +1,7 @@ -# Copyright 1998-2012 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals from portage.dbapi import dbapi from portage.dbapi.dep_expand import dep_expand @@ -74,30 +75,55 @@ class fakedbapi(dbapi): @param metadata: dict """ self._clear_cache() - if not hasattr(mycpv, 'cp'): + + try: + mycp = mycpv.cp + except AttributeError: + mycp = None + try: + myslot = mycpv.slot + except AttributeError: + myslot = None + + if mycp is None or \ + (myslot is None and metadata is not None and metadata.get('SLOT')): if metadata is None: mycpv = _pkg_str(mycpv) else: - mycpv = _pkg_str(mycpv, slot=metadata.get('SLOT'), - repo=metadata.get('repository')) - mycp = mycpv.cp + mycpv = _pkg_str(mycpv, metadata=metadata, + settings=self.settings) + + mycp = mycpv.cp + try: + myslot = mycpv.slot + except AttributeError: + pass + self.cpvdict[mycpv] = metadata - myslot = None - if self._exclusive_slots and metadata: - myslot = metadata.get("SLOT", None) + if not self._exclusive_slots: + myslot = None if myslot and mycp in self.cpdict: # If necessary, remove another package in the same SLOT. for cpv in self.cpdict[mycp]: if mycpv != cpv: - other_metadata = self.cpvdict[cpv] - if other_metadata: - if myslot == other_metadata.get("SLOT", None): + try: + other_slot = cpv.slot + except AttributeError: + pass + else: + if myslot == other_slot: self.cpv_remove(cpv) break - if mycp not in self.cpdict: - self.cpdict[mycp] = [] - if not mycpv in self.cpdict[mycp]: - self.cpdict[mycp].append(mycpv) + + cp_list = self.cpdict.get(mycp) + if cp_list is None: + cp_list = [] + self.cpdict[mycp] = cp_list + try: + cp_list.remove(mycpv) + except ValueError: + pass + cp_list.append(mycpv) def cpv_remove(self,mycpv): """Removes a cpv from the list of available packages.""" diff --git a/portage_with_autodep/pym/portage/dbapi/virtual.pyo b/portage_with_autodep/pym/portage/dbapi/virtual.pyo Binary files differindex 9f7c667..ca52263 100644 --- a/portage_with_autodep/pym/portage/dbapi/virtual.pyo +++ b/portage_with_autodep/pym/portage/dbapi/virtual.pyo diff --git a/portage_with_autodep/pym/portage/debug.pyo b/portage_with_autodep/pym/portage/debug.pyo Binary files differindex 82a5e8f..a3c437d 100644 --- a/portage_with_autodep/pym/portage/debug.pyo +++ b/portage_with_autodep/pym/portage/debug.pyo diff --git a/portage_with_autodep/pym/portage/dep/__init__.py b/portage_with_autodep/pym/portage/dep/__init__.py index 152af0a..798903f 100644 --- a/portage_with_autodep/pym/portage/dep/__init__.py +++ b/portage_with_autodep/pym/portage/dep/__init__.py @@ -1,7 +1,9 @@ # deps.py -- Portage dependency resolution functions -# Copyright 2003-2012 Gentoo Foundation +# Copyright 2003-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = [ 'Atom', 'best_match_to_list', 'cpvequal', 'dep_getcpv', 'dep_getkey', 'dep_getslot', @@ -13,20 +15,6 @@ __all__ = [ '_repo_separator', '_slot_separator', ] -# DEPEND SYNTAX: -# -# 'use?' only affects the immediately following word! -# Nesting is the only legal way to form multiple '[!]use?' requirements. -# -# Where: 'a' and 'b' are use flags, and 'z' is a depend atom. -# -# "a? z" -- If 'a' in [use], then b is valid. -# "a? ( z )" -- Syntax with parenthesis. -# "a? b? z" -- Deprecated. -# "a? ( b? z )" -- Valid -# "a? ( b? ( z ) ) -- Valid -# - import re, sys import warnings from itertools import chain @@ -36,23 +24,164 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.util:cmp_sort_key,writemsg', ) -from portage import _unicode_decode -from portage.eapi import eapi_has_slot_deps, eapi_has_src_uri_arrows, \ - eapi_has_use_deps, eapi_has_strong_blocks, eapi_has_use_dep_defaults, \ - eapi_has_repo_deps, eapi_allows_dots_in_PN, eapi_allows_dots_in_use_flags +from portage import _encodings, _unicode_decode, _unicode_encode +from portage.eapi import _get_eapi_attrs from portage.exception import InvalidAtom, InvalidData, InvalidDependString from portage.localization import _ from portage.versions import catpkgsplit, catsplit, \ - vercmp, ververify, _cp, _cpv, _pkg_str, _unknown_repo + vercmp, ververify, _cp, _cpv, _pkg_str, _slot, _unknown_repo, _vr import portage.cache.mappings if sys.hexversion >= 0x3000000: basestring = str + _unicode = str +else: + _unicode = unicode + +# \w is [a-zA-Z0-9_] + +# PMS 3.1.3: A slot name may contain any of the characters [A-Za-z0-9+_.-]. +# It must not begin with a hyphen or a dot. +_slot_separator = ":" +# loosly match SLOT, which may have an optional ABI part +_slot_loose = r'([\w+./*=-]+)' + +_use = r'\[.*\]' +_op = r'([=~]|[><]=?)' + +_repo_separator = "::" +_repo_name = r'[\w][\w-]*' +_repo_name_re = re.compile('^' + _repo_name + '$', re.UNICODE) +_repo = r'(?:' + _repo_separator + '(' + _repo_name + ')' + ')?' + +_extended_cat = r'[\w+*][\w+.*-]*' + +_slot_dep_re_cache = {} + +def _get_slot_dep_re(eapi_attrs): + cache_key = eapi_attrs.slot_operator + slot_re = _slot_dep_re_cache.get(cache_key) + if slot_re is not None: + return slot_re + + if eapi_attrs.slot_operator: + slot_re = _slot + r'?(\*|=|/' + _slot + r'=?)?' + else: + slot_re = _slot + + slot_re = re.compile('^' + slot_re + '$', re.VERBOSE | re.UNICODE) + + _slot_dep_re_cache[cache_key] = slot_re + return slot_re + +def _match_slot(atom, pkg): + if pkg.slot == atom.slot: + if not atom.sub_slot: + return True + elif atom.sub_slot == pkg.sub_slot: + return True + return False + +_atom_re_cache = {} + +def _get_atom_re(eapi_attrs): + cache_key = eapi_attrs.dots_in_PN + atom_re = _atom_re_cache.get(cache_key) + if atom_re is not None: + return atom_re + + if eapi_attrs.dots_in_PN: + cp_re = _cp['dots_allowed_in_PN'] + cpv_re = _cpv['dots_allowed_in_PN'] + else: + cp_re = _cp['dots_disallowed_in_PN'] + cpv_re = _cpv['dots_disallowed_in_PN'] + + atom_re = re.compile('^(?P<without_use>(?:' + + '(?P<op>' + _op + cpv_re + ')|' + + '(?P<star>=' + cpv_re + r'\*)|' + + '(?P<simple>' + cp_re + '))' + + '(' + _slot_separator + _slot_loose + ')?' + + _repo + ')(' + _use + ')?$', re.VERBOSE | re.UNICODE) + + _atom_re_cache[cache_key] = atom_re + return atom_re + +_atom_wildcard_re_cache = {} + +def _get_atom_wildcard_re(eapi_attrs): + cache_key = eapi_attrs.dots_in_PN + atom_re = _atom_wildcard_re_cache.get(cache_key) + if atom_re is not None: + return atom_re + + if eapi_attrs.dots_in_PN: + pkg_re = r'[\w+*][\w+.*-]*?' + else: + pkg_re = r'[\w+*][\w+*-]*?' + + atom_re = re.compile(r'((?P<simple>(' + + _extended_cat + r')/(' + pkg_re + r'(-' + _vr + ')?))' + \ + '|(?P<star>=((' + _extended_cat + r')/(' + pkg_re + r'))-(?P<version>\*\w+\*)))' + \ + '(:(?P<slot>' + _slot_loose + r'))?(' + + _repo_separator + r'(?P<repo>' + _repo_name + r'))?$', re.UNICODE) + + _atom_wildcard_re_cache[cache_key] = atom_re + return atom_re + +_usedep_re_cache = {} + +def _get_usedep_re(eapi_attrs): + """ + @param eapi_attrs: The EAPI attributes from _get_eapi_attrs + @type eapi_attrs: _eapi_attrs + @rtype: regular expression object + @return: A regular expression object that matches valid USE deps for the + given eapi. + """ + cache_key = eapi_attrs.dots_in_use_flags + usedep_re = _usedep_re_cache.get(cache_key) + if usedep_re is not None: + return usedep_re + + if eapi_attrs.dots_in_use_flags: + _flag_re = r'[A-Za-z0-9][A-Za-z0-9+_@.-]*' + else: + _flag_re = r'[A-Za-z0-9][A-Za-z0-9+_@-]*' + + usedep_re = re.compile(r'^(?P<prefix>[!-]?)(?P<flag>' + + _flag_re + r')(?P<default>(\(\+\)|\(\-\))?)(?P<suffix>[?=]?)$') + + _usedep_re_cache[cache_key] = usedep_re + return usedep_re -# Api consumers included in portage should set this to True. -# Once the relevant api changes are in a portage release with -# stable keywords, make these warnings unconditional. -_internal_warnings = False +_useflag_re_cache = {} + +def _get_useflag_re(eapi): + """ + When eapi is None then validation is not as strict, since we want the + same to work for multiple EAPIs that may have slightly different rules. + @param eapi: The EAPI + @type eapi: String or None + @rtype: regular expression object + @return: A regular expression object that matches valid USE flags for the + given eapi. + """ + eapi_attrs = _get_eapi_attrs(eapi) + cache_key = eapi_attrs.dots_in_use_flags + useflag_re = _useflag_re_cache.get(cache_key) + if useflag_re is not None: + return useflag_re + + if eapi_attrs.dots_in_use_flags: + flag_re = r'[A-Za-z0-9][A-Za-z0-9+_@.-]*' + else: + flag_re = r'[A-Za-z0-9][A-Za-z0-9+_@-]*' + + useflag_re = re.compile(r'^' + flag_re + r'$') + + _useflag_re_cache[cache_key] = useflag_re + return useflag_re def cpvequal(cpv1, cpv2): """ @@ -109,7 +238,7 @@ def strip_empty(myarr): ('portage.dep.strip_empty',), DeprecationWarning, stacklevel=2) return [x for x in myarr if x] -def paren_reduce(mystr): +def paren_reduce(mystr, _deprecation_warn=True): """ Take a string and convert all paren enclosed entities into sublists and split the list elements by spaces. All redundant brackets are removed. @@ -123,7 +252,7 @@ def paren_reduce(mystr): @rtype: Array @return: The reduced string in an array """ - if _internal_warnings: + if portage._internal_caller and _deprecation_warn: warnings.warn(_("%s is deprecated and will be removed without replacement.") % \ ('portage.dep.paren_reduce',), DeprecationWarning, stacklevel=2) mysplit = mystr.split() @@ -215,7 +344,7 @@ class paren_normalize(list): """Take a dependency structure as returned by paren_reduce or use_reduce and generate an equivalent structure that has no redundant lists.""" def __init__(self, src): - if _internal_warnings: + if portage._internal_caller: warnings.warn(_("%s is deprecated and will be removed without replacement.") % \ ('portage.dep.paren_normalize',), DeprecationWarning, stacklevel=2) list.__init__(self) @@ -250,7 +379,7 @@ class paren_normalize(list): self._zap_parens(x, dest) return dest -def paren_enclose(mylist, unevaluated_atom=False): +def paren_enclose(mylist, unevaluated_atom=False, opconvert=False): """ Convert a list to a string with sublists enclosed with parens. @@ -267,7 +396,10 @@ def paren_enclose(mylist, unevaluated_atom=False): mystrparts = [] for x in mylist: if isinstance(x, list): - mystrparts.append("( "+paren_enclose(x)+" )") + if opconvert and x and x[0] == "||": + mystrparts.append("%s ( %s )" % (x[0], paren_enclose(x[1:]))) + else: + mystrparts.append("( %s )" % paren_enclose(x)) else: if unevaluated_atom: x = getattr(x, 'unevaluated_atom', x) @@ -308,7 +440,7 @@ def use_reduce(depstr, uselist=[], masklist=[], matchall=False, excludeall=[], i @return: The use reduced depend array """ if isinstance(depstr, list): - if _internal_warnings: + if portage._internal_caller: warnings.warn(_("Passing paren_reduced dep arrays to %s is deprecated. " + \ "Pass the original dep string instead.") % \ ('portage.dep.use_reduce',), DeprecationWarning, stacklevel=2) @@ -320,6 +452,7 @@ def use_reduce(depstr, uselist=[], masklist=[], matchall=False, excludeall=[], i if matchall and matchnone: raise ValueError("portage.dep.use_reduce: 'matchall' and 'matchnone' are mutually exclusive") + eapi_attrs = _get_eapi_attrs(eapi) useflag_re = _get_useflag_re(eapi) def is_active(conditional): @@ -535,7 +668,7 @@ def use_reduce(depstr, uselist=[], masklist=[], matchall=False, excludeall=[], i if not is_src_uri: raise InvalidDependString( _("SRC_URI arrow are only allowed in SRC_URI: token %s") % (pos+1,)) - if eapi is None or not eapi_has_src_uri_arrows(eapi): + if not eapi_attrs.src_uri_arrows: raise InvalidDependString( _("SRC_URI arrow not allowed in EAPI %s: token %s") % (eapi, pos+1)) need_simple_token = True @@ -608,7 +741,7 @@ def dep_opconvert(deplist): @return: The new list with the new ordering """ - if _internal_warnings: + if portage._internal_caller: warnings.warn(_("%s is deprecated. Use %s with the opconvert parameter set to True instead.") % \ ('portage.dep.dep_opconvert', 'portage.dep.use_reduce'), DeprecationWarning, stacklevel=2) @@ -639,7 +772,7 @@ def flatten(mylist): @rtype: List @return: A single list containing only non-list elements. """ - if _internal_warnings: + if portage._internal_caller: warnings.warn(_("%s is deprecated and will be removed without replacement.") % \ ('portage.dep.flatten',), DeprecationWarning, stacklevel=2) @@ -651,30 +784,9 @@ def flatten(mylist): newlist.append(x) return newlist - -_usedep_re = { - "dots_disallowed_in_use_flags": re.compile("^(?P<prefix>[!-]?)(?P<flag>[A-Za-z0-9][A-Za-z0-9+_@-]*)(?P<default>(\(\+\)|\(\-\))?)(?P<suffix>[?=]?)$"), - "dots_allowed_in_use_flags": re.compile("^(?P<prefix>[!-]?)(?P<flag>[A-Za-z0-9][A-Za-z0-9+_@.-]*)(?P<default>(\(\+\)|\(\-\))?)(?P<suffix>[?=]?)$"), -} - -def _get_usedep_re(eapi): - """ - When eapi is None then validation is not as strict, since we want the - same to work for multiple EAPIs that may have slightly different rules. - @param eapi: The EAPI - @type eapi: String or None - @rtype: regular expression object - @return: A regular expression object that matches valid USE deps for the - given eapi. - """ - if eapi is None or eapi_allows_dots_in_use_flags(eapi): - return _usedep_re["dots_allowed_in_use_flags"] - else: - return _usedep_re["dots_disallowed_in_use_flags"] - class _use_dep(object): - __slots__ = ("__weakref__", "eapi", "conditional", "missing_enabled", "missing_disabled", + __slots__ = ("_eapi_attrs", "conditional", "missing_enabled", "missing_disabled", "disabled", "enabled", "tokens", "required") class _conditionals_class(object): @@ -700,10 +812,10 @@ class _use_dep(object): 'not_equal': '!%s=', } - def __init__(self, use, eapi, enabled_flags=None, disabled_flags=None, missing_enabled=None, \ + def __init__(self, use, eapi_attrs, enabled_flags=None, disabled_flags=None, missing_enabled=None, missing_disabled=None, conditional=None, required=None): - self.eapi = eapi + self._eapi_attrs = eapi_attrs if enabled_flags is not None: #A shortcut for the classe's own methods. @@ -732,7 +844,7 @@ class _use_dep(object): no_default = set() conditional = {} - usedep_re = _get_usedep_re(self.eapi) + usedep_re = _get_usedep_re(self._eapi_attrs) for x in use: m = usedep_re.match(x) @@ -800,6 +912,14 @@ class _use_dep(object): return "" return "[%s]" % (",".join(self.tokens),) + if sys.hexversion < 0x3000000: + + __unicode__ = __str__ + + def __str__(self): + return _unicode_encode(self.__unicode__(), + encoding=_encodings['content'], errors='backslashreplace') + def __repr__(self): return "portage.dep._use_dep(%s)" % repr(self.tokens) @@ -835,7 +955,7 @@ class _use_dep(object): disabled_flags = set(self.disabled) tokens = [] - usedep_re = _get_usedep_re(self.eapi) + usedep_re = _get_usedep_re(self._eapi_attrs) for x in self.tokens: m = usedep_re.match(x) @@ -871,7 +991,7 @@ class _use_dep(object): else: tokens.append(x) - return _use_dep(tokens, self.eapi, enabled_flags=enabled_flags, disabled_flags=disabled_flags, \ + return _use_dep(tokens, self._eapi_attrs, enabled_flags=enabled_flags, disabled_flags=disabled_flags, missing_enabled=self.missing_enabled, missing_disabled=self.missing_disabled, required=self.required) def violated_conditionals(self, other_use, is_valid_flag, parent_use=None): @@ -893,7 +1013,7 @@ class _use_dep(object): def validate_flag(flag): return is_valid_flag(flag) or flag in all_defaults - usedep_re = _get_usedep_re(self.eapi) + usedep_re = _get_usedep_re(self._eapi_attrs) for x in self.tokens: m = usedep_re.match(x) @@ -985,7 +1105,7 @@ class _use_dep(object): tokens.append(x) conditional.setdefault("disabled", set()).add(flag) - return _use_dep(tokens, self.eapi, enabled_flags=enabled_flags, disabled_flags=disabled_flags, \ + return _use_dep(tokens, self._eapi_attrs, enabled_flags=enabled_flags, disabled_flags=disabled_flags, missing_enabled=self.missing_enabled, missing_disabled=self.missing_disabled, \ conditional=conditional, required=self.required) @@ -1005,7 +1125,7 @@ class _use_dep(object): missing_disabled = self.missing_disabled tokens = [] - usedep_re = _get_usedep_re(self.eapi) + usedep_re = _get_usedep_re(self._eapi_attrs) for x in self.tokens: m = usedep_re.match(x) @@ -1041,15 +1161,10 @@ class _use_dep(object): else: tokens.append(x) - return _use_dep(tokens, self.eapi, enabled_flags=enabled_flags, disabled_flags=disabled_flags, \ + return _use_dep(tokens, self._eapi_attrs, enabled_flags=enabled_flags, disabled_flags=disabled_flags, missing_enabled=missing_enabled, missing_disabled=missing_disabled, required=self.required) -if sys.hexversion < 0x3000000: - _atom_base = unicode -else: - _atom_base = str - -class Atom(_atom_base): +class Atom(_unicode): """ For compatibility with existing atom string manipulation code, this @@ -1068,51 +1183,72 @@ class Atom(_atom_base): def __init__(self, forbid_overlap=False): self.overlap = self._overlap(forbid=forbid_overlap) - def __new__(cls, s, unevaluated_atom=None, allow_wildcard=False, allow_repo=False, + def __new__(cls, s, unevaluated_atom=None, allow_wildcard=False, allow_repo=None, _use=None, eapi=None, is_valid_flag=None): - return _atom_base.__new__(cls, s) + return _unicode.__new__(cls, s) - def __init__(self, s, unevaluated_atom=None, allow_wildcard=False, allow_repo=False, + def __init__(self, s, unevaluated_atom=None, allow_wildcard=False, allow_repo=None, _use=None, eapi=None, is_valid_flag=None): if isinstance(s, Atom): # This is an efficiency assertion, to ensure that the Atom # constructor is not called redundantly. raise TypeError(_("Expected %s, got %s") % \ - (_atom_base, type(s))) + (_unicode, type(s))) - if not isinstance(s, _atom_base): - # Avoid TypeError from _atom_base.__init__ with PyPy. + if not isinstance(s, _unicode): + # Avoid TypeError from _unicode.__init__ with PyPy. s = _unicode_decode(s) - _atom_base.__init__(s) + _unicode.__init__(s) + + eapi_attrs = _get_eapi_attrs(eapi) + atom_re = _get_atom_re(eapi_attrs) - atom_re = _get_atom_re(eapi) - if eapi_has_repo_deps(eapi): - allow_repo = True + self.__dict__['eapi'] = eapi + if eapi is not None: + # Ignore allow_repo when eapi is specified. + allow_repo = eapi_attrs.repo_deps + else: + if allow_repo is None: + allow_repo = True + blocker_prefix = "" if "!" == s[:1]: blocker = self._blocker(forbid_overlap=("!" == s[1:2])) if blocker.overlap.forbid: + blocker_prefix = s[:2] s = s[2:] else: + blocker_prefix = s[:1] s = s[1:] else: blocker = False self.__dict__['blocker'] = blocker m = atom_re.match(s) extended_syntax = False + extended_version = None if m is None: if allow_wildcard: - m = _get_atom_wildcard_re(eapi).match(s) + atom_re = _get_atom_wildcard_re(eapi_attrs) + m = atom_re.match(s) if m is None: raise InvalidAtom(self) - op = None gdict = m.groupdict() - cpv = cp = gdict['simple'] + if m.group('star') is not None: + op = '=*' + base = atom_re.groupindex['star'] + cp = m.group(base + 1) + cpv = m.group('star')[1:] + extended_version = m.group(base + 4) + else: + op = None + cpv = cp = m.group('simple') + if m.group(atom_re.groupindex['simple'] + 3) is not None: + raise InvalidAtom(self) if cpv.find("**") != -1: raise InvalidAtom(self) - slot = gdict['slot'] - repo = gdict['repo'] + slot = m.group('slot') + repo = m.group('repo') use_str = None extended_syntax = True else: @@ -1155,9 +1291,38 @@ class Atom(_atom_base): except InvalidData: # plain cp, wildcard, or something self.__dict__['cpv'] = cpv - self.__dict__['version'] = None + self.__dict__['version'] = extended_version self.__dict__['repo'] = repo - self.__dict__['slot'] = slot + if slot is None: + self.__dict__['slot'] = None + self.__dict__['sub_slot'] = None + self.__dict__['slot_operator'] = None + else: + slot_re = _get_slot_dep_re(eapi_attrs) + slot_match = slot_re.match(slot) + if slot_match is None: + raise InvalidAtom(self) + if eapi_attrs.slot_operator: + self.__dict__['slot'] = slot_match.group(1) + sub_slot = slot_match.group(2) + if sub_slot is not None: + sub_slot = sub_slot.lstrip("/") + if sub_slot in ("*", "="): + self.__dict__['sub_slot'] = None + self.__dict__['slot_operator'] = sub_slot + else: + slot_operator = None + if sub_slot is not None and sub_slot[-1:] == "=": + slot_operator = sub_slot[-1:] + sub_slot = sub_slot[:-1] + self.__dict__['sub_slot'] = sub_slot + self.__dict__['slot_operator'] = slot_operator + if self.slot is not None and self.slot_operator == "*": + raise InvalidAtom(self) + else: + self.__dict__['slot'] = slot + self.__dict__['sub_slot'] = None + self.__dict__['slot_operator'] = None self.__dict__['operator'] = op self.__dict__['extended_syntax'] = extended_syntax @@ -1168,16 +1333,19 @@ class Atom(_atom_base): if _use is not None: use = _use else: - use = _use_dep(use_str[1:-1].split(","), eapi) - without_use = Atom(m.group('without_use'), allow_repo=allow_repo) + use = _use_dep(use_str[1:-1].split(","), eapi_attrs) + without_use = Atom(blocker_prefix + m.group('without_use'), + allow_repo=allow_repo) else: use = None if unevaluated_atom is not None and \ unevaluated_atom.use is not None: # unevaluated_atom.use is used for IUSE checks when matching # packages, so it must not propagate to without_use - without_use = Atom(s, allow_wildcard=allow_wildcard, - allow_repo=allow_repo) + without_use = Atom(_unicode(self), + allow_wildcard=allow_wildcard, + allow_repo=allow_repo, + eapi=eapi) else: without_use = self @@ -1193,16 +1361,16 @@ class Atom(_atom_base): if not isinstance(eapi, basestring): raise TypeError('expected eapi argument of ' + \ '%s, got %s: %s' % (basestring, type(eapi), eapi,)) - if self.slot and not eapi_has_slot_deps(eapi): + if self.slot and not eapi_attrs.slot_deps: raise InvalidAtom( _("Slot deps are not allowed in EAPI %s: '%s'") \ % (eapi, self), category='EAPI.incompatible') if self.use: - if not eapi_has_use_deps(eapi): + if not eapi_attrs.use_deps: raise InvalidAtom( _("Use deps are not allowed in EAPI %s: '%s'") \ % (eapi, self), category='EAPI.incompatible') - elif not eapi_has_use_dep_defaults(eapi) and \ + elif not eapi_attrs.use_dep_defaults and \ (self.use.missing_enabled or self.use.missing_disabled): raise InvalidAtom( _("Use dep defaults are not allowed in EAPI %s: '%s'") \ @@ -1225,12 +1393,21 @@ class Atom(_atom_base): "conditional '%s' in atom '%s' is not in IUSE") \ % (flag, conditional_str % flag, self) raise InvalidAtom(msg, category='IUSE.missing') - if self.blocker and self.blocker.overlap.forbid and not eapi_has_strong_blocks(eapi): + if self.blocker and self.blocker.overlap.forbid and not eapi_attrs.strong_blocks: raise InvalidAtom( _("Strong blocks are not allowed in EAPI %s: '%s'") \ % (eapi, self), category='EAPI.incompatible') @property + def slot_operator_built(self): + """ + Returns True if slot_operator == "=" and sub_slot is not None. + NOTE: foo/bar:2= is unbuilt and returns False, whereas foo/bar:2/2= + is built and returns True. + """ + return self.slot_operator == "=" and self.sub_slot is not None + + @property def without_repo(self): if self.repo is None: return self @@ -1239,18 +1416,29 @@ class Atom(_atom_base): @property def without_slot(self): - if self.slot is None: + if self.slot is None and self.slot_operator is None: return self - return Atom(self.replace(_slot_separator + self.slot, '', 1), + atom = remove_slot(self) + if self.repo is not None: + atom += _repo_separator + self.repo + if self.use is not None: + atom += _unicode(self.use) + return Atom(atom, allow_repo=True, allow_wildcard=True) def with_repo(self, repo): atom = remove_slot(self) - if self.slot is not None: - atom += _slot_separator + self.slot + if self.slot is not None or self.slot_operator is not None: + atom += _slot_separator + if self.slot is not None: + atom += self.slot + if self.sub_slot is not None: + atom += "/%s" % self.sub_slot + if self.slot_operator is not None: + atom += self.slot_operator atom += _repo_separator + repo if self.use is not None: - atom += str(self.use) + atom += _unicode(self.use) return Atom(atom, allow_repo=True, allow_wildcard=True) def with_slot(self, slot): @@ -1258,7 +1446,7 @@ class Atom(_atom_base): if self.repo is not None: atom += _repo_separator + self.repo if self.use is not None: - atom += str(self.use) + atom += _unicode(self.use) return Atom(atom, allow_repo=True, allow_wildcard=True) def __setattr__(self, name, value): @@ -1307,10 +1495,16 @@ class Atom(_atom_base): if not (self.use and self.use.conditional): return self atom = remove_slot(self) - if self.slot: - atom += ":%s" % self.slot + if self.slot is not None or self.slot_operator is not None: + atom += _slot_separator + if self.slot is not None: + atom += self.slot + if self.sub_slot is not None: + atom += "/%s" % self.sub_slot + if self.slot_operator is not None: + atom += self.slot_operator use_dep = self.use.evaluate_conditionals(use) - atom += str(use_dep) + atom += _unicode(use_dep) return Atom(atom, unevaluated_atom=self, allow_repo=(self.repo is not None), _use=use_dep) def violated_conditionals(self, other_use, is_valid_flag, parent_use=None): @@ -1329,20 +1523,32 @@ class Atom(_atom_base): if not self.use: return self atom = remove_slot(self) - if self.slot: - atom += ":%s" % self.slot + if self.slot is not None or self.slot_operator is not None: + atom += _slot_separator + if self.slot is not None: + atom += self.slot + if self.sub_slot is not None: + atom += "/%s" % self.sub_slot + if self.slot_operator is not None: + atom += self.slot_operator use_dep = self.use.violated_conditionals(other_use, is_valid_flag, parent_use) - atom += str(use_dep) + atom += _unicode(use_dep) return Atom(atom, unevaluated_atom=self, allow_repo=(self.repo is not None), _use=use_dep) def _eval_qa_conditionals(self, use_mask, use_force): if not (self.use and self.use.conditional): return self atom = remove_slot(self) - if self.slot: - atom += ":%s" % self.slot + if self.slot is not None or self.slot_operator is not None: + atom += _slot_separator + if self.slot is not None: + atom += self.slot + if self.sub_slot is not None: + atom += "/%s" % self.sub_slot + if self.slot_operator is not None: + atom += self.slot_operator use_dep = self.use._eval_qa_conditionals(use_mask, use_force) - atom += str(use_dep) + atom += _unicode(use_dep) return Atom(atom, unevaluated_atom=self, allow_repo=(self.repo is not None), _use=use_dep) def __copy__(self): @@ -1366,7 +1572,7 @@ def extended_cp_match(extended_cp, other_cp): extended_cp_re = _extended_cp_re_cache.get(extended_cp) if extended_cp_re is None: extended_cp_re = re.compile("^" + re.escape(extended_cp).replace( - r'\*', '[^/]*') + "$") + r'\*', '[^/]*') + "$", re.UNICODE) _extended_cp_re_cache[extended_cp] = extended_cp_re return extended_cp_re.match(other_cp) is not None @@ -1644,77 +1850,8 @@ def dep_getusedeps( depend ): open_bracket = depend.find( '[', open_bracket+1 ) return tuple(use_list) -# \w is [a-zA-Z0-9_] - -# 2.1.3 A slot name may contain any of the characters [A-Za-z0-9+_.-]. -# It must not begin with a hyphen or a dot. -_slot_separator = ":" -_slot = r'([\w+][\w+.-]*)' -_slot_re = re.compile('^' + _slot + '$', re.VERBOSE) - -_use = r'\[.*\]' -_op = r'([=~]|[><]=?)' -_repo_separator = "::" -_repo_name = r'[\w][\w-]*' -_repo = r'(?:' + _repo_separator + '(' + _repo_name + ')' + ')?' - -_atom_re = { - "dots_disallowed_in_PN": re.compile('^(?P<without_use>(?:' + - '(?P<op>' + _op + _cpv['dots_disallowed_in_PN'] + ')|' + - '(?P<star>=' + _cpv['dots_disallowed_in_PN'] + r'\*)|' + - '(?P<simple>' + _cp['dots_disallowed_in_PN'] + '))' + - '(' + _slot_separator + _slot + ')?' + _repo + ')(' + _use + ')?$', re.VERBOSE), - "dots_allowed_in_PN": re.compile('^(?P<without_use>(?:' + - '(?P<op>' + _op + _cpv['dots_allowed_in_PN'] + ')|' + - '(?P<star>=' + _cpv['dots_allowed_in_PN'] + r'\*)|' + - '(?P<simple>' + _cp['dots_allowed_in_PN'] + '))' + - '(' + _slot_separator + _slot + ')?' + _repo + ')(' + _use + ')?$', re.VERBOSE), -} - -def _get_atom_re(eapi): - if eapi is None or eapi_allows_dots_in_PN(eapi): - return _atom_re["dots_allowed_in_PN"] - else: - return _atom_re["dots_disallowed_in_PN"] - -_extended_cat = r'[\w+*][\w+.*-]*' -_extended_pkg = { - "dots_disallowed_in_PN": r'[\w+*][\w+*-]*?', - "dots_allowed_in_PN": r'[\w+*][\w+.*-]*?', -} - -_atom_wildcard_re = { - "dots_disallowed_in_PN": re.compile('(?P<simple>(' + _extended_cat + ')/(' + _extended_pkg['dots_disallowed_in_PN'] + '))(:(?P<slot>' + _slot + '))?(' + _repo_separator + '(?P<repo>' + _repo_name + '))?$'), - "dots_allowed_in_PN": re.compile('(?P<simple>(' + _extended_cat + ')/(' + _extended_pkg['dots_allowed_in_PN'] + '))(:(?P<slot>' + _slot + '))?(' + _repo_separator + '(?P<repo>' + _repo_name + '))?$'), -} - -def _get_atom_wildcard_re(eapi): - if eapi is None or eapi_allows_dots_in_PN(eapi): - return _atom_wildcard_re["dots_allowed_in_PN"] - else: - return _atom_wildcard_re["dots_disallowed_in_PN"] - -_useflag_re = { - "dots_disallowed_in_use_flags": re.compile(r'^[A-Za-z0-9][A-Za-z0-9+_@-]*$'), - "dots_allowed_in_use_flags": re.compile(r'^[A-Za-z0-9][A-Za-z0-9+_@.-]*$'), -} - -def _get_useflag_re(eapi): - """ - When eapi is None then validation is not as strict, since we want the - same to work for multiple EAPIs that may have slightly different rules. - @param eapi: The EAPI - @type eapi: String or None - @rtype: regular expression object - @return: A regular expression object that matches valid USE flags for the - given eapi. - """ - if eapi is None or eapi_allows_dots_in_use_flags(eapi): - return _useflag_re["dots_allowed_in_use_flags"] - else: - return _useflag_re["dots_disallowed_in_use_flags"] - -def isvalidatom(atom, allow_blockers=False, allow_wildcard=False, allow_repo=False): +def isvalidatom(atom, allow_blockers=False, allow_wildcard=False, + allow_repo=False, eapi=None): """ Check to see if a depend atom is valid @@ -1731,9 +1868,15 @@ def isvalidatom(atom, allow_blockers=False, allow_wildcard=False, allow_repo=Fal 1) False if the atom is invalid 2) True if the atom is valid """ + + if eapi is not None and isinstance(atom, Atom) and atom.eapi != eapi: + # We'll construct a new atom with the given eapi. + atom = _unicode(atom) + try: if not isinstance(atom, Atom): - atom = Atom(atom, allow_wildcard=allow_wildcard, allow_repo=allow_repo) + atom = Atom(atom, allow_wildcard=allow_wildcard, + allow_repo=allow_repo, eapi=eapi) if not allow_blockers and atom.blocker: return False return True @@ -1859,19 +2002,23 @@ def best_match_to_list(mypkg, mylist): """ operator_values = {'=':6, '~':5, '=*':4, '>':2, '<':2, '>=':2, '<=':2, None:1} - maxvalue = -2 + maxvalue = -99 bestm = None mypkg_cpv = None for x in match_to_list(mypkg, mylist): if x.extended_syntax: - if dep_getslot(x) is not None: + if x.operator == '=*': if maxvalue < 0: maxvalue = 0 bestm = x - else: + elif x.slot is not None: if maxvalue < -1: maxvalue = -1 bestm = x + else: + if maxvalue < -2: + maxvalue = -2 + bestm = x continue if dep_getslot(x) is not None: if maxvalue < 3: @@ -1955,7 +2102,8 @@ def match_from_list(mydep, candidate_list): mylist = [] - if operator is None: + if mydep.extended_syntax: + for x in candidate_list: cp = getattr(x, "cp", None) if cp is None: @@ -1966,8 +2114,38 @@ def match_from_list(mydep, candidate_list): if cp is None: continue - if cp == mycpv or (mydep.extended_syntax and \ - extended_cp_match(mydep.cp, cp)): + if cp == mycpv or extended_cp_match(mydep.cp, cp): + mylist.append(x) + + if mylist and mydep.operator == "=*": + + candidate_list = mylist + mylist = [] + # Currently, only \*\w+\* is supported. + ver = mydep.version[1:-1] + + for x in candidate_list: + x_ver = getattr(x, "version", None) + if x_ver is None: + xs = catpkgsplit(remove_slot(x)) + if xs is None: + continue + x_ver = "-".join(xs[-2:]) + if ver in x_ver: + mylist.append(x) + + elif operator is None: + for x in candidate_list: + cp = getattr(x, "cp", None) + if cp is None: + mysplit = catpkgsplit(remove_slot(x)) + if mysplit is not None: + cp = mysplit[0] + '/' + mysplit[1] + + if cp is None: + continue + + if cp == mydep.cp: mylist.append(x) elif operator == "=": # Exact match @@ -1983,19 +2161,40 @@ def match_from_list(mydep, candidate_list): # XXX: Nasty special casing for leading zeros # Required as =* is a literal prefix match, so can't # use vercmp - mysplit = catpkgsplit(mycpv) - myver = mysplit[2].lstrip("0") + myver = mycpv_cps[2].lstrip("0") if not myver or not myver[0].isdigit(): myver = "0"+myver - mycpv_cmp = mysplit[0]+"/"+mysplit[1]+"-"+myver + if myver == mycpv_cps[2]: + mycpv_cmp = mycpv + else: + # Use replace to preserve the revision part if it exists + # (mycpv_cps[3] can't be trusted because in contains r0 + # even when the input has no revision part). + mycpv_cmp = mycpv.replace( + mydep.cp + "-" + mycpv_cps[2], + mydep.cp + "-" + myver, 1) for x in candidate_list: - xs = getattr(x, "cpv_split", None) - if xs is None: - xs = catpkgsplit(remove_slot(x)) + try: + x.cp + except AttributeError: + try: + pkg = _pkg_str(remove_slot(x)) + except InvalidData: + continue + else: + pkg = x + + xs = pkg.cpv_split myver = xs[2].lstrip("0") if not myver or not myver[0].isdigit(): myver = "0"+myver - xcpv = xs[0]+"/"+xs[1]+"-"+myver + if myver == xs[2]: + xcpv = pkg.cpv + else: + # Use replace to preserve the revision part if it exists. + xcpv = pkg.cpv.replace( + pkg.cp + "-" + xs[2], + pkg.cp + "-" + myver, 1) if xcpv.startswith(mycpv_cmp): mylist.append(x) @@ -2048,16 +2247,33 @@ def match_from_list(mydep, candidate_list): else: raise KeyError(_("Unknown operator: %s") % mydep) - if slot is not None and not mydep.extended_syntax: + if mydep.slot is not None: candidate_list = mylist mylist = [] for x in candidate_list: - xslot = getattr(x, "slot", False) - if xslot is False: + x_pkg = None + try: + x.cpv + except AttributeError: xslot = dep_getslot(x) - if xslot is not None and xslot != slot: - continue - mylist.append(x) + if xslot is not None: + try: + x_pkg = _pkg_str(remove_slot(x), slot=xslot) + except InvalidData: + continue + else: + x_pkg = x + + if x_pkg is None: + mylist.append(x) + else: + try: + x_pkg.slot + except AttributeError: + mylist.append(x) + else: + if _match_slot(mydep, x_pkg): + mylist.append(x) if mydep.unevaluated_atom.use: candidate_list = mylist @@ -2071,26 +2287,26 @@ def match_from_list(mydep, candidate_list): continue if mydep.use: - - missing_enabled = mydep.use.missing_enabled.difference(x.iuse.all) - missing_disabled = mydep.use.missing_disabled.difference(x.iuse.all) + is_valid_flag = x.iuse.is_valid_flag + missing_enabled = frozenset(flag for flag in + mydep.use.missing_enabled if not is_valid_flag(flag)) + missing_disabled = frozenset(flag for flag in + mydep.use.missing_disabled if not is_valid_flag(flag)) if mydep.use.enabled: - if mydep.use.enabled.intersection(missing_disabled): + if any(f in mydep.use.enabled for f in missing_disabled): continue need_enabled = mydep.use.enabled.difference(use.enabled) if need_enabled: - need_enabled = need_enabled.difference(missing_enabled) - if need_enabled: + if any(f not in missing_enabled for f in need_enabled): continue if mydep.use.disabled: - if mydep.use.disabled.intersection(missing_enabled): + if any(f in mydep.use.disabled for f in missing_enabled): continue need_disabled = mydep.use.disabled.intersection(use.enabled) if need_disabled: - need_disabled = need_disabled.difference(missing_disabled) - if need_disabled: + if any(f not in missing_disabled for f in need_disabled): continue mylist.append(x) @@ -2110,9 +2326,9 @@ def match_from_list(mydep, candidate_list): return mylist def human_readable_required_use(required_use): - return required_use.replace("^^", "exactly-one-of").replace("||", "any-of") + return required_use.replace("^^", "exactly-one-of").replace("||", "any-of").replace("??", "at-most-one-of") -def get_required_use_flags(required_use): +def get_required_use_flags(required_use, eapi=None): """ Returns a set of use flags that are used in the given REQUIRED_USE string @@ -2122,6 +2338,12 @@ def get_required_use_flags(required_use): @return: Set of use flags that are used in the given REQUIRED_USE string """ + eapi_attrs = _get_eapi_attrs(eapi) + if eapi_attrs.required_use_at_most_one_of: + valid_operators = ("||", "^^", "??") + else: + valid_operators = ("||", "^^") + mysplit = required_use.split() level = 0 stack = [[]] @@ -2150,7 +2372,7 @@ def get_required_use_flags(required_use): l = stack.pop() ignore = False if stack[level]: - if stack[level][-1] in ("||", "^^") or \ + if stack[level][-1] in valid_operators or \ (not isinstance(stack[level][-1], bool) and \ stack[level][-1][-1] == "?"): ignore = True @@ -2162,15 +2384,14 @@ def get_required_use_flags(required_use): else: raise InvalidDependString( _("malformed syntax: '%s'") % required_use) - elif token in ("||", "^^"): + elif token in valid_operators: if need_bracket: raise InvalidDependString( _("malformed syntax: '%s'") % required_use) need_bracket = True stack[level].append(token) else: - if need_bracket or "(" in token or ")" in token or \ - "|" in token or "^" in token: + if need_bracket: raise InvalidDependString( _("malformed syntax: '%s'") % required_use) @@ -2225,7 +2446,7 @@ class _RequiredUseBranch(object): complex_nesting = False node = self while node != None and not complex_nesting: - if node._operator in ("||", "^^"): + if node._operator in ("||", "^^", "??"): complex_nesting = True else: node = node._parent @@ -2246,7 +2467,7 @@ class _RequiredUseBranch(object): if sys.hexversion < 0x3000000: __nonzero__ = __bool__ -def check_required_use(required_use, use, iuse_match): +def check_required_use(required_use, use, iuse_match, eapi=None): """ Checks if the use flags listed in 'use' satisfy all constraints specified in 'constraints'. @@ -2262,6 +2483,12 @@ def check_required_use(required_use, use, iuse_match): @return: Indicates if REQUIRED_USE constraints are satisfied """ + eapi_attrs = _get_eapi_attrs(eapi) + if eapi_attrs.required_use_at_most_one_of: + valid_operators = ("||", "^^", "??") + else: + valid_operators = ("||", "^^") + def is_active(token): if token.startswith("!"): flag = token[1:] @@ -2271,6 +2498,11 @@ def check_required_use(required_use, use, iuse_match): is_negated = False if not flag or not iuse_match(flag): + if not eapi_attrs.required_use_at_most_one_of and flag == "?": + msg = _("Operator '??' is not supported with EAPI '%s'") \ + % (eapi,) + e = InvalidData(msg, category='EAPI.incompatible') + raise InvalidDependString(msg, errors=(e,)) msg = _("USE flag '%s' is not in IUSE") \ % (flag,) e = InvalidData(msg, category='IUSE.missing') @@ -2288,6 +2520,8 @@ def check_required_use(required_use, use, iuse_match): return (True in argument) elif operator == "^^": return (argument.count(True) == 1) + elif operator == "??": + return (argument.count(True) <= 1) elif operator[-1] == "?": return (False not in argument) @@ -2317,7 +2551,7 @@ def check_required_use(required_use, use, iuse_match): l = stack.pop() op = None if stack[level]: - if stack[level][-1] in ("||", "^^"): + if stack[level][-1] in valid_operators: op = stack[level].pop() satisfied = is_satisfied(op, l) stack[level].append(satisfied) @@ -2346,7 +2580,7 @@ def check_required_use(required_use, use, iuse_match): stack[level].append(satisfied) if len(node._children) <= 1 or \ - node._parent._operator not in ("||", "^^"): + node._parent._operator not in valid_operators: last_node = node._parent._children.pop() if last_node is not node: raise AssertionError( @@ -2362,7 +2596,7 @@ def check_required_use(required_use, use, iuse_match): raise AssertionError( "node is not last child of parent") - elif len(node._children) == 1 and op in ("||", "^^"): + elif len(node._children) == 1 and op in valid_operators: last_node = node._parent._children.pop() if last_node is not node: raise AssertionError( @@ -2372,7 +2606,7 @@ def check_required_use(required_use, use, iuse_match): node._children[0]._parent = node._parent node = node._children[0] if node._operator is None and \ - node._parent._operator not in ("||", "^^"): + node._parent._operator not in valid_operators: last_node = node._parent._children.pop() if last_node is not node: raise AssertionError( @@ -2386,7 +2620,7 @@ def check_required_use(required_use, use, iuse_match): else: raise InvalidDependString( _("malformed syntax: '%s'") % required_use) - elif token in ("||", "^^"): + elif token in valid_operators: if need_bracket: raise InvalidDependString( _("malformed syntax: '%s'") % required_use) @@ -2396,8 +2630,7 @@ def check_required_use(required_use, use, iuse_match): node._children.append(child) node = child else: - if need_bracket or "(" in token or ")" in token or \ - "|" in token or "^" in token: + if need_bracket: raise InvalidDependString( _("malformed syntax: '%s'") % required_use) @@ -2425,16 +2658,16 @@ def extract_affecting_use(mystr, atom, eapi=None): that decide if the given atom is in effect. Example usage: - >>> extract_use_cond('sasl? ( dev-libs/cyrus-sasl ) \ + >>> extract_affecting_use('sasl? ( dev-libs/cyrus-sasl ) \ !minimal? ( cxx? ( dev-libs/cyrus-sasl ) )', 'dev-libs/cyrus-sasl') - (['sasl', 'minimal', 'cxx']) + {'cxx', 'minimal', 'sasl'} - @param dep: The dependency string + @param mystr: The dependency string @type mystr: String @param atom: The atom to get into effect @type atom: String - @rtype: Tuple of two lists of strings - @return: List of use flags that need to be enabled, List of use flag that need to be disabled + @rtype: Set of strings + @return: Set of use flags affecting given atom """ useflag_re = _get_useflag_re(eapi) mysplit = mystr.split() @@ -2540,3 +2773,48 @@ def extract_affecting_use(mystr, atom, eapi=None): _("malformed syntax: '%s'") % mystr) return affecting_use + +def extract_unpack_dependencies(src_uri, unpackers): + """ + Return unpack dependencies string for given SRC_URI string. + + @param src_uri: SRC_URI string + @type src_uri: String + @param unpackers: Dictionary mapping archive suffixes to dependency strings + @type unpackers: Dictionary + @rtype: String + @return: Dependency string specifying packages required to unpack archives. + """ + src_uri = src_uri.split() + + depend = [] + for i in range(len(src_uri)): + if src_uri[i][-1] == "?" or src_uri[i] in ("(", ")"): + depend.append(src_uri[i]) + elif (i+1 < len(src_uri) and src_uri[i+1] == "->") or src_uri[i] == "->": + continue + else: + for suffix in sorted(unpackers, key=lambda x: len(x), reverse=True): + suffix = suffix.lower() + if src_uri[i].lower().endswith(suffix): + depend.append(unpackers[suffix]) + break + + while True: + cleaned_depend = depend[:] + for i in range(len(cleaned_depend)): + if cleaned_depend[i] is None: + continue + elif cleaned_depend[i] == "(" and cleaned_depend[i+1] == ")": + cleaned_depend[i] = None + cleaned_depend[i+1] = None + elif cleaned_depend[i][-1] == "?" and cleaned_depend[i+1] == "(" and cleaned_depend[i+2] == ")": + cleaned_depend[i] = None + cleaned_depend[i+1] = None + cleaned_depend[i+2] = None + if depend == cleaned_depend: + break + else: + depend = [x for x in cleaned_depend if x is not None] + + return " ".join(depend) diff --git a/portage_with_autodep/pym/portage/dep/__init__.pyo b/portage_with_autodep/pym/portage/dep/__init__.pyo Binary files differindex c78bb23..515ec01 100644 --- a/portage_with_autodep/pym/portage/dep/__init__.pyo +++ b/portage_with_autodep/pym/portage/dep/__init__.pyo diff --git a/portage_with_autodep/pym/portage/dep/dep_check.py b/portage_with_autodep/pym/portage/dep/dep_check.py index 99a5eb0..b5ace3d 100644 --- a/portage_with_autodep/pym/portage/dep/dep_check.py +++ b/portage_with_autodep/pym/portage/dep/dep_check.py @@ -1,16 +1,19 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ['dep_check', 'dep_eval', 'dep_wordreduce', 'dep_zapdeps'] import logging +import operator import portage -from portage import _unicode_decode from portage.dep import Atom, match_from_list, use_reduce from portage.exception import InvalidDependString, ParseError from portage.localization import _ from portage.util import writemsg, writemsg_level +from portage.util.SlotObject import SlotObject from portage.versions import vercmp, _pkg_str def _expand_new_virtuals(mysplit, edebug, mydbapi, mysettings, myroot="/", @@ -160,7 +163,7 @@ def _expand_new_virtuals(mysplit, edebug, mydbapi, mysettings, myroot="/", # According to GLEP 37, RDEPEND is the only dependency # type that is valid for new-style virtuals. Repoman # should enforce this. - depstring = pkg.metadata['RDEPEND'] + depstring = pkg._metadata['RDEPEND'] pkg_kwargs = kwargs.copy() pkg_kwargs["myuse"] = pkg_use_enabled(pkg) if edebug: @@ -183,7 +186,7 @@ def _expand_new_virtuals(mysplit, edebug, mydbapi, mysettings, myroot="/", del mytrees["virt_parent"] if not mycheck[0]: - raise ParseError(_unicode_decode("%s: %s '%s'") % \ + raise ParseError("%s: %s '%s'" % \ (pkg, mycheck[1], depstring)) # pull in the new-style virtual @@ -254,6 +257,10 @@ def dep_eval(deplist): return 0 return 1 +class _dep_choice(SlotObject): + __slots__ = ('atoms', 'slot_map', 'cp_map', 'all_available', + 'all_installed_slots') + def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): """ Takes an unreduced and reduced deplist and removes satisfied dependencies. @@ -316,6 +323,7 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): priority = trees[myroot].get("priority") graph_db = trees[myroot].get("graph_db") graph = trees[myroot].get("graph") + want_update_pkg = trees[myroot].get("want_update_pkg") vardb = None if "vartree" in trees[myroot]: vardb = trees[myroot]["vartree"].dbapi @@ -324,6 +332,13 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): else: mydbapi = trees[myroot]["porttree"].dbapi + try: + mydbapi_match_pkgs = mydbapi.match_pkgs + except AttributeError: + def mydbapi_match_pkgs(atom): + return [mydbapi._pkg_str(cpv, atom.repo) + for cpv in mydbapi.match(atom)] + # Sort the deps into installed, not installed but already # in the graph and other, not installed and not in the graph # and other, with values of [[required_atom], availablility] @@ -347,24 +362,17 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): continue # Ignore USE dependencies here since we don't want USE # settings to adversely affect || preference evaluation. - avail_pkg = mydbapi.match(atom.without_use) + avail_pkg = mydbapi_match_pkgs(atom.without_use) if avail_pkg: avail_pkg = avail_pkg[-1] # highest (ascending order) - try: - slot = avail_pkg.slot - except AttributeError: - eapi, slot, repo = mydbapi.aux_get(avail_pkg, - ["EAPI", "SLOT", "repository"]) - avail_pkg = _pkg_str(avail_pkg, eapi=eapi, - slot=slot, repo=repo) - avail_slot = Atom("%s:%s" % (atom.cp, slot)) + avail_slot = Atom("%s:%s" % (atom.cp, avail_pkg.slot)) if not avail_pkg: all_available = False all_use_satisfied = False break if atom.use: - avail_pkg_use = mydbapi.match(atom) + avail_pkg_use = mydbapi_match_pkgs(atom) if not avail_pkg_use: all_use_satisfied = False else: @@ -372,13 +380,7 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): avail_pkg_use = avail_pkg_use[-1] if avail_pkg_use != avail_pkg: avail_pkg = avail_pkg_use - try: - slot = avail_pkg.slot - except AttributeError: - eapi, slot, repo = mydbapi.aux_get(avail_pkg, - ["EAPI", "SLOT", "repository"]) - avail_pkg = _pkg_str(avail_pkg, - eapi=eapi, slot=slot, repo=repo) + avail_slot = Atom("%s:%s" % (atom.cp, avail_pkg.slot)) slot_map[avail_slot] = avail_pkg highest_cpv = cp_map.get(avail_pkg.cp) @@ -386,7 +388,9 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): vercmp(avail_pkg.version, highest_cpv.version) > 0: cp_map[avail_pkg.cp] = avail_pkg - this_choice = (atoms, slot_map, cp_map, all_available) + this_choice = _dep_choice(atoms=atoms, slot_map=slot_map, + cp_map=cp_map, all_available=all_available, + all_installed_slots=False) if all_available: # The "all installed" criterion is not version or slot specific. # If any version of a package is already in the graph then we @@ -407,6 +411,7 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): not slot_atom.startswith("virtual/"): all_installed_slots = False break + this_choice.all_installed_slots = all_installed_slots if graph_db is None: if all_use_satisfied: if all_installed: @@ -468,8 +473,27 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): elif all_installed: if all_installed_slots: preferred_installed.append(this_choice) - else: + elif parent is None or want_update_pkg is None: preferred_any_slot.append(this_choice) + else: + # When appropriate, prefer a slot that is not + # installed yet for bug #478188. + want_update = True + for slot_atom, avail_pkg in slot_map.items(): + if avail_pkg in graph: + continue + # New-style virtuals have zero cost to install. + if slot_atom.startswith("virtual/") or \ + vardb.match(slot_atom): + continue + if not want_update_pkg(parent, avail_pkg): + want_update = False + break + + if want_update: + preferred_installed.append(this_choice) + else: + preferred_any_slot.append(this_choice) else: preferred_non_installed.append(this_choice) else: @@ -490,6 +514,7 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): all_installed = False if all_installed: + this_choice.all_installed_slots = True other_installed.append(this_choice) elif some_installed: other_installed_some.append(this_choice) @@ -506,22 +531,23 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): for choices in choice_bins: if len(choices) < 2: continue + # Prefer choices with all_installed_slots for bug #480736. + choices.sort(key=operator.attrgetter('all_installed_slots'), + reverse=True) for choice_1 in choices[1:]: - atoms_1, slot_map_1, cp_map_1, all_available_1 = choice_1 - cps = set(cp_map_1) + cps = set(choice_1.cp_map) for choice_2 in choices: if choice_1 is choice_2: # choice_1 will not be promoted, so move on break - atoms_2, slot_map_2, cp_map_2, all_available_2 = choice_2 - intersecting_cps = cps.intersection(cp_map_2) + intersecting_cps = cps.intersection(choice_2.cp_map) if not intersecting_cps: continue has_upgrade = False has_downgrade = False for cp in intersecting_cps: - version_1 = cp_map_1[cp] - version_2 = cp_map_2[cp] + version_1 = choice_1.cp_map[cp] + version_2 = choice_2.cp_map[cp] difference = vercmp(version_1.version, version_2.version) if difference != 0: if difference > 0: @@ -538,9 +564,9 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): for allow_masked in (False, True): for choices in choice_bins: - for atoms, slot_map, cp_map, all_available in choices: - if all_available or allow_masked: - return atoms + for choice in choices: + if choice.all_available or allow_masked: + return choice.atoms assert(False) # This point should not be reachable @@ -575,18 +601,15 @@ def dep_check(depstring, mydbapi, mysettings, use="yes", mode=None, myuse=None, mymasks = set() useforce = set() - useforce.add(mysettings["ARCH"]) if use == "all": - # This masking/forcing is only for repoman. In other cases, relevant - # masking/forcing should have already been applied via - # config.regenerate(). Also, binary or installed packages may have - # been built with flags that are now masked, and it would be - # inconsistent to mask them now. Additionally, myuse may consist of - # flags from a parent package that is being merged to a $ROOT that is - # different from the one that mysettings represents. + # This is only for repoman, in order to constrain the use_reduce + # matchall behavior to account for profile use.mask/force. The + # ARCH/archlist code here may be redundant, since the profile + # really should be handling ARCH masking/forcing itself. mymasks.update(mysettings.usemask) mymasks.update(mysettings.archlist()) mymasks.discard(mysettings["ARCH"]) + useforce.add(mysettings["ARCH"]) useforce.update(mysettings.useforce) useforce.difference_update(mymasks) @@ -609,14 +632,17 @@ def dep_check(depstring, mydbapi, mysettings, use="yes", mode=None, myuse=None, # dependencies so that things like --depclean work as well as possible # in spite of partial invalidity. if not current_parent.installed: - eapi = current_parent.metadata['EAPI'] + eapi = current_parent.eapi - try: - mysplit = use_reduce(depstring, uselist=myusesplit, masklist=mymasks, \ - matchall=(use=="all"), excludeall=useforce, opconvert=True, \ - token_class=Atom, eapi=eapi) - except InvalidDependString as e: - return [0, _unicode_decode("%s") % (e,)] + if isinstance(depstring, list): + mysplit = depstring + else: + try: + mysplit = use_reduce(depstring, uselist=myusesplit, + masklist=mymasks, matchall=(use=="all"), excludeall=useforce, + opconvert=True, token_class=Atom, eapi=eapi) + except InvalidDependString as e: + return [0, "%s" % (e,)] if mysplit == []: #dependencies were reduced to nothing @@ -630,10 +656,10 @@ def dep_check(depstring, mydbapi, mysettings, use="yes", mode=None, myuse=None, use_force=useforce, use_mask=mymasks, use_cache=use_cache, use_binaries=use_binaries, myroot=myroot, trees=trees) except ParseError as e: - return [0, _unicode_decode("%s") % (e,)] + return [0, "%s" % (e,)] - mysplit2=mysplit[:] - mysplit2=dep_wordreduce(mysplit2,mysettings,mydbapi,mode,use_cache=use_cache) + mysplit2 = dep_wordreduce(mysplit, + mysettings, mydbapi, mode, use_cache=use_cache) if mysplit2 is None: return [0, _("Invalid token")] diff --git a/portage_with_autodep/pym/portage/dep/dep_check.pyo b/portage_with_autodep/pym/portage/dep/dep_check.pyo Binary files differindex 1b9e03f..feec00e 100644 --- a/portage_with_autodep/pym/portage/dep/dep_check.pyo +++ b/portage_with_autodep/pym/portage/dep/dep_check.pyo diff --git a/portage_with_autodep/pym/portage/dispatch_conf.py b/portage_with_autodep/pym/portage/dispatch_conf.py index 4c68dfc..79daa9f 100644 --- a/portage_with_autodep/pym/portage/dispatch_conf.py +++ b/portage_with_autodep/pym/portage/dispatch_conf.py @@ -1,5 +1,5 @@ # archive_conf.py -- functionality common to archive-conf and dispatch-conf -# Copyright 2003-2012 Gentoo Foundation +# Copyright 2003-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 @@ -31,9 +31,17 @@ def diffstatusoutput(cmd, file1, file2): # Use Popen to emulate getstatusoutput(), since getstatusoutput() may # raise a UnicodeDecodeError which makes the output inaccessible. args = shlex_split(cmd % (file1, file2)) - if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000: - # Python 3.1 does not support bytes in Popen args. - args = [portage._unicode_encode(x, errors='strict') for x in args] + + if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \ + not os.path.isabs(args[0]): + # Python 3.1 _execvp throws TypeError for non-absolute executable + # path passed as bytes (see http://bugs.python.org/issue8513). + fullname = portage.process.find_binary(args[0]) + if fullname is None: + raise portage.exception.CommandNotFound(args[0]) + args[0] = fullname + + args = [portage._unicode_encode(x, errors='strict') for x in args] proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output = portage._unicode_decode(proc.communicate()[0]) @@ -43,7 +51,7 @@ def diffstatusoutput(cmd, file1, file2): return (proc.wait(), output) def read_config(mandatory_opts): - eprefix = portage.const.EPREFIX + eprefix = portage.settings["EPREFIX"] config_path = os.path.join(eprefix or os.sep, "etc/dispatch-conf.conf") loader = KeyValuePairFileLoader(config_path, None) opts, errors = loader.load() diff --git a/portage_with_autodep/pym/portage/dispatch_conf.pyo b/portage_with_autodep/pym/portage/dispatch_conf.pyo Binary files differindex 6239859..2241d20 100644 --- a/portage_with_autodep/pym/portage/dispatch_conf.pyo +++ b/portage_with_autodep/pym/portage/dispatch_conf.pyo diff --git a/portage_with_autodep/pym/portage/eapi.py b/portage_with_autodep/pym/portage/eapi.py index 79cf891..4f77910 100644 --- a/portage_with_autodep/pym/portage/eapi.py +++ b/portage_with_autodep/pym/portage/eapi.py @@ -1,12 +1,22 @@ # Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +import collections + +from portage import eapi_is_supported + def eapi_has_iuse_defaults(eapi): return eapi != "0" +def eapi_has_iuse_effective(eapi): + return eapi not in ("0", "1", "2", "3", "4", "4-python", "4-slot-abi") + def eapi_has_slot_deps(eapi): return eapi != "0" +def eapi_has_slot_operator(eapi): + return eapi not in ("0", "1", "2", "3", "4", "4-python") + def eapi_has_src_uri_arrows(eapi): return eapi not in ("0", "1") @@ -34,8 +44,11 @@ def eapi_exports_merge_type(eapi): def eapi_exports_replace_vars(eapi): return eapi not in ("0", "1", "2", "3") +def eapi_exports_EBUILD_PHASE_FUNC(eapi): + return eapi not in ("0", "1", "2", "3", "4", "4-python", "4-slot-abi") + def eapi_exports_REPOSITORY(eapi): - return eapi in ("4-python",) + return eapi in ("4-python", "5-progress") def eapi_has_pkg_pretend(eapi): return eapi not in ("0", "1", "2", "3") @@ -49,14 +62,83 @@ def eapi_has_dosed_dohard(eapi): def eapi_has_required_use(eapi): return eapi not in ("0", "1", "2", "3") +def eapi_has_required_use_at_most_one_of(eapi): + return eapi not in ("0", "1", "2", "3", "4", "4-python", "4-slot-abi") + def eapi_has_use_dep_defaults(eapi): return eapi not in ("0", "1", "2", "3") def eapi_has_repo_deps(eapi): - return eapi in ("4-python",) + return eapi in ("4-python", "5-progress") def eapi_allows_dots_in_PN(eapi): - return eapi in ("4-python",) + return eapi in ("4-python", "5-progress") def eapi_allows_dots_in_use_flags(eapi): - return eapi in ("4-python",) + return eapi in ("4-python", "5-progress") + +def eapi_supports_stable_use_forcing_and_masking(eapi): + return eapi not in ("0", "1", "2", "3", "4", "4-python", "4-slot-abi") + +def eapi_allows_directories_on_profile_level_and_repository_level(eapi): + return eapi in ("4-python", "5-progress") + +def eapi_has_use_aliases(eapi): + return eapi in ("4-python", "5-progress") + +def eapi_has_automatic_unpack_dependencies(eapi): + return eapi in ("5-progress",) + +def eapi_has_hdepend(eapi): + return eapi in ("5-hdepend",) + +def eapi_has_targetroot(eapi): + return eapi in ("5-hdepend",) + +_eapi_attrs = collections.namedtuple('_eapi_attrs', + 'dots_in_PN dots_in_use_flags exports_EBUILD_PHASE_FUNC ' + 'feature_flag_test feature_flag_targetroot ' + 'hdepend iuse_defaults iuse_effective ' + 'repo_deps required_use required_use_at_most_one_of slot_operator slot_deps ' + 'src_uri_arrows strong_blocks use_deps use_dep_defaults') + +_eapi_attrs_cache = {} + +def _get_eapi_attrs(eapi): + """ + When eapi is None then validation is not as strict, since we want the + same to work for multiple EAPIs that may have slightly different rules. + An unsupported eapi is handled the same as when eapi is None, which may + be helpful for handling of corrupt EAPI metadata in essential functions + such as pkgsplit. + """ + eapi_attrs = _eapi_attrs_cache.get(eapi) + if eapi_attrs is not None: + return eapi_attrs + + orig_eapi = eapi + if eapi is not None and not eapi_is_supported(eapi): + eapi = None + + eapi_attrs = _eapi_attrs( + dots_in_PN = (eapi is None or eapi_allows_dots_in_PN(eapi)), + dots_in_use_flags = (eapi is None or eapi_allows_dots_in_use_flags(eapi)), + exports_EBUILD_PHASE_FUNC = (eapi is None or eapi_exports_EBUILD_PHASE_FUNC(eapi)), + feature_flag_test = True, + feature_flag_targetroot = (eapi is not None and eapi_has_targetroot(eapi)), + hdepend = (eapi is not None and eapi_has_hdepend(eapi)), + iuse_defaults = (eapi is None or eapi_has_iuse_defaults(eapi)), + iuse_effective = (eapi is not None and eapi_has_iuse_effective(eapi)), + repo_deps = (eapi is None or eapi_has_repo_deps(eapi)), + required_use = (eapi is None or eapi_has_required_use(eapi)), + required_use_at_most_one_of = (eapi is None or eapi_has_required_use_at_most_one_of(eapi)), + slot_deps = (eapi is None or eapi_has_slot_deps(eapi)), + slot_operator = (eapi is None or eapi_has_slot_operator(eapi)), + src_uri_arrows = (eapi is None or eapi_has_src_uri_arrows(eapi)), + strong_blocks = (eapi is None or eapi_has_strong_blocks(eapi)), + use_deps = (eapi is None or eapi_has_use_deps(eapi)), + use_dep_defaults = (eapi is None or eapi_has_use_dep_defaults(eapi)) + ) + + _eapi_attrs_cache[orig_eapi] = eapi_attrs + return eapi_attrs diff --git a/portage_with_autodep/pym/portage/eapi.pyo b/portage_with_autodep/pym/portage/eapi.pyo Binary files differindex ce67ab1..cfce6bd 100644 --- a/portage_with_autodep/pym/portage/eapi.pyo +++ b/portage_with_autodep/pym/portage/eapi.pyo diff --git a/portage_with_autodep/pym/portage/eclass_cache.py b/portage_with_autodep/pym/portage/eclass_cache.py index cb2cf8a..8c209c8 100644 --- a/portage_with_autodep/pym/portage/eclass_cache.py +++ b/portage_with_autodep/pym/portage/eclass_cache.py @@ -1,7 +1,9 @@ -# Copyright 2005-2011 Gentoo Foundation +# Copyright 2005-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # Author(s): Nicholas Carpaski (carpaski@gentoo.org), Brian Harring (ferringb@gentoo.org) +from __future__ import unicode_literals + __all__ = ["cache"] import stat @@ -12,6 +14,7 @@ import errno from portage.exception import FileNotFound, PermissionDenied from portage import os from portage import checksum +from portage import _shell_quote if sys.hexversion >= 0x3000000: long = int @@ -60,6 +63,7 @@ class cache(object): self.eclasses = {} # {"Name": hashed_path} self._eclass_locations = {} + self._eclass_locations_str = None # screw with the porttree ordering, w/out having bash inherit match it, and I'll hurt you. # ~harring @@ -98,6 +102,7 @@ class cache(object): self.porttrees = self.porttrees + other.porttrees self.eclasses.update(other.eclasses) self._eclass_locations.update(other._eclass_locations) + self._eclass_locations_str = None def update_eclasses(self): self.eclasses = {} @@ -169,3 +174,10 @@ class cache(object): ec_dict[x] = self.eclasses[x] return ec_dict + + @property + def eclass_locations_string(self): + if self._eclass_locations_str is None: + self._eclass_locations_str = " ".join(_shell_quote(x) + for x in reversed(self.porttrees)) + return self._eclass_locations_str diff --git a/portage_with_autodep/pym/portage/eclass_cache.pyo b/portage_with_autodep/pym/portage/eclass_cache.pyo Binary files differindex ebe3463..ffd56f7 100644 --- a/portage_with_autodep/pym/portage/eclass_cache.pyo +++ b/portage_with_autodep/pym/portage/eclass_cache.pyo diff --git a/portage_with_autodep/pym/portage/elog/__init__.pyo b/portage_with_autodep/pym/portage/elog/__init__.pyo Binary files differindex 39dc595..6bf9898 100644 --- a/portage_with_autodep/pym/portage/elog/__init__.pyo +++ b/portage_with_autodep/pym/portage/elog/__init__.pyo diff --git a/portage_with_autodep/pym/portage/elog/filtering.pyo b/portage_with_autodep/pym/portage/elog/filtering.pyo Binary files differindex 3a040cc..c67c7bb 100644 --- a/portage_with_autodep/pym/portage/elog/filtering.pyo +++ b/portage_with_autodep/pym/portage/elog/filtering.pyo diff --git a/portage_with_autodep/pym/portage/elog/messages.pyo b/portage_with_autodep/pym/portage/elog/messages.pyo Binary files differindex c033f55..0e3239f 100644 --- a/portage_with_autodep/pym/portage/elog/messages.pyo +++ b/portage_with_autodep/pym/portage/elog/messages.pyo diff --git a/portage_with_autodep/pym/portage/elog/mod_custom.pyo b/portage_with_autodep/pym/portage/elog/mod_custom.pyo Binary files differindex 317fab4..cc95c24 100644 --- a/portage_with_autodep/pym/portage/elog/mod_custom.pyo +++ b/portage_with_autodep/pym/portage/elog/mod_custom.pyo diff --git a/portage_with_autodep/pym/portage/elog/mod_echo.pyo b/portage_with_autodep/pym/portage/elog/mod_echo.pyo Binary files differindex 6a00d4c..be383e2 100644 --- a/portage_with_autodep/pym/portage/elog/mod_echo.pyo +++ b/portage_with_autodep/pym/portage/elog/mod_echo.pyo diff --git a/portage_with_autodep/pym/portage/elog/mod_mail.pyo b/portage_with_autodep/pym/portage/elog/mod_mail.pyo Binary files differindex 5d87aa6..bfed1d3 100644 --- a/portage_with_autodep/pym/portage/elog/mod_mail.pyo +++ b/portage_with_autodep/pym/portage/elog/mod_mail.pyo diff --git a/portage_with_autodep/pym/portage/elog/mod_mail_summary.pyo b/portage_with_autodep/pym/portage/elog/mod_mail_summary.pyo Binary files differindex d7306b5..17c2135 100644 --- a/portage_with_autodep/pym/portage/elog/mod_mail_summary.pyo +++ b/portage_with_autodep/pym/portage/elog/mod_mail_summary.pyo diff --git a/portage_with_autodep/pym/portage/elog/mod_save.py b/portage_with_autodep/pym/portage/elog/mod_save.py index c69f4a3..7b1cd46 100644 --- a/portage_with_autodep/pym/portage/elog/mod_save.py +++ b/portage_with_autodep/pym/portage/elog/mod_save.py @@ -1,7 +1,8 @@ # elog/mod_save.py - elog dispatch module -# Copyright 2006-2011 Gentoo Foundation +# Copyright 2006-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +import errno import io import time import portage @@ -47,11 +48,22 @@ def process(mysettings, key, logentries, fulltext): elogfilename = os.path.join(log_subdir, cat + ':' + elogfilename) _ensure_log_subdirs(logdir, log_subdir) - elogfile = io.open(_unicode_encode(elogfilename, - encoding=_encodings['fs'], errors='strict'), - mode='w', encoding=_encodings['content'], errors='backslashreplace') - elogfile.write(_unicode_decode(fulltext)) - elogfile.close() + try: + with io.open(_unicode_encode(elogfilename, + encoding=_encodings['fs'], errors='strict'), mode='w', + encoding=_encodings['content'], + errors='backslashreplace') as elogfile: + elogfile.write(_unicode_decode(fulltext)) + except IOError as e: + func_call = "open('%s', 'w')" % elogfilename + if e.errno == errno.EACCES: + raise portage.exception.PermissionDenied(func_call) + elif e.errno == errno.EPERM: + raise portage.exception.OperationNotPermitted(func_call) + elif e.errno == errno.EROFS: + raise portage.exception.ReadOnlyFileSystem(func_call) + else: + raise # Copy group permission bits from parent directory. elogdir_st = os.stat(log_subdir) diff --git a/portage_with_autodep/pym/portage/elog/mod_save.pyo b/portage_with_autodep/pym/portage/elog/mod_save.pyo Binary files differindex fb28b76..82d683d 100644 --- a/portage_with_autodep/pym/portage/elog/mod_save.pyo +++ b/portage_with_autodep/pym/portage/elog/mod_save.pyo diff --git a/portage_with_autodep/pym/portage/elog/mod_save_summary.py b/portage_with_autodep/pym/portage/elog/mod_save_summary.py index 347f66e..786f894 100644 --- a/portage_with_autodep/pym/portage/elog/mod_save_summary.py +++ b/portage_with_autodep/pym/portage/elog/mod_save_summary.py @@ -1,8 +1,12 @@ # elog/mod_save_summary.py - elog dispatch module -# Copyright 2006-2011 Gentoo Foundation +# Copyright 2006-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + +import errno import io +import sys import time import portage from portage import os @@ -37,9 +41,21 @@ def process(mysettings, key, logentries, fulltext): # TODO: Locking elogfilename = elogdir+"/summary.log" - elogfile = io.open(_unicode_encode(elogfilename, - encoding=_encodings['fs'], errors='strict'), - mode='a', encoding=_encodings['content'], errors='backslashreplace') + try: + elogfile = io.open(_unicode_encode(elogfilename, + encoding=_encodings['fs'], errors='strict'), + mode='a', encoding=_encodings['content'], + errors='backslashreplace') + except IOError as e: + func_call = "open('%s', 'a')" % elogfilename + if e.errno == errno.EACCES: + raise portage.exception.PermissionDenied(func_call) + elif e.errno == errno.EPERM: + raise portage.exception.OperationNotPermitted(func_call) + elif e.errno == errno.EROFS: + raise portage.exception.ReadOnlyFileSystem(func_call) + else: + raise # Copy group permission bits from parent directory. elogdir_st = os.stat(elogdir) @@ -58,17 +74,19 @@ def process(mysettings, key, logentries, fulltext): apply_permissions(elogfilename, uid=logfile_uid, gid=elogdir_gid, mode=elogdir_grp_mode, mask=0) - time_str = time.strftime("%Y-%m-%d %H:%M:%S %Z", - time.localtime(time.time())) - # Avoid potential UnicodeDecodeError later. + time_fmt = "%Y-%m-%d %H:%M:%S %Z" + if sys.hexversion < 0x3000000: + time_fmt = _unicode_encode(time_fmt) + time_str = time.strftime(time_fmt, time.localtime(time.time())) + # Avoid potential UnicodeDecodeError in Python 2, since strftime + # returns bytes in Python 2, and %Z may contain non-ascii chars. time_str = _unicode_decode(time_str, encoding=_encodings['content'], errors='replace') - elogfile.write(_unicode_decode( - _(">>> Messages generated by process " + + elogfile.write(_(">>> Messages generated by process " "%(pid)d on %(time)s for package %(pkg)s:\n\n") % - {"pid": os.getpid(), "time": time_str, "pkg": key})) + {"pid": os.getpid(), "time": time_str, "pkg": key}) elogfile.write(_unicode_decode(fulltext)) - elogfile.write(_unicode_decode("\n")) + elogfile.write("\n") elogfile.close() return elogfilename diff --git a/portage_with_autodep/pym/portage/elog/mod_save_summary.pyo b/portage_with_autodep/pym/portage/elog/mod_save_summary.pyo Binary files differindex 8f99c51..19e4422 100644 --- a/portage_with_autodep/pym/portage/elog/mod_save_summary.pyo +++ b/portage_with_autodep/pym/portage/elog/mod_save_summary.pyo diff --git a/portage_with_autodep/pym/portage/elog/mod_syslog.pyo b/portage_with_autodep/pym/portage/elog/mod_syslog.pyo Binary files differindex c7b4248..bb01208 100644 --- a/portage_with_autodep/pym/portage/elog/mod_syslog.pyo +++ b/portage_with_autodep/pym/portage/elog/mod_syslog.pyo diff --git a/portage_with_autodep/pym/portage/env/__init__.pyo b/portage_with_autodep/pym/portage/env/__init__.pyo Binary files differindex 846aea3..b45054e 100644 --- a/portage_with_autodep/pym/portage/env/__init__.pyo +++ b/portage_with_autodep/pym/portage/env/__init__.pyo diff --git a/portage_with_autodep/pym/portage/env/config.pyo b/portage_with_autodep/pym/portage/env/config.pyo Binary files differindex 13c2e86..0a7faa3 100644 --- a/portage_with_autodep/pym/portage/env/config.pyo +++ b/portage_with_autodep/pym/portage/env/config.pyo diff --git a/portage_with_autodep/pym/portage/env/loaders.py b/portage_with_autodep/pym/portage/env/loaders.py index 372bc12..f869884 100644 --- a/portage_with_autodep/pym/portage/env/loaders.py +++ b/portage_with_autodep/pym/portage/env/loaders.py @@ -1,10 +1,14 @@ # config.py -- Portage Config -# Copyright 2007-2011 Gentoo Foundation +# Copyright 2007-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import errno import io import stat +import portage +portage.proxy.lazyimport.lazyimport(globals(), + 'portage.util:writemsg', +) from portage import os from portage import _encodings from portage import _unicode_decode @@ -149,17 +153,21 @@ class FileLoader(DataLoader): func = self.lineParser for fn in RecursiveFileLoader(self.fname): try: - f = io.open(_unicode_encode(fn, + with io.open(_unicode_encode(fn, encoding=_encodings['fs'], errors='strict'), mode='r', - encoding=_encodings['content'], errors='replace') + encoding=_encodings['content'], errors='replace') as f: + lines = f.readlines() except EnvironmentError as e: - if e.errno not in (errno.ENOENT, errno.ESTALE): + if e.errno == errno.EACCES: + writemsg(_("Permission denied: '%s'\n") % fn, noiselevel=-1) + del e + elif e.errno in (errno.ENOENT, errno.ESTALE): + del e + else: raise - del e - continue - for line_num, line in enumerate(f): - func(line, line_num, data, errors) - f.close() + else: + for line_num, line in enumerate(lines): + func(line, line_num, data, errors) return (data, errors) def lineParser(self, line, line_num, data, errors): diff --git a/portage_with_autodep/pym/portage/env/loaders.pyo b/portage_with_autodep/pym/portage/env/loaders.pyo Binary files differindex 2622a9f..91c3efa 100644 --- a/portage_with_autodep/pym/portage/env/loaders.pyo +++ b/portage_with_autodep/pym/portage/env/loaders.pyo diff --git a/portage_with_autodep/pym/portage/env/validators.pyo b/portage_with_autodep/pym/portage/env/validators.pyo Binary files differindex cd18adb..24e1510 100644 --- a/portage_with_autodep/pym/portage/env/validators.pyo +++ b/portage_with_autodep/pym/portage/env/validators.pyo diff --git a/portage_with_autodep/pym/portage/exception.py b/portage_with_autodep/pym/portage/exception.py index 5ccd750..1388c49 100644 --- a/portage_with_autodep/pym/portage/exception.py +++ b/portage_with_autodep/pym/portage/exception.py @@ -1,4 +1,4 @@ -# Copyright 1998-2011 Gentoo Foundation +# Copyright 1998-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import signal @@ -11,26 +11,35 @@ if sys.hexversion >= 0x3000000: class PortageException(Exception): """General superclass for portage exceptions""" - def __init__(self,value): - self.value = value[:] - if isinstance(self.value, basestring): - self.value = _unicode_decode(self.value, - encoding=_encodings['content'], errors='replace') + if sys.hexversion >= 0x3000000: + def __init__(self,value): + self.value = value[:] - def __str__(self): - if isinstance(self.value, basestring): - return self.value - else: - return _unicode_decode(repr(self.value), - encoding=_encodings['content'], errors='replace') - - if sys.hexversion < 0x3000000: - - __unicode__ = __str__ + def __str__(self): + if isinstance(self.value, str): + return self.value + else: + return repr(self.value) + else: + def __init__(self,value): + self.value = value[:] + if isinstance(self.value, basestring): + self.value = _unicode_decode(self.value, + encoding=_encodings['content'], errors='replace') + + def __unicode__(self): + if isinstance(self.value, unicode): + return self.value + else: + return _unicode_decode(repr(self.value), + encoding=_encodings['content'], errors='replace') def __str__(self): - return _unicode_encode(self.__unicode__(), - encoding=_encodings['content'], errors='backslashreplace') + if isinstance(self.value, unicode): + return _unicode_encode(self.value, + encoding=_encodings['content'], errors='backslashreplace') + else: + return repr(self.value) class CorruptionError(PortageException): """Corruption indication""" diff --git a/portage_with_autodep/pym/portage/exception.pyo b/portage_with_autodep/pym/portage/exception.pyo Binary files differindex 3a60e7c..c60c3a0 100644 --- a/portage_with_autodep/pym/portage/exception.pyo +++ b/portage_with_autodep/pym/portage/exception.pyo diff --git a/portage_with_autodep/pym/portage/getbinpkg.py b/portage_with_autodep/pym/portage/getbinpkg.py index 212f788..ff656ba 100644 --- a/portage_with_autodep/pym/portage/getbinpkg.py +++ b/portage_with_autodep/pym/portage/getbinpkg.py @@ -1,7 +1,9 @@ # getbinpkg.py -- Portage binary-package helper functions -# Copyright 2003-2012 Gentoo Foundation +# Copyright 2003-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + from portage.output import colorize from portage.cache.mappings import slot_dict_class from portage.localization import _ @@ -18,6 +20,7 @@ import socket import time import tempfile import base64 +import warnings _all_errors = [NotImplementedError, ValueError, socket.error] @@ -65,6 +68,10 @@ if sys.hexversion >= 0x3000000: long = int def make_metadata_dict(data): + + warnings.warn("portage.getbinpkg.make_metadata_dict() is deprecated", + DeprecationWarning, stacklevel=2) + myid,myglob = data mydict = {} @@ -84,6 +91,10 @@ class ParseLinks(html_parser_HTMLParser): """Parser class that overrides HTMLParser to grab all anchors from an html page and provide suffix and prefix limitors""" def __init__(self): + + warnings.warn("portage.getbinpkg.ParseLinks is deprecated", + DeprecationWarning, stacklevel=2) + self.PL_anchors = [] html_parser_HTMLParser.__init__(self) @@ -122,6 +133,9 @@ def create_conn(baseurl,conn=None): optional connection. If connection is already active, it is passed on. baseurl is reduced to address and is returned in tuple (conn,address)""" + warnings.warn("portage.getbinpkg.create_conn() is deprecated", + DeprecationWarning, stacklevel=2) + parts = baseurl.split("://",1) if len(parts) != 2: raise ValueError(_("Provided URI does not " @@ -221,6 +235,10 @@ def create_conn(baseurl,conn=None): def make_ftp_request(conn, address, rest=None, dest=None): """(conn,address,rest) --- uses the conn object to request the data from address and issuing a rest if it is passed.""" + + warnings.warn("portage.getbinpkg.make_ftp_request() is deprecated", + DeprecationWarning, stacklevel=2) + try: if dest: @@ -270,6 +288,9 @@ def make_http_request(conn, address, params={}, headers={}, dest=None): the data from address, performing Location forwarding and using the optional params and headers.""" + warnings.warn("portage.getbinpkg.make_http_request() is deprecated", + DeprecationWarning, stacklevel=2) + rc = 0 response = None while (rc == 0) or (rc == 301) or (rc == 302): @@ -312,6 +333,10 @@ def make_http_request(conn, address, params={}, headers={}, dest=None): def match_in_array(array, prefix="", suffix="", match_both=1, allow_overlap=0): + + warnings.warn("portage.getbinpkg.match_in_array() is deprecated", + DeprecationWarning, stacklevel=2) + myarray = [] if not (prefix and suffix): @@ -352,6 +377,9 @@ def dir_get_list(baseurl,conn=None): URI should be in the form <proto>://<site>[:port]<path> Connection is used for persistent connection instances.""" + warnings.warn("portage.getbinpkg.dir_get_list() is deprecated", + DeprecationWarning, stacklevel=2) + if not conn: keepconnection = 0 else: @@ -400,6 +428,9 @@ def file_get_metadata(baseurl,conn=None, chunk_size=3000): URI should be in the form <proto>://<site>[:port]<path> Connection is used for persistent connection instances.""" + warnings.warn("portage.getbinpkg.file_get_metadata() is deprecated", + DeprecationWarning, stacklevel=2) + if not conn: keepconnection = 0 else: @@ -446,30 +477,53 @@ def file_get_metadata(baseurl,conn=None, chunk_size=3000): return myid -def file_get(baseurl,dest,conn=None,fcmd=None,filename=None): +def file_get(baseurl=None, dest=None, conn=None, fcmd=None, filename=None, + fcmd_vars=None): """(baseurl,dest,fcmd=) -- Takes a base url to connect to and read from. URI should be in the form <proto>://[user[:pass]@]<site>[:port]<path>""" if not fcmd: + + warnings.warn("Use of portage.getbinpkg.file_get() without the fcmd " + "parameter is deprecated", DeprecationWarning, stacklevel=2) + return file_get_lib(baseurl,dest,conn) - if not filename: - filename = os.path.basename(baseurl) - variables = { - "DISTDIR": dest, - "URI": baseurl, - "FILE": filename - } + variables = {} + + if fcmd_vars is not None: + variables.update(fcmd_vars) + + if "DISTDIR" not in variables: + if dest is None: + raise portage.exception.MissingParameter( + _("%s is missing required '%s' key") % + ("fcmd_vars", "DISTDIR")) + variables["DISTDIR"] = dest + + if "URI" not in variables: + if baseurl is None: + raise portage.exception.MissingParameter( + _("%s is missing required '%s' key") % + ("fcmd_vars", "URI")) + variables["URI"] = baseurl + + if "FILE" not in variables: + if filename is None: + filename = os.path.basename(variables["URI"]) + variables["FILE"] = filename from portage.util import varexpand from portage.process import spawn myfetch = portage.util.shlex_split(fcmd) myfetch = [varexpand(x, mydict=variables) for x in myfetch] fd_pipes= { - 0:sys.stdin.fileno(), - 1:sys.stdout.fileno(), - 2:sys.stdout.fileno() + 0:portage._get_stdin().fileno(), + 1:sys.__stdout__.fileno(), + 2:sys.__stdout__.fileno() } + sys.__stdout__.flush() + sys.__stderr__.flush() retval = spawn(myfetch, env=os.environ.copy(), fd_pipes=fd_pipes) if retval != os.EX_OK: sys.stderr.write(_("Fetcher exited with a failure condition.\n")) @@ -481,6 +535,9 @@ def file_get_lib(baseurl,dest,conn=None): URI should be in the form <proto>://<site>[:port]<path> Connection is used for persistent connection instances.""" + warnings.warn("portage.getbinpkg.file_get_lib() is deprecated", + DeprecationWarning, stacklevel=2) + if not conn: keepconnection = 0 else: @@ -524,6 +581,10 @@ def file_get_lib(baseurl,dest,conn=None): def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache=1, makepickle=None): """(baseurl,conn,chunk_size,verbose) -- """ + + warnings.warn("portage.getbinpkg.dir_get_metadata() is deprecated", + DeprecationWarning, stacklevel=2) + if not conn: keepconnection = 0 else: @@ -557,7 +618,9 @@ def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache= out.write(_("Loaded metadata pickle.\n")) out.flush() metadatafile.close() - except (AttributeError, EOFError, EnvironmentError, ValueError, pickle.UnpicklingError): + except (SystemExit, KeyboardInterrupt): + raise + except Exception: metadata = {} if baseurl not in metadata: metadata[baseurl]={} diff --git a/portage_with_autodep/pym/portage/getbinpkg.pyo b/portage_with_autodep/pym/portage/getbinpkg.pyo Binary files differindex 53ec2e9..578acd9 100644 --- a/portage_with_autodep/pym/portage/getbinpkg.pyo +++ b/portage_with_autodep/pym/portage/getbinpkg.pyo diff --git a/portage_with_autodep/pym/portage/glsa.py b/portage_with_autodep/pym/portage/glsa.py index 1857695..cac0f1a 100644 --- a/portage_with_autodep/pym/portage/glsa.py +++ b/portage_with_autodep/pym/portage/glsa.py @@ -1,7 +1,7 @@ # Copyright 2003-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals import io import sys @@ -9,8 +9,12 @@ try: from urllib.request import urlopen as urllib_request_urlopen except ImportError: from urllib import urlopen as urllib_request_urlopen +import codecs import re +import operator import xml.dom.minidom +from io import StringIO +from functools import reduce import portage from portage import os @@ -19,13 +23,13 @@ from portage import _unicode_decode from portage import _unicode_encode from portage.versions import pkgsplit, vercmp, best from portage.util import grabfile -from portage.const import CACHE_PATH +from portage.const import PRIVATE_PATH from portage.localization import _ from portage.dep import _slot_separator # Note: the space for rgt and rlt is important !! # FIXME: use slot deps instead, requires GLSA format versioning -opMapping = {"le": "<=", "lt": "<", "eq": "=", "gt": ">", "ge": ">=", +opMapping = {"le": "<=", "lt": "<", "eq": "=", "gt": ">", "ge": ">=", "rge": ">=~", "rle": "<=~", "rgt": " >~", "rlt": " <~"} NEWLINE_ESCAPE = "!;\\n" # some random string to mark newlines that should be preserved SPACE_ESCAPE = "!;_" # some random string to mark spaces that should be preserved @@ -39,22 +43,22 @@ def get_applied_glsas(settings): @rtype: list @return: list of glsa IDs """ - return grabfile(os.path.join(settings["EROOT"], CACHE_PATH, "glsa")) + return grabfile(os.path.join(settings["EROOT"], PRIVATE_PATH, "glsa_injected")) # TODO: use the textwrap module instead def wrap(text, width, caption=""): """ Wraps the given text at column I{width}, optionally indenting - it so that no text is under I{caption}. It's possible to encode + it so that no text is under I{caption}. It's possible to encode hard linebreaks in I{text} with L{NEWLINE_ESCAPE}. - + @type text: String @param text: the text to be wrapped @type width: Integer @param width: the column at which the text should be wrapped @type caption: String - @param caption: this string is inserted at the beginning of the + @param caption: this string is inserted at the beginning of the return value and the paragraph is indented up to C{len(caption)}. @rtype: String @@ -65,7 +69,7 @@ def wrap(text, width, caption=""): text = text.replace(2*NEWLINE_ESCAPE, NEWLINE_ESCAPE+" "+NEWLINE_ESCAPE) words = text.split() indentLevel = len(caption)+1 - + for w in words: if line != "" and line[-1] == "\n": rValue += line @@ -94,10 +98,10 @@ def get_glsa_list(myconfig): Returns a list of all available GLSAs in the given repository by comparing the filelist there with the pattern described in the config. - + @type myconfig: portage.config @param myconfig: Portage settings instance - + @rtype: List of Strings @return: a list of GLSA IDs in this repository """ @@ -113,10 +117,10 @@ def get_glsa_list(myconfig): dirlist = os.listdir(repository) prefix = "glsa-" suffix = ".xml" - + for f in dirlist: try: - if f[:len(prefix)] == prefix: + if f[:len(prefix)] == prefix and f[-1*len(suffix):] == suffix: rValue.append(f[len(prefix):-1*len(suffix)]) except IndexError: pass @@ -125,22 +129,20 @@ def get_glsa_list(myconfig): def getListElements(listnode): """ Get all <li> elements for a given <ol> or <ul> node. - + @type listnode: xml.dom.Node @param listnode: <ul> or <ol> list to get the elements for @rtype: List of Strings @return: a list that contains the value of the <li> elements """ - rValue = [] if not listnode.nodeName in ["ul", "ol"]: raise GlsaFormatException("Invalid function call: listnode is not <ul> or <ol>") - for li in listnode.childNodes: - if li.nodeType != xml.dom.Node.ELEMENT_NODE: - continue - rValue.append(getText(li, format="strip")) + rValue = [getText(li, format="strip") \ + for li in listnode.childNodes \ + if li.nodeType == xml.dom.Node.ELEMENT_NODE] return rValue -def getText(node, format): +def getText(node, format, textfd = None): """ This is the main parser function. It takes a node and traverses recursive over the subnodes, getting the text of each (and the @@ -148,7 +150,7 @@ def getText(node, format): parameter the text might be formatted by adding/removing newlines, tabs and spaces. This function is only useful for the GLSA DTD, it's not applicable for other DTDs. - + @type node: xml.dom.Node @param node: the root node to start with the parsing @type format: String @@ -158,45 +160,54 @@ def getText(node, format): replaces multiple spaces with one space. I{xml} does some more formatting, depending on the type of the encountered nodes. + @type textfd: writable file-like object + @param textfd: the file-like object to write the output to @rtype: String @return: the (formatted) content of the node and its subnodes + except if textfd was not none """ - rValue = "" + if not textfd: + textfd = StringIO() + returnNone = False + else: + returnNone = True if format in ["strip", "keep"]: if node.nodeName in ["uri", "mail"]: - rValue += node.childNodes[0].data+": "+node.getAttribute("link") + textfd.write(node.childNodes[0].data+": "+node.getAttribute("link")) else: for subnode in node.childNodes: if subnode.nodeName == "#text": - rValue += subnode.data + textfd.write(subnode.data) else: - rValue += getText(subnode, format) - else: + getText(subnode, format, textfd) + else: # format = "xml" for subnode in node.childNodes: if subnode.nodeName == "p": for p_subnode in subnode.childNodes: if p_subnode.nodeName == "#text": - rValue += p_subnode.data.strip() + textfd.write(p_subnode.data.strip()) elif p_subnode.nodeName in ["uri", "mail"]: - rValue += p_subnode.childNodes[0].data - rValue += " ( "+p_subnode.getAttribute("link")+" )" - rValue += NEWLINE_ESCAPE + textfd.write(p_subnode.childNodes[0].data) + textfd.write(" ( "+p_subnode.getAttribute("link")+" )") + textfd.write(NEWLINE_ESCAPE) elif subnode.nodeName == "ul": for li in getListElements(subnode): - rValue += "-"+SPACE_ESCAPE+li+NEWLINE_ESCAPE+" " + textfd.write("-"+SPACE_ESCAPE+li+NEWLINE_ESCAPE+" ") elif subnode.nodeName == "ol": i = 0 for li in getListElements(subnode): i = i+1 - rValue += str(i)+"."+SPACE_ESCAPE+li+NEWLINE_ESCAPE+" " + textfd.write(str(i)+"."+SPACE_ESCAPE+li+NEWLINE_ESCAPE+" ") elif subnode.nodeName == "code": - rValue += getText(subnode, format="keep").replace("\n", NEWLINE_ESCAPE) - if rValue[-1*len(NEWLINE_ESCAPE):] != NEWLINE_ESCAPE: - rValue += NEWLINE_ESCAPE + textfd.write(getText(subnode, format="keep").lstrip().replace("\n", NEWLINE_ESCAPE)) + textfd.write(NEWLINE_ESCAPE) elif subnode.nodeName == "#text": - rValue += subnode.data + textfd.write(subnode.data) else: raise GlsaFormatException(_("Invalid Tag found: "), subnode.nodeName) + if returnNone: + return None + rValue = textfd.getvalue() if format == "strip": rValue = rValue.strip(" \n\t") rValue = re.sub("[\s]{2,}", " ", rValue) @@ -206,7 +217,7 @@ def getMultiTagsText(rootnode, tagname, format): """ Returns a list with the text of all subnodes of type I{tagname} under I{rootnode} (which itself is not parsed) using the given I{format}. - + @type rootnode: xml.dom.Node @param rootnode: the node to search for I{tagname} @type tagname: String @@ -216,16 +227,15 @@ def getMultiTagsText(rootnode, tagname, format): @rtype: List of Strings @return: a list containing the text of all I{tagname} childnodes """ - rValue = [] - for e in rootnode.getElementsByTagName(tagname): - rValue.append(getText(e, format)) + rValue = [getText(e, format) \ + for e in rootnode.getElementsByTagName(tagname)] return rValue def makeAtom(pkgname, versionNode): """ - creates from the given package name and information in the + creates from the given package name and information in the I{versionNode} a (syntactical) valid portage atom. - + @type pkgname: String @param pkgname: the name of the package for this atom @type versionNode: xml.dom.Node @@ -248,9 +258,9 @@ def makeAtom(pkgname, versionNode): def makeVersion(versionNode): """ - creates from the information in the I{versionNode} a + creates from the information in the I{versionNode} a version string (format <op><version>). - + @type versionNode: xml.dom.Node @param versionNode: a <vulnerable> or <unaffected> Node that contains the version information for this atom @@ -270,17 +280,17 @@ def makeVersion(versionNode): def match(atom, dbapi, match_type="default"): """ - wrapper that calls revisionMatch() or portage.dbapi.dbapi.match() depending on + wrapper that calls revisionMatch() or portage.dbapi.dbapi.match() depending on the given atom. - + @type atom: string @param atom: a <~ or >~ atom or a normal portage atom that contains the atom to match against @type dbapi: portage.dbapi.dbapi @param dbapi: one of the portage databases to use as information source @type match_type: string - @param match_type: if != "default" passed as first argument to dbapi.xmatch + @param match_type: if != "default" passed as first argument to dbapi.xmatch to apply the wanted visibility filters - + @rtype: list of strings @return: a list with the matching versions """ @@ -296,15 +306,15 @@ def revisionMatch(revisionAtom, dbapi, match_type="default"): handler for the special >~, >=~, <=~ and <~ atoms that are supposed to behave as > and < except that they are limited to the same version, the range only applies to the revision part. - + @type revisionAtom: string @param revisionAtom: a <~ or >~ atom that contains the atom to match against @type dbapi: portage.dbapi.dbapi @param dbapi: one of the portage databases to use as information source @type match_type: string - @param match_type: if != "default" passed as first argument to portdb.xmatch + @param match_type: if != "default" passed as first argument to portdb.xmatch to apply the wanted visibility filters - + @rtype: list of strings @return: a list with the matching versions """ @@ -325,18 +335,19 @@ def revisionMatch(revisionAtom, dbapi, match_type="default"): if eval(r1+" "+revisionAtom[0:2]+" "+r2): rValue.append(v) return rValue - + def getMinUpgrade(vulnerableList, unaffectedList, portdbapi, vardbapi, minimize=True): """ Checks if the systemstate is matching an atom in I{vulnerableList} and returns string describing - the lowest version for the package that matches an atom in + the lowest version for the package that matches an atom in I{unaffectedList} and is greater than the currently installed - version or None if the system is not affected. Both - I{vulnerableList} and I{unaffectedList} should have the + version. It will return an empty list if the system is affected, + and no upgrade is possible or None if the system is not affected. + Both I{vulnerableList} and I{unaffectedList} should have the same base package. - + @type vulnerableList: List of Strings @param vulnerableList: atoms matching vulnerable package versions @type unaffectedList: List of Strings @@ -347,46 +358,51 @@ def getMinUpgrade(vulnerableList, unaffectedList, portdbapi, vardbapi, minimize= @param vardbapi: Installed package repository @type minimize: Boolean @param minimize: True for a least-change upgrade, False for emerge-like algorithm - + @rtype: String | None @return: the lowest unaffected version that is greater than the installed version. - """ - rValue = None - v_installed = [] - u_installed = [] - for v in vulnerableList: - v_installed += match(v, vardbapi) + """ + rValue = "" + v_installed = reduce(operator.add, [match(v, vardbapi) for v in vulnerableList], []) + u_installed = reduce(operator.add, [match(u, vardbapi) for u in unaffectedList], []) - for u in unaffectedList: - u_installed += match(u, vardbapi) - - install_unaffected = True - for i in v_installed: - if i not in u_installed: - install_unaffected = False + # remove all unaffected atoms from vulnerable list + v_installed = list(set(v_installed).difference(set(u_installed))) - if install_unaffected: - return rValue - + if not v_installed: + return None + + # this tuple holds all vulnerable atoms, and the related upgrade atom + vuln_update = [] + avail_updates = set() for u in unaffectedList: - mylist = match(u, portdbapi, match_type="match-all") - for c in mylist: - i = best(v_installed) - if vercmp(c.version, i.version) > 0 \ - and (rValue == None \ - or not match("="+rValue, portdbapi) \ - or (minimize ^ (vercmp(c.version, rValue.version) > 0)) \ - and match("="+c, portdbapi)) \ - and portdbapi.aux_get(c, ["SLOT"]) == vardbapi.aux_get(best(v_installed), ["SLOT"]): - rValue = c - return rValue + # TODO: This had match_type="match-all" before. I don't think it should + # since we disregarded masked items later anyway (match(=rValue, "porttree")) + avail_updates.update(match(u, portdbapi)) + # if an atom is already installed, we should not consider it for upgrades + avail_updates.difference_update(u_installed) + + for vuln in v_installed: + update = "" + for c in avail_updates: + c_pv = portage.catpkgsplit(c) + if vercmp(c.version, vuln.version) > 0 \ + and (update == "" \ + or (minimize ^ (vercmp(c.version, update.version) > 0))) \ + and portdbapi._pkg_str(c, None).slot == vardbapi._pkg_str(vuln, None).slot: + update = c_pv[0]+"/"+c_pv[1]+"-"+c_pv[2] + if c_pv[3] != "r0": # we don't like -r0 for display + update += "-"+c_pv[3] + vuln_update.append([vuln, update]) + + return vuln_update def format_date(datestr): """ Takes a date (announced, revised) date from a GLSA and formats it as readable text (i.e. "January 1, 2008"). - + @type date: String @param date: the date string to reformat @rtype: String @@ -396,16 +412,16 @@ def format_date(datestr): splitdate = datestr.split("-", 2) if len(splitdate) != 3: return datestr - + # This cannot raise an error as we use () instead of [] splitdate = (int(x) for x in splitdate) - + from datetime import date try: d = date(*splitdate) except ValueError: return datestr - + # TODO We could format to local date format '%x' here? return _unicode_decode(d.strftime("%B %d, %Y"), encoding=_encodings['content'], errors='replace') @@ -417,7 +433,7 @@ class GlsaTypeException(Exception): class GlsaFormatException(Exception): pass - + class GlsaArgumentException(Exception): pass @@ -429,9 +445,9 @@ class Glsa: """ def __init__(self, myid, myconfig, vardbapi, portdbapi): """ - Simple constructor to set the ID, store the config and gets the + Simple constructor to set the ID, store the config and gets the XML data by calling C{self.read()}. - + @type myid: String @param myid: String describing the id for the GLSA object (standard GLSAs have an ID of the form YYYYMM-nn) or an existing @@ -461,7 +477,7 @@ class Glsa: """ Here we build the filename from the config and the ID and pass it to urllib to fetch it from the filesystem or a remote server. - + @rtype: None @return: None """ @@ -473,15 +489,21 @@ class Glsa: myurl = "file://"+self.nr else: myurl = repository + "glsa-%s.xml" % str(self.nr) - self.parse(urllib_request_urlopen(myurl)) + + f = urllib_request_urlopen(myurl) + try: + self.parse(f) + finally: + f.close() + return None def parse(self, myfile): """ - This method parses the XML file and sets up the internal data + This method parses the XML file and sets up the internal data structures by calling the different helper functions in this module. - + @type myfile: String @param myfile: Filename to grab the XML data from @rtype: None @@ -504,27 +526,27 @@ class Glsa: self.title = getText(myroot.getElementsByTagName("title")[0], format="strip") self.synopsis = getText(myroot.getElementsByTagName("synopsis")[0], format="strip") self.announced = format_date(getText(myroot.getElementsByTagName("announced")[0], format="strip")) - - count = 1 + # Support both formats of revised: # <revised>December 30, 2007: 02</revised> # <revised count="2">2007-12-30</revised> revisedEl = myroot.getElementsByTagName("revised")[0] self.revised = getText(revisedEl, format="strip") - if ((sys.hexversion >= 0x3000000 and "count" in revisedEl.attributes) or - (sys.hexversion < 0x3000000 and revisedEl.attributes.has_key("count"))): - count = revisedEl.getAttribute("count") - elif (self.revised.find(":") >= 0): - (self.revised, count) = self.revised.split(":") - + count = revisedEl.attributes.get("count") + if count is None: + if self.revised.find(":") >= 0: + (self.revised, count) = self.revised.split(":") + else: + count = 1 + self.revised = format_date(self.revised) - + try: self.count = int(count) except ValueError: # TODO should this raise a GlsaFormatException? self.count = 1 - + # now the optional and 0-n toplevel, #PCDATA tags and references try: self.access = getText(myroot.getElementsByTagName("access")[0], format="strip") @@ -532,7 +554,7 @@ class Glsa: self.access = "" self.bugs = getMultiTagsText(myroot, "bug", format="strip") self.references = getMultiTagsText(myroot.getElementsByTagName("references")[0], "uri", format="keep") - + # and now the formatted text elements self.description = getText(myroot.getElementsByTagName("description")[0], format="xml") self.workaround = getText(myroot.getElementsByTagName("workaround")[0], format="xml") @@ -542,7 +564,7 @@ class Glsa: try: self.background = getText(myroot.getElementsByTagName("background")[0], format="xml") except IndexError: - self.background = "" + self.background = "" # finally the interesting tags (product, affected, package) self.glsatype = myroot.getElementsByTagName("product")[0].getAttribute("type") @@ -572,16 +594,18 @@ class Glsa: self.services = self.affected.getElementsByTagName("service") return None - def dump(self, outstream=sys.stdout): + def dump(self, outstream=sys.stdout, encoding="utf-8"): """ - Dumps a plaintext representation of this GLSA to I{outfile} or + Dumps a plaintext representation of this GLSA to I{outfile} or B{stdout} if it is ommitted. You can specify an alternate - I{encoding} if needed (default is latin1). - + I{encoding} if needed (default is utf-8). + @type outstream: File @param outfile: Stream that should be used for writing (defaults to sys.stdout) """ + outstream = getattr(outstream, "buffer", outstream) + outstream = codecs.getwriter(encoding)(outstream) width = 76 outstream.write(("GLSA %s: \n%s" % (self.nr, self.title)).center(width)+"\n") outstream.write((width*"=")+"\n") @@ -606,30 +630,24 @@ class Glsa: pass if len(self.bugs) > 0: outstream.write(_("\nRelated bugs: ")) - for i in range(0, len(self.bugs)): - outstream.write(self.bugs[i]) - if i < len(self.bugs)-1: - outstream.write(", ") - else: - outstream.write("\n") + outstream.write(", ".join(self.bugs)) + outstream.write("\n") if self.background: outstream.write("\n"+wrap(self.background, width, caption=_("Background: "))) outstream.write("\n"+wrap(self.description, width, caption=_("Description: "))) outstream.write("\n"+wrap(self.impact_text, width, caption=_("Impact: "))) outstream.write("\n"+wrap(self.workaround, width, caption=_("Workaround: "))) outstream.write("\n"+wrap(self.resolution, width, caption=_("Resolution: "))) - myreferences = "" - for r in self.references: - myreferences += (r.replace(" ", SPACE_ESCAPE)+NEWLINE_ESCAPE+" ") + myreferences = " ".join(r.replace(" ", SPACE_ESCAPE)+NEWLINE_ESCAPE for r in self.references) outstream.write("\n"+wrap(myreferences, width, caption=_("References: "))) outstream.write("\n") - + def isVulnerable(self): """ Tests if the system is affected by this GLSA by checking if any vulnerable package versions are installed. Also checks for affected architectures. - + @rtype: Boolean @return: True if the system is affected, False if not """ @@ -641,56 +659,67 @@ class Glsa: for v in path["vul_atoms"]: rValue = rValue \ or (len(match(v, self.vardbapi)) > 0 \ - and getMinUpgrade(path["vul_atoms"], path["unaff_atoms"], \ + and None != getMinUpgrade(path["vul_atoms"], path["unaff_atoms"], \ self.portdbapi, self.vardbapi)) return rValue - - def isApplied(self): + + def isInjected(self): """ - Looks if the GLSA IDis in the GLSA checkfile to check if this - GLSA was already applied. - + Looks if the GLSA ID is in the GLSA checkfile to check if this + GLSA should be marked as applied. + @rtype: Boolean - @return: True if the GLSA was applied, False if not + @returns: True if the GLSA is in the inject file, False if not """ + if not os.access(os.path.join(self.config["EROOT"], + PRIVATE_PATH, "glsa_injected"), os.R_OK): + return False return (self.nr in get_applied_glsas(self.config)) def inject(self): """ Puts the ID of this GLSA into the GLSA checkfile, so it won't - show up on future checks. Should be called after a GLSA is + show up on future checks. Should be called after a GLSA is applied or on explicit user request. @rtype: None @return: None """ - if not self.isApplied(): + if not self.isInjected(): checkfile = io.open( _unicode_encode(os.path.join(self.config["EROOT"], - CACHE_PATH, "glsa"), - encoding=_encodings['fs'], errors='strict'), + PRIVATE_PATH, "glsa_injected"), + encoding=_encodings['fs'], errors='strict'), mode='a+', encoding=_encodings['content'], errors='strict') checkfile.write(_unicode_decode(self.nr + "\n")) checkfile.close() return None - + def getMergeList(self, least_change=True): """ Returns the list of package-versions that have to be merged to - apply this GLSA properly. The versions are as low as possible + apply this GLSA properly. The versions are as low as possible while avoiding downgrades (see L{getMinUpgrade}). - + @type least_change: Boolean @param least_change: True if the smallest possible upgrade should be selected, False for an emerge-like algorithm @rtype: List of Strings @return: list of package-versions that have to be merged """ - rValue = [] - for pkg in self.packages: + return list(set(update for (vuln, update) in self.getAffectionTable(least_change) if update)) + + def getAffectionTable(self, least_change=True): + """ + Will initialize the self.systemAffection list of + atoms installed on the system that are affected + by this GLSA, and the atoms that are minimal upgrades. + """ + systemAffection = [] + for pkg in self.packages.keys(): for path in self.packages[pkg]: - update = getMinUpgrade(path["vul_atoms"], path["unaff_atoms"], \ + update = getMinUpgrade(path["vul_atoms"], path["unaff_atoms"], self.portdbapi, self.vardbapi, minimize=least_change) if update: - rValue.append(update) - return rValue + systemAffection.extend(update) + return systemAffection diff --git a/portage_with_autodep/pym/portage/glsa.pyo b/portage_with_autodep/pym/portage/glsa.pyo Binary files differindex 65162f1..dd97e49 100644 --- a/portage_with_autodep/pym/portage/glsa.pyo +++ b/portage_with_autodep/pym/portage/glsa.pyo diff --git a/portage_with_autodep/pym/portage/localization.py b/portage_with_autodep/pym/portage/localization.py index d16c4b1..2815ef5 100644 --- a/portage_with_autodep/pym/portage/localization.py +++ b/portage_with_autodep/pym/portage/localization.py @@ -1,12 +1,18 @@ # localization.py -- Code to manage/help portage localization. -# Copyright 2004 Gentoo Foundation +# Copyright 2004-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from portage import _unicode_decode # We define this to make the transition easier for us. def _(mystr): - return mystr - + """ + Always returns unicode, regardless of the input type. This is + helpful for avoiding UnicodeDecodeError from __str__() with + Python 2, by ensuring that string format operations invoke + __unicode__() instead of __str__(). + """ + return _unicode_decode(mystr) def localization_example(): # Dict references allow translators to rearrange word order. diff --git a/portage_with_autodep/pym/portage/localization.pyo b/portage_with_autodep/pym/portage/localization.pyo Binary files differindex e992e3a..50a2805 100644 --- a/portage_with_autodep/pym/portage/localization.pyo +++ b/portage_with_autodep/pym/portage/localization.pyo diff --git a/portage_with_autodep/pym/portage/locks.py b/portage_with_autodep/pym/portage/locks.py index 59fbc6e..8571d8c 100644 --- a/portage_with_autodep/pym/portage/locks.py +++ b/portage_with_autodep/pym/portage/locks.py @@ -1,5 +1,5 @@ # portage: Lock management code -# Copyright 2004-2012 Gentoo Foundation +# Copyright 2004-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ["lockdir", "unlockdir", "lockfile", "unlockfile", \ @@ -17,7 +17,6 @@ import portage from portage import os, _encodings, _unicode_decode from portage.exception import DirectoryNotFound, FileNotFound, \ InvalidData, TryAgain, OperationNotPermitted, PermissionDenied -from portage.data import portage_gid from portage.util import writemsg from portage.localization import _ @@ -64,6 +63,9 @@ def lockfile(mypath, wantnewlockfile=0, unlinkfile=0, if not mypath: raise InvalidData(_("Empty path given")) + # Since Python 3.4, chown requires int type (no proxies). + portage_gid = int(portage.data.portage_gid) + # Support for file object or integer file descriptor parameters is # deprecated due to ambiguity in whether or not it's safe to close # the file descriptor, making it prone to "Bad file descriptor" errors @@ -148,7 +150,7 @@ def lockfile(mypath, wantnewlockfile=0, unlinkfile=0, except IOError as e: if not hasattr(e, "errno"): raise - if e.errno in (errno.EACCES, errno.EAGAIN): + if e.errno in (errno.EACCES, errno.EAGAIN, errno.ENOLCK): # resource temp unavailable; eg, someone beat us to the lock. if flags & os.O_NONBLOCK: os.close(myfd) @@ -163,19 +165,43 @@ def lockfile(mypath, wantnewlockfile=0, unlinkfile=0, if isinstance(mypath, int): waiting_msg = _("waiting for lock on fd %i") % myfd else: - waiting_msg = _("waiting for lock on %s\n") % lockfilename + waiting_msg = _("waiting for lock on %s") % lockfilename if out is not None: out.ebegin(waiting_msg) # try for the exclusive lock now. - try: - locking_method(myfd, fcntl.LOCK_EX) - except EnvironmentError as e: - if out is not None: - out.eend(1, str(e)) - raise + enolock_msg_shown = False + while True: + try: + locking_method(myfd, fcntl.LOCK_EX) + except EnvironmentError as e: + if e.errno == errno.ENOLCK: + # This is known to occur on Solaris NFS (see + # bug #462694). Assume that the error is due + # to temporary exhaustion of record locks, + # and loop until one becomes available. + if not enolock_msg_shown: + enolock_msg_shown = True + if isinstance(mypath, int): + context_desc = _("Error while waiting " + "to lock fd %i") % myfd + else: + context_desc = _("Error while waiting " + "to lock '%s'") % lockfilename + writemsg("\n!!! %s: %s\n" % (context_desc, e), + noiselevel=-1) + + time.sleep(_HARDLINK_POLL_LATENCY) + continue + + if out is not None: + out.eend(1, str(e)) + raise + else: + break + if out is not None: out.eend(os.EX_OK) - elif e.errno in (errno.ENOSYS, errno.ENOLCK): + elif e.errno in (errno.ENOSYS,): # We're not allowed to lock on this FS. if not isinstance(lockfilename, int): # If a file object was passed in, it's not safe @@ -207,6 +233,17 @@ def lockfile(mypath, wantnewlockfile=0, unlinkfile=0, waiting_msg=waiting_msg, flags=flags) if myfd != HARDLINK_FD: + + # FD_CLOEXEC is enabled by default in Python >=3.4. + if sys.hexversion < 0x3040000: + try: + fcntl.FD_CLOEXEC + except AttributeError: + pass + else: + fcntl.fcntl(myfd, fcntl.F_SETFD, + fcntl.fcntl(myfd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC) + _open_fds.add(myfd) writemsg(str((lockfilename,myfd,unlinkfile))+"\n",1) @@ -339,6 +376,9 @@ def hardlink_lockfile(lockfilename, max_wait=DeprecationWarning, preexisting = os.path.exists(lockfilename) myhardlock = hardlock_name(lockfilename) + # Since Python 3.4, chown requires int type (no proxies). + portage_gid = int(portage.data.portage_gid) + # myhardlock must not exist prior to our link() call, and we can # safely unlink it since its file name is unique to our PID try: diff --git a/portage_with_autodep/pym/portage/locks.pyo b/portage_with_autodep/pym/portage/locks.pyo Binary files differindex 9c90a2f..c2e24fb 100644 --- a/portage_with_autodep/pym/portage/locks.pyo +++ b/portage_with_autodep/pym/portage/locks.pyo diff --git a/portage_with_autodep/pym/portage/mail.pyo b/portage_with_autodep/pym/portage/mail.pyo Binary files differindex bc3a76d..f0ef5df 100644 --- a/portage_with_autodep/pym/portage/mail.pyo +++ b/portage_with_autodep/pym/portage/mail.pyo diff --git a/portage_with_autodep/pym/portage/manifest.py b/portage_with_autodep/pym/portage/manifest.py index 90324ee..510e203 100644 --- a/portage_with_autodep/pym/portage/manifest.py +++ b/portage_with_autodep/pym/portage/manifest.py @@ -1,14 +1,19 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + import errno import io import re +import sys import warnings import portage portage.proxy.lazyimport.lazyimport(globals(), - 'portage.checksum:hashfunc_map,perform_multiple_checksums,verify_all', + 'portage.checksum:hashfunc_map,perform_multiple_checksums,' + \ + 'verify_all,_apply_hash_filter,_filter_unaccelarated_hashes', + 'portage.repository.config:_find_invalid_path_char', 'portage.util:write_atomic', ) @@ -23,8 +28,15 @@ from portage.const import (MANIFEST1_HASH_FUNCTIONS, MANIFEST2_HASH_DEFAULTS, MANIFEST2_HASH_FUNCTIONS, MANIFEST2_IDENTIFIERS, MANIFEST2_REQUIRED_HASH) from portage.localization import _ -# Characters prohibited by repoman's file.name check. -_prohibited_filename_chars_re = re.compile(r'[^a-zA-Z0-9._\-+:]') +_manifest_re = re.compile( + r'^(' + '|'.join(MANIFEST2_IDENTIFIERS) + r') (.*)( \d+( \S+ \S+)+)$', + re.UNICODE) + +if sys.hexversion >= 0x3000000: + _unicode = str + basestring = str +else: + _unicode = unicode class FileNotInManifestException(PortageException): pass @@ -37,15 +49,10 @@ def manifest2AuxfileFilter(filename): for x in mysplit: if x[:1] == '.': return False - if _prohibited_filename_chars_re.search(x) is not None: - return False return not filename[:7] == 'digest-' def manifest2MiscfileFilter(filename): - filename = filename.strip(os.sep) - if _prohibited_filename_chars_re.search(filename) is not None: - return False - return not (filename in ["CVS", ".svn", "files", "Manifest"] or filename.endswith(".ebuild")) + return not (filename == "Manifest" or filename.endswith(".ebuild")) def guessManifestFileType(filename): """ Perform a best effort guess of which type the given filename is, avoid using this if possible """ @@ -66,18 +73,17 @@ def guessThinManifestFileType(filename): return None return "DIST" -def parseManifest2(mysplit): +def parseManifest2(line): + if not isinstance(line, basestring): + line = ' '.join(line) myentry = None - if len(mysplit) > 4 and mysplit[0] in MANIFEST2_IDENTIFIERS: - mytype = mysplit[0] - myname = mysplit[1] - try: - mysize = int(mysplit[2]) - except ValueError: - return None - myhashes = dict(zip(mysplit[3::2], mysplit[4::2])) - myhashes["size"] = mysize - myentry = Manifest2Entry(type=mytype, name=myname, hashes=myhashes) + match = _manifest_re.match(line) + if match is not None: + tokens = match.group(3).split() + hashes = dict(zip(tokens[1::2], tokens[2::2])) + hashes["size"] = int(tokens[0]) + myentry = Manifest2Entry(type=match.group(1), + name=match.group(2), hashes=hashes) return myentry class ManifestEntry(object): @@ -107,11 +113,20 @@ class Manifest2Entry(ManifestEntry): def __ne__(self, other): return not self.__eq__(other) + if sys.hexversion < 0x3000000: + + __unicode__ = __str__ + + def __str__(self): + return _unicode_encode(self.__unicode__(), + encoding=_encodings['repo.content'], errors='strict') + class Manifest(object): parsers = (parseManifest2,) - def __init__(self, pkgdir, distdir, fetchlist_dict=None, + def __init__(self, pkgdir, distdir=None, fetchlist_dict=None, manifest1_compat=DeprecationWarning, from_scratch=False, thin=False, - allow_missing=False, allow_create=True, hashes=None): + allow_missing=False, allow_create=True, hashes=None, + find_invalid_path_char=None): """ Create new Manifest instance for package in pkgdir. Do not parse Manifest file if from_scratch == True (only for internal use) The fetchlist_dict parameter is required only for generation of @@ -124,6 +139,9 @@ class Manifest(object): "portage.manifest.Manifest constructor is deprecated.", DeprecationWarning, stacklevel=2) + if find_invalid_path_char is None: + find_invalid_path_char = _find_invalid_path_char + self._find_invalid_path_char = find_invalid_path_char self.pkgdir = _unicode_decode(pkgdir).rstrip(os.sep) + os.sep self.fhashdict = {} self.hashes = set() @@ -172,13 +190,12 @@ class Manifest(object): """Parse a manifest. If myhashdict is given then data will be added too it. Otherwise, a new dict will be created and returned.""" try: - fd = io.open(_unicode_encode(file_path, + with io.open(_unicode_encode(file_path, encoding=_encodings['fs'], errors='strict'), mode='r', - encoding=_encodings['repo.content'], errors='replace') - if myhashdict is None: - myhashdict = {} - self._parseDigests(fd, myhashdict=myhashdict, **kwargs) - fd.close() + encoding=_encodings['repo.content'], errors='replace') as f: + if myhashdict is None: + myhashdict = {} + self._parseDigests(f, myhashdict=myhashdict, **kwargs) return myhashdict except (OSError, IOError) as e: if e.errno == errno.ENOENT: @@ -197,9 +214,8 @@ class Manifest(object): """Parse manifest lines and return a list of manifest entries.""" for myline in mylines: myentry = None - mysplit = myline.split() for parser in self.parsers: - myentry = parser(mysplit) + myentry = parser(myline) if myentry is not None: yield myentry break # go to the next line @@ -254,9 +270,12 @@ class Manifest(object): (MANIFEST2_REQUIRED_HASH, t, f)) def write(self, sign=False, force=False): - """ Write Manifest instance to disk, optionally signing it """ + """ Write Manifest instance to disk, optionally signing it. Returns + True if the Manifest is actually written, and False if the write + is skipped due to existing Manifest being identical.""" + rval = False if not self.allow_create: - return + return rval self.checkIntegrity() try: myentries = list(self._createManifestEntries()) @@ -288,7 +307,8 @@ class Manifest(object): # thin manifests with no DIST entries, myentries is # non-empty for all currently known use cases. write_atomic(self.getFullname(), "".join("%s\n" % - str(myentry) for myentry in myentries)) + _unicode(myentry) for myentry in myentries)) + rval = True else: # With thin manifest, there's no need to have # a Manifest file if there are no DIST entries. @@ -297,6 +317,7 @@ class Manifest(object): except OSError as e: if e.errno != errno.ENOENT: raise + rval = True if sign: self.sign() @@ -304,6 +325,7 @@ class Manifest(object): if e.errno == errno.EACCES: raise PermissionDenied(str(e)) raise + return rval def sign(self): """ Sign the Manifest """ @@ -362,10 +384,11 @@ class Manifest(object): distfilehashes = self.fhashdict["DIST"] else: distfilehashes = {} - self.__init__(self.pkgdir, self.distdir, + self.__init__(self.pkgdir, distdir=self.distdir, fetchlist_dict=self.fetchlist_dict, from_scratch=True, thin=self.thin, allow_missing=self.allow_missing, - allow_create=self.allow_create, hashes=self.hashes) + allow_create=self.allow_create, hashes=self.hashes, + find_invalid_path_char=self._find_invalid_path_char) pn = os.path.basename(self.pkgdir.rstrip(os.path.sep)) cat = self._pkgdir_category() @@ -460,7 +483,8 @@ class Manifest(object): if pf is not None: mytype = "EBUILD" cpvlist.append(pf) - elif manifest2MiscfileFilter(f): + elif self._find_invalid_path_char(f) == -1 and \ + manifest2MiscfileFilter(f): mytype = "MISC" else: continue @@ -479,7 +503,8 @@ class Manifest(object): full_path = os.path.join(parentdir, f) recursive_files.append(full_path[cut_len:]) for f in recursive_files: - if not manifest2AuxfileFilter(f): + if self._find_invalid_path_char(f) != -1 or \ + not manifest2AuxfileFilter(f): continue self.fhashdict["AUX"][f] = perform_multiple_checksums( os.path.join(self.pkgdir, "files", f.lstrip(os.sep)), self.hashes) @@ -501,14 +526,17 @@ class Manifest(object): for t in MANIFEST2_IDENTIFIERS: self.checkTypeHashes(t, ignoreMissingFiles=ignoreMissingFiles) - def checkTypeHashes(self, idtype, ignoreMissingFiles=False): + def checkTypeHashes(self, idtype, ignoreMissingFiles=False, hash_filter=None): for f in self.fhashdict[idtype]: - self.checkFileHashes(idtype, f, ignoreMissing=ignoreMissingFiles) + self.checkFileHashes(idtype, f, ignoreMissing=ignoreMissingFiles, + hash_filter=hash_filter) - def checkFileHashes(self, ftype, fname, ignoreMissing=False): - myhashes = self.fhashdict[ftype][fname] + def checkFileHashes(self, ftype, fname, ignoreMissing=False, hash_filter=None): + digests = _filter_unaccelarated_hashes(self.fhashdict[ftype][fname]) + if hash_filter is not None: + digests = _apply_hash_filter(digests, hash_filter) try: - ok,reason = verify_all(self._getAbsname(ftype, fname), self.fhashdict[ftype][fname]) + ok, reason = verify_all(self._getAbsname(ftype, fname), digests) if not ok: raise DigestException(tuple([self._getAbsname(ftype, fname)]+list(reason))) return ok, reason diff --git a/portage_with_autodep/pym/portage/manifest.pyo b/portage_with_autodep/pym/portage/manifest.pyo Binary files differindex d482bbd..40d96e1 100644 --- a/portage_with_autodep/pym/portage/manifest.pyo +++ b/portage_with_autodep/pym/portage/manifest.pyo diff --git a/portage_with_autodep/pym/portage/news.py b/portage_with_autodep/pym/portage/news.py index bbd9325..408fb5c 100644 --- a/portage_with_autodep/pym/portage/news.py +++ b/portage_with_autodep/pym/portage/news.py @@ -1,8 +1,8 @@ # portage: news management code -# Copyright 2006-2011 Gentoo Foundation +# Copyright 2006-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import print_function +from __future__ import print_function, unicode_literals __all__ = ["NewsManager", "NewsItem", "DisplayRestriction", "DisplayProfileRestriction", "DisplayKeywordRestriction", @@ -13,6 +13,7 @@ import io import logging import os as _os import re +import portage from portage import OrderedDict from portage import os from portage import _encodings @@ -241,7 +242,8 @@ class NewsItem(object): for values in self.restrictions.values(): any_match = False for restriction in values: - if restriction.checkRestriction(**kwargs): + if restriction.checkRestriction( + **portage._native_kwargs(kwargs)): any_match = True if not any_match: all_match = False @@ -388,7 +390,7 @@ def count_unread_news(portdb, vardb, repos=None, update=True): # NOTE: The NewsManager typically handles permission errors by # returning silently, so PermissionDenied won't necessarily be # raised even if we do trigger a permission error above. - msg = _unicode_decode("Permission denied: '%s'\n") % (e,) + msg = "Permission denied: '%s'\n" % (e,) if msg in permission_msgs: pass else: diff --git a/portage_with_autodep/pym/portage/news.pyo b/portage_with_autodep/pym/portage/news.pyo Binary files differindex bbd247c..0a1d4ab 100644 --- a/portage_with_autodep/pym/portage/news.pyo +++ b/portage_with_autodep/pym/portage/news.pyo diff --git a/portage_with_autodep/pym/portage/output.py b/portage_with_autodep/pym/portage/output.py index 98bec81..fc1b042 100644 --- a/portage_with_autodep/pym/portage/output.py +++ b/portage_with_autodep/pym/portage/output.py @@ -1,4 +1,4 @@ -# Copyright 1998-2011 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __docformat__ = "epytext" @@ -7,6 +7,7 @@ import errno import io import formatter import re +import subprocess import sys import portage @@ -163,15 +164,12 @@ def _parse_color_map(config_root='/', onerror=None): token = token[1:-1] return token - f = None try: - f = io.open(_unicode_encode(myfile, + with io.open(_unicode_encode(myfile, encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['content'], errors='replace') - lineno = 0 - for line in f: - lineno += 1 - + mode='r', encoding=_encodings['content'], errors='replace') as f: + lines = f.readlines() + for lineno, line in enumerate(lines): commenter_pos = line.find("#") line = line[:commenter_pos].strip() @@ -229,9 +227,6 @@ def _parse_color_map(config_root='/', onerror=None): elif e.errno == errno.EACCES: raise PermissionDenied(myfile) raise - finally: - if f is not None: - f.close() def nc_len(mystr): tmp = re.sub(esc_seq + "^m]+m", "", mystr); @@ -244,7 +239,7 @@ _max_xtermTitle_len = 253 def xtermTitle(mystr, raw=False): global _disable_xtermTitle if _disable_xtermTitle is None: - _disable_xtermTitle = not (sys.stderr.isatty() and \ + _disable_xtermTitle = not (sys.__stderr__.isatty() and \ 'TERM' in os.environ and \ _legal_terms_re.match(os.environ['TERM']) is not None) @@ -277,15 +272,18 @@ def xtermTitleReset(): if dotitles and \ 'TERM' in os.environ and \ _legal_terms_re.match(os.environ['TERM']) is not None and \ - sys.stderr.isatty(): + sys.__stderr__.isatty(): from portage.process import find_binary, spawn shell = os.environ.get("SHELL") if not shell or not os.access(shell, os.EX_OK): shell = find_binary("sh") if shell: spawn([shell, "-c", prompt_command], env=os.environ, - fd_pipes={0:sys.stdin.fileno(),1:sys.stderr.fileno(), - 2:sys.stderr.fileno()}) + fd_pipes={ + 0: portage._get_stdin().fileno(), + 1: sys.__stderr__.fileno(), + 2: sys.__stderr__.fileno() + }) else: os.system(prompt_command) return @@ -425,28 +423,41 @@ class StyleWriter(formatter.DumbWriter): if self.style_listener: self.style_listener(styles) -def get_term_size(): +def get_term_size(fd=None): """ Get the number of lines and columns of the tty that is connected to - stdout. Returns a tuple of (lines, columns) or (0, 0) if an error + fd. Returns a tuple of (lines, columns) or (0, 0) if an error occurs. The curses module is used if available, otherwise the output of `stty size` is parsed. The lines and columns values are guaranteed to be greater than or equal to zero, since a negative COLUMNS variable is known to prevent some commands from working (see bug #394091). """ - if not sys.stdout.isatty(): + if fd is None: + fd = sys.stdout + if not hasattr(fd, 'isatty') or not fd.isatty(): return (0, 0) try: import curses try: - curses.setupterm() + curses.setupterm(term=os.environ.get("TERM", "unknown"), + fd=fd.fileno()) return curses.tigetnum('lines'), curses.tigetnum('cols') except curses.error: pass except ImportError: pass - st, out = portage.subprocess_getstatusoutput('stty size') - if st == os.EX_OK: + + try: + proc = subprocess.Popen(["stty", "size"], + stdout=subprocess.PIPE, stderr=fd) + except EnvironmentError as e: + if e.errno != errno.ENOENT: + raise + # stty command not found + return (0, 0) + + out = _unicode_decode(proc.communicate()[0]) + if proc.wait() == os.EX_OK: out = out.split() if len(out) == 2: try: @@ -631,11 +642,14 @@ class EOutput(object): class ProgressBar(object): """The interface is copied from the ProgressBar class from the EasyDialogs module (which is Mac only).""" - def __init__(self, title=None, maxval=0, label=None): - self._title = title + def __init__(self, title=None, maxval=0, label=None, max_desc_length=25): + self._title = title or "" self._maxval = maxval - self._label = maxval + self._label = label or "" self._curval = 0 + self._desc = "" + self._desc_max_length = max_desc_length + self._set_desc() @property def curval(self): @@ -659,10 +673,23 @@ class ProgressBar(object): def title(self, newstr): """Sets the text in the title bar of the progress dialog to newstr.""" self._title = newstr + self._set_desc() def label(self, newstr): """Sets the text in the progress box of the progress dialog to newstr.""" self._label = newstr + self._set_desc() + + def _set_desc(self): + self._desc = "%s%s" % ( + "%s: " % self._title if self._title else "", + "%s" % self._label if self._label else "" + ) + if len(self._desc) > self._desc_max_length: # truncate if too long + self._desc = "%s..." % self._desc[:self._desc_max_length - 3] + if len(self._desc): + self._desc = self._desc.ljust(self._desc_max_length) + def set(self, value, maxval=None): """ @@ -691,10 +718,10 @@ class ProgressBar(object): class TermProgressBar(ProgressBar): """A tty progress bar similar to wget's.""" - def __init__(self, **kwargs): + def __init__(self, fd=sys.stdout, **kwargs): ProgressBar.__init__(self, **kwargs) - lines, self.term_columns = get_term_size() - self.file = sys.stdout + lines, self.term_columns = get_term_size(fd) + self.file = fd self._min_columns = 11 self._max_columns = 80 # for indeterminate mode, ranges from 0.0 to 1.0 @@ -717,16 +744,18 @@ class TermProgressBar(ProgressBar): curval = self._curval maxval = self._maxval position = self._position - percentage_str_width = 4 + percentage_str_width = 5 square_brackets_width = 2 if cols < percentage_str_width: return "" - bar_space = cols - percentage_str_width - square_brackets_width + bar_space = cols - percentage_str_width - square_brackets_width - 1 + if self._desc: + bar_space -= self._desc_max_length if maxval == 0: max_bar_width = bar_space-3 - image = " " + _percent = "".ljust(percentage_str_width) if cols < min_columns: - return image + return "" if position <= 0.5: offset = 2 * position else: @@ -742,16 +771,16 @@ class TermProgressBar(ProgressBar): position = 0.5 self._position = position bar_width = int(offset * max_bar_width) - image = image + "[" + (bar_width * " ") + \ - "<=>" + ((max_bar_width - bar_width) * " ") + "]" + image = "%s%s%s" % (self._desc, _percent, + "[" + (bar_width * " ") + \ + "<=>" + ((max_bar_width - bar_width) * " ") + "]") return image else: percentage = int(100 * float(curval) / maxval) - if percentage == 100: - percentage_str_width += 1 - bar_space -= 1 max_bar_width = bar_space - 1 - image = ("%d%% " % percentage).rjust(percentage_str_width) + _percent = ("%d%% " % percentage).rjust(percentage_str_width) + image = "%s%s" % (self._desc, _percent) + if cols < min_columns: return image offset = float(curval) / maxval diff --git a/portage_with_autodep/pym/portage/output.pyo b/portage_with_autodep/pym/portage/output.pyo Binary files differindex 993a2de..2bb98b0 100644 --- a/portage_with_autodep/pym/portage/output.pyo +++ b/portage_with_autodep/pym/portage/output.pyo diff --git a/portage_with_autodep/pym/portage/package/__init__.pyo b/portage_with_autodep/pym/portage/package/__init__.pyo Binary files differindex 9d8f30c..a17fc87 100644 --- a/portage_with_autodep/pym/portage/package/__init__.pyo +++ b/portage_with_autodep/pym/portage/package/__init__.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/__init__.pyo b/portage_with_autodep/pym/portage/package/ebuild/__init__.pyo Binary files differindex 927b4bc..ff34626 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/__init__.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/__init__.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.py b/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.py index 0c613ce..af606f1 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.py @@ -11,7 +11,7 @@ from portage.dep import ExtendedAtomDict, _repo_separator, _slot_separator from portage.localization import _ from portage.package.ebuild._config.helper import ordered_by_atom_specificity from portage.util import grabdict_package, stack_lists, writemsg -from portage.versions import cpv_getkey, _pkg_str +from portage.versions import _pkg_str class KeywordsManager(object): """Manager class to handle keywords processing and validation""" @@ -77,7 +77,9 @@ class KeywordsManager(object): def getKeywords(self, cpv, slot, keywords, repo): - if not hasattr(cpv, 'slot'): + try: + cpv.slot + except AttributeError: pkg = _pkg_str(cpv, slot=slot, repo=repo) else: pkg = cpv @@ -91,6 +93,47 @@ class KeywordsManager(object): keywords.extend(pkg_keywords) return stack_lists(keywords, incremental=True) + def isStable(self, pkg, global_accept_keywords, backuped_accept_keywords): + mygroups = self.getKeywords(pkg, None, pkg._metadata["KEYWORDS"], None) + pgroups = global_accept_keywords.split() + + unmaskgroups = self.getPKeywords(pkg, None, None, + global_accept_keywords) + pgroups.extend(unmaskgroups) + + egroups = backuped_accept_keywords.split() + + if unmaskgroups or egroups: + pgroups = self._getEgroups(egroups, pgroups) + else: + pgroups = set(pgroups) + + if self._getMissingKeywords(pkg, pgroups, mygroups): + return False + + if pkg.cpv._settings.local_config: + # If replacing all keywords with unstable variants would mask the + # package, then it's considered stable. + unstable = [] + for kw in mygroups: + if kw[:1] != "~": + kw = "~" + kw + unstable.append(kw) + + return bool(self._getMissingKeywords(pkg, pgroups, set(unstable))) + else: + # For repoman, if the package has an effective stable keyword that + # intersects with the effective ACCEPT_KEYWORDS for the current + # profile, then consider it stable. + for kw in pgroups: + if kw[:1] != "~": + if kw in mygroups or '*' in mygroups: + return True + if kw == '*': + for x in mygroups: + if x[:1] != "~": + return True + return False def getMissingKeywords(self, cpv, @@ -237,7 +280,7 @@ class KeywordsManager(object): if not mygroups: # If KEYWORDS is empty then we still have to return something # in order to distinguish from the case of "none missing". - mygroups.append("**") + mygroups = ["**"] missing = mygroups return missing @@ -261,9 +304,11 @@ class KeywordsManager(object): """ pgroups = global_accept_keywords.split() - if not hasattr(cpv, 'slot'): + try: + cpv.slot + except AttributeError: cpv = _pkg_str(cpv, slot=slot, repo=repo) - cp = cpv_getkey(cpv) + cp = cpv.cp unmaskgroups = [] if self._p_accept_keywords: @@ -288,4 +333,3 @@ class KeywordsManager(object): for x in pkg_accept_keywords: unmaskgroups.extend(x) return unmaskgroups - diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.pyo Binary files differindex 15043f0..b922211 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/LicenseManager.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/LicenseManager.pyo Binary files differindex 4a38298..4ddda5e 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/LicenseManager.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/LicenseManager.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.py b/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.py index f7a1177..80b6a70 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.py @@ -1,6 +1,8 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ( 'LocationsManager', ) @@ -13,10 +15,12 @@ import portage from portage import os, eapi_is_supported, _encodings, _unicode_encode from portage.const import CUSTOM_PROFILE_PATH, GLOBAL_CONFIG_PATH, \ PROFILE_PATH, USER_CONFIG_PATH +from portage.eapi import eapi_allows_directories_on_profile_level_and_repository_level from portage.exception import DirectoryNotFound, ParseError from portage.localization import _ from portage.util import ensure_dirs, grabfile, \ normalize_path, shlex_split, writemsg +from portage.util._path import exists_raise_eaccess, isdir_raise_eaccess from portage.repository.config import parse_layout_conf, \ _portage1_profiles_allow_directories @@ -27,7 +31,7 @@ _PORTAGE1_DIRECTORIES = frozenset([ 'use.mask', 'use.force']) _profile_node = collections.namedtuple('_profile_node', - 'location portage1_directories') + 'location portage1_directories user_config') _allow_parent_colon = frozenset( ["portage-2"]) @@ -45,9 +49,13 @@ class LocationsManager(object): if self.eprefix is None: self.eprefix = portage.const.EPREFIX + elif self.eprefix: + self.eprefix = normalize_path(self.eprefix) + if self.eprefix == os.sep: + self.eprefix = "" if self.config_root is None: - self.config_root = self.eprefix + os.sep + self.config_root = portage.const.EPREFIX + os.sep self.config_root = normalize_path(os.path.abspath( self.config_root)).rstrip(os.path.sep) + os.path.sep @@ -72,14 +80,26 @@ class LocationsManager(object): known_repos = tuple(known_repos) if self.config_profile_path is None: + deprecated_profile_path = os.path.join( + self.config_root, 'etc', 'make.profile') self.config_profile_path = \ os.path.join(self.config_root, PROFILE_PATH) - if os.path.isdir(self.config_profile_path): + if isdir_raise_eaccess(self.config_profile_path): self.profile_path = self.config_profile_path + if isdir_raise_eaccess(deprecated_profile_path) and not \ + os.path.samefile(self.profile_path, + deprecated_profile_path): + # Don't warn if they refer to the same path, since + # that can be used for backward compatibility with + # old software. + writemsg("!!! %s\n" % + _("Found 2 make.profile dirs: " + "using '%s', ignoring '%s'") % + (self.profile_path, deprecated_profile_path), + noiselevel=-1) else: - self.config_profile_path = \ - os.path.join(self.abs_user_config, 'make.profile') - if os.path.isdir(self.config_profile_path): + self.config_profile_path = deprecated_profile_path + if isdir_raise_eaccess(self.config_profile_path): self.profile_path = self.config_profile_path else: self.profile_path = None @@ -99,9 +119,9 @@ class LocationsManager(object): self._addProfile(os.path.realpath(self.profile_path), repositories, known_repos) except ParseError as e: - writemsg(_("!!! Unable to parse profile: '%s'\n") % \ - self.profile_path, noiselevel=-1) - writemsg("!!! ParseError: %s\n" % str(e), noiselevel=-1) + if not portage._sync_disabled_warnings: + writemsg(_("!!! Unable to parse profile: '%s'\n") % self.profile_path, noiselevel=-1) + writemsg("!!! ParseError: %s\n" % str(e), noiselevel=-1) self.profiles = [] self.profiles_complex = [] @@ -111,14 +131,15 @@ class LocationsManager(object): if os.path.exists(custom_prof): self.user_profile_dir = custom_prof self.profiles.append(custom_prof) - self.profiles_complex.append(_profile_node(custom_prof, True)) + self.profiles_complex.append( + _profile_node(custom_prof, True, True)) del custom_prof self.profiles = tuple(self.profiles) self.profiles_complex = tuple(self.profiles_complex) def _check_var_directory(self, varname, var): - if not os.path.isdir(var): + if not isdir_raise_eaccess(var): writemsg(_("!!! Error: %s='%s' is not a directory. " "Please correct this.\n") % (varname, var), noiselevel=-1) @@ -130,33 +151,9 @@ class LocationsManager(object): allow_parent_colon = True repo_loc = None compat_mode = False - intersecting_repos = [x for x in known_repos if current_abs_path.startswith(x[0])] - if intersecting_repos: - # protect against nested repositories. Insane configuration, but the longest - # path will be the correct one. - repo_loc, layout_data = max(intersecting_repos, key=lambda x:len(x[0])) - allow_directories = any(x in _portage1_profiles_allow_directories - for x in layout_data['profile-formats']) - compat_mode = layout_data['profile-formats'] == ('portage-1-compat',) - allow_parent_colon = any(x in _allow_parent_colon - for x in layout_data['profile-formats']) - if compat_mode: - offenders = _PORTAGE1_DIRECTORIES.intersection(os.listdir(currentPath)) - offenders = sorted(x for x in offenders - if os.path.isdir(os.path.join(currentPath, x))) - if offenders: - warnings.warn(_("Profile '%(profile_path)s' in repository " - "'%(repo_name)s' is implicitly using 'portage-1' profile format, but " - "the repository profiles are not marked as that format. This will break " - "in the future. Please either convert the following paths " - "to files, or add\nprofile-formats = portage-1\nto the " - "repositories layout.conf. Files: '%(files)s'\n") - % dict(profile_path=currentPath, repo_name=repo_loc, - files=', '.join(offenders))) - - parentsFile = os.path.join(currentPath, "parent") eapi_file = os.path.join(currentPath, "eapi") + eapi = "0" f = None try: f = io.open(_unicode_encode(eapi_file, @@ -174,7 +171,38 @@ class LocationsManager(object): finally: if f is not None: f.close() - if os.path.exists(parentsFile): + + intersecting_repos = [x for x in known_repos if current_abs_path.startswith(x[0])] + if intersecting_repos: + # protect against nested repositories. Insane configuration, but the longest + # path will be the correct one. + repo_loc, layout_data = max(intersecting_repos, key=lambda x:len(x[0])) + allow_directories = eapi_allows_directories_on_profile_level_and_repository_level(eapi) or \ + any(x in _portage1_profiles_allow_directories for x in layout_data['profile-formats']) + compat_mode = not eapi_allows_directories_on_profile_level_and_repository_level(eapi) and \ + layout_data['profile-formats'] == ('portage-1-compat',) + allow_parent_colon = any(x in _allow_parent_colon + for x in layout_data['profile-formats']) + + if compat_mode: + offenders = _PORTAGE1_DIRECTORIES.intersection(os.listdir(currentPath)) + offenders = sorted(x for x in offenders + if os.path.isdir(os.path.join(currentPath, x))) + if offenders: + warnings.warn(_( + "\nThe selected profile is implicitly using the 'portage-1' format:\n" + "\tprofile = %(profile_path)s\n" + "But this repository is not using that format:\n" + "\trepo = %(repo_name)s\n" + "This will break in the future. Please convert these dirs to files:\n" + "\t%(files)s\n" + "Or, add this line to the repository's layout.conf:\n" + "\tprofile-formats = portage-1") + % dict(profile_path=currentPath, repo_name=repo_loc, + files='\n\t'.join(offenders))) + + parentsFile = os.path.join(currentPath, "parent") + if exists_raise_eaccess(parentsFile): parents = grabfile(parentsFile) if not parents: raise ParseError( @@ -196,7 +224,7 @@ class LocationsManager(object): # of the current repo, so realpath it. parentPath = os.path.realpath(parentPath) - if os.path.exists(parentPath): + if exists_raise_eaccess(parentPath): self._addProfile(parentPath, repositories, known_repos) else: raise ParseError( @@ -205,7 +233,7 @@ class LocationsManager(object): self.profiles.append(currentPath) self.profiles_complex.append( - _profile_node(currentPath, allow_directories)) + _profile_node(currentPath, allow_directories, False)) def _expand_parent_colon(self, parentsFile, parentPath, repo_loc, repositories): @@ -253,29 +281,10 @@ class LocationsManager(object): self.eroot = self.target_root.rstrip(os.sep) + self.eprefix + os.sep - # make.globals should not be relative to config_root - # because it only contains constants. However, if EPREFIX - # is set then there are two possible scenarios: - # 1) If $ROOT == "/" then make.globals should be - # relative to EPREFIX. - # 2) If $ROOT != "/" then the correct location of - # make.globals needs to be specified in the constructor - # parameters, since it's a property of the host system - # (and the current config represents the target system). self.global_config_path = GLOBAL_CONFIG_PATH - if self.eprefix: - if self.target_root == "/": - # case (1) above - self.global_config_path = os.path.join(self.eprefix, - GLOBAL_CONFIG_PATH.lstrip(os.sep)) - else: - # case (2) above - # For now, just assume make.globals is relative - # to EPREFIX. - # TODO: Pass in more info to the constructor, - # so we know the host system configuration. - self.global_config_path = os.path.join(self.eprefix, - GLOBAL_CONFIG_PATH.lstrip(os.sep)) + if portage.const.EPREFIX: + self.global_config_path = os.path.join(portage.const.EPREFIX, + GLOBAL_CONFIG_PATH.lstrip(os.sep)) def set_port_dirs(self, portdir, portdir_overlay): self.portdir = portdir @@ -287,7 +296,7 @@ class LocationsManager(object): for ov in shlex_split(self.portdir_overlay): ov = normalize_path(ov) profiles_dir = os.path.join(ov, "profiles") - if os.path.isdir(profiles_dir): + if isdir_raise_eaccess(profiles_dir): self.overlay_profiles.append(profiles_dir) self.profile_locations = [os.path.join(portdir, "profiles")] + self.overlay_profiles diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.pyo Binary files differindex c64d313..5ef52ec 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.py b/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.py index bce1152..aeb04d7 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ( @@ -8,11 +8,10 @@ __all__ = ( import warnings from portage import os -from portage.dep import ExtendedAtomDict, match_from_list, _repo_separator, _slot_separator +from portage.dep import ExtendedAtomDict, match_from_list from portage.localization import _ from portage.util import append_repo, grabfile_package, stack_lists, writemsg -from portage.versions import cpv_getkey -from _emerge.Package import Package +from portage.versions import _pkg_str class MaskManager(object): @@ -47,7 +46,7 @@ class MaskManager(object): "the repository profiles are not marked as that format. This will break " "in the future. Please either convert the following paths " "to files, or add\nprofile-formats = portage-1\nto the " - "repositories layout.conf.\n") + "repository's layout.conf.\n") % dict(repo_name=repo_config.name)) return pmask_cache[loc] @@ -185,12 +184,15 @@ class MaskManager(object): @return: A matching atom string or None if one is not found. """ - cp = cpv_getkey(cpv) - mask_atoms = self._pmaskdict.get(cp) + try: + cpv.slot + except AttributeError: + pkg = _pkg_str(cpv, slot=slot, repo=repo) + else: + pkg = cpv + + mask_atoms = self._pmaskdict.get(pkg.cp) if mask_atoms: - pkg = "".join((cpv, _slot_separator, slot)) - if repo and repo != Package.UNKNOWN_REPO: - pkg = "".join((pkg, _repo_separator, repo)) pkg_list = [pkg] for x in mask_atoms: if not match_from_list(x, pkg_list): @@ -219,8 +221,15 @@ class MaskManager(object): @return: A matching atom string or None if one is not found. """ - cp = cpv_getkey(cpv) - return self._getMaskAtom(cpv, slot, repo, self._punmaskdict.get(cp)) + try: + cpv.slot + except AttributeError: + pkg = _pkg_str(cpv, slot=slot, repo=repo) + else: + pkg = cpv + + return self._getMaskAtom(pkg, slot, repo, + self._punmaskdict.get(pkg.cp)) def getRawMaskAtom(self, cpv, slot, repo): diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.pyo Binary files differindex f48eb47..abc46de 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.py b/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.py index e1ec7f4..0d00810 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.py @@ -1,4 +1,4 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ( @@ -7,36 +7,49 @@ __all__ = ( from _emerge.Package import Package from portage import os -from portage.dep import dep_getrepo, dep_getslot, ExtendedAtomDict, remove_slot, _get_useflag_re +from portage.dep import Atom, dep_getrepo, dep_getslot, ExtendedAtomDict, remove_slot, _get_useflag_re, _repo_separator +from portage.eapi import eapi_has_use_aliases, eapi_supports_stable_use_forcing_and_masking +from portage.exception import InvalidAtom from portage.localization import _ -from portage.util import grabfile, grabdict_package, read_corresponding_eapi_file, stack_lists, writemsg -from portage.versions import cpv_getkey, _pkg_str +from portage.util import grabfile, grabdict, grabdict_package, read_corresponding_eapi_file, stack_lists, writemsg +from portage.versions import _pkg_str from portage.package.ebuild._config.helper import ordered_by_atom_specificity class UseManager(object): - def __init__(self, repositories, profiles, abs_user_config, user_config=True): + def __init__(self, repositories, profiles, abs_user_config, is_stable, + user_config=True): # file variable #-------------------------------- # repositories #-------------------------------- # use.mask _repo_usemask_dict + # use.stable.mask _repo_usestablemask_dict # use.force _repo_useforce_dict + # use.stable.force _repo_usestableforce_dict + # use.aliases _repo_usealiases_dict # package.use.mask _repo_pusemask_dict + # package.use.stable.mask _repo_pusestablemask_dict # package.use.force _repo_puseforce_dict + # package.use.stable.force _repo_pusestableforce_dict + # package.use.aliases _repo_pusealiases_dict #-------------------------------- # profiles #-------------------------------- # use.mask _usemask_list + # use.stable.mask _usestablemask_list # use.force _useforce_list + # use.stable.force _usestableforce_list # package.use.mask _pusemask_list + # package.use.stable.mask _pusestablemask_list # package.use _pkgprofileuse # package.use.force _puseforce_list + # package.use.stable.force _pusestableforce_list #-------------------------------- # user config #-------------------------------- - # package.use _pusedict + # package.use _pusedict # Dynamic variables tracked by the config class #-------------------------------- @@ -49,26 +62,61 @@ class UseManager(object): #-------------------------------- # puse + self._user_config = user_config + self._is_stable = is_stable self._repo_usemask_dict = self._parse_repository_files_to_dict_of_tuples("use.mask", repositories) + self._repo_usestablemask_dict = \ + self._parse_repository_files_to_dict_of_tuples("use.stable.mask", + repositories, eapi_filter=eapi_supports_stable_use_forcing_and_masking) self._repo_useforce_dict = self._parse_repository_files_to_dict_of_tuples("use.force", repositories) + self._repo_usestableforce_dict = \ + self._parse_repository_files_to_dict_of_tuples("use.stable.force", + repositories, eapi_filter=eapi_supports_stable_use_forcing_and_masking) self._repo_pusemask_dict = self._parse_repository_files_to_dict_of_dicts("package.use.mask", repositories) + self._repo_pusestablemask_dict = \ + self._parse_repository_files_to_dict_of_dicts("package.use.stable.mask", + repositories, eapi_filter=eapi_supports_stable_use_forcing_and_masking) self._repo_puseforce_dict = self._parse_repository_files_to_dict_of_dicts("package.use.force", repositories) + self._repo_pusestableforce_dict = \ + self._parse_repository_files_to_dict_of_dicts("package.use.stable.force", + repositories, eapi_filter=eapi_supports_stable_use_forcing_and_masking) self._repo_puse_dict = self._parse_repository_files_to_dict_of_dicts("package.use", repositories) self._usemask_list = self._parse_profile_files_to_tuple_of_tuples("use.mask", profiles) + self._usestablemask_list = \ + self._parse_profile_files_to_tuple_of_tuples("use.stable.mask", + profiles, eapi_filter=eapi_supports_stable_use_forcing_and_masking) self._useforce_list = self._parse_profile_files_to_tuple_of_tuples("use.force", profiles) + self._usestableforce_list = \ + self._parse_profile_files_to_tuple_of_tuples("use.stable.force", + profiles, eapi_filter=eapi_supports_stable_use_forcing_and_masking) self._pusemask_list = self._parse_profile_files_to_tuple_of_dicts("package.use.mask", profiles) + self._pusestablemask_list = \ + self._parse_profile_files_to_tuple_of_dicts("package.use.stable.mask", + profiles, eapi_filter=eapi_supports_stable_use_forcing_and_masking) self._pkgprofileuse = self._parse_profile_files_to_tuple_of_dicts("package.use", profiles, juststrings=True) self._puseforce_list = self._parse_profile_files_to_tuple_of_dicts("package.use.force", profiles) + self._pusestableforce_list = \ + self._parse_profile_files_to_tuple_of_dicts("package.use.stable.force", + profiles, eapi_filter=eapi_supports_stable_use_forcing_and_masking) self._pusedict = self._parse_user_files_to_extatomdict("package.use", abs_user_config, user_config) + self._repo_usealiases_dict = self._parse_repository_usealiases(repositories) + self._repo_pusealiases_dict = self._parse_repository_packageusealiases(repositories) + self.repositories = repositories - - def _parse_file_to_tuple(self, file_name, recursive=True): + + def _parse_file_to_tuple(self, file_name, recursive=True, eapi_filter=None): ret = [] lines = grabfile(file_name, recursive=recursive) eapi = read_corresponding_eapi_file(file_name) + if eapi_filter is not None and not eapi_filter(eapi): + if lines: + writemsg(_("--- EAPI '%s' does not support '%s': '%s'\n") % + (eapi, os.path.basename(file_name), file_name), + noiselevel=-1) + return () useflag_re = _get_useflag_re(eapi) for prefixed_useflag in lines: if prefixed_useflag[:1] == "-": @@ -82,11 +130,26 @@ class UseManager(object): ret.append(prefixed_useflag) return tuple(ret) - def _parse_file_to_dict(self, file_name, juststrings=False, recursive=True): + def _parse_file_to_dict(self, file_name, juststrings=False, recursive=True, + eapi_filter=None, user_config=False): ret = {} location_dict = {} - file_dict = grabdict_package(file_name, recursive=recursive, verify_eapi=True) - eapi = read_corresponding_eapi_file(file_name) + eapi = read_corresponding_eapi_file(file_name, default=None) + if eapi is None and not user_config: + eapi = "0" + if eapi is None: + ret = ExtendedAtomDict(dict) + else: + ret = {} + file_dict = grabdict_package(file_name, recursive=recursive, + allow_wildcard=(eapi is None), allow_repo=(eapi is None), + verify_eapi=(eapi is not None)) + if eapi is not None and eapi_filter is not None and not eapi_filter(eapi): + if file_dict: + writemsg(_("--- EAPI '%s' does not support '%s': '%s'\n") % + (eapi, os.path.basename(file_name), file_name), + noiselevel=-1) + return ret useflag_re = _get_useflag_re(eapi) for k, v in file_dict.items(): useflags = [] @@ -119,31 +182,116 @@ class UseManager(object): return ret - def _parse_repository_files_to_dict_of_tuples(self, file_name, repositories): + def _parse_repository_files_to_dict_of_tuples(self, file_name, repositories, eapi_filter=None): ret = {} for repo in repositories.repos_with_profiles(): - ret[repo.name] = self._parse_file_to_tuple(os.path.join(repo.location, "profiles", file_name)) + ret[repo.name] = self._parse_file_to_tuple(os.path.join(repo.location, "profiles", file_name), eapi_filter=eapi_filter) return ret - def _parse_repository_files_to_dict_of_dicts(self, file_name, repositories): + def _parse_repository_files_to_dict_of_dicts(self, file_name, repositories, eapi_filter=None): ret = {} for repo in repositories.repos_with_profiles(): - ret[repo.name] = self._parse_file_to_dict(os.path.join(repo.location, "profiles", file_name)) + ret[repo.name] = self._parse_file_to_dict(os.path.join(repo.location, "profiles", file_name), eapi_filter=eapi_filter) return ret - def _parse_profile_files_to_tuple_of_tuples(self, file_name, locations): + def _parse_profile_files_to_tuple_of_tuples(self, file_name, locations, + eapi_filter=None): return tuple(self._parse_file_to_tuple( os.path.join(profile.location, file_name), - recursive=profile.portage1_directories) + recursive=profile.portage1_directories, eapi_filter=eapi_filter) for profile in locations) - def _parse_profile_files_to_tuple_of_dicts(self, file_name, locations, juststrings=False): + def _parse_profile_files_to_tuple_of_dicts(self, file_name, locations, + juststrings=False, eapi_filter=None): return tuple(self._parse_file_to_dict( os.path.join(profile.location, file_name), juststrings, - recursive=profile.portage1_directories) + recursive=profile.portage1_directories, eapi_filter=eapi_filter, + user_config=profile.user_config) for profile in locations) - def getUseMask(self, pkg=None): + def _parse_repository_usealiases(self, repositories): + ret = {} + for repo in repositories.repos_with_profiles(): + file_name = os.path.join(repo.location, "profiles", "use.aliases") + eapi = read_corresponding_eapi_file(file_name) + useflag_re = _get_useflag_re(eapi) + raw_file_dict = grabdict(file_name, recursive=True) + file_dict = {} + for real_flag, aliases in raw_file_dict.items(): + if useflag_re.match(real_flag) is None: + writemsg(_("--- Invalid real USE flag in '%s': '%s'\n") % (file_name, real_flag), noiselevel=-1) + else: + for alias in aliases: + if useflag_re.match(alias) is None: + writemsg(_("--- Invalid USE flag alias for '%s' real USE flag in '%s': '%s'\n") % + (real_flag, file_name, alias), noiselevel=-1) + else: + if any(alias in v for k, v in file_dict.items() if k != real_flag): + writemsg(_("--- Duplicated USE flag alias in '%s': '%s'\n") % + (file_name, alias), noiselevel=-1) + else: + file_dict.setdefault(real_flag, []).append(alias) + ret[repo.name] = file_dict + return ret + + def _parse_repository_packageusealiases(self, repositories): + ret = {} + for repo in repositories.repos_with_profiles(): + file_name = os.path.join(repo.location, "profiles", "package.use.aliases") + eapi = read_corresponding_eapi_file(file_name) + useflag_re = _get_useflag_re(eapi) + lines = grabfile(file_name, recursive=True) + file_dict = {} + for line in lines: + elements = line.split() + atom = elements[0] + try: + atom = Atom(atom, eapi=eapi) + except InvalidAtom: + writemsg(_("--- Invalid atom in '%s': '%s'\n") % (file_name, atom)) + continue + if len(elements) == 1: + writemsg(_("--- Missing real USE flag for '%s' in '%s'\n") % (atom, file_name), noiselevel=-1) + continue + real_flag = elements[1] + if useflag_re.match(real_flag) is None: + writemsg(_("--- Invalid real USE flag for '%s' in '%s': '%s'\n") % (atom, file_name, real_flag), noiselevel=-1) + else: + for alias in elements[2:]: + if useflag_re.match(alias) is None: + writemsg(_("--- Invalid USE flag alias for '%s' real USE flag for '%s' in '%s': '%s'\n") % + (real_flag, atom, file_name, alias), noiselevel=-1) + else: + # Duplicated USE flag aliases in entries for different atoms + # matching the same package version are detected in getUseAliases(). + if any(alias in v for k, v in file_dict.get(atom.cp, {}).get(atom, {}).items() if k != real_flag): + writemsg(_("--- Duplicated USE flag alias for '%s' in '%s': '%s'\n") % + (atom, file_name, alias), noiselevel=-1) + else: + file_dict.setdefault(atom.cp, {}).setdefault(atom, {}).setdefault(real_flag, []).append(alias) + ret[repo.name] = file_dict + return ret + + def _isStable(self, pkg): + if self._user_config: + try: + return pkg.stable + except AttributeError: + # KEYWORDS is unavailable (prior to "depend" phase) + return False + + try: + pkg._metadata + except AttributeError: + # KEYWORDS is unavailable (prior to "depend" phase) + return False + + # Since repoman uses different config instances for + # different profiles, we have to be careful to do the + # stable check against the correct profile here. + return self._is_stable(pkg) + + def getUseMask(self, pkg=None, stable=None): if pkg is None: return frozenset(stack_lists( self._usemask_list, incremental=True)) @@ -155,7 +303,12 @@ class UseManager(object): repo = dep_getrepo(pkg) pkg = _pkg_str(remove_slot(pkg), slot=slot, repo=repo) cp = pkg.cp + + if stable is None: + stable = self._isStable(pkg) + usemask = [] + if hasattr(pkg, "repo") and pkg.repo != Package.UNKNOWN_REPO: repos = [] try: @@ -166,30 +319,56 @@ class UseManager(object): repos.append(pkg.repo) for repo in repos: usemask.append(self._repo_usemask_dict.get(repo, {})) + if stable: + usemask.append(self._repo_usestablemask_dict.get(repo, {})) cpdict = self._repo_pusemask_dict.get(repo, {}).get(cp) if cpdict: pkg_usemask = ordered_by_atom_specificity(cpdict, pkg) if pkg_usemask: usemask.extend(pkg_usemask) + if stable: + cpdict = self._repo_pusestablemask_dict.get(repo, {}).get(cp) + if cpdict: + pkg_usemask = ordered_by_atom_specificity(cpdict, pkg) + if pkg_usemask: + usemask.extend(pkg_usemask) + for i, pusemask_dict in enumerate(self._pusemask_list): if self._usemask_list[i]: usemask.append(self._usemask_list[i]) + if stable and self._usestablemask_list[i]: + usemask.append(self._usestablemask_list[i]) cpdict = pusemask_dict.get(cp) if cpdict: pkg_usemask = ordered_by_atom_specificity(cpdict, pkg) if pkg_usemask: usemask.extend(pkg_usemask) + if stable: + cpdict = self._pusestablemask_list[i].get(cp) + if cpdict: + pkg_usemask = ordered_by_atom_specificity(cpdict, pkg) + if pkg_usemask: + usemask.extend(pkg_usemask) + return frozenset(stack_lists(usemask, incremental=True)) - def getUseForce(self, pkg=None): + def getUseForce(self, pkg=None, stable=None): if pkg is None: return frozenset(stack_lists( self._useforce_list, incremental=True)) cp = getattr(pkg, "cp", None) if cp is None: - cp = cpv_getkey(remove_slot(pkg)) + slot = dep_getslot(pkg) + repo = dep_getrepo(pkg) + pkg = _pkg_str(remove_slot(pkg), slot=slot, repo=repo) + cp = pkg.cp + + if stable is None: + stable = self._isStable(pkg) + useforce = [] + if hasattr(pkg, "repo") and pkg.repo != Package.UNKNOWN_REPO: repos = [] try: @@ -200,25 +379,90 @@ class UseManager(object): repos.append(pkg.repo) for repo in repos: useforce.append(self._repo_useforce_dict.get(repo, {})) + if stable: + useforce.append(self._repo_usestableforce_dict.get(repo, {})) cpdict = self._repo_puseforce_dict.get(repo, {}).get(cp) if cpdict: pkg_useforce = ordered_by_atom_specificity(cpdict, pkg) if pkg_useforce: useforce.extend(pkg_useforce) + if stable: + cpdict = self._repo_pusestableforce_dict.get(repo, {}).get(cp) + if cpdict: + pkg_useforce = ordered_by_atom_specificity(cpdict, pkg) + if pkg_useforce: + useforce.extend(pkg_useforce) + for i, puseforce_dict in enumerate(self._puseforce_list): if self._useforce_list[i]: useforce.append(self._useforce_list[i]) + if stable and self._usestableforce_list[i]: + useforce.append(self._usestableforce_list[i]) cpdict = puseforce_dict.get(cp) if cpdict: pkg_useforce = ordered_by_atom_specificity(cpdict, pkg) if pkg_useforce: useforce.extend(pkg_useforce) + if stable: + cpdict = self._pusestableforce_list[i].get(cp) + if cpdict: + pkg_useforce = ordered_by_atom_specificity(cpdict, pkg) + if pkg_useforce: + useforce.extend(pkg_useforce) + return frozenset(stack_lists(useforce, incremental=True)) + def getUseAliases(self, pkg): + if hasattr(pkg, "eapi") and not eapi_has_use_aliases(pkg.eapi): + return {} + + cp = getattr(pkg, "cp", None) + if cp is None: + slot = dep_getslot(pkg) + repo = dep_getrepo(pkg) + pkg = _pkg_str(remove_slot(pkg), slot=slot, repo=repo) + cp = pkg.cp + + usealiases = {} + + if hasattr(pkg, "repo") and pkg.repo != Package.UNKNOWN_REPO: + repos = [] + try: + repos.extend(repo.name for repo in + self.repositories[pkg.repo].masters) + except KeyError: + pass + repos.append(pkg.repo) + for repo in repos: + usealiases_dict = self._repo_usealiases_dict.get(repo, {}) + for real_flag, aliases in usealiases_dict.items(): + for alias in aliases: + if any(alias in v for k, v in usealiases.items() if k != real_flag): + writemsg(_("--- Duplicated USE flag alias for '%s%s%s': '%s'\n") % + (pkg.cpv, _repo_separator, pkg.repo, alias), noiselevel=-1) + else: + usealiases.setdefault(real_flag, []).append(alias) + cp_usealiases_dict = self._repo_pusealiases_dict.get(repo, {}).get(cp) + if cp_usealiases_dict: + usealiases_dict_list = ordered_by_atom_specificity(cp_usealiases_dict, pkg) + for usealiases_dict in usealiases_dict_list: + for real_flag, aliases in usealiases_dict.items(): + for alias in aliases: + if any(alias in v for k, v in usealiases.items() if k != real_flag): + writemsg(_("--- Duplicated USE flag alias for '%s%s%s': '%s'\n") % + (pkg.cpv, _repo_separator, pkg.repo, alias), noiselevel=-1) + else: + usealiases.setdefault(real_flag, []).append(alias) + + return usealiases + def getPUSE(self, pkg): cp = getattr(pkg, "cp", None) if cp is None: - cp = cpv_getkey(remove_slot(pkg)) + slot = dep_getslot(pkg) + repo = dep_getrepo(pkg) + pkg = _pkg_str(remove_slot(pkg), slot=slot, repo=repo) + cp = pkg.cp ret = "" cpdict = self._pusedict.get(cp) if cpdict: diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.pyo Binary files differindex 2c9a609..16fa78e 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/VirtualsManager.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/VirtualsManager.pyo Binary files differindex b2ebd21..88010cc 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/VirtualsManager.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/VirtualsManager.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/__init__.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/__init__.pyo Binary files differindex b03cc29..f3c8238 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/__init__.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/__init__.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/env_var_validation.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/env_var_validation.pyo Binary files differindex aeee789..d78d7d2 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/env_var_validation.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/env_var_validation.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/features_set.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/features_set.pyo Binary files differindex 9854444..ef59bc0 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/features_set.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/features_set.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/helper.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/helper.pyo Binary files differindex f2b9261..d2a012f 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/helper.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/helper.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.py b/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.py index 1a75de9..f241e65 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.py @@ -1,6 +1,8 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ( 'case_insensitive_vars', 'default_globals', 'env_blacklist', \ 'environ_filter', 'environ_whitelist', 'environ_whitelist_re', @@ -13,14 +15,17 @@ import re # configuration files. env_blacklist = frozenset(( "A", "AA", "CATEGORY", "DEPEND", "DESCRIPTION", "EAPI", - "EBUILD_FORCE_TEST", "EBUILD_PHASE", "EBUILD_SKIP_MANIFEST", + "EBUILD_FORCE_TEST", "EBUILD_PHASE", + "EBUILD_PHASE_FUNC", "EBUILD_SKIP_MANIFEST", "ED", "EMERGE_FROM", "EPREFIX", "EROOT", - "GREP_OPTIONS", "HOMEPAGE", "INHERITED", "IUSE", + "GREP_OPTIONS", "HDEPEND", "HOMEPAGE", + "INHERITED", "IUSE", "IUSE_EFFECTIVE", "KEYWORDS", "LICENSE", "MERGE_TYPE", "PDEPEND", "PF", "PKGUSE", "PORTAGE_BACKGROUND", - "PORTAGE_BACKGROUND_UNMERGE", "PORTAGE_BUILDIR_LOCKED", - "PORTAGE_BUILT_USE", "PORTAGE_CONFIGROOT", "PORTAGE_IUSE", - "PORTAGE_NONFATAL", "PORTAGE_REPO_NAME", + "PORTAGE_BACKGROUND_UNMERGE", "PORTAGE_BUILDDIR_LOCKED", + "PORTAGE_BUILT_USE", "PORTAGE_CONFIGROOT", + "PORTAGE_INTERNAL_CALLER", "PORTAGE_IUSE", + "PORTAGE_NONFATAL", "PORTAGE_PIPE_FD", "PORTAGE_REPO_NAME", "PORTAGE_USE", "PROPERTIES", "PROVIDE", "RDEPEND", "REPOSITORY", "RESTRICT", "ROOT", "SLOT", "SRC_URI" )) @@ -39,7 +44,7 @@ environ_whitelist += [ "ACCEPT_LICENSE", "BASH_ENV", "BUILD_PREFIX", "COLUMNS", "D", "DISTDIR", "DOC_SYMLINKS_DIR", "EAPI", "EBUILD", "EBUILD_FORCE_TEST", - "EBUILD_PHASE", "ECLASSDIR", "ECLASS_DEPTH", "ED", + "EBUILD_PHASE", "EBUILD_PHASE_FUNC", "ECLASSDIR", "ECLASS_DEPTH", "ED", "EMERGE_FROM", "EPREFIX", "EROOT", "FEATURES", "FILESDIR", "HOME", "MERGE_TYPE", "NOCOLOR", "PATH", "PKGDIR", @@ -49,19 +54,26 @@ environ_whitelist += [ "PORTAGE_BINPKG_FILE", "PORTAGE_BINPKG_TAR_OPTS", "PORTAGE_BINPKG_TMPFILE", "PORTAGE_BIN_PATH", - "PORTAGE_BUILDDIR", "PORTAGE_BUNZIP2_COMMAND", "PORTAGE_BZIP2_COMMAND", - "PORTAGE_COLORMAP", "PORTAGE_COMPRESS_EXCLUDE_SUFFIXES", + "PORTAGE_BUILDDIR", "PORTAGE_BUILD_GROUP", "PORTAGE_BUILD_USER", + "PORTAGE_BUNZIP2_COMMAND", "PORTAGE_BZIP2_COMMAND", + "PORTAGE_COLORMAP", "PORTAGE_COMPRESS", + "PORTAGE_COMPRESS_EXCLUDE_SUFFIXES", "PORTAGE_CONFIGROOT", "PORTAGE_DEBUG", "PORTAGE_DEPCACHEDIR", + "PORTAGE_DOHTML_UNWARNED_SKIPPED_EXTENSIONS", + "PORTAGE_DOHTML_UNWARNED_SKIPPED_FILES", + "PORTAGE_DOHTML_WARN_ON_SKIPPED_FILES", "PORTAGE_EBUILD_EXIT_FILE", "PORTAGE_FEATURES", "PORTAGE_GID", "PORTAGE_GRPNAME", + "PORTAGE_INTERNAL_CALLER", "PORTAGE_INST_GID", "PORTAGE_INST_UID", - "PORTAGE_IPC_DAEMON", "PORTAGE_IUSE", - "PORTAGE_LOG_FILE", "PORTAGE_OVERRIDE_EPREFIX", - "PORTAGE_PYM_PATH", "PORTAGE_PYTHON", "PORTAGE_QUIET", - "PORTAGE_REPO_NAME", "PORTAGE_RESTRICT", + "PORTAGE_IPC_DAEMON", "PORTAGE_IUSE", "PORTAGE_ECLASS_LOCATIONS", + "PORTAGE_LOG_FILE", "PORTAGE_OVERRIDE_EPREFIX", "PORTAGE_PIPE_FD", + "PORTAGE_PYM_PATH", "PORTAGE_PYTHON", + "PORTAGE_PYTHONPATH", "PORTAGE_QUIET", + "PORTAGE_REPO_NAME", "PORTAGE_REPOSITORIES", "PORTAGE_RESTRICT", "PORTAGE_SIGPIPE_STATUS", "PORTAGE_TMPDIR", "PORTAGE_UPDATE_ENV", "PORTAGE_USERNAME", - "PORTAGE_VERBOSE", "PORTAGE_WORKDIR_MODE", + "PORTAGE_VERBOSE", "PORTAGE_WORKDIR_MODE", "PORTAGE_XATTR_EXCLUDE", "PORTDIR", "PORTDIR_OVERLAY", "PREROOTPATH", "PROFILE_PATHS", "REPLACING_VERSIONS", "REPLACED_BY_VERSION", "ROOT", "ROOTPATH", "T", "TMP", "TMPDIR", @@ -102,7 +114,7 @@ environ_whitelist += [ environ_whitelist += [ "CVS_RSH", "ECHANGELOG_USER", "GPG_AGENT_INFO", "LOG_SOCKET", - "SSH_AGENT_PID", "SSH_AUTH_SOCK" + "SSH_AGENT_PID", "SSH_AUTH_SOCK", "STY", "WINDOW", "XAUTHORITY", ] @@ -133,9 +145,11 @@ environ_filter += [ # portage config variables and variables set directly by portage environ_filter += [ - "ACCEPT_CHOSTS", "ACCEPT_KEYWORDS", "ACCEPT_PROPERTIES", "AUTOCLEAN", + "ACCEPT_CHOSTS", "ACCEPT_KEYWORDS", "ACCEPT_PROPERTIES", + "ACCEPT_RESTRICT", "AUTOCLEAN", "CLEAN_DELAY", "COLLISION_IGNORE", "CONFIG_PROTECT", "CONFIG_PROTECT_MASK", + "DCO_SIGNED_OFF_BY", "EGENCACHE_DEFAULT_OPTS", "EMERGE_DEFAULT_OPTS", "EMERGE_LOG_DIR", "EMERGE_WARNING_DELAY", @@ -144,8 +158,9 @@ environ_filter += [ "FETCHCOMMAND_RSYNC", "FETCHCOMMAND_SFTP", "GENTOO_MIRRORS", "NOCONFMEM", "O", "PORTAGE_BACKGROUND", "PORTAGE_BACKGROUND_UNMERGE", - "PORTAGE_BINHOST", - "PORTAGE_BINHOST_CHUNKSIZE", "PORTAGE_BUILDIR_LOCKED", + "PORTAGE_BINHOST", "PORTAGE_BINPKG_FORMAT", + "PORTAGE_BUILDDIR_LOCKED", + "PORTAGE_CHECKSUM_FILTER", "PORTAGE_ELOG_CLASSES", "PORTAGE_ELOG_MAILFROM", "PORTAGE_ELOG_MAILSUBJECT", "PORTAGE_ELOG_MAILURI", "PORTAGE_ELOG_SYSTEM", @@ -157,13 +172,20 @@ environ_filter += [ "PORTAGE_REPO_DUPLICATE_WARN", "PORTAGE_RO_DISTDIRS", "PORTAGE_RSYNC_EXTRA_OPTS", "PORTAGE_RSYNC_OPTS", - "PORTAGE_RSYNC_RETRIES", "PORTAGE_SYNC_STALE", - "PORTAGE_USE", "PORT_LOGDIR", "PORT_LOGDIR_CLEAN", + "PORTAGE_RSYNC_RETRIES", "PORTAGE_SSH_OPTS", "PORTAGE_SYNC_STALE", + "PORTAGE_USE", + "PORT_LOGDIR", "PORT_LOGDIR_CLEAN", "QUICKPKG_DEFAULT_OPTS", "REPOMAN_DEFAULT_OPTS", "RESUMECOMMAND", "RESUMECOMMAND_FTP", "RESUMECOMMAND_HTTP", "RESUMECOMMAND_HTTPS", "RESUMECOMMAND_RSYNC", "RESUMECOMMAND_SFTP", - "SYNC", "USE_EXPAND_HIDDEN", "USE_ORDER", + "UNINSTALL_IGNORE", "USE_EXPAND_HIDDEN", "USE_ORDER", + "__PORTAGE_HELPER" +] + +# No longer supported variables +environ_filter += [ + "SYNC" ] environ_filter = frozenset(environ_filter) diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.pyo Binary files differindex 06ea37e..92c5d32 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_eapi_invalid.py b/portage_with_autodep/pym/portage/package/ebuild/_eapi_invalid.py deleted file mode 100644 index d23677d..0000000 --- a/portage_with_autodep/pym/portage/package/ebuild/_eapi_invalid.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2012 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -import textwrap - -import portage -from portage.dep import _repo_separator -from portage.elog import elog_process -from portage.elog.messages import eerror - -def eapi_invalid(self, cpv, repo_name, settings, - eapi_var, eapi_parsed, eapi_lineno): - - msg = [] - msg.extend(textwrap.wrap(("EAPI assignment in ebuild '%s%s%s' does not" - " conform with PMS section 7.3.1 (see bug #402167):") % - (cpv, _repo_separator, repo_name), 70)) - - if not eapi_parsed: - # None means the assignment was not found, while an - # empty string indicates an (invalid) empty assingment. - msg.append( - "\tvalid EAPI assignment must" - " occur on or before line: %s" % - eapi_lineno) - else: - msg.append(("\tbash returned EAPI '%s' which does not match " - "assignment on line: %s") % - (eapi_var, eapi_lineno)) - - if 'parse-eapi-ebuild-head' in settings.features: - msg.extend(textwrap.wrap(("NOTE: This error will soon" - " become unconditionally fatal in a future version of Portage," - " but at this time, it can by made non-fatal by setting" - " FEATURES=-parse-eapi-ebuild-head in" - " make.conf."), 70)) - else: - msg.extend(textwrap.wrap(("NOTE: This error will soon" - " become unconditionally fatal in a future version of Portage." - " At the earliest opportunity, please enable" - " FEATURES=parse-eapi-ebuild-head in make.conf in order to" - " make this error fatal."), 70)) - - if portage.data.secpass >= 2: - # TODO: improve elog permission error handling (bug #416231) - for line in msg: - eerror(line, phase="other", key=cpv) - elog_process(cpv, settings, - phasefilter=("other",)) - - else: - out = portage.output.EOutput() - for line in msg: - out.eerror(line) diff --git a/portage_with_autodep/pym/portage/package/ebuild/_eapi_invalid.pyo b/portage_with_autodep/pym/portage/package/ebuild/_eapi_invalid.pyo Binary files differdeleted file mode 100644 index 0181c03..0000000 --- a/portage_with_autodep/pym/portage/package/ebuild/_eapi_invalid.pyo +++ /dev/null diff --git a/portage_with_autodep/pym/portage/package/ebuild/_ipc/ExitCommand.pyo b/portage_with_autodep/pym/portage/package/ebuild/_ipc/ExitCommand.pyo Binary files differindex 315cb0f..744bb83 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_ipc/ExitCommand.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_ipc/ExitCommand.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_ipc/IpcCommand.pyo b/portage_with_autodep/pym/portage/package/ebuild/_ipc/IpcCommand.pyo Binary files differindex 9f75518..ae66c3e 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_ipc/IpcCommand.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_ipc/IpcCommand.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.py b/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.py index 7bbb0e8..351c956 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.py @@ -1,12 +1,13 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + import io import portage from portage import os -from portage import _unicode_decode -from portage.dep import Atom +from portage.dep import Atom, _repo_name_re from portage.eapi import eapi_has_repo_deps from portage.elog import messages as elog_messages from portage.exception import InvalidAtom @@ -20,6 +21,12 @@ class QueryCommand(IpcCommand): _db = None + @classmethod + def get_db(cls): + if cls._db is not None: + return cls._db + return portage.db + def __init__(self, settings, phase): IpcCommand.__init__(self) self.settings = settings @@ -30,41 +37,46 @@ class QueryCommand(IpcCommand): @return: tuple of (stdout, stderr, returncode) """ - cmd, root, atom_str = argv + # Python 3: + # cmd, root, *args = argv + cmd = argv[0] + root = argv[1] + args = argv[2:] + + warnings = [] + warnings_str = '' + db = self.get_db() eapi = self.settings.get('EAPI') - allow_repo = eapi_has_repo_deps(eapi) - try: - atom = Atom(atom_str, allow_repo=allow_repo) - except InvalidAtom: - return ('', 'invalid atom: %s\n' % atom_str, 2) - warnings = [] - try: - atom = Atom(atom_str, allow_repo=allow_repo, eapi=eapi) - except InvalidAtom as e: - warnings.append(_unicode_decode("QA Notice: %s: %s") % (cmd, e)) + root = normalize_path(root).rstrip(os.path.sep) + os.path.sep + if root not in db: + return ('', '%s: Invalid ROOT: %s\n' % (cmd, root), 3) - use = self.settings.get('PORTAGE_BUILT_USE') - if use is None: - use = self.settings['PORTAGE_USE'] + portdb = db[root]["porttree"].dbapi + vardb = db[root]["vartree"].dbapi - use = frozenset(use.split()) - atom = atom.evaluate_conditionals(use) + if cmd in ('best_version', 'has_version'): + allow_repo = eapi_has_repo_deps(eapi) + try: + atom = Atom(args[0], allow_repo=allow_repo) + except InvalidAtom: + return ('', '%s: Invalid atom: %s\n' % (cmd, args[0]), 2) - db = self._db - if db is None: - db = portage.db + try: + atom = Atom(args[0], allow_repo=allow_repo, eapi=eapi) + except InvalidAtom as e: + warnings.append("QA Notice: %s: %s" % (cmd, e)) - warnings_str = '' - if warnings: - warnings_str = self._elog('eqawarn', warnings) + use = self.settings.get('PORTAGE_BUILT_USE') + if use is None: + use = self.settings['PORTAGE_USE'] - root = normalize_path(root).rstrip(os.path.sep) + os.path.sep - if root not in db: - return ('', 'invalid ROOT: %s\n' % root, 2) + use = frozenset(use.split()) + atom = atom.evaluate_conditionals(use) - vardb = db[root]["vartree"].dbapi + if warnings: + warnings_str = self._elog('eqawarn', warnings) if cmd == 'has_version': if vardb.match(atom): @@ -75,8 +87,35 @@ class QueryCommand(IpcCommand): elif cmd == 'best_version': m = best(vardb.match(atom)) return ('%s\n' % m, warnings_str, 0) + elif cmd in ('master_repositories', 'repository_path', 'available_eclasses', 'eclass_path', 'license_path'): + repo = _repo_name_re.match(args[0]) + if repo is None: + return ('', '%s: Invalid repository: %s\n' % (cmd, args[0]), 2) + try: + repo = portdb.repositories[args[0]] + except KeyError: + return ('', warnings_str, 1) + + if cmd == 'master_repositories': + return ('%s\n' % ' '.join(x.name for x in repo.masters), warnings_str, 0) + elif cmd == 'repository_path': + return ('%s\n' % repo.location, warnings_str, 0) + elif cmd == 'available_eclasses': + return ('%s\n' % ' '.join(sorted(repo.eclass_db.eclasses)), warnings_str, 0) + elif cmd == 'eclass_path': + try: + eclass = repo.eclass_db.eclasses[args[1]] + except KeyError: + return ('', warnings_str, 1) + return ('%s\n' % eclass.location, warnings_str, 0) + elif cmd == 'license_path': + paths = reversed([os.path.join(x.location, 'licenses', args[1]) for x in list(repo.masters) + [repo]]) + for path in paths: + if os.path.exists(path): + return ('%s\n' % path, warnings_str, 0) + return ('', warnings_str, 1) else: - return ('', 'invalid command: %s\n' % cmd, 2) + return ('', 'Invalid command: %s\n' % cmd, 3) def _elog(self, elog_funcname, lines): """ diff --git a/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.pyo b/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.pyo Binary files differindex 0e9ee96..3087272 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_ipc/__init__.pyo b/portage_with_autodep/pym/portage/package/ebuild/_ipc/__init__.pyo Binary files differindex d9f8d25..270e321 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_ipc/__init__.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_ipc/__init__.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.py b/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.py index 94f8c79..0fc53c8 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.py @@ -1,8 +1,9 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import tempfile +import portage from portage import os from portage import shutil from portage.const import EBUILD_PHASES @@ -10,10 +11,12 @@ from portage.elog import elog_process from portage.package.ebuild.config import config from portage.package.ebuild.doebuild import doebuild_environment from portage.package.ebuild.prepare_build_dirs import prepare_build_dirs +from portage.util._async.SchedulerInterface import SchedulerInterface +from portage.util._eventloop.EventLoop import EventLoop +from portage.util._eventloop.global_event_loop import global_event_loop from _emerge.EbuildPhase import EbuildPhase -from _emerge.PollScheduler import PollScheduler -def spawn_nofetch(portdb, ebuild_path, settings=None): +def spawn_nofetch(portdb, ebuild_path, settings=None, fd_pipes=None): """ This spawns pkg_nofetch if appropriate. The settings parameter is useful only if setcpv has already been called in order @@ -47,7 +50,7 @@ def spawn_nofetch(portdb, ebuild_path, settings=None): settings = config(clone=settings) if 'PORTAGE_PARALLEL_FETCHONLY' in settings: - return + return os.EX_OK # We must create our private PORTAGE_TMPDIR before calling # doebuild_environment(), since lots of variables such @@ -59,7 +62,7 @@ def spawn_nofetch(portdb, ebuild_path, settings=None): settings['PORTAGE_TMPDIR'] = private_tmpdir settings.backup_changes('PORTAGE_TMPDIR') # private temp dir was just created, so it's not locked yet - settings.pop('PORTAGE_BUILDIR_LOCKED', None) + settings.pop('PORTAGE_BUILDDIR_LOCKED', None) try: doebuild_environment(ebuild_path, 'nofetch', @@ -73,14 +76,18 @@ def spawn_nofetch(portdb, ebuild_path, settings=None): if 'fetch' not in restrict and \ 'nofetch' not in defined_phases: - return + return os.EX_OK prepare_build_dirs(settings=settings) ebuild_phase = EbuildPhase(background=False, - phase='nofetch', scheduler=PollScheduler().sched_iface, - settings=settings) + phase='nofetch', + scheduler=SchedulerInterface(portage._internal_caller and + global_event_loop() or EventLoop(main=False)), + fd_pipes=fd_pipes, settings=settings) ebuild_phase.start() ebuild_phase.wait() elog_process(settings.mycpv, settings) finally: shutil.rmtree(private_tmpdir) + + return ebuild_phase.returncode diff --git a/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.pyo b/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.pyo Binary files differindex ac449ea..d4c597c 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/config.py b/portage_with_autodep/pym/portage/package/ebuild/config.py index 97cbd99..92e6c3f 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/config.py +++ b/portage_with_autodep/pym/portage/package/ebuild/config.py @@ -1,6 +1,8 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = [ 'autouse', 'best_from_dict', 'check_config_instance', 'config', ] @@ -19,6 +21,8 @@ from _emerge.Package import Package import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.data:portage_gid', + 'portage.dbapi.vartree:vartree', + 'portage.package.ebuild.doebuild:_phase_func_map', ) from portage import bsd_chflags, \ load_mod, os, selinux, _unicode_decode @@ -29,10 +33,9 @@ from portage.const import CACHE_PATH, \ USER_VIRTUALS_FILE from portage.dbapi import dbapi from portage.dbapi.porttree import portdbapi -from portage.dbapi.vartree import vartree from portage.dep import Atom, isvalidatom, match_from_list, use_reduce, _repo_separator, _slot_separator from portage.eapi import eapi_exports_AA, eapi_exports_merge_type, \ - eapi_supports_prefix, eapi_exports_replace_vars + eapi_supports_prefix, eapi_exports_replace_vars, _get_eapi_attrs from portage.env.loaders import KeyValuePairFileLoader from portage.exception import InvalidDependString, PortageException from portage.localization import _ @@ -42,7 +45,8 @@ from portage.repository.config import load_repository_config from portage.util import ensure_dirs, getconfig, grabdict, \ grabdict_package, grabfile, grabfile_package, LazyItemsDict, \ normalize_path, shlex_split, stack_dictlist, stack_dicts, stack_lists, \ - writemsg, writemsg_level + writemsg, writemsg_level, _eapi_cache +from portage.util._path import exists_raise_eaccess, isdir_raise_eaccess from portage.versions import catpkgsplit, catsplit, cpv_getkey, _pkg_str from portage.package.ebuild._config import special_env_vars @@ -55,10 +59,29 @@ from portage.package.ebuild._config.LocationsManager import LocationsManager from portage.package.ebuild._config.MaskManager import MaskManager from portage.package.ebuild._config.VirtualsManager import VirtualsManager from portage.package.ebuild._config.helper import ordered_by_atom_specificity, prune_incremental +from portage.package.ebuild._config.unpack_dependencies import load_unpack_dependencies_configuration if sys.hexversion >= 0x3000000: basestring = str +_feature_flags_cache = {} + +def _get_feature_flags(eapi_attrs): + cache_key = (eapi_attrs.feature_flag_test, eapi_attrs.feature_flag_targetroot) + flags = _feature_flags_cache.get(cache_key) + if flags is not None: + return flags + + flags = [] + if eapi_attrs.feature_flag_test: + flags.append("test") + if eapi_attrs.feature_flag_targetroot: + flags.append("targetroot") + + flags = frozenset(flags) + _feature_flags_cache[cache_key] = flags + return flags + def autouse(myvartree, use_cache=1, mysettings=None): warnings.warn("portage.autouse() is deprecated", DeprecationWarning, stacklevel=2) @@ -123,9 +146,9 @@ class config(object): """ _constant_keys = frozenset(['PORTAGE_BIN_PATH', 'PORTAGE_GID', - 'PORTAGE_PYM_PATH']) + 'PORTAGE_PYM_PATH', 'PORTAGE_PYTHONPATH']) - _setcpv_aux_keys = ('DEFINED_PHASES', 'DEPEND', 'EAPI', + _setcpv_aux_keys = ('DEFINED_PHASES', 'DEPEND', 'EAPI', 'HDEPEND', 'INHERITED', 'IUSE', 'REQUIRED_USE', 'KEYWORDS', 'LICENSE', 'PDEPEND', 'PROPERTIES', 'PROVIDE', 'RDEPEND', 'SLOT', 'repository', 'RESTRICT', 'LICENSE',) @@ -146,7 +169,7 @@ class config(object): def __init__(self, clone=None, mycpv=None, config_profile_path=None, config_incrementals=None, config_root=None, target_root=None, eprefix=None, local_config=True, env=None, - _unmatched_removal=False): + _unmatched_removal=False, repositories=None): """ @param clone: If provided, init will use deepcopy to copy by value the instance. @type clone: Instance of config class. @@ -160,7 +183,8 @@ class config(object): @type config_incrementals: List @param config_root: path to read local config from (defaults to "/", see PORTAGE_CONFIGROOT) @type config_root: String - @param target_root: __init__ override of $ROOT env variable. + @param target_root: the target root, which typically corresponds to the + value of the $ROOT env variable (default is /) @type target_root: String @param eprefix: set the EPREFIX variable (default is portage.const.EPREFIX) @type eprefix: String @@ -173,14 +197,21 @@ class config(object): @param _unmatched_removal: Enabled by repoman when the --unmatched-removal option is given. @type _unmatched_removal: Boolean + @param repositories: Configuration of repositories. + Defaults to portage.repository.config.load_repository_config(). + @type repositories: Instance of portage.repository.config.RepoConfigLoader class. """ + # This is important when config is reloaded after emerge --sync. + _eapi_cache.clear() + # When initializing the global portage.settings instance, avoid # raising exceptions whenever possible since exceptions thrown # from 'import portage' or 'import portage.exceptions' statements # can practically render the api unusable for api consumers. tolerant = hasattr(portage, '_initializing_globals') self._tolerant = tolerant + self._unmatched_removal = _unmatched_removal self.locked = 0 self.mycpv = None @@ -191,8 +222,10 @@ class config(object): self.uvlist = [] self._accept_chost_re = None self._accept_properties = None + self._accept_restrict = None self._features_overrides = [] self._make_defaults = None + self._parent_stable = None # _unknown_features records unknown features that # have triggered warning messages, and ensures that @@ -205,6 +238,7 @@ class config(object): # For immutable attributes, use shallow copy for # speed and memory conservation. self._tolerant = clone._tolerant + self._unmatched_removal = clone._unmatched_removal self.categories = clone.categories self.depcachedir = clone.depcachedir self.incrementals = clone.incrementals @@ -213,6 +247,8 @@ class config(object): self.profiles = clone.profiles self.packages = clone.packages self.repositories = clone.repositories + self.unpack_dependencies = clone.unpack_dependencies + self._iuse_effective = clone._iuse_effective self._iuse_implicit_match = clone._iuse_implicit_match self._non_user_variables = clone._non_user_variables self._env_d_blacklist = clone._env_d_blacklist @@ -227,9 +263,12 @@ class config(object): self._setcpv_args_hash = clone._setcpv_args_hash # immutable attributes (internal policy ensures lack of mutation) - self._keywords_manager = clone._keywords_manager + self._locations_manager = clone._locations_manager self._use_manager = clone._use_manager - self._mask_manager = clone._mask_manager + # force instantiation of lazy immutable objects when cloning, so + # that they're not instantiated more than once + self._keywords_manager_obj = clone._keywords_manager + self._mask_manager_obj = clone._mask_manager # shared mutable attributes self._unknown_features = clone._unknown_features @@ -266,30 +305,55 @@ class config(object): #all LicenseManager instances. self._license_manager = clone._license_manager - self._virtuals_manager = copy.deepcopy(clone._virtuals_manager) + # force instantiation of lazy objects when cloning, so + # that they're not instantiated more than once + self._virtuals_manager_obj = copy.deepcopy(clone._virtuals_manager) self._accept_properties = copy.deepcopy(clone._accept_properties) self._ppropertiesdict = copy.deepcopy(clone._ppropertiesdict) + self._accept_restrict = copy.deepcopy(clone._accept_restrict) + self._paccept_restrict = copy.deepcopy(clone._paccept_restrict) self._penvdict = copy.deepcopy(clone._penvdict) self._expand_map = copy.deepcopy(clone._expand_map) else: + # lazily instantiated objects + self._keywords_manager_obj = None + self._mask_manager_obj = None + self._virtuals_manager_obj = None + locations_manager = LocationsManager(config_root=config_root, config_profile_path=config_profile_path, eprefix=eprefix, local_config=local_config, target_root=target_root) + self._locations_manager = locations_manager eprefix = locations_manager.eprefix config_root = locations_manager.config_root abs_user_config = locations_manager.abs_user_config + make_conf_paths = [ + os.path.join(config_root, 'etc', 'make.conf'), + os.path.join(config_root, MAKE_CONF_FILE) + ] + try: + if os.path.samefile(*make_conf_paths): + make_conf_paths.pop() + except OSError: + pass - make_conf = getconfig( - os.path.join(config_root, MAKE_CONF_FILE), - tolerant=tolerant, allow_sourcing=True) or {} - - make_conf.update(getconfig( - os.path.join(abs_user_config, 'make.conf'), - tolerant=tolerant, allow_sourcing=True, - expand=make_conf) or {}) + make_conf_count = 0 + make_conf = {} + for x in make_conf_paths: + mygcfg = getconfig(x, + tolerant=tolerant, allow_sourcing=True, + expand=make_conf, recursive=True) + if mygcfg is not None: + make_conf.update(mygcfg) + make_conf_count += 1 + + if make_conf_count == 2: + writemsg("!!! %s\n" % + _("Found 2 make.conf files, using both '%s' and '%s'") % + tuple(make_conf_paths), noiselevel=-1) # Allow ROOT setting to come from make.conf if it's not overridden # by the constructor argument (from the calling environment). @@ -315,15 +379,30 @@ class config(object): # lead to unexpected results. env_d = getconfig(os.path.join(eroot, "etc", "profile.env"), - expand=False) or {} + tolerant=tolerant, expand=False) or {} expand_map = env_d.copy() self._expand_map = expand_map # Allow make.globals to set default paths relative to ${EPREFIX}. expand_map["EPREFIX"] = eprefix - make_globals = getconfig(os.path.join( - self.global_config_path, 'make.globals'), expand=expand_map) + make_globals_path = os.path.join( + self.global_config_path, 'make.globals') + old_make_globals = os.path.join(config_root, + 'etc', 'make.globals') + if os.path.isfile(old_make_globals) and \ + not os.path.samefile(make_globals_path, old_make_globals): + # Don't warn if they refer to the same path, since + # that can be used for backward compatibility with + # old software. + writemsg("!!! %s\n" % + _("Found obsolete make.globals file: " + "'%s', (using '%s' instead)") % + (old_make_globals, make_globals_path), + noiselevel=-1) + + make_globals = getconfig(make_globals_path, + tolerant=tolerant, expand=expand_map) if make_globals is None: make_globals = {} @@ -412,6 +491,7 @@ class config(object): known_repos = [] portdir = "" portdir_overlay = "" + portdir_sync = None for confs in [make_globals, make_conf, self.configdict["env"]]: v = confs.get("PORTDIR") if v is not None: @@ -421,12 +501,52 @@ class config(object): if v is not None: portdir_overlay = v known_repos.extend(shlex_split(v)) + v = confs.get("SYNC") + if v is not None: + portdir_sync = v + known_repos = frozenset(known_repos) self["PORTDIR"] = portdir self["PORTDIR_OVERLAY"] = portdir_overlay + if portdir_sync: + self["SYNC"] = portdir_sync self.lookuplist = [self.configdict["env"]] - self.repositories = load_repository_config(self) + if repositories is None: + self.repositories = load_repository_config(self) + else: + self.repositories = repositories + + self['PORTAGE_REPOSITORIES'] = self.repositories.config_string() + self.backup_changes('PORTAGE_REPOSITORIES') + + #filling PORTDIR and PORTDIR_OVERLAY variable for compatibility + main_repo = self.repositories.mainRepo() + if main_repo is not None: + self["PORTDIR"] = main_repo.user_location + self.backup_changes("PORTDIR") + expand_map["PORTDIR"] = self["PORTDIR"] + + # repoman controls PORTDIR_OVERLAY via the environment, so no + # special cases are needed here. + portdir_overlay = list(self.repositories.repoUserLocationList()) + if portdir_overlay and portdir_overlay[0] == self["PORTDIR"]: + portdir_overlay = portdir_overlay[1:] + + new_ov = [] + if portdir_overlay: + for ov in portdir_overlay: + ov = normalize_path(ov) + if isdir_raise_eaccess(ov) or portage._sync_disabled_warnings: + new_ov.append(portage._shell_quote(ov)) + else: + writemsg(_("!!! Invalid PORTDIR_OVERLAY" + " (not a dir): '%s'\n") % ov, noiselevel=-1) + self["PORTDIR_OVERLAY"] = " ".join(new_ov) + self.backup_changes("PORTDIR_OVERLAY") + expand_map["PORTDIR_OVERLAY"] = self["PORTDIR_OVERLAY"] + + locations_manager.set_port_dirs(self["PORTDIR"], self["PORTDIR_OVERLAY"]) locations_manager.load_profiles(self.repositories, known_repos) profiles_complex = locations_manager.profiles_complex @@ -446,11 +566,13 @@ class config(object): x = Atom(x.lstrip('*')) self.prevmaskdict.setdefault(x.cp, []).append(x) + self.unpack_dependencies = load_unpack_dependencies_configuration(self.repositories) mygcfg = {} - if self.profiles: - mygcfg_dlists = [getconfig(os.path.join(x, "make.defaults"), - expand=expand_map) for x in self.profiles] + if profiles_complex: + mygcfg_dlists = [getconfig(os.path.join(x.location, "make.defaults"), + tolerant=tolerant, expand=expand_map, recursive=x.portage1_directories) + for x in profiles_complex] self._make_defaults = mygcfg_dlists mygcfg = stack_dicts(mygcfg_dlists, incrementals=self.incrementals) @@ -459,15 +581,11 @@ class config(object): self.configlist.append(mygcfg) self.configdict["defaults"]=self.configlist[-1] - mygcfg = getconfig( - os.path.join(config_root, MAKE_CONF_FILE), - tolerant=tolerant, allow_sourcing=True, - expand=expand_map) or {} - - mygcfg.update(getconfig( - os.path.join(abs_user_config, 'make.conf'), - tolerant=tolerant, allow_sourcing=True, - expand=expand_map) or {}) + mygcfg = {} + for x in make_conf_paths: + mygcfg.update(getconfig(x, + tolerant=tolerant, allow_sourcing=True, + expand=expand_map, recursive=True) or {}) # Don't allow the user to override certain variables in make.conf profile_only_variables = self.configdict["defaults"].get( @@ -520,66 +638,34 @@ class config(object): self.backup_changes("PORTAGE_CONFIGROOT") self["ROOT"] = target_root self.backup_changes("ROOT") - - # The PORTAGE_OVERRIDE_EPREFIX variable propagates the EPREFIX - # of this config instance to any portage commands or API - # consumers running in subprocesses. self["EPREFIX"] = eprefix self.backup_changes("EPREFIX") - self["PORTAGE_OVERRIDE_EPREFIX"] = eprefix - self.backup_changes("PORTAGE_OVERRIDE_EPREFIX") self["EROOT"] = eroot self.backup_changes("EROOT") + # The prefix of the running portage instance is used in the + # ebuild environment to implement the --host-root option for + # best_version and has_version. + self["PORTAGE_OVERRIDE_EPREFIX"] = portage.const.EPREFIX + self.backup_changes("PORTAGE_OVERRIDE_EPREFIX") + self._ppropertiesdict = portage.dep.ExtendedAtomDict(dict) + self._paccept_restrict = portage.dep.ExtendedAtomDict(dict) self._penvdict = portage.dep.ExtendedAtomDict(dict) - #filling PORTDIR and PORTDIR_OVERLAY variable for compatibility - main_repo = self.repositories.mainRepo() - if main_repo is not None: - self["PORTDIR"] = main_repo.user_location - self.backup_changes("PORTDIR") - - # repoman controls PORTDIR_OVERLAY via the environment, so no - # special cases are needed here. - portdir_overlay = list(self.repositories.repoUserLocationList()) - if portdir_overlay and portdir_overlay[0] == self["PORTDIR"]: - portdir_overlay = portdir_overlay[1:] - - new_ov = [] - if portdir_overlay: - shell_quote_re = re.compile(r"[\s\\\"'$`]") - for ov in portdir_overlay: - ov = normalize_path(ov) - if os.path.isdir(ov): - if shell_quote_re.search(ov) is not None: - ov = portage._shell_quote(ov) - new_ov.append(ov) - else: - writemsg(_("!!! Invalid PORTDIR_OVERLAY" - " (not a dir): '%s'\n") % ov, noiselevel=-1) - - self["PORTDIR_OVERLAY"] = " ".join(new_ov) - self.backup_changes("PORTDIR_OVERLAY") - - locations_manager.set_port_dirs(self["PORTDIR"], self["PORTDIR_OVERLAY"]) - self._repo_make_defaults = {} for repo in self.repositories.repos_with_profiles(): d = getconfig(os.path.join(repo.location, "profiles", "make.defaults"), - expand=self.configdict["globals"].copy()) or {} + tolerant=tolerant, expand=self.configdict["globals"].copy(), recursive=repo.portage1_profiles) or {} if d: for k in chain(self._env_blacklist, profile_only_variables, self._global_only_vars): d.pop(k, None) self._repo_make_defaults[repo.name] = d - #Read package.keywords and package.accept_keywords. - self._keywords_manager = KeywordsManager(profiles_complex, abs_user_config, \ - local_config, global_accept_keywords=self.configdict["defaults"].get("ACCEPT_KEYWORDS", "")) - #Read all USE related files from profiles and optionally from user config. - self._use_manager = UseManager(self.repositories, profiles_complex, abs_user_config, user_config=local_config) + self._use_manager = UseManager(self.repositories, profiles_complex, + abs_user_config, self._isStable, user_config=local_config) #Initialize all USE related variables we track ourselves. self.usemask = self._use_manager.getUseMask() self.useforce = self._use_manager.getUseForce() @@ -595,13 +681,6 @@ class config(object): self._license_manager.extract_global_changes( \ self.configdict["conf"].get("ACCEPT_LICENSE", "")) - #Read package.mask and package.unmask from profiles and optionally from user config - self._mask_manager = MaskManager(self.repositories, profiles_complex, - abs_user_config, user_config=local_config, - strict_umatched_removal=_unmatched_removal) - - self._virtuals_manager = VirtualsManager(self.profiles) - if local_config: #package.properties propdict = grabdict_package(os.path.join( @@ -616,6 +695,20 @@ class config(object): for k, v in propdict.items(): self._ppropertiesdict.setdefault(k.cp, {})[k] = v + # package.accept_restrict + d = grabdict_package(os.path.join( + abs_user_config, "package.accept_restrict"), + recursive=True, allow_wildcard=True, + allow_repo=True, verify_eapi=False) + v = d.pop("*/*", None) + if v is not None: + if "ACCEPT_RESTRICT" in self.configdict["conf"]: + self.configdict["conf"]["ACCEPT_RESTRICT"] += " " + " ".join(v) + else: + self.configdict["conf"]["ACCEPT_RESTRICT"] = " ".join(v) + for k, v in d.items(): + self._paccept_restrict.setdefault(k.cp, {})[k] = v + #package.env penvdict = grabdict_package(os.path.join( abs_user_config, "package.env"), recursive=1, allow_wildcard=True, \ @@ -702,21 +795,9 @@ class config(object): self.backupenv["USE_ORDER"] = "env:pkg:conf:defaults:pkginternal:repo:env.d" self.depcachedir = DEPCACHE_PATH - if eprefix: - # See comments about make.globals and EPREFIX - # above. DEPCACHE_PATH is similar. - if target_root == "/": - # case (1) above - self.depcachedir = os.path.join(eprefix, - DEPCACHE_PATH.lstrip(os.sep)) - else: - # case (2) above - # For now, just assume DEPCACHE_PATH is relative - # to EPREFIX. - # TODO: Pass in more info to the constructor, - # so we know the host system configuration. - self.depcachedir = os.path.join(eprefix, - DEPCACHE_PATH.lstrip(os.sep)) + if portage.const.EPREFIX: + self.depcachedir = os.path.join(portage.const.EPREFIX, + DEPCACHE_PATH.lstrip(os.sep)) if self.get("PORTAGE_DEPCACHEDIR", None): self.depcachedir = self["PORTAGE_DEPCACHEDIR"] @@ -783,12 +864,17 @@ class config(object): self[var] = default_val self.backup_changes(var) + if portage._internal_caller: + self["PORTAGE_INTERNAL_CALLER"] = "1" + self.backup_changes("PORTAGE_INTERNAL_CALLER") + # initialize self.features self.regenerate() if bsd_chflags: self.features.add('chflags') + self._iuse_effective = self._calc_iuse_effective() self._iuse_implicit_match = _iuse_implicit_match_cache(self) self._validate_commands() @@ -798,11 +884,6 @@ class config(object): self[k] = self[k].lower() self.backup_changes(k) - if main_repo is not None and not main_repo.sync: - main_repo_sync = self.get("SYNC") - if main_repo_sync: - main_repo.sync = main_repo_sync - # The first constructed config object initializes these modules, # and subsequent calls to the _init() functions have no effect. portage.output._init(config_root=self['PORTAGE_CONFIGROOT']) @@ -882,6 +963,32 @@ class config(object): noiselevel=-1) @property + def _keywords_manager(self): + if self._keywords_manager_obj is None: + self._keywords_manager_obj = KeywordsManager( + self._locations_manager.profiles_complex, + self._locations_manager.abs_user_config, + self.local_config, + global_accept_keywords=self.configdict["defaults"].get("ACCEPT_KEYWORDS", "")) + return self._keywords_manager_obj + + @property + def _mask_manager(self): + if self._mask_manager_obj is None: + self._mask_manager_obj = MaskManager(self.repositories, + self._locations_manager.profiles_complex, + self._locations_manager.abs_user_config, + user_config=self.local_config, + strict_umatched_removal=self._unmatched_removal) + return self._mask_manager_obj + + @property + def _virtuals_manager(self): + if self._virtuals_manager_obj is None: + self._virtuals_manager_obj = VirtualsManager(self.profiles) + return self._virtuals_manager_obj + + @property def pkeywordsdict(self): result = self._keywords_manager.pkeywordsdict.copy() for k, v in result.items(): @@ -919,13 +1026,23 @@ class config(object): writemsg(_("!!! INVALID ACCEPT_KEYWORDS: %s\n") % str(group), noiselevel=-1) - profile_broken = not self.profile_path or \ - not os.path.exists(os.path.join(self.profile_path, "parent")) and \ - os.path.exists(os.path.join(self["PORTDIR"], "profiles")) + profile_broken = False + + if not self.profile_path: + profile_broken = True + else: + # If any one of these files exists, then + # the profile is considered valid. + for x in ("make.defaults", "parent", + "packages", "use.force", "use.mask"): + if exists_raise_eaccess(os.path.join(self.profile_path, x)): + break + else: + profile_broken = True - if profile_broken: + if profile_broken and not portage._sync_disabled_warnings: abs_profile_path = None - for x in (PROFILE_PATH, 'etc/portage/make.profile'): + for x in (PROFILE_PATH, 'etc/make.profile'): x = os.path.join(self["PORTAGE_CONFIGROOT"], x) try: os.lstat(x) @@ -1091,8 +1208,11 @@ class config(object): the previously calculated USE settings. """ - def __init__(self, use, usemask, iuse_implicit, + def __init__(self, settings, unfiltered_use, + use, usemask, iuse_implicit, use_expand_split, use_expand_dict): + self._settings = settings + self._unfiltered_use = unfiltered_use self._use = use self._usemask = usemask self._iuse_implicit = iuse_implicit @@ -1147,13 +1267,32 @@ class config(object): # Don't export empty USE_EXPAND vars unless the user config # exports them as empty. This is required for vars such as # LINGUAS, where unset and empty have different meanings. + # The special '*' token is understood by ebuild.sh, which + # will unset the variable so that things like LINGUAS work + # properly (see bug #459350). if has_wildcard: - # ebuild.sh will see this and unset the variable so - # that things like LINGUAS work properly value = '*' else: if has_iuse: - value = '' + already_set = False + # Skip the first 'env' configdict, in order to + # avoid infinite recursion here, since that dict's + # __getitem__ calls the current __getitem__. + for d in self._settings.lookuplist[1:]: + if key in d: + already_set = True + break + + if not already_set: + for x in self._unfiltered_use: + if x[:prefix_len] == prefix: + already_set = True + break + + if already_set: + value = '' + else: + value = '*' else: # It's not in IUSE, so just allow the variable content # to pass through if it is defined somewhere. This @@ -1189,7 +1328,7 @@ class config(object): if not isinstance(mycpv, basestring): pkg = mycpv mycpv = pkg.cpv - mydb = pkg.metadata + mydb = pkg._metadata explicit_iuse = pkg.iuse.all args_hash = (mycpv, id(pkg)) if pkg.built: @@ -1210,6 +1349,7 @@ class config(object): iuse = "" pkg_configdict = self.configdict["pkg"] previous_iuse = pkg_configdict.get("IUSE") + previous_iuse_effective = pkg_configdict.get("IUSE_EFFECTIVE") previous_features = pkg_configdict.get("FEATURES") aux_keys = self._setcpv_aux_keys @@ -1221,6 +1361,7 @@ class config(object): pkg_configdict["CATEGORY"] = cat pkg_configdict["PF"] = pf repository = None + eapi = None if mydb: if not hasattr(mydb, "aux_get"): for k in aux_keys: @@ -1247,14 +1388,16 @@ class config(object): # Empty USE means this dbapi instance does not contain # built packages. built_use = None + eapi = pkg_configdict['EAPI'] repository = pkg_configdict.pop("repository", None) if repository is not None: pkg_configdict["PORTAGE_REPO_NAME"] = repository - slot = pkg_configdict["SLOT"] iuse = pkg_configdict["IUSE"] if pkg is None: - cpv_slot = _pkg_str(self.mycpv, slot=slot, repo=repository) + self.mycpv = _pkg_str(self.mycpv, metadata=pkg_configdict, + settings=self) + cpv_slot = self.mycpv else: cpv_slot = pkg pkginternaluse = [] @@ -1264,6 +1407,9 @@ class config(object): elif x.startswith("-"): pkginternaluse.append(x) pkginternaluse = " ".join(pkginternaluse) + + eapi_attrs = _get_eapi_attrs(eapi) + if pkginternaluse != self.configdict["pkginternal"].get("USE", ""): self.configdict["pkginternal"]["USE"] = pkginternaluse has_changed = True @@ -1394,30 +1540,70 @@ class config(object): # If reset() has not been called, it's safe to return # early if IUSE has not changed. - if not has_changed and previous_iuse == iuse: + if not has_changed and previous_iuse == iuse and \ + (previous_iuse_effective is not None == eapi_attrs.iuse_effective): return # Filter out USE flags that aren't part of IUSE. This has to # be done for every setcpv() call since practically every # package has different IUSE. use = set(self["USE"].split()) + unfiltered_use = frozenset(use) if explicit_iuse is None: explicit_iuse = frozenset(x.lstrip("+-") for x in iuse.split()) - iuse_implicit_match = self._iuse_implicit_match - portage_iuse = self._get_implicit_iuse() - portage_iuse.update(explicit_iuse) + + if eapi_attrs.iuse_effective: + iuse_implicit_match = self._iuse_effective_match + portage_iuse = set(self._iuse_effective) + portage_iuse.update(explicit_iuse) + self.configdict["pkg"]["IUSE_EFFECTIVE"] = \ + " ".join(sorted(portage_iuse)) + else: + iuse_implicit_match = self._iuse_implicit_match + portage_iuse = self._get_implicit_iuse() + portage_iuse.update(explicit_iuse) # PORTAGE_IUSE is not always needed so it's lazily evaluated. self.configdict["env"].addLazySingleton( "PORTAGE_IUSE", _lazy_iuse_regex, portage_iuse) - ebuild_force_test = self.get("EBUILD_FORCE_TEST") == "1" + if pkg is None: + raw_restrict = pkg_configdict.get("RESTRICT") + else: + raw_restrict = pkg._raw_metadata["RESTRICT"] + + restrict_test = False + if raw_restrict: + try: + if built_use is not None: + restrict = use_reduce(raw_restrict, + uselist=built_use, flat=True) + else: + # Use matchnone=True to ignore USE conditional parts + # of RESTRICT, since we want to know whether to mask + # the "test" flag _before_ we know the USE values + # that would be needed to evaluate the USE + # conditionals (see bug #273272). + restrict = use_reduce(raw_restrict, + matchnone=True, flat=True) + except PortageException: + pass + else: + restrict_test = "test" in restrict + + ebuild_force_test = not restrict_test and \ + self.get("EBUILD_FORCE_TEST") == "1" + if ebuild_force_test and \ not hasattr(self, "_ebuild_force_test_msg_shown"): self._ebuild_force_test_msg_shown = True writemsg(_("Forcing test.\n"), noiselevel=-1) - if "test" in self.features: - if "test" in self.usemask and not ebuild_force_test: + + if "test" in explicit_iuse or iuse_implicit_match("test"): + if "test" not in self.features: + use.discard("test") + elif restrict_test or \ + ("test" in self.usemask and not ebuild_force_test): # "test" is in IUSE and USE=test is masked, so execution # of src_test() probably is not reliable. Therefore, # temporarily disable FEATURES=test just for this package. @@ -1430,6 +1616,13 @@ class config(object): self.usemask = \ frozenset(x for x in self.usemask if x != "test") + if eapi_attrs.feature_flag_targetroot and \ + ("targetroot" in explicit_iuse or iuse_implicit_match("targetroot")): + if self["ROOT"] != "/": + use.add("targetroot") + else: + use.discard("targetroot") + # Allow _* flags from USE_EXPAND wildcards to pass through here. use.difference_update([x for x in use \ if (x not in explicit_iuse and \ @@ -1440,7 +1633,8 @@ class config(object): # comparison instead of startswith(). use_expand_split = set(x.lower() for \ x in self.get('USE_EXPAND', '').split()) - lazy_use_expand = self._lazy_use_expand(use, self.usemask, + lazy_use_expand = self._lazy_use_expand( + self, unfiltered_use, use, self.usemask, portage_iuse, use_expand_split, self._use_expand_dict) use_expand_iuses = {} @@ -1470,6 +1664,14 @@ class config(object): self.configdict['env'].addLazySingleton(k, lazy_use_expand.__getitem__, k) + for k in self.get("USE_EXPAND_UNPREFIXED", "").split(): + var_split = self.get(k, '').split() + var_split = [ x for x in var_split if x in use ] + if var_split: + self.configlist[-1][k] = ' '.join(var_split) + elif k in self: + self.configlist[-1][k] = '' + # Filtered for the ebuild environment. Store this in a separate # attribute since we still want to be able to see global USE # settings for things like emerge --info. @@ -1477,6 +1679,10 @@ class config(object): self.configdict["env"]["PORTAGE_USE"] = \ " ".join(sorted(x for x in use if x[-2:] != '_*')) + # Clear the eapi cache here rather than in the constructor, since + # setcpv triggers lazy instantiation of things like _use_manager. + _eapi_cache.clear() + def _grab_pkg_env(self, penv, container, protected_keys=None): if protected_keys is None: protected_keys = () @@ -1510,9 +1716,42 @@ class config(object): else: container[k] = v + def _iuse_effective_match(self, flag): + return flag in self._iuse_effective + + def _calc_iuse_effective(self): + """ + Beginning with EAPI 5, IUSE_EFFECTIVE is defined by PMS. + """ + iuse_effective = [] + iuse_effective.extend(self.get("IUSE_IMPLICIT", "").split()) + + # USE_EXPAND_IMPLICIT should contain things like ARCH, ELIBC, + # KERNEL, and USERLAND. + use_expand_implicit = frozenset( + self.get("USE_EXPAND_IMPLICIT", "").split()) + + # USE_EXPAND_UNPREFIXED should contain at least ARCH, and + # USE_EXPAND_VALUES_ARCH should contain all valid ARCH flags. + for v in self.get("USE_EXPAND_UNPREFIXED", "").split(): + if v not in use_expand_implicit: + continue + iuse_effective.extend( + self.get("USE_EXPAND_VALUES_" + v, "").split()) + + use_expand = frozenset(self.get("USE_EXPAND", "").split()) + for v in use_expand_implicit: + if v not in use_expand: + continue + lower_v = v.lower() + for x in self.get("USE_EXPAND_VALUES_" + v, "").split(): + iuse_effective.append(lower_v + "_" + x) + + return frozenset(iuse_effective) + def _get_implicit_iuse(self): """ - Some flags are considered to + Prior to EAPI 5, these flags are considered to be implicit members of IUSE: * Flags derived from ARCH * Flags derived from USE_EXPAND_HIDDEN variables @@ -1549,11 +1788,11 @@ class config(object): return iuse_implicit - def _getUseMask(self, pkg): - return self._use_manager.getUseMask(pkg) + def _getUseMask(self, pkg, stable=None): + return self._use_manager.getUseMask(pkg, stable=stable) - def _getUseForce(self, pkg): - return self._use_manager.getUseForce(pkg) + def _getUseForce(self, pkg, stable=None): + return self._use_manager.getUseForce(pkg, stable=stable) def _getMaskAtom(self, cpv, metadata): """ @@ -1618,6 +1857,11 @@ class config(object): return x return None + def _isStable(self, pkg): + return self._keywords_manager.isStable(pkg, + self.get("ACCEPT_KEYWORDS", ""), + self.configdict["backupenv"].get("ACCEPT_KEYWORDS", "")) + def _getKeywords(self, cpv, metadata): return self._keywords_manager.getKeywords(cpv, metadata["SLOT"], \ metadata.get("KEYWORDS", ""), metadata.get("repository")) @@ -1706,9 +1950,10 @@ class config(object): @return: A list of properties that have not been accepted. """ accept_properties = self._accept_properties - if not hasattr(cpv, 'slot'): - cpv = _pkg_str(cpv, slot=metadata["SLOT"], - repo=metadata.get("repository")) + try: + cpv.slot + except AttributeError: + cpv = _pkg_str(cpv, metadata=metadata, settings=self) cp = cpv_getkey(cpv) cpdict = self._ppropertiesdict.get(cp) if cpdict: @@ -1720,7 +1965,6 @@ class config(object): properties_str = metadata.get("PROPERTIES", "") properties = set(use_reduce(properties_str, matchall=1, flat=True)) - properties.discard('||') acceptable_properties = set() for x in accept_properties: @@ -1738,40 +1982,58 @@ class config(object): else: use = [] - properties_struct = use_reduce(properties_str, uselist=use, opconvert=True) - return self._getMaskedProperties(properties_struct, acceptable_properties) - - def _getMaskedProperties(self, properties_struct, acceptable_properties): - if not properties_struct: - return [] - if properties_struct[0] == "||": - ret = [] - for element in properties_struct[1:]: - if isinstance(element, list): - if element: - tmp = self._getMaskedProperties( - element, acceptable_properties) - if not tmp: - return [] - ret.extend(tmp) - else: - if element in acceptable_properties: - return[] - ret.append(element) - # Return all masked properties, since we don't know which combination - # (if any) the user will decide to unmask - return ret - - ret = [] - for element in properties_struct: - if isinstance(element, list): - if element: - ret.extend(self._getMaskedProperties(element, - acceptable_properties)) + return [x for x in use_reduce(properties_str, uselist=use, flat=True) + if x not in acceptable_properties] + + def _getMissingRestrict(self, cpv, metadata): + """ + Take a RESTRICT string and return a list of any tokens the user + may need to accept for the given package. The returned list will not + contain any tokens that have already been accepted. This method + can throw an InvalidDependString exception. + + @param cpv: The package name (for package.accept_restrict support) + @type cpv: String + @param metadata: A dictionary of raw package metadata + @type metadata: dict + @rtype: List + @return: A list of tokens that have not been accepted. + """ + accept_restrict = self._accept_restrict + try: + cpv.slot + except AttributeError: + cpv = _pkg_str(cpv, metadata=metadata, settings=self) + cp = cpv_getkey(cpv) + cpdict = self._paccept_restrict.get(cp) + if cpdict: + paccept_restrict_list = ordered_by_atom_specificity(cpdict, cpv) + if paccept_restrict_list: + accept_restrict = list(self._accept_restrict) + for x in paccept_restrict_list: + accept_restrict.extend(x) + + restrict_str = metadata.get("RESTRICT", "") + all_restricts = set(use_reduce(restrict_str, matchall=1, flat=True)) + + acceptable_restricts = set() + for x in accept_restrict: + if x == '*': + acceptable_restricts.update(all_restricts) + elif x == '-*': + acceptable_restricts.clear() + elif x[:1] == '-': + acceptable_restricts.discard(x[1:]) else: - if element not in acceptable_properties: - ret.append(element) - return ret + acceptable_restricts.add(x) + + if "?" in restrict_str: + use = metadata["USE"].split() + else: + use = [] + + return [x for x in use_reduce(restrict_str, uselist=use, flat=True) + if x not in acceptable_restricts] def _accept_chost(self, cpv, metadata): """ @@ -1840,7 +2102,8 @@ class config(object): """Reload things like /etc/profile.env that can change during runtime.""" env_d_filename = os.path.join(self["EROOT"], "etc", "profile.env") self.configdict["env.d"].clear() - env_d = getconfig(env_d_filename, expand=False) + env_d = getconfig(env_d_filename, + tolerant=self._tolerant, expand=False) if env_d: # env_d will be None if profile.env doesn't exist. for k in self._env_d_blacklist: @@ -1909,6 +2172,18 @@ class config(object): # repoman will accept any property self._accept_properties = ('*',) + if self.local_config: + mysplit = [] + for curdb in mydbs: + mysplit.extend(curdb.get('ACCEPT_RESTRICT', '').split()) + mysplit = prune_incremental(mysplit) + self.configlist[-1]['ACCEPT_RESTRICT'] = ' '.join(mysplit) + if tuple(mysplit) != self._accept_restrict: + self._accept_restrict = tuple(mysplit) + else: + # repoman will accept any property + self._accept_restrict = ('*',) + increment_lists = {} for k in myincrementals: incremental_list = [] @@ -1963,6 +2238,8 @@ class config(object): if v is not None: use_expand_dict[k] = v + use_expand_unprefixed = self.get("USE_EXPAND_UNPREFIXED", "").split() + # In order to best accomodate the long-standing practice of # setting default USE_EXPAND variables in the profile's # make.defaults, we translate these variables into their @@ -1976,6 +2253,12 @@ class config(object): continue use = cfg.get("USE", "") expand_use = [] + + for k in use_expand_unprefixed: + v = cfg.get(k) + if v is not None: + expand_use.extend(v.split()) + for k in use_expand_dict: v = cfg.get(k) if v is None: @@ -2013,6 +2296,17 @@ class config(object): iuse = [x.lstrip("+-") for x in iuse.split()] myflags = set() for curdb in self.uvlist: + + for k in use_expand_unprefixed: + v = curdb.get(k) + if v is None: + continue + for x in v.split(): + if x[:1] == "-": + myflags.discard(x[1:]) + else: + myflags.add(x) + cur_use_expand = [x for x in use_expand if x in curdb] mysplit = curdb.get("USE", "").split() if not mysplit and not cur_use_expand: @@ -2129,6 +2423,14 @@ class config(object): elif k in self: self.configlist[-1][k] = '' + for k in use_expand_unprefixed: + var_split = self.get(k, '').split() + var_split = [ x for x in var_split if x in myflags ] + if var_split: + self.configlist[-1][k] = ' '.join(var_split) + elif k in self: + self.configlist[-1][k] = '' + @property def virts_p(self): warnings.warn("portage config.virts_p attribute " + \ @@ -2189,8 +2491,22 @@ class config(object): elif mykey == "PORTAGE_PYM_PATH": return portage._pym_path + elif mykey == "PORTAGE_PYTHONPATH": + value = [x for x in \ + self.backupenv.get("PYTHONPATH", "").split(":") if x] + need_pym_path = True + if value: + try: + need_pym_path = not os.path.samefile(value[0], + portage._pym_path) + except OSError: + pass + if need_pym_path: + value.insert(0, portage._pym_path) + return ":".join(value) + elif mykey == "PORTAGE_GID": - return _unicode_decode(str(portage_gid)) + return "%s" % portage_gid for d in self.lookuplist: try: @@ -2277,6 +2593,7 @@ class config(object): environ_filter = self._environ_filter eapi = self.get('EAPI') + eapi_attrs = _get_eapi_attrs(eapi) phase = self.get('EBUILD_PHASE') filter_calling_env = False if self.mycpv is not None and \ @@ -2358,14 +2675,20 @@ class config(object): not eapi_exports_replace_vars(eapi): mydict.pop("REPLACED_BY_VERSION", None) + if phase is not None and eapi_attrs.exports_EBUILD_PHASE_FUNC: + phase_func = _phase_func_map.get(phase) + if phase_func is not None: + mydict["EBUILD_PHASE_FUNC"] = phase_func + return mydict def thirdpartymirrors(self): if getattr(self, "_thirdpartymirrors", None) is None: - profileroots = [os.path.join(self["PORTDIR"], "profiles")] - for x in shlex_split(self.get("PORTDIR_OVERLAY", "")): - profileroots.insert(0, os.path.join(x, "profiles")) - thirdparty_lists = [grabdict(os.path.join(x, "thirdpartymirrors")) for x in profileroots] + thirdparty_lists = [] + for repo_name in reversed(self.repositories.prepos_order): + thirdparty_lists.append(grabdict(os.path.join( + self.repositories[repo_name].location, + "profiles", "thirdpartymirrors"))) self._thirdpartymirrors = stack_dictlist(thirdparty_lists, incremental=True) return self._thirdpartymirrors diff --git a/portage_with_autodep/pym/portage/package/ebuild/config.pyo b/portage_with_autodep/pym/portage/package/ebuild/config.pyo Binary files differindex 742ee2b..f4123ed 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/config.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/config.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/deprecated_profile_check.py b/portage_with_autodep/pym/portage/package/ebuild/deprecated_profile_check.py index 3fab4da..fdb19b4 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/deprecated_profile_check.py +++ b/portage_with_autodep/pym/portage/package/ebuild/deprecated_profile_check.py @@ -1,10 +1,11 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ['deprecated_profile_check'] import io +import portage from portage import os, _encodings, _unicode_encode from portage.const import DEPRECATED_PROFILE_FILE from portage.localization import _ @@ -12,16 +13,32 @@ from portage.output import colorize from portage.util import writemsg def deprecated_profile_check(settings=None): - config_root = "/" + config_root = None + eprefix = None + deprecated_profile_file = None if settings is not None: config_root = settings["PORTAGE_CONFIGROOT"] - deprecated_profile_file = os.path.join(config_root, - DEPRECATED_PROFILE_FILE) - if not os.access(deprecated_profile_file, os.R_OK): - return False - dcontent = io.open(_unicode_encode(deprecated_profile_file, + eprefix = settings["EPREFIX"] + for x in reversed(settings.profiles): + deprecated_profile_file = os.path.join(x, "deprecated") + if os.access(deprecated_profile_file, os.R_OK): + break + else: + deprecated_profile_file = None + + if deprecated_profile_file is None: + deprecated_profile_file = os.path.join(config_root or "/", + DEPRECATED_PROFILE_FILE) + if not os.access(deprecated_profile_file, os.R_OK): + deprecated_profile_file = os.path.join(config_root or "/", + 'etc', 'make.profile', 'deprecated') + if not os.access(deprecated_profile_file, os.R_OK): + return + + with io.open(_unicode_encode(deprecated_profile_file, encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['content'], errors='replace').readlines() + mode='r', encoding=_encodings['content'], errors='replace') as f: + dcontent = f.readlines() writemsg(colorize("BAD", _("\n!!! Your current profile is " "deprecated and not supported anymore.")) + "\n", noiselevel=-1) writemsg(colorize("BAD", _("!!! Use eselect profile to update your " @@ -30,13 +47,37 @@ def deprecated_profile_check(settings=None): writemsg(colorize("BAD", _("!!! Please refer to the " "Gentoo Upgrading Guide.")) + "\n", noiselevel=-1) return True - newprofile = dcontent[0] + newprofile = dcontent[0].rstrip("\n") writemsg(colorize("BAD", _("!!! Please upgrade to the " - "following profile if possible:")) + "\n", noiselevel=-1) - writemsg(8*" " + colorize("GOOD", newprofile) + "\n", noiselevel=-1) + "following profile if possible:")) + "\n\n", noiselevel=-1) + writemsg(8*" " + colorize("GOOD", newprofile) + "\n\n", noiselevel=-1) if len(dcontent) > 1: writemsg(_("To upgrade do the following steps:\n"), noiselevel=-1) for myline in dcontent[1:]: writemsg(myline, noiselevel=-1) writemsg("\n\n", noiselevel=-1) + else: + writemsg(_("You may use the following command to upgrade:\n\n"), noiselevel=-1) + writemsg(8*" " + colorize("INFORM", 'eselect profile set ' + + newprofile) + "\n\n", noiselevel=-1) + + if settings is not None: + main_repo_loc = settings.repositories.mainRepoLocation() + new_profile_path = os.path.join(main_repo_loc, + "profiles", newprofile.rstrip("\n")) + + if os.path.isdir(new_profile_path): + new_config = portage.config(config_root=config_root, + config_profile_path=new_profile_path, + eprefix=eprefix) + + if not new_config.profiles: + writemsg("\n %s %s\n" % (colorize("WARN", "*"), + _("You must update portage before you " + "can migrate to the above profile.")), noiselevel=-1) + writemsg(" %s %s\n\n" % (colorize("WARN", "*"), + _("In order to update portage, " + "run 'emerge --oneshot portage'.")), + noiselevel=-1) + return True diff --git a/portage_with_autodep/pym/portage/package/ebuild/deprecated_profile_check.pyo b/portage_with_autodep/pym/portage/package/ebuild/deprecated_profile_check.pyo Binary files differindex 2b9362b..df43aab 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/deprecated_profile_check.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/deprecated_profile_check.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/digestcheck.py b/portage_with_autodep/pym/portage/package/ebuild/digestcheck.py index 8705639..e207ba8 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/digestcheck.py +++ b/portage_with_autodep/pym/portage/package/ebuild/digestcheck.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ['digestcheck'] @@ -6,6 +6,7 @@ __all__ = ['digestcheck'] import warnings from portage import os, _encodings, _unicode_decode +from portage.checksum import _hash_filter from portage.exception import DigestException, FileNotFound from portage.localization import _ from portage.output import EOutput @@ -28,6 +29,9 @@ def digestcheck(myfiles, mysettings, strict=False, justmanifest=None, mf=None): if mysettings.get("EBUILD_SKIP_MANIFEST") == "1": return 1 pkgdir = mysettings["O"] + hash_filter = _hash_filter(mysettings.get("PORTAGE_CHECKSUM_FILTER", "")) + if hash_filter.transparent: + hash_filter = None if mf is None: mf = mysettings.repositories.get_repo_for_location( os.path.dirname(os.path.dirname(pkgdir))) @@ -38,15 +42,16 @@ def digestcheck(myfiles, mysettings, strict=False, justmanifest=None, mf=None): if not mf.thin and strict and "PORTAGE_PARALLEL_FETCHONLY" not in mysettings: if mf.fhashdict.get("EBUILD"): eout.ebegin(_("checking ebuild checksums ;-)")) - mf.checkTypeHashes("EBUILD") + mf.checkTypeHashes("EBUILD", hash_filter=hash_filter) eout.eend(0) if mf.fhashdict.get("AUX"): eout.ebegin(_("checking auxfile checksums ;-)")) - mf.checkTypeHashes("AUX") + mf.checkTypeHashes("AUX", hash_filter=hash_filter) eout.eend(0) if mf.fhashdict.get("MISC"): eout.ebegin(_("checking miscfile checksums ;-)")) - mf.checkTypeHashes("MISC", ignoreMissingFiles=True) + mf.checkTypeHashes("MISC", ignoreMissingFiles=True, + hash_filter=hash_filter) eout.eend(0) for f in myfiles: eout.ebegin(_("checking %s ;-)") % f) @@ -58,7 +63,7 @@ def digestcheck(myfiles, mysettings, strict=False, justmanifest=None, mf=None): writemsg(_("\n!!! Missing digest for '%s'\n") % (f,), noiselevel=-1) return 0 - mf.checkFileHashes(ftype, f) + mf.checkFileHashes(ftype, f, hash_filter=hash_filter) eout.eend(0) except FileNotFound as e: eout.eend(1) diff --git a/portage_with_autodep/pym/portage/package/ebuild/digestcheck.pyo b/portage_with_autodep/pym/portage/package/ebuild/digestcheck.pyo Binary files differindex 66987a2..c6a8d4e 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/digestcheck.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/digestcheck.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/digestgen.py b/portage_with_autodep/pym/portage/package/ebuild/digestgen.py index 6ad3397..95d02db 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/digestgen.py +++ b/portage_with_autodep/pym/portage/package/ebuild/digestgen.py @@ -112,67 +112,64 @@ def digestgen(myarchives=None, mysettings=None, myportdb=None): missing_files.append(myfile) continue - if missing_files: - for myfile in missing_files: - uris = set() - all_restrict = set() - for cpv in distfiles_map[myfile]: - uris.update(myportdb.getFetchMap( - cpv, mytree=mytree)[myfile]) - restrict = myportdb.aux_get(cpv, ['RESTRICT'], - mytree=mytree)[0] - # Here we ignore conditional parts of RESTRICT since - # they don't apply unconditionally. Assume such - # conditionals only apply on the client side where - # digestgen() does not need to be called. - all_restrict.update(use_reduce(restrict, - flat=True, matchnone=True)) - - # fetch() uses CATEGORY and PF to display a message - # when fetch restriction is triggered. - cat, pf = catsplit(cpv) - mysettings["CATEGORY"] = cat - mysettings["PF"] = pf - - # fetch() uses PORTAGE_RESTRICT to control fetch - # restriction, which is only applied to files that - # are not fetchable via a mirror:// URI. - mysettings["PORTAGE_RESTRICT"] = " ".join(all_restrict) - - try: - st = os.stat(os.path.join( - mysettings["DISTDIR"],myfile)) - except OSError: - st = None - - if not fetch({myfile : uris}, mysettings): - myebuild = os.path.join(mysettings["O"], - catsplit(cpv)[1] + ".ebuild") - spawn_nofetch(myportdb, myebuild) - writemsg(_("!!! Fetch failed for %s, can't update " - "Manifest\n") % myfile, noiselevel=-1) - if myfile in dist_hashes and \ - st is not None and st.st_size > 0: - # stat result is obtained before calling fetch(), - # since fetch may rename the existing file if the - # digest does not match. - writemsg(_("!!! If you would like to " - "forcefully replace the existing " - "Manifest entry\n!!! for %s, use " - "the following command:\n") % myfile + \ - "!!! " + colorize("INFORM", - "ebuild --force %s manifest" % \ - os.path.basename(myebuild)) + "\n", - noiselevel=-1) - return 0 + for myfile in missing_files: + uris = set() + all_restrict = set() + for cpv in distfiles_map[myfile]: + uris.update(myportdb.getFetchMap( + cpv, mytree=mytree)[myfile]) + restrict = myportdb.aux_get(cpv, ['RESTRICT'], mytree=mytree)[0] + # Here we ignore conditional parts of RESTRICT since + # they don't apply unconditionally. Assume such + # conditionals only apply on the client side where + # digestgen() does not need to be called. + all_restrict.update(use_reduce(restrict, + flat=True, matchnone=True)) + + # fetch() uses CATEGORY and PF to display a message + # when fetch restriction is triggered. + cat, pf = catsplit(cpv) + mysettings["CATEGORY"] = cat + mysettings["PF"] = pf + + # fetch() uses PORTAGE_RESTRICT to control fetch + # restriction, which is only applied to files that + # are not fetchable via a mirror:// URI. + mysettings["PORTAGE_RESTRICT"] = " ".join(all_restrict) + + try: + st = os.stat(os.path.join(mysettings["DISTDIR"], myfile)) + except OSError: + st = None + + if not fetch({myfile : uris}, mysettings): + myebuild = os.path.join(mysettings["O"], + catsplit(cpv)[1] + ".ebuild") + spawn_nofetch(myportdb, myebuild) + writemsg(_("!!! Fetch failed for %s, can't update Manifest\n") + % myfile, noiselevel=-1) + if myfile in dist_hashes and \ + st is not None and st.st_size > 0: + # stat result is obtained before calling fetch(), + # since fetch may rename the existing file if the + # digest does not match. + cmd = colorize("INFORM", "ebuild --force %s manifest" % + os.path.basename(myebuild)) + writemsg((_( + "!!! If you would like to forcefully replace the existing Manifest entry\n" + "!!! for %s, use the following command:\n") % myfile) + + "!!! %s\n" % cmd, + noiselevel=-1) + return 0 + writemsg_stdout(_(">>> Creating Manifest for %s\n") % mysettings["O"]) try: mf.create(assumeDistHashesSometimes=True, assumeDistHashesAlways=( "assume-digests" in mysettings.features)) except FileNotFound as e: - writemsg(_("!!! File %s doesn't exist, can't update " - "Manifest\n") % e, noiselevel=-1) + writemsg(_("!!! File %s doesn't exist, can't update Manifest\n") + % e, noiselevel=-1) return 0 except PortagePackageException as e: writemsg(("!!! %s\n") % (e,), noiselevel=-1) diff --git a/portage_with_autodep/pym/portage/package/ebuild/digestgen.pyo b/portage_with_autodep/pym/portage/package/ebuild/digestgen.pyo Binary files differindex 66876ec..a4e9b62 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/digestgen.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/digestgen.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/doebuild.py b/portage_with_autodep/pym/portage/package/ebuild/doebuild.py index 610172f..5351c52 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/doebuild.py +++ b/portage_with_autodep/pym/portage/package/ebuild/doebuild.py @@ -1,14 +1,19 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ['doebuild', 'doebuild_environment', 'spawn', 'spawnebuild'] +import grp import gzip import errno import io from itertools import chain import logging import os as _os +import platform +import pwd import re import signal import stat @@ -25,7 +30,13 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.package.ebuild.digestcheck:digestcheck', 'portage.package.ebuild.digestgen:digestgen', 'portage.package.ebuild.fetch:fetch', + 'portage.package.ebuild._ipc.QueryCommand:QueryCommand', + 'portage.dep._slot_operator:evaluate_slot_operator_equal_deps', 'portage.package.ebuild._spawn_nofetch:spawn_nofetch', + 'portage.util._desktop_entry:validate_desktop_entry', + 'portage.util._async.SchedulerInterface:SchedulerInterface', + 'portage.util._eventloop.EventLoop:EventLoop', + 'portage.util._eventloop.global_event_loop:global_event_loop', 'portage.util.ExtractKernelVersion:ExtractKernelVersion' ) @@ -43,7 +54,7 @@ from portage.dep import Atom, check_required_use, \ from portage.eapi import eapi_exports_KV, eapi_exports_merge_type, \ eapi_exports_replace_vars, eapi_exports_REPOSITORY, \ eapi_has_required_use, eapi_has_src_prepare_and_src_configure, \ - eapi_has_pkg_pretend + eapi_has_pkg_pretend, _get_eapi_attrs from portage.elog import elog_process, _preload_elog_modules from portage.elog.messages import eerror, eqawarn from portage.exception import DigestException, FileNotFound, \ @@ -55,14 +66,13 @@ from portage.package.ebuild.prepare_build_dirs import prepare_build_dirs from portage.util import apply_recursive_permissions, \ apply_secpass_permissions, noiselimit, normalize_path, \ writemsg, writemsg_stdout, write_atomic -from portage.util.lafilefixer import rewrite_lafile +from portage.util.lafilefixer import rewrite_lafile from portage.versions import _pkgsplit from _emerge.BinpkgEnvExtractor import BinpkgEnvExtractor from _emerge.EbuildBuildDir import EbuildBuildDir from _emerge.EbuildPhase import EbuildPhase from _emerge.EbuildSpawnProcess import EbuildSpawnProcess from _emerge.Package import Package -from _emerge.PollScheduler import PollScheduler from _emerge.RootConfig import RootConfig _unsandboxed_phases = frozenset([ @@ -72,6 +82,40 @@ _unsandboxed_phases = frozenset([ "prerm", "setup" ]) +# phases in which IPC with host is allowed +_ipc_phases = frozenset([ + "setup", "pretend", + "preinst", "postinst", "prerm", "postrm", +]) + +# phases in which networking access is allowed +_networked_phases = frozenset([ + # for VCS fetching + "unpack", + # + for network-bound IPC +] + list(_ipc_phases)) + +_phase_func_map = { + "config": "pkg_config", + "setup": "pkg_setup", + "nofetch": "pkg_nofetch", + "unpack": "src_unpack", + "prepare": "src_prepare", + "configure": "src_configure", + "compile": "src_compile", + "test": "src_test", + "install": "src_install", + "preinst": "pkg_preinst", + "postinst": "pkg_postinst", + "prerm": "pkg_prerm", + "postrm": "pkg_postrm", + "info": "pkg_info", + "pretend": "pkg_pretend", +} + +_vdb_use_conditional_keys = Package._dep_keys + \ + ('LICENSE', 'PROPERTIES', 'PROVIDE', 'RESTRICT',) + def _doebuild_spawn(phase, settings, actionmap=None, **kwargs): """ All proper ebuild phases which execute ebuild.sh are spawned @@ -81,8 +125,18 @@ def _doebuild_spawn(phase, settings, actionmap=None, **kwargs): if phase in _unsandboxed_phases: kwargs['free'] = True + kwargs['ipc'] = 'ipc-sandbox' not in settings.features or \ + phase in _ipc_phases + kwargs['networked'] = 'network-sandbox' not in settings.features or \ + phase in _networked_phases + if phase == 'depend': kwargs['droppriv'] = 'userpriv' in settings.features + # It's not necessary to close_fds for this phase, since + # it should not spawn any daemons, and close_fds is + # best avoided since it can interact badly with some + # garbage collectors (see _setup_pipes docstring). + kwargs['close_fds'] = False if actionmap is not None and phase in actionmap: kwargs.update(actionmap[phase]["args"]) @@ -100,17 +154,24 @@ def _doebuild_spawn(phase, settings, actionmap=None, **kwargs): settings['EBUILD_PHASE'] = phase try: - return spawn(cmd, settings, **kwargs) + return spawn(cmd, settings, **portage._native_kwargs(kwargs)) finally: settings.pop('EBUILD_PHASE', None) -def _spawn_phase(phase, settings, actionmap=None, **kwargs): - if kwargs.get('returnpid'): - return _doebuild_spawn(phase, settings, actionmap=actionmap, **kwargs) +def _spawn_phase(phase, settings, actionmap=None, returnpid=False, + logfile=None, **kwargs): + + if returnpid: + return _doebuild_spawn(phase, settings, actionmap=actionmap, + returnpid=returnpid, logfile=logfile, **kwargs) + # The logfile argument is unused here, since EbuildPhase uses + # the PORTAGE_LOG_FILE variable if set. ebuild_phase = EbuildPhase(actionmap=actionmap, background=False, - phase=phase, scheduler=PollScheduler().sched_iface, - settings=settings) + phase=phase, scheduler=SchedulerInterface(portage._internal_caller and + global_event_loop() or EventLoop(main=False)), + settings=settings, **kwargs) + ebuild_phase.start() ebuild_phase.wait() return ebuild_phase.returncode @@ -123,19 +184,28 @@ def _doebuild_path(settings, eapi=None): # Note: PORTAGE_BIN_PATH may differ from the global constant # when portage is reinstalling itself. portage_bin_path = settings["PORTAGE_BIN_PATH"] - eprefix = settings["EPREFIX"] + eprefix = portage.const.EPREFIX prerootpath = [x for x in settings.get("PREROOTPATH", "").split(":") if x] rootpath = [x for x in settings.get("ROOTPATH", "").split(":") if x] + overrides = [x for x in settings.get( + "__PORTAGE_TEST_PATH_OVERRIDE", "").split(":") if x] prefixes = [] if eprefix: prefixes.append(eprefix) prefixes.append("/") - path = [] + path = overrides + + if "xattr" in settings.features: + path.append(os.path.join(portage_bin_path, "ebuild-helpers", "xattr")) - if eapi not in (None, "0", "1", "2", "3"): - path.append(os.path.join(portage_bin_path, "ebuild-helpers", "4")) + if eprefix and uid != 0 and "fakeroot" not in settings.features: + path.append(os.path.join(portage_bin_path, + "ebuild-helpers", "unprivileged")) + + if settings.get("USERLAND", "GNU") != "GNU": + path.append(os.path.join(portage_bin_path, "ebuild-helpers", "bsd")) path.append(os.path.join(portage_bin_path, "ebuild-helpers")) path.extend(prerootpath) @@ -254,10 +324,11 @@ def doebuild_environment(myebuild, mydo, myroot=None, settings=None, if hasattr(mydbapi, 'repositories'): repo = mydbapi.repositories.get_repo_for_location(mytree) mysettings['PORTDIR'] = repo.eclass_db.porttrees[0] - mysettings['PORTDIR_OVERLAY'] = ' '.join(repo.eclass_db.porttrees[1:]) + mysettings['PORTAGE_ECLASS_LOCATIONS'] = repo.eclass_db.eclass_locations_string mysettings.configdict["pkg"]["PORTAGE_REPO_NAME"] = repo.name mysettings["PORTDIR"] = os.path.realpath(mysettings["PORTDIR"]) + mysettings.pop("PORTDIR_OVERLAY", None) mysettings["DISTDIR"] = os.path.realpath(mysettings["DISTDIR"]) mysettings["RPMDIR"] = os.path.realpath(mysettings["RPMDIR"]) @@ -414,14 +485,14 @@ _doebuild_commands_without_builddir = ( 'fetch', 'fetchall', 'help', 'manifest' ) -def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, - fetchonly=0, cleanup=0, dbkey=None, use_cache=1, fetchall=0, tree=None, +def doebuild(myebuild, mydo, _unused=DeprecationWarning, settings=None, debug=0, listonly=0, + fetchonly=0, cleanup=0, dbkey=DeprecationWarning, use_cache=1, fetchall=0, tree=None, mydbapi=None, vartree=None, prev_mtimes=None, fd_pipes=None, returnpid=False): """ Wrapper function that invokes specific ebuild phases through the spawning of ebuild.sh - + @param myebuild: name of the ebuild to invoke the phase on (CPV) @type myebuild: String @param mydo: Phase to run @@ -464,13 +535,13 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, @return: 1. 0 for success 2. 1 for error - + Most errors have an accompanying error message. - + listonly and fetchonly are only really necessary for operations involving 'fetch' prev_mtimes are only necessary for merge operations. Other variables may not be strictly required, many have defaults that are set inside of doebuild. - + """ if settings is None: @@ -478,17 +549,22 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, mysettings = settings myroot = settings['EROOT'] - if _unused is not None and _unused != mysettings['EROOT']: + if _unused is not DeprecationWarning: warnings.warn("The third parameter of the " - "portage.doebuild() is now unused. Use " - "settings['ROOT'] instead.", + "portage.doebuild() is deprecated. Instead " + "settings['EROOT'] is used.", + DeprecationWarning, stacklevel=2) + + if dbkey is not DeprecationWarning: + warnings.warn("portage.doebuild() called " + "with deprecated dbkey argument.", DeprecationWarning, stacklevel=2) if not tree: writemsg("Warning: tree not specified to doebuild\n") tree = "porttree" - - # chunked out deps for each phase, so that ebuild binary can use it + + # chunked out deps for each phase, so that ebuild binary can use it # to collapse targets down. actionmap_deps={ "pretend" : [], @@ -503,7 +579,7 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, "package":["install"], "merge" :["install"], } - + if mydbapi is None: mydbapi = portage.db[myroot][tree].dbapi @@ -518,7 +594,7 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, "fetch", "fetchall", "digest", "unpack", "prepare", "configure", "compile", "test", "install", "rpm", "qmerge", "merge", - "package","unmerge", "manifest"] + "package", "unmerge", "manifest", "nofetch"] if mydo not in validcommands: validcommands.sort() @@ -532,8 +608,11 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, return 1 if returnpid and mydo != 'depend': - warnings.warn("portage.doebuild() called " + \ - "with returnpid parameter enabled. This usage will " + \ + # This case is not supported, since it bypasses the EbuildPhase class + # which implements important functionality (including post phase hooks + # and IPC for things like best/has_version and die). + warnings.warn("portage.doebuild() called " + "with returnpid parameter enabled. This usage will " "not be supported in the future.", DeprecationWarning, stacklevel=2) @@ -541,9 +620,6 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, fetchall = 1 mydo = "fetch" - parallel_fetchonly = mydo in ("fetch", "fetchall") and \ - "PORTAGE_PARALLEL_FETCHONLY" in mysettings - if mydo not in clean_phases and not os.path.exists(myebuild): writemsg("!!! doebuild: %s not found for %s\n" % (myebuild, mydo), noiselevel=-1) @@ -650,7 +726,7 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, # we can temporarily override PORTAGE_TMPDIR with a random temp dir # so that there's no need for locking and it can be used even if the # user isn't in the portage group. - if mydo in ("info",): + if not returnpid and mydo in ("info",): tmpdir = tempfile.mkdtemp() tmpdir_orig = mysettings["PORTAGE_TMPDIR"] mysettings["PORTAGE_TMPDIR"] = tmpdir @@ -661,9 +737,10 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, if mydo in clean_phases: builddir_lock = None if not returnpid and \ - 'PORTAGE_BUILDIR_LOCKED' not in mysettings: + 'PORTAGE_BUILDDIR_LOCKED' not in mysettings: builddir_lock = EbuildBuildDir( - scheduler=PollScheduler().sched_iface, + scheduler=(portage._internal_caller and + global_event_loop() or EventLoop(main=False)), settings=mysettings) builddir_lock.lock() try: @@ -679,42 +756,7 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, if returnpid: return _spawn_phase(mydo, mysettings, fd_pipes=fd_pipes, returnpid=returnpid) - elif isinstance(dbkey, dict): - warnings.warn("portage.doebuild() called " + \ - "with dict dbkey argument. This usage will " + \ - "not be supported in the future.", - DeprecationWarning, stacklevel=2) - mysettings["dbkey"] = "" - pr, pw = os.pipe() - fd_pipes = { - 0:sys.stdin.fileno(), - 1:sys.stdout.fileno(), - 2:sys.stderr.fileno(), - 9:pw} - mypids = _spawn_phase(mydo, mysettings, returnpid=True, - fd_pipes=fd_pipes) - os.close(pw) # belongs exclusively to the child process now - f = os.fdopen(pr, 'rb', 0) - for k, v in zip(auxdbkeys, - (_unicode_decode(line).rstrip('\n') for line in f)): - dbkey[k] = v - f.close() - retval = os.waitpid(mypids[0], 0)[1] - portage.process.spawned_pids.remove(mypids[0]) - # If it got a signal, return the signal that was sent, but - # shift in order to distinguish it from a return value. (just - # like portage.process.spawn() would do). - if retval & 0xff: - retval = (retval & 0xff) << 8 - else: - # Otherwise, return its exit code. - retval = retval >> 8 - if retval == os.EX_OK and len(dbkey) != len(auxdbkeys): - # Don't trust bash's returncode if the - # number of lines is incorrect. - retval = 1 - return retval - elif dbkey: + elif dbkey and dbkey is not DeprecationWarning: mysettings["dbkey"] = dbkey else: mysettings["dbkey"] = \ @@ -723,14 +765,25 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, return _spawn_phase(mydo, mysettings, fd_pipes=fd_pipes, returnpid=returnpid) - # Validate dependency metadata here to ensure that ebuilds with invalid - # data are never installed via the ebuild command. Don't bother when - # returnpid == True since there's no need to do this every time emerge - # executes a phase. + elif mydo == "nofetch": + + if returnpid: + writemsg("!!! doebuild: %s\n" % + _("returnpid is not supported for phase '%s'\n" % mydo), + noiselevel=-1) + + return spawn_nofetch(mydbapi, myebuild, settings=mysettings, + fd_pipes=fd_pipes) + if tree == "porttree": - rval = _validate_deps(mysettings, myroot, mydo, mydbapi) - if rval != os.EX_OK: - return rval + + if not returnpid: + # Validate dependency metadata here to ensure that ebuilds with + # invalid data are never installed via the ebuild command. Skip + # this when returnpid is True (assume the caller handled it). + rval = _validate_deps(mysettings, myroot, mydo, mydbapi) + if rval != os.EX_OK: + return rval else: # FEATURES=noauto only makes sense for porttree, and we don't want @@ -739,20 +792,25 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, if "noauto" in mysettings.features: mysettings.features.discard("noauto") - # The info phase is special because it uses mkdtemp so and - # user (not necessarily in the portage group) can run it. - if mydo not in ('info',) and \ + # If we are not using a private temp dir, then check access + # to the global temp dir. + if tmpdir is None and \ mydo not in _doebuild_commands_without_builddir: rval = _check_temp_dir(mysettings) if rval != os.EX_OK: return rval if mydo == "unmerge": + if returnpid: + writemsg("!!! doebuild: %s\n" % + _("returnpid is not supported for phase '%s'\n" % mydo), + noiselevel=-1) return unmerge(mysettings["CATEGORY"], mysettings["PF"], myroot, mysettings, vartree=vartree) phases_to_run = set() - if "noauto" in mysettings.features or \ + if returnpid or \ + "noauto" in mysettings.features or \ mydo not in actionmap_deps: phases_to_run.add(mydo) else: @@ -803,9 +861,10 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, if newstuff: if builddir_lock is None and \ - 'PORTAGE_BUILDIR_LOCKED' not in mysettings: + 'PORTAGE_BUILDDIR_LOCKED' not in mysettings: builddir_lock = EbuildBuildDir( - scheduler=PollScheduler().sched_iface, + scheduler=(portage._internal_caller and + global_event_loop() or EventLoop(main=False)), settings=mysettings) builddir_lock.lock() try: @@ -823,12 +882,12 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, # in order to satisfy the sane $PWD requirement (from bug #239560) # when pkg_nofetch is spawned. have_build_dirs = False - if not parallel_fetchonly and \ - mydo not in ('digest', 'fetch', 'help', 'manifest'): + if mydo not in ('digest', 'fetch', 'help', 'manifest'): if not returnpid and \ - 'PORTAGE_BUILDIR_LOCKED' not in mysettings: + 'PORTAGE_BUILDDIR_LOCKED' not in mysettings: builddir_lock = EbuildBuildDir( - scheduler=PollScheduler().sched_iface, + scheduler=(portage._internal_caller and + global_event_loop() or EventLoop(main=False)), settings=mysettings) builddir_lock.lock() mystatus = prepare_build_dirs(myroot, mysettings, cleanup) @@ -871,9 +930,8 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, else: vardb = vartree.dbapi cpv = mysettings.mycpv - cp = portage.versions.cpv_getkey(cpv) - slot = mysettings["SLOT"] - cpv_slot = cp + ":" + slot + cpv_slot = "%s%s%s" % \ + (cpv.cp, portage.dep._slot_separator, cpv.slot) mysettings["REPLACING_VERSIONS"] = " ".join( set(portage.versions.cpv_getversion(match) \ for match in vardb.match(cpv_slot) + \ @@ -883,8 +941,16 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, # the sandbox -- and stop now. if mydo in ("config", "help", "info", "postinst", "preinst", "pretend", "postrm", "prerm"): - return _spawn_phase(mydo, mysettings, - fd_pipes=fd_pipes, logfile=logfile, returnpid=returnpid) + if mydo in ("preinst", "postinst"): + env_file = os.path.join(os.path.dirname(mysettings["EBUILD"]), + "environment.bz2") + if os.path.isfile(env_file): + mysettings["PORTAGE_UPDATE_ENV"] = env_file + try: + return _spawn_phase(mydo, mysettings, + fd_pipes=fd_pipes, logfile=logfile, returnpid=returnpid) + finally: + mysettings.pop("PORTAGE_UPDATE_ENV", None) mycpv = "/".join((mysettings["CATEGORY"], mysettings["PF"])) @@ -925,7 +991,8 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, if not fetch(fetchme, mysettings, listonly=listonly, fetchonly=fetchonly, allow_missing_digests=True, digests=dist_digests): - spawn_nofetch(mydbapi, myebuild, settings=mysettings) + spawn_nofetch(mydbapi, myebuild, settings=mysettings, + fd_pipes=fd_pipes) if listonly: # The convention for listonly mode is to report # success in any case, even though fetch() may @@ -957,11 +1024,7 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, mf = None _doebuild_manifest_cache = None return not digestgen(mysettings=mysettings, myportdb=mydbapi) - elif mydo != 'fetch' and \ - "digest" in mysettings.features: - # Don't do this when called by emerge or when called just - # for fetch (especially parallel-fetch) since it's not needed - # and it can interfere with parallel tasks. + elif "digest" in mysettings.features: mf = None _doebuild_manifest_cache = None digestgen(mysettings=mysettings, myportdb=mydbapi) @@ -970,14 +1033,17 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, if mydo in ("digest", "manifest"): return 1 + if mydo == "fetch": + # Return after digestgen for FEATURES=digest support. + # Return before digestcheck, since fetch() already + # checked any relevant digests. + return 0 + # See above comment about fetching only when needed if tree == 'porttree' and \ not digestcheck(checkme, mysettings, "strict" in features, mf=mf): return 1 - if mydo == "fetch": - return 0 - # remove PORTAGE_ACTUAL_DISTDIR once cvs/svn is supported via SRC_URI if tree == 'porttree' and \ ((mydo != "setup" and "noauto" not in features) \ @@ -993,7 +1059,9 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, if len(actionmap_deps.get(x, [])): actionmap[x]["dep"] = ' '.join(actionmap_deps[x]) - if mydo in actionmap: + regular_actionmap_phase = mydo in actionmap + + if regular_actionmap_phase: bintree = None if mydo == "package": # Make sure the package directory exists before executing @@ -1017,6 +1085,9 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, actionmap, mysettings, debug, logfile=logfile, fd_pipes=fd_pipes, returnpid=returnpid) + if returnpid and isinstance(retval, list): + return retval + if retval == os.EX_OK: if mydo == "package" and bintree is not None: bintree.inject(mysettings.mycpv, @@ -1028,7 +1099,15 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, except OSError: pass - elif mydo=="qmerge": + elif returnpid: + writemsg("!!! doebuild: %s\n" % + _("returnpid is not supported for phase '%s'\n" % mydo), + noiselevel=-1) + + if regular_actionmap_phase: + # handled above + pass + elif mydo == "qmerge": # check to ensure install was run. this *only* pops up when users # forget it and are using ebuild if not os.path.exists( @@ -1045,7 +1124,8 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, mysettings["CATEGORY"], mysettings["PF"], mysettings["D"], os.path.join(mysettings["PORTAGE_BUILDDIR"], "build-info"), myroot, mysettings, myebuild=mysettings["EBUILD"], mytree=tree, - mydbapi=mydbapi, vartree=vartree, prev_mtimes=prev_mtimes) + mydbapi=mydbapi, vartree=vartree, prev_mtimes=prev_mtimes, + fd_pipes=fd_pipes) elif mydo=="merge": retval = spawnebuild("install", actionmap, mysettings, debug, alwaysdep=1, logfile=logfile, fd_pipes=fd_pipes, @@ -1061,7 +1141,9 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, mysettings["D"], os.path.join(mysettings["PORTAGE_BUILDDIR"], "build-info"), myroot, mysettings, myebuild=mysettings["EBUILD"], mytree=tree, mydbapi=mydbapi, - vartree=vartree, prev_mtimes=prev_mtimes) + vartree=vartree, prev_mtimes=prev_mtimes, + fd_pipes=fd_pipes) + else: writemsg_stdout(_("!!! Unknown mydo: %s\n") % mydo, noiselevel=-1) return 1 @@ -1161,7 +1243,9 @@ def _prepare_env_file(settings): """ env_extractor = BinpkgEnvExtractor(background=False, - scheduler=PollScheduler().sched_iface, settings=settings) + scheduler=(portage._internal_caller and + global_event_loop() or EventLoop(main=False)), + settings=settings) if env_extractor.dest_env_exists(): # There are lots of possible states when doebuild() @@ -1243,7 +1327,7 @@ def _spawn_actionmap(settings): misc_sh_binary = os.path.join(portage_bin_path, os.path.basename(MISC_SH_BINARY)) ebuild_sh = _shell_quote(ebuild_sh_binary) + " %s" - misc_sh = _shell_quote(misc_sh_binary) + " dyn_%s" + misc_sh = _shell_quote(misc_sh_binary) + " __dyn_%s" # args are for the to spawn function actionmap = { @@ -1299,10 +1383,10 @@ def _validate_deps(mysettings, myroot, mydo, mydbapi): if not pkg.built and \ mydo not in ("digest", "help", "manifest") and \ - pkg.metadata["REQUIRED_USE"] and \ - eapi_has_required_use(pkg.metadata["EAPI"]): - result = check_required_use(pkg.metadata["REQUIRED_USE"], - pkg.use.enabled, pkg.iuse.is_valid_flag) + pkg._metadata["REQUIRED_USE"] and \ + eapi_has_required_use(pkg.eapi): + result = check_required_use(pkg._metadata["REQUIRED_USE"], + pkg.use.enabled, pkg.iuse.is_valid_flag, eapi=pkg.eapi) if not result: reduced_noise = result.tounicode() writemsg("\n %s\n" % _("The following REQUIRED_USE flag" + \ @@ -1310,7 +1394,7 @@ def _validate_deps(mysettings, myroot, mydo, mydbapi): writemsg(" %s\n" % reduced_noise, noiselevel=-1) normalized_required_use = \ - " ".join(pkg.metadata["REQUIRED_USE"].split()) + " ".join(pkg._metadata["REQUIRED_USE"].split()) if reduced_noise != normalized_required_use: writemsg("\n %s\n" % _("The above constraints " + \ "are a subset of the following complete expression:"), @@ -1325,7 +1409,8 @@ def _validate_deps(mysettings, myroot, mydo, mydbapi): # XXX This would be to replace getstatusoutput completely. # XXX Issue: cannot block execution. Deadlock condition. -def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakeroot=0, **keywords): +def spawn(mystring, mysettings, debug=False, free=False, droppriv=False, + sesandbox=False, fakeroot=False, networked=True, ipc=True, **keywords): """ Spawn a subprocess with extra portage-specific options. Optiosn include: @@ -1355,6 +1440,10 @@ def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakero @type sesandbox: Boolean @param fakeroot: Run this command with faked root privileges @type fakeroot: Boolean + @param networked: Run this command with networking access enabled + @type networked: Boolean + @param ipc: Run this command with host IPC access enabled + @type ipc: Boolean @param keywords: Extra options encoded as a dict, to be passed to spawn @type keywords: Dictionary @rtype: Integer @@ -1367,29 +1456,90 @@ def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakero fd_pipes = keywords.get("fd_pipes") if fd_pipes is None: fd_pipes = { - 0:sys.stdin.fileno(), - 1:sys.stdout.fileno(), - 2:sys.stderr.fileno(), + 0:portage._get_stdin().fileno(), + 1:sys.__stdout__.fileno(), + 2:sys.__stderr__.fileno(), } # In some cases the above print statements don't flush stdout, so # it needs to be flushed before allowing a child process to use it # so that output always shows in the correct order. - stdout_filenos = (sys.stdout.fileno(), sys.stderr.fileno()) + stdout_filenos = (sys.__stdout__.fileno(), sys.__stderr__.fileno()) for fd in fd_pipes.values(): if fd in stdout_filenos: - sys.stdout.flush() - sys.stderr.flush() + sys.__stdout__.flush() + sys.__stderr__.flush() break features = mysettings.features + + # Use Linux namespaces if available + if uid == 0 and platform.system() == 'Linux': + keywords['unshare_net'] = not networked + keywords['unshare_ipc'] = not ipc + # TODO: Enable fakeroot to be used together with droppriv. The # fake ownership/permissions will have to be converted to real # permissions in the merge phase. fakeroot = fakeroot and uid != 0 and portage.process.fakeroot_capable - if droppriv and uid == 0 and portage_gid and portage_uid and \ - hasattr(os, "setgroups"): - keywords.update({"uid":portage_uid,"gid":portage_gid, - "groups":userpriv_groups,"umask":0o02}) + portage_build_uid = os.getuid() + portage_build_gid = os.getgid() + if uid == 0 and portage_uid and portage_gid and hasattr(os, "setgroups"): + if droppriv: + keywords.update({ + "uid": portage_uid, + "gid": portage_gid, + "groups": userpriv_groups, + "umask": 0o02 + }) + + # Adjust pty ownership so that subprocesses + # can directly access /dev/fd/{1,2}. + stdout_fd = fd_pipes.get(1) + if stdout_fd is not None: + try: + subprocess_tty = _os.ttyname(stdout_fd) + except OSError: + pass + else: + try: + parent_tty = _os.ttyname(sys.__stdout__.fileno()) + except OSError: + parent_tty = None + + if subprocess_tty != parent_tty: + _os.chown(subprocess_tty, + int(portage_uid), int(portage_gid)) + + if "userpriv" in features and "userpriv" not in mysettings["PORTAGE_RESTRICT"].split() and secpass >= 2: + # Since Python 3.4, getpwuid and getgrgid + # require int type (no proxies). + portage_build_uid = int(portage_uid) + portage_build_gid = int(portage_gid) + + if "PORTAGE_BUILD_USER" not in mysettings: + user = None + try: + user = pwd.getpwuid(portage_build_uid).pw_name + except KeyError: + if portage_build_uid == 0: + user = "root" + elif portage_build_uid == portage_uid: + user = portage.data._portage_username + if user is not None: + mysettings["PORTAGE_BUILD_USER"] = user + + if "PORTAGE_BUILD_GROUP" not in mysettings: + group = None + try: + group = grp.getgrgid(portage_build_gid).gr_name + except KeyError: + if portage_build_gid == 0: + group = "root" + elif portage_build_gid == portage_gid: + group = portage.data._portage_grpname + if group is not None: + mysettings["PORTAGE_BUILD_GROUP"] = group + if not free: free=((droppriv and "usersandbox" not in features) or \ (not droppriv and "sandbox" not in features and \ @@ -1423,12 +1573,15 @@ def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakero mysettings["PORTAGE_SANDBOX_T"]) if keywords.get("returnpid"): - return spawn_func(mystring, env=mysettings.environ(), **keywords) + return spawn_func(mystring, env=mysettings.environ(), + **portage._native_kwargs(keywords)) proc = EbuildSpawnProcess( background=False, args=mystring, - scheduler=PollScheduler().sched_iface, spawn_func=spawn_func, - settings=mysettings, **keywords) + scheduler=SchedulerInterface(portage._internal_caller and + global_event_loop() or EventLoop(main=False)), + spawn_func=spawn_func, + settings=mysettings, **portage._native_kwargs(keywords)) proc.start() proc.wait() @@ -1440,8 +1593,8 @@ def spawnebuild(mydo, actionmap, mysettings, debug, alwaysdep=0, logfile=None, fd_pipes=None, returnpid=False): if returnpid: - warnings.warn("portage.spawnebuild() called " + \ - "with returnpid parameter enabled. This usage will " + \ + warnings.warn("portage.spawnebuild() called " + "with returnpid parameter enabled. This usage will " "not be supported in the future.", DeprecationWarning, stacklevel=2) @@ -1534,7 +1687,52 @@ def _check_build_log(mysettings, out=None): configure_opts_warn = [] configure_opts_warn_re = re.compile( - r'^configure: WARNING: [Uu]nrecognized options: ') + r'^configure: WARNING: [Uu]nrecognized options: (.*)') + + qa_configure_opts = "" + try: + with io.open(_unicode_encode(os.path.join( + mysettings["PORTAGE_BUILDDIR"], + "build-info", "QA_CONFIGURE_OPTIONS"), + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') as qa_configure_opts_f: + qa_configure_opts = qa_configure_opts_f.read() + except IOError as e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + raise + + qa_configure_opts = qa_configure_opts.split() + if qa_configure_opts: + if len(qa_configure_opts) > 1: + qa_configure_opts = "|".join("(%s)" % x for x in qa_configure_opts) + qa_configure_opts = "^(%s)$" % qa_configure_opts + else: + qa_configure_opts = "^%s$" % qa_configure_opts[0] + qa_configure_opts = re.compile(qa_configure_opts) + + qa_am_maintainer_mode = [] + try: + with io.open(_unicode_encode(os.path.join( + mysettings["PORTAGE_BUILDDIR"], + "build-info", "QA_AM_MAINTAINER_MODE"), + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') as qa_am_maintainer_mode_f: + qa_am_maintainer_mode = [x for x in + qa_am_maintainer_mode_f.read().splitlines() if x] + except IOError as e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + raise + + if qa_am_maintainer_mode: + if len(qa_am_maintainer_mode) > 1: + qa_am_maintainer_mode = \ + "|".join("(%s)" % x for x in qa_am_maintainer_mode) + qa_am_maintainer_mode = "^(%s)$" % qa_am_maintainer_mode + else: + qa_am_maintainer_mode = "^%s$" % qa_am_maintainer_mode[0] + qa_am_maintainer_mode = re.compile(qa_am_maintainer_mode) # Exclude output from dev-libs/yaz-3.0.47 which looks like this: # @@ -1556,7 +1754,9 @@ def _check_build_log(mysettings, out=None): for line in f: line = _unicode_decode(line) if am_maintainer_mode_re.search(line) is not None and \ - am_maintainer_mode_exclude_re.search(line) is None: + am_maintainer_mode_exclude_re.search(line) is None and \ + (not qa_am_maintainer_mode or + qa_am_maintainer_mode.search(line) is None): am_maintainer_mode.append(line.rstrip("\n")) if bash_command_not_found_re.match(line) is not None and \ @@ -1566,8 +1766,11 @@ def _check_build_log(mysettings, out=None): if helper_missing_file_re.match(line) is not None: helper_missing_file.append(line.rstrip("\n")) - if configure_opts_warn_re.match(line) is not None: - configure_opts_warn.append(line.rstrip("\n")) + m = configure_opts_warn_re.match(line) + if m is not None: + for x in m.group(1).split(", "): + if not qa_configure_opts or qa_configure_opts.match(x) is None: + configure_opts_warn.append(x) if make_jobserver_re.match(line) is not None: make_jobserver.append(line.rstrip("\n")) @@ -1616,7 +1819,7 @@ def _check_build_log(mysettings, out=None): if configure_opts_warn: msg = [_("QA Notice: Unrecognized configure options:")] msg.append("") - msg.extend("\t" + line for line in configure_opts_warn) + msg.extend("\t%s" % x for x in configure_opts_warn) _eqawarn(msg) if make_jobserver: @@ -1629,7 +1832,7 @@ def _check_build_log(mysettings, out=None): if f_real is not None: f_real.close() -def _post_src_install_chost_fix(settings): +def _post_src_install_write_metadata(settings): """ It's possible that the ebuild has changed the CHOST variable, so revert it to the initial @@ -1637,10 +1840,16 @@ def _post_src_install_chost_fix(settings): due to local environment settings like in bug #386829. """ + eapi_attrs = _get_eapi_attrs(settings.configdict['pkg']['EAPI']) + build_info_dir = os.path.join(settings['PORTAGE_BUILDDIR'], 'build-info') - for k in ('IUSE',): - v = settings.get(k) + metadata_keys = ['IUSE'] + if eapi_attrs.iuse_effective: + metadata_keys.append('IUSE_EFFECTIVE') + + for k in metadata_keys: + v = settings.configdict['pkg'].get(k) if v is not None: write_atomic(os.path.join(build_info_dir, k), v + '\n') @@ -1652,9 +1861,59 @@ def _post_src_install_chost_fix(settings): if v is not None: write_atomic(os.path.join(build_info_dir, k), v + '\n') -_vdb_use_conditional_keys = ('DEPEND', 'LICENSE', 'PDEPEND', - 'PROPERTIES', 'PROVIDE', 'RDEPEND', 'RESTRICT',) -_vdb_use_conditional_atoms = frozenset(['DEPEND', 'PDEPEND', 'RDEPEND']) + with io.open(_unicode_encode(os.path.join(build_info_dir, + 'BUILD_TIME'), encoding=_encodings['fs'], errors='strict'), + mode='w', encoding=_encodings['repo.content'], + errors='strict') as f: + f.write("%.0f\n" % (time.time(),)) + + use = frozenset(settings['PORTAGE_USE'].split()) + for k in _vdb_use_conditional_keys: + v = settings.configdict['pkg'].get(k) + filename = os.path.join(build_info_dir, k) + if v is None: + try: + os.unlink(filename) + except OSError: + pass + continue + + if k.endswith('DEPEND'): + if eapi_attrs.slot_operator: + continue + token_class = Atom + else: + token_class = None + + v = use_reduce(v, uselist=use, token_class=token_class) + v = paren_enclose(v) + if not v: + try: + os.unlink(filename) + except OSError: + pass + continue + with io.open(_unicode_encode(os.path.join(build_info_dir, + k), encoding=_encodings['fs'], errors='strict'), + mode='w', encoding=_encodings['repo.content'], + errors='strict') as f: + f.write('%s\n' % v) + + if eapi_attrs.slot_operator: + deps = evaluate_slot_operator_equal_deps(settings, use, QueryCommand.get_db()) + for k, v in deps.items(): + filename = os.path.join(build_info_dir, k) + if not v: + try: + os.unlink(filename) + except OSError: + pass + continue + with io.open(_unicode_encode(os.path.join(build_info_dir, + k), encoding=_encodings['fs'], errors='strict'), + mode='w', encoding=_encodings['repo.content'], + errors='strict') as f: + f.write('%s\n' % v) def _preinst_bsdflags(mysettings): if bsd_chflags: @@ -1696,6 +1955,33 @@ def _post_src_install_uid_fix(mysettings, out): destdir = mysettings["D"] ed_len = len(mysettings["ED"]) unicode_errors = [] + desktop_file_validate = \ + portage.process.find_binary("desktop-file-validate") is not None + xdg_dirs = mysettings.get('XDG_DATA_DIRS', '/usr/share').split(':') + xdg_dirs = tuple(os.path.join(i, "applications") + os.sep + for i in xdg_dirs if i) + + qa_desktop_file = "" + try: + with io.open(_unicode_encode(os.path.join( + mysettings["PORTAGE_BUILDDIR"], + "build-info", "QA_DESKTOP_FILE"), + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') as f: + qa_desktop_file = f.read() + except IOError as e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + raise + + qa_desktop_file = qa_desktop_file.split() + if qa_desktop_file: + if len(qa_desktop_file) > 1: + qa_desktop_file = "|".join("(%s)" % x for x in qa_desktop_file) + qa_desktop_file = "^(%s)$" % qa_desktop_file + else: + qa_desktop_file = "^%s$" % qa_desktop_file[0] + qa_desktop_file = re.compile(qa_desktop_file) while True: @@ -1704,6 +1990,7 @@ def _post_src_install_uid_fix(mysettings, out): counted_inodes = set() fixlafiles_announced = False fixlafiles = "fixlafiles" in mysettings.features + desktopfile_errors = [] for parent, dirs, files in os.walk(destdir): try: @@ -1743,6 +2030,16 @@ def _post_src_install_uid_fix(mysettings, out): else: fpath = os.path.join(parent, fname) + fpath_relative = fpath[ed_len - 1:] + if desktop_file_validate and fname.endswith(".desktop") and \ + os.path.isfile(fpath) and \ + fpath_relative.startswith(xdg_dirs) and \ + not (qa_desktop_file and qa_desktop_file.match(fpath_relative.strip(os.sep)) is not None): + + desktop_validate = validate_desktop_entry(fpath) + if desktop_validate: + desktopfile_errors.extend(desktop_validate) + if fixlafiles and \ fname.endswith(".la") and os.path.isfile(fpath): f = open(_unicode_encode(fpath, @@ -1809,6 +2106,11 @@ def _post_src_install_uid_fix(mysettings, out): if not unicode_error: break + if desktopfile_errors: + for l in _merge_desktopfile_error(desktopfile_errors): + l = l.replace(mysettings["ED"], '/') + eqawarn(l, phase='install', key=mysettings.mycpv, out=out) + if unicode_errors: for l in _merge_unicode_error(unicode_errors): eqawarn(l, phase='install', key=mysettings.mycpv, out=out) @@ -1820,47 +2122,9 @@ def _post_src_install_uid_fix(mysettings, out): 'SIZE'), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['repo.content'], errors='strict') - f.write(_unicode_decode(str(size) + '\n')) + f.write('%d\n' % size) f.close() - f = io.open(_unicode_encode(os.path.join(build_info_dir, - 'BUILD_TIME'), encoding=_encodings['fs'], errors='strict'), - mode='w', encoding=_encodings['repo.content'], - errors='strict') - f.write(_unicode_decode("%.0f\n" % (time.time(),))) - f.close() - - use = frozenset(mysettings['PORTAGE_USE'].split()) - for k in _vdb_use_conditional_keys: - v = mysettings.configdict['pkg'].get(k) - filename = os.path.join(build_info_dir, k) - if v is None: - try: - os.unlink(filename) - except OSError: - pass - continue - - if k.endswith('DEPEND'): - token_class = Atom - else: - token_class = None - - v = use_reduce(v, uselist=use, token_class=token_class) - v = paren_enclose(v) - if not v: - try: - os.unlink(filename) - except OSError: - pass - continue - f = io.open(_unicode_encode(os.path.join(build_info_dir, - k), encoding=_encodings['fs'], errors='strict'), - mode='w', encoding=_encodings['repo.content'], - errors='strict') - f.write(_unicode_decode(v + '\n')) - f.close() - _reapply_bsdflags_to_image(mysettings) def _reapply_bsdflags_to_image(mysettings): @@ -2009,6 +2273,20 @@ def _post_src_install_soname_symlinks(mysettings, out): for line in qa_msg: eqawarn(line, key=mysettings.mycpv, out=out) +def _merge_desktopfile_error(errors): + lines = [] + + msg = _("QA Notice: This package installs one or more .desktop files " + "that do not pass validation.") + lines.extend(wrap(msg, 72)) + + lines.append("") + errors.sort() + lines.extend("\t" + x for x in errors) + lines.append("") + + return lines + def _merge_unicode_error(errors): lines = [] @@ -2065,11 +2343,6 @@ def _handle_self_update(settings, vardb): if settings["ROOT"] == "/" and \ portage.dep.match_from_list( portage.const.PORTAGE_PACKAGE_ATOM, [cpv]): - inherited = frozenset(settings.get('INHERITED', '').split()) - if not vardb.cpv_exists(cpv) or \ - '9999' in cpv or \ - 'git' in inherited or \ - 'git-2' in inherited: - _prepare_self_update(settings) - return True + _prepare_self_update(settings) + return True return False diff --git a/portage_with_autodep/pym/portage/package/ebuild/doebuild.pyo b/portage_with_autodep/pym/portage/package/ebuild/doebuild.pyo Binary files differindex a6ebb1d..846d99a 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/doebuild.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/doebuild.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/fetch.py b/portage_with_autodep/pym/portage/package/ebuild/fetch.py index b795b28..5316f03 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/fetch.py +++ b/portage_with_autodep/pym/portage/package/ebuild/fetch.py @@ -1,4 +1,4 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -14,6 +14,10 @@ import stat import sys import tempfile +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse import portage portage.proxy.lazyimport.lazyimport(globals(), @@ -25,7 +29,8 @@ portage.proxy.lazyimport.lazyimport(globals(), from portage import OrderedDict, os, selinux, shutil, _encodings, \ _shell_quote, _unicode_encode -from portage.checksum import hashfunc_map, perform_md5, verify_all +from portage.checksum import (hashfunc_map, perform_md5, verify_all, + _filter_unaccelarated_hashes, _hash_filter, _apply_hash_filter) from portage.const import BASH_BINARY, CUSTOM_MIRRORS_FILE, \ GLOBAL_CONFIG_PATH from portage.data import portage_gid, portage_uid, secpass, userpriv_groups @@ -63,9 +68,9 @@ def _spawn_fetch(settings, args, **kwargs): if "fd_pipes" not in kwargs: kwargs["fd_pipes"] = { - 0 : sys.stdin.fileno(), - 1 : sys.stdout.fileno(), - 2 : sys.stdout.fileno(), + 0 : portage._get_stdin().fileno(), + 1 : sys.__stdout__.fileno(), + 2 : sys.__stdout__.fileno(), } if "userfetch" in settings.features and \ @@ -184,7 +189,7 @@ def _check_digests(filename, digests, show_errors=1): return False return True -def _check_distfile(filename, digests, eout, show_errors=1): +def _check_distfile(filename, digests, eout, show_errors=1, hash_filter=None): """ @return a tuple of (match, stat_obj) where match is True if filename matches all given digests (if any) and stat_obj is a stat result, or @@ -210,6 +215,9 @@ def _check_distfile(filename, digests, eout, show_errors=1): # Zero-byte distfiles are always invalid. return (False, st) else: + digests = _filter_unaccelarated_hashes(digests) + if hash_filter is not None: + digests = _apply_hash_filter(digests, hash_filter) if _check_digests(filename, digests, show_errors=show_errors): eout.ebegin("%s %s ;-)" % (os.path.basename(filename), " ".join(sorted(digests)))) @@ -339,7 +347,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, _("!!! For fetching to a read-only filesystem, " "locking should be turned off.\n")), noiselevel=-1) writemsg(_("!!! This can be done by adding -distlocks to " - "FEATURES in /etc/make.conf\n"), noiselevel=-1) + "FEATURES in /etc/portage/make.conf\n"), noiselevel=-1) # use_locks = 0 # local mirrors are always added @@ -353,6 +361,9 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, if try_mirrors: mymirrors += [x.rstrip("/") for x in mysettings["GENTOO_MIRRORS"].split() if x] + hash_filter = _hash_filter(mysettings.get("PORTAGE_CHECKSUM_FILTER", "")) + if hash_filter.transparent: + hash_filter = None skip_manifest = mysettings.get("EBUILD_SKIP_MANIFEST") == "1" if skip_manifest: allow_missing_digests = True @@ -395,12 +406,16 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, for myfile, uri_set in myuris.items(): for myuri in uri_set: file_uri_tuples.append((myfile, myuri)) + if not uri_set: + file_uri_tuples.append((myfile, None)) else: for myuri in myuris: - file_uri_tuples.append((os.path.basename(myuri), myuri)) + if urlparse(myuri).scheme: + file_uri_tuples.append((os.path.basename(myuri), myuri)) + else: + file_uri_tuples.append((os.path.basename(myuri), None)) filedict = OrderedDict() - primaryuri_indexes={} primaryuri_dict = {} thirdpartymirror_uris = {} for myfile, myuri in file_uri_tuples: @@ -408,6 +423,8 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, filedict[myfile]=[] for y in range(0,len(locations)): filedict[myfile].append(locations[y]+"/distfiles/"+myfile) + if myuri is None: + continue if myuri[:9]=="mirror://": eidx = myuri.find("/", 9) if eidx != -1: @@ -422,10 +439,9 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, # now try the official mirrors if mirrorname in thirdpartymirrors: - random.shuffle(thirdpartymirrors[mirrorname]) - uris = [locmirr.rstrip("/") + "/" + path \ for locmirr in thirdpartymirrors[mirrorname]] + random.shuffle(uris) filedict[myfile].extend(uris) thirdpartymirror_uris.setdefault(myfile, []).extend(uris) @@ -438,26 +454,30 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, if restrict_fetch or force_mirror: # Only fetch from specific mirrors is allowed. continue - if "primaryuri" in restrict: - # Use the source site first. - if myfile in primaryuri_indexes: - primaryuri_indexes[myfile] += 1 - else: - primaryuri_indexes[myfile] = 0 - filedict[myfile].insert(primaryuri_indexes[myfile], myuri) - else: - filedict[myfile].append(myuri) primaryuris = primaryuri_dict.get(myfile) if primaryuris is None: primaryuris = [] primaryuri_dict[myfile] = primaryuris primaryuris.append(myuri) + # Order primaryuri_dict values to match that in SRC_URI. + for uris in primaryuri_dict.values(): + uris.reverse() + # Prefer thirdpartymirrors over normal mirrors in cases when # the file does not yet exist on the normal mirrors. for myfile, uris in thirdpartymirror_uris.items(): primaryuri_dict.setdefault(myfile, []).extend(uris) + # Now merge primaryuri values into filedict (includes mirrors + # explicitly referenced in SRC_URI). + if "primaryuri" in restrict: + for myfile, uris in filedict.items(): + filedict[myfile] = primaryuri_dict.get(myfile, []) + uris + else: + for myfile in filedict: + filedict[myfile] += primaryuri_dict.get(myfile, []) + can_fetch=True if listonly: @@ -635,7 +655,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, eout = EOutput() eout.quiet = mysettings.get("PORTAGE_QUIET") == "1" match, mystat = _check_distfile( - myfile_path, pruned_digests, eout) + myfile_path, pruned_digests, eout, hash_filter=hash_filter) if match: # Skip permission adjustment for symlinks, since we don't # want to modify anything outside of the primary DISTDIR, @@ -707,7 +727,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, for x in ro_distdirs: filename = os.path.join(x, myfile) match, mystat = _check_distfile( - filename, pruned_digests, eout) + filename, pruned_digests, eout, hash_filter=hash_filter) if match: readonly_file = filename break @@ -732,7 +752,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, "remaining space.\n"), noiselevel=-1) if userfetch: writemsg(_("!!! You may set FEATURES=\"-userfetch\"" - " in /etc/make.conf in order to fetch with\n" + " in /etc/portage/make.conf in order to fetch with\n" "!!! superuser privileges.\n"), noiselevel=-1) if fsmirrors and not os.path.exists(myfile_path) and has_space: @@ -793,8 +813,10 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, eout.eend(0) continue else: - verified_ok, reason = verify_all( - myfile_path, mydigests[myfile]) + digests = _filter_unaccelarated_hashes(mydigests[myfile]) + if hash_filter is not None: + digests = _apply_hash_filter(digests, hash_filter) + verified_ok, reason = verify_all(myfile_path, digests) if not verified_ok: writemsg(_("!!! Previously fetched" " file: '%s'\n") % myfile, noiselevel=-1) @@ -816,7 +838,6 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, eout = EOutput() eout.quiet = \ mysettings.get("PORTAGE_QUIET", None) == "1" - digests = mydigests.get(myfile) if digests: digests = list(digests) digests.sort() @@ -844,8 +865,8 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, protocol = loc[0:loc.find("://")] global_config_path = GLOBAL_CONFIG_PATH - if mysettings['EPREFIX']: - global_config_path = os.path.join(mysettings['EPREFIX'], + if portage.const.EPREFIX: + global_config_path = os.path.join(portage.const.EPREFIX, GLOBAL_CONFIG_PATH.lstrip(os.sep)) missing_file_param = False @@ -954,11 +975,16 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, writemsg_stdout(_(">>> Downloading '%s'\n") % \ _hide_url_passwd(loc)) variables = { - "DISTDIR": mysettings["DISTDIR"], "URI": loc, "FILE": myfile } + for k in ("DISTDIR", "PORTAGE_SSH_OPTS"): + try: + variables[k] = mysettings[k] + except KeyError: + pass + myfetch = shlex_split(locfetch) myfetch = [varexpand(x, mydict=variables) for x in myfetch] myret = -1 @@ -1051,7 +1077,10 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, # file NOW, for those users who don't have a stable/continuous # net connection. This way we have a chance to try to download # from another mirror... - verified_ok,reason = verify_all(mysettings["DISTDIR"]+"/"+myfile, mydigests[myfile]) + digests = _filter_unaccelarated_hashes(mydigests[myfile]) + if hash_filter is not None: + digests = _apply_hash_filter(digests, hash_filter) + verified_ok, reason = verify_all(myfile_path, digests) if not verified_ok: writemsg(_("!!! Fetched file: %s VERIFY FAILED!\n") % myfile, noiselevel=-1) @@ -1085,7 +1114,6 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, else: eout = EOutput() eout.quiet = mysettings.get("PORTAGE_QUIET", None) == "1" - digests = mydigests.get(myfile) if digests: eout.ebegin("%s %s ;-)" % \ (myfile, " ".join(sorted(digests)))) diff --git a/portage_with_autodep/pym/portage/package/ebuild/fetch.pyo b/portage_with_autodep/pym/portage/package/ebuild/fetch.pyo Binary files differindex 3bd81df..6e2c242 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/fetch.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/fetch.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.py b/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.py index 8a88c2f..70a6bf2 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.py +++ b/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ['getmaskingreason'] @@ -6,13 +6,12 @@ __all__ = ['getmaskingreason'] import portage from portage import os from portage.const import USER_CONFIG_PATH -from portage.dep import Atom, match_from_list, _slot_separator, _repo_separator +from portage.dep import Atom, match_from_list from portage.exception import InvalidAtom from portage.localization import _ from portage.repository.config import _gen_valid_repo from portage.util import grablines, normalize_path -from portage.versions import catpkgsplit -from _emerge.Package import Package +from portage.versions import catpkgsplit, _pkg_str def getmaskingreason(mycpv, metadata=None, settings=None, portdb=None, return_location=False, myrepo=None): @@ -60,23 +59,20 @@ def getmaskingreason(mycpv, metadata=None, settings=None, # Sometimes we can't access SLOT or repository due to corruption. pkg = mycpv - if metadata is not None: - pkg = "".join((mycpv, _slot_separator, metadata["SLOT"])) - # At this point myrepo should be None, a valid name, or - # Package.UNKNOWN_REPO which we ignore. - if myrepo is not None and myrepo != Package.UNKNOWN_REPO: - pkg = "".join((pkg, _repo_separator, myrepo)) + try: + pkg.slot + except AttributeError: + pkg = _pkg_str(mycpv, metadata=metadata, repo=myrepo) + cpv_slot_list = [pkg] - mycp=mysplit[0]+"/"+mysplit[1] + mycp = pkg.cp - # XXX- This is a temporary duplicate of code from the config constructor. - locations = [os.path.join(settings["PORTDIR"], "profiles")] + locations = [] + if pkg.repo in settings.repositories: + for repo in settings.repositories[pkg.repo].masters + (settings.repositories[pkg.repo],): + locations.append(os.path.join(repo.location, "profiles")) locations.extend(settings.profiles) - for ov in settings["PORTDIR_OVERLAY"].split(): - profdir = os.path.join(normalize_path(ov), "profiles") - if os.path.isdir(profdir): - locations.append(profdir) locations.append(os.path.join(settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH)) locations.reverse() diff --git a/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.pyo b/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.pyo Binary files differindex 1614244..6c0073d 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.py b/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.py index 9bf605d..c8954aa 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.py +++ b/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.py @@ -1,12 +1,15 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ['getmaskingstatus'] import sys import portage from portage import eapi_is_supported, _eapi_is_deprecated +from portage.exception import InvalidDependString from portage.localization import _ from portage.package.ebuild.config import config from portage.versions import catpkgsplit, _pkg_str @@ -48,7 +51,7 @@ def _getmaskingstatus(mycpv, settings, portdb, myrepo=None): # emerge passed in a Package instance pkg = mycpv mycpv = pkg.cpv - metadata = pkg.metadata + metadata = pkg._metadata installed = pkg.installed if metadata is None: @@ -65,10 +68,11 @@ def _getmaskingstatus(mycpv, settings, portdb, myrepo=None): else: metadata["USE"] = "" - if not hasattr(mycpv, 'slot'): + try: + mycpv.slot + except AttributeError: try: - mycpv = _pkg_str(mycpv, slot=metadata['SLOT'], - repo=metadata.get('repository')) + mycpv = _pkg_str(mycpv, metadata=metadata, settings=settings) except portage.exception.InvalidData: raise ValueError(_("invalid CPV: %s") % mycpv) @@ -83,6 +87,7 @@ def _getmaskingstatus(mycpv, settings, portdb, myrepo=None): mygroups = settings._getKeywords(mycpv, metadata) licenses = metadata["LICENSE"] properties = metadata["PROPERTIES"] + restrict = metadata["RESTRICT"] if not eapi_is_supported(eapi): return [_MaskReason("EAPI", "EAPI %s" % eapi)] elif _eapi_is_deprecated(eapi) and not installed: @@ -122,6 +127,13 @@ def _getmaskingstatus(mycpv, settings, portdb, myrepo=None): if gp=="*": kmask=None break + elif gp == "~*": + for x in pgroups: + if x[:1] == "~": + kmask = None + break + if kmask is None: + break elif gp=="-"+myarch and myarch in pgroups: kmask="-"+myarch break @@ -161,6 +173,15 @@ def _getmaskingstatus(mycpv, settings, portdb, myrepo=None): except portage.exception.InvalidDependString as e: rValue.append(_MaskReason("invalid", "PROPERTIES: "+str(e))) + try: + missing_restricts = settings._getMissingRestrict(mycpv, metadata) + if missing_restricts: + msg = list(missing_restricts) + msg.append("in RESTRICT") + rValue.append(_MaskReason("RESTRICT", " ".join(msg))) + except InvalidDependString as e: + rValue.append(_MaskReason("invalid", "RESTRICT: %s" % (e,))) + # Only show KEYWORDS masks for installed packages # if they're not masked for any other reason. if kmask and (not installed or not rValue): diff --git a/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.pyo b/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.pyo Binary files differindex 9cf1d9d..27bb3c7 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.py b/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.py index b8fbdc5..6782160 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.py +++ b/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.py @@ -1,6 +1,8 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ['prepare_build_dirs'] import errno @@ -338,12 +340,12 @@ def _prepare_workdir(mysettings): try: _ensure_log_subdirs(logdir, log_subdir) except PortageException as e: - writemsg(_unicode_decode("!!! %s\n") % (e,), noiselevel=-1) + writemsg("!!! %s\n" % (e,), noiselevel=-1) if os.access(log_subdir, os.W_OK): logdir_subdir_ok = True else: - writemsg(_unicode_decode("!!! %s: %s\n") % + writemsg("!!! %s: %s\n" % (_("Permission Denied"), log_subdir), noiselevel=-1) tmpdir_log_path = os.path.join( diff --git a/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.pyo b/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.pyo Binary files differindex 2dcfaea..fd6d446 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.pyo diff --git a/portage_with_autodep/pym/portage/process.py b/portage_with_autodep/pym/portage/process.py index d7d1037..7334c00 100644 --- a/portage_with_autodep/pym/portage/process.py +++ b/portage_with_autodep/pym/portage/process.py @@ -1,25 +1,30 @@ # portage.py -- core Portage functionality -# Copyright 1998-2012 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import atexit import errno +import fcntl import platform import signal +import socket +import struct import sys import traceback +import os as _os from portage import os from portage import _encodings from portage import _unicode_encode import portage portage.proxy.lazyimport.lazyimport(globals(), - 'portage.util:dump_traceback', + 'portage.util:dump_traceback,writemsg', ) from portage.const import BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY, AUTODEP_LIBRARY from portage.exception import CommandNotFound +from portage.util._ctypes import find_library, LoadLibrary, ctypes try: import resource @@ -30,10 +35,32 @@ except ImportError: if sys.hexversion >= 0x3000000: basestring = str -if os.path.isdir("/proc/%i/fd" % os.getpid()): +# Support PEP 446 for Python >=3.4 +try: + _set_inheritable = _os.set_inheritable +except AttributeError: + _set_inheritable = None + +try: + _FD_CLOEXEC = fcntl.FD_CLOEXEC +except AttributeError: + _FD_CLOEXEC = None + +# Prefer /proc/self/fd if available (/dev/fd +# doesn't work on solaris, see bug #474536). +for _fd_dir in ("/proc/self/fd", "/dev/fd"): + if os.path.isdir(_fd_dir): + break + else: + _fd_dir = None + +# /dev/fd does not work on FreeBSD, see bug #478446 +if platform.system() in ('FreeBSD',) and _fd_dir == '/dev/fd': + _fd_dir = None + +if _fd_dir is not None: def get_open_fds(): - return (int(fd) for fd in os.listdir("/proc/%i/fd" % os.getpid()) \ - if fd.isdigit()) + return (int(fd) for fd in os.listdir(_fd_dir) if fd.isdigit()) if platform.python_implementation() == 'PyPy': # EAGAIN observed with PyPy 1.8. @@ -46,6 +73,13 @@ if os.path.isdir("/proc/%i/fd" % os.getpid()): raise return range(max_fd_limit) +elif os.path.isdir("/proc/%s/fd" % os.getpid()): + # In order for this function to work in forked subprocesses, + # os.getpid() must be called from inside the function. + def get_open_fds(): + return (int(fd) for fd in os.listdir("/proc/%s/fd" % os.getpid()) + if fd.isdigit()) + else: def get_open_fds(): return range(max_fd_limit) @@ -62,7 +96,7 @@ fakeroot_capable = (os.path.isfile(FAKEROOT_BINARY) and def spawn_bash(mycommand, debug=False, opt_name=None, **keywords): """ Spawns a bash shell running a specific commands - + @param mycommand: The command for bash to run @type mycommand: String @param debug: Turn bash debugging on (set -x) @@ -91,7 +125,7 @@ def spawn_autodep(mycommand, opt_name=None, **keywords): # Core part: tell the loader to preload logging library keywords["env"]["LD_PRELOAD"]=AUTODEP_LIBRARY - return spawn_bash(mycommand, opt_name=opt_name, **keywords) + return spawn_bash(mycommand, opt_name=opt_name, **keywords) def spawn_sandbox(mycommand, opt_name=None, **keywords): if not sandbox_capable: @@ -154,33 +188,31 @@ def run_exitfuncs(): atexit.register(run_exitfuncs) -# We need to make sure that any processes spawned are killed off when -# we exit. spawn() takes care of adding and removing pids to this list -# as it creates and cleans up processes. -spawned_pids = [] -def cleanup(): - while spawned_pids: - pid = spawned_pids.pop() +# It used to be necessary for API consumers to remove pids from spawned_pids, +# since otherwise it would accumulate a pids endlessly. Now, spawned_pids is +# just an empty dummy list, so for backward compatibility, ignore ValueError +# for removal on non-existent items. +class _dummy_list(list): + def remove(self, item): + # TODO: Trigger a DeprecationWarning here, after stable portage + # has dummy spawned_pids. try: - # With waitpid and WNOHANG, only check the - # first element of the tuple since the second - # element may vary (bug #337465). - if os.waitpid(pid, os.WNOHANG)[0] == 0: - os.kill(pid, signal.SIGTERM) - os.waitpid(pid, 0) - except OSError: - # This pid has been cleaned up outside - # of spawn(). + list.remove(self, item) + except ValueError: pass -atexit_register(cleanup) +spawned_pids = _dummy_list() + +def cleanup(): + pass def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, uid=None, gid=None, groups=None, umask=None, logfile=None, - path_lookup=True, pre_exec=None): + path_lookup=True, pre_exec=None, close_fds=True, unshare_net=False, + unshare_ipc=False, cgroup=None): """ Spawns a given command. - + @param mycommand: the command to execute @type mycommand: String or List (Popen style list) @param env: A dict of Key=Value pairs for env variables @@ -188,6 +220,7 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, @param opt_name: an optional name for the spawn'd process (defaults to the binary name) @type opt_name: String @param fd_pipes: A dict of mapping for pipes, { '0': stdin, '1': stdout } for example + (default is {0:stdin, 1:stdout, 2:stderr}) @type fd_pipes: Dictionary @param returnpid: Return the Process IDs for a successful spawn. NOTE: This requires the caller clean up all the PIDs, otherwise spawn will clean them. @@ -206,10 +239,19 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, @type path_lookup: Boolean @param pre_exec: A function to be called with no arguments just prior to the exec call. @type pre_exec: callable - + @param close_fds: If True, then close all file descriptors except those + referenced by fd_pipes (default is True). + @type close_fds: Boolean + @param unshare_net: If True, networking will be unshared from the spawned process + @type unshare_net: Boolean + @param unshare_ipc: If True, IPC will be unshared from the spawned process + @type unshare_ipc: Boolean + @param cgroup: CGroup path to bind the process to + @type cgroup: String + logfile requires stdout and stderr to be assigned to this process (ie not pointed somewhere else.) - + """ # mycommand is either a str or a list @@ -239,9 +281,9 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, # default to propagating our stdin, stdout and stderr. if fd_pipes is None: fd_pipes = { - 0:sys.stdin.fileno(), - 1:sys.stdout.fileno(), - 2:sys.stderr.fileno(), + 0:portage._get_stdin().fileno(), + 1:sys.__stdout__.fileno(), + 2:sys.__stderr__.fileno(), } # mypids will hold the pids of all processes created. @@ -269,21 +311,40 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, fd_pipes[1] = pw fd_pipes[2] = pw - pid = os.fork() + # This caches the libc library lookup in the current + # process, so that it's only done once rather than + # for each child process. + if unshare_net or unshare_ipc: + find_library("c") - if pid == 0: - try: - _exec(binary, mycommand, opt_name, fd_pipes, - env, gid, groups, uid, umask, pre_exec) - except SystemExit: - raise - except Exception as e: - # We need to catch _any_ exception so that it doesn't - # propagate out of this function and cause exiting - # with anything other than os._exit() - sys.stderr.write("%s:\n %s\n" % (e, " ".join(mycommand))) - traceback.print_exc() - sys.stderr.flush() + parent_pid = os.getpid() + pid = None + try: + pid = os.fork() + + if pid == 0: + try: + _exec(binary, mycommand, opt_name, fd_pipes, + env, gid, groups, uid, umask, pre_exec, close_fds, + unshare_net, unshare_ipc, cgroup) + except SystemExit: + raise + except Exception as e: + # We need to catch _any_ exception so that it doesn't + # propagate out of this function and cause exiting + # with anything other than os._exit() + writemsg("%s:\n %s\n" % (e, " ".join(mycommand)), + noiselevel=-1) + traceback.print_exc() + sys.stderr.flush() + + finally: + if pid == 0 or (pid is None and os.getpid() != parent_pid): + # Call os._exit() from a finally block in order + # to suppress any finally blocks from earlier + # in the call stack (see bug #345289). This + # finally block has to be setup before the fork + # in order to avoid a race condition. os._exit(1) if not isinstance(pid, int): @@ -291,7 +352,6 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, # Add the pid to our local and the global pid lists. mypids.append(pid) - spawned_pids.append(pid) # If we started a tee process the write side of the pipe is no # longer needed, so close it. @@ -314,10 +374,6 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, # and wait for it. retval = os.waitpid(pid, 0)[1] - # When it's done, we can remove it from the - # global pid list as well. - spawned_pids.remove(pid) - if retval: # If it failed, kill off anything else that # isn't dead yet. @@ -328,7 +384,6 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, if os.waitpid(pid, os.WNOHANG)[0] == 0: os.kill(pid, signal.SIGTERM) os.waitpid(pid, 0) - spawned_pids.remove(pid) # If it got a signal, return the signal that was sent. if (retval & 0xff): @@ -341,11 +396,11 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, return 0 def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, - pre_exec): + pre_exec, close_fds, unshare_net, unshare_ipc, cgroup): """ Execute a given binary with options - + @param binary: Name of program to execute @type binary: String @param mycommand: Options for program @@ -366,10 +421,16 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, @type umask: Integer @param pre_exec: A function to be called with no arguments just prior to the exec call. @type pre_exec: callable + @param unshare_net: If True, networking will be unshared from the spawned process + @type unshare_net: Boolean + @param unshare_ipc: If True, IPC will be unshared from the spawned process + @type unshare_ipc: Boolean + @param cgroup: CGroup path to bind the process to + @type cgroup: String @rtype: None @return: Never returns (calls os.execve) """ - + # If the process we're creating hasn't been given a name # assign it the name of the executable. if not opt_name: @@ -384,6 +445,10 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, myargs = [opt_name] myargs.extend(mycommand[1:]) + # Avoid a potential UnicodeEncodeError from os.execve(). + myargs = [_unicode_encode(x, encoding=_encodings['fs'], + errors='strict') for x in myargs] + # Use default signal handlers in order to avoid problems # killing subprocesses as reported in bug #353239. signal.signal(signal.SIGINT, signal.SIG_DFL) @@ -396,15 +461,63 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, # the parent process (see bug #289486). signal.signal(signal.SIGQUIT, signal.SIG_DFL) - _setup_pipes(fd_pipes) + _setup_pipes(fd_pipes, close_fds=close_fds, inheritable=True) + + # Add to cgroup + # it's better to do it from the child since we can guarantee + # it is done before we start forking children + if cgroup: + with open(os.path.join(cgroup, 'cgroup.procs'), 'a') as f: + f.write('%d\n' % os.getpid()) + + # Unshare (while still uid==0) + if unshare_net or unshare_ipc: + filename = find_library("c") + if filename is not None: + libc = LoadLibrary(filename) + if libc is not None: + CLONE_NEWIPC = 0x08000000 + CLONE_NEWNET = 0x40000000 + + flags = 0 + if unshare_net: + flags |= CLONE_NEWNET + if unshare_ipc: + flags |= CLONE_NEWIPC + + try: + if libc.unshare(flags) != 0: + writemsg("Unable to unshare: %s\n" % ( + errno.errorcode.get(ctypes.get_errno(), '?')), + noiselevel=-1) + else: + if unshare_net: + # 'up' the loopback + IFF_UP = 0x1 + ifreq = struct.pack('16sh', b'lo', IFF_UP) + SIOCSIFFLAGS = 0x8914 + + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, 0) + try: + fcntl.ioctl(sock, SIOCSIFFLAGS, ifreq) + except IOError as e: + writemsg("Unable to enable loopback interface: %s\n" % ( + errno.errorcode.get(e.errno, '?')), + noiselevel=-1) + sock.close() + except AttributeError: + # unshare() not supported by libc + pass # Set requested process permissions. if gid: - os.setgid(gid) + # Cast proxies to int, in case it matters. + os.setgid(int(gid)) if groups: os.setgroups(groups) if uid: - os.setuid(uid) + # Cast proxies to int, in case it matters. + os.setuid(int(uid)) if umask: os.umask(umask) if pre_exec: @@ -413,9 +526,16 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, # And switch to the new process. os.execve(binary, myargs, env) -def _setup_pipes(fd_pipes, close_fds=True): +def _setup_pipes(fd_pipes, close_fds=True, inheritable=None): """Setup pipes for a forked process. + Even when close_fds is False, file descriptors referenced as + values in fd_pipes are automatically closed if they do not also + occur as keys in fd_pipes. It is assumed that the caller will + explicitly add them to the fd_pipes keys if they are intended + to remain open. This allows for convenient elimination of + unnecessary duplicate file descriptors. + WARNING: When not followed by exec, the close_fds behavior can trigger interference from destructors that close file descriptors. This interference happens when the garbage @@ -426,22 +546,92 @@ def _setup_pipes(fd_pipes, close_fds=True): and also with CPython under some circumstances (as triggered by xmpppy in bug #374335). In order to close a safe subset of file descriptors, see portage.locks._close_fds(). + + NOTE: When not followed by exec, even when close_fds is False, + it's still possible for dup2() calls to cause interference in a + way that's similar to the way that close_fds interferes (since + dup2() has to close the target fd if it happens to be open). + It's possible to avoid such interference by using allocated + file descriptors as the keys in fd_pipes. For example: + + pr, pw = os.pipe() + fd_pipes[pw] = pw + + By using the allocated pw file descriptor as the key in fd_pipes, + it's not necessary for dup2() to close a file descriptor (it + actually does nothing in this case), which avoids possible + interference. """ - my_fds = {} + + reverse_map = {} # To protect from cases where direct assignment could - # clobber needed fds ({1:2, 2:1}) we first dupe the fds - # into unused fds. - for fd in fd_pipes: - my_fds[fd] = os.dup(fd_pipes[fd]) - # Then assign them to what they should be. - for fd in my_fds: - os.dup2(my_fds[fd], fd) + # clobber needed fds ({1:2, 2:1}) we create a reverse map + # in order to know when it's necessary to create temporary + # backup copies with os.dup(). + for newfd, oldfd in fd_pipes.items(): + newfds = reverse_map.get(oldfd) + if newfds is None: + newfds = [] + reverse_map[oldfd] = newfds + newfds.append(newfd) + + # Assign newfds via dup2(), making temporary backups when + # necessary, and closing oldfd if the caller has not + # explicitly requested for it to remain open by adding + # it to the keys of fd_pipes. + while reverse_map: + + oldfd, newfds = reverse_map.popitem() + old_fdflags = None + + for newfd in newfds: + if newfd in reverse_map: + # Make a temporary backup before re-assignment, assuming + # that backup_fd won't collide with a key in reverse_map + # (since all of the keys correspond to open file + # descriptors, and os.dup() only allocates a previously + # unused file discriptors). + backup_fd = os.dup(newfd) + reverse_map[backup_fd] = reverse_map.pop(newfd) + + if oldfd != newfd: + os.dup2(oldfd, newfd) + if _set_inheritable is not None: + # Don't do this unless _set_inheritable is available, + # since it's used below to ensure correct state, and + # otherwise /dev/null stdin fails to inherit (at least + # with Python versions from 3.1 to 3.3). + if old_fdflags is None: + old_fdflags = fcntl.fcntl(oldfd, fcntl.F_GETFD) + fcntl.fcntl(newfd, fcntl.F_SETFD, old_fdflags) + + if _set_inheritable is not None: + + inheritable_state = None + if not (old_fdflags is None or _FD_CLOEXEC is None): + inheritable_state = not bool(old_fdflags & _FD_CLOEXEC) + + if inheritable is not None: + if inheritable_state is not inheritable: + _set_inheritable(newfd, inheritable) + + elif newfd in (0, 1, 2): + if inheritable_state is not True: + _set_inheritable(newfd, True) + + if oldfd not in fd_pipes: + # If oldfd is not a key in fd_pipes, then it's safe + # to close now, since we've already made all of the + # requested duplicates. This also closes every + # backup_fd that may have been created on previous + # iterations of this loop. + os.close(oldfd) if close_fds: # Then close _all_ fds that haven't been explicitly # requested to be kept open. for fd in get_open_fds(): - if fd not in my_fds: + if fd not in fd_pipes: try: os.close(fd) except OSError: @@ -450,14 +640,22 @@ def _setup_pipes(fd_pipes, close_fds=True): def find_binary(binary): """ Given a binary name, find the binary in PATH - + @param binary: Name of the binary to find @type string @rtype: None or string @return: full path to binary or None if the binary could not be located. """ - for path in os.environ.get("PATH", "").split(":"): - filename = "%s/%s" % (path, binary) - if os.access(filename, os.X_OK) and os.path.isfile(filename): + paths = os.environ.get("PATH", "") + if sys.hexversion >= 0x3000000 and isinstance(binary, bytes): + # return bytes when input is bytes + paths = paths.encode(sys.getfilesystemencoding(), 'surrogateescape') + paths = paths.split(b':') + else: + paths = paths.split(':') + + for path in paths: + filename = _os.path.join(path, binary) + if _os.access(filename, os.X_OK) and _os.path.isfile(filename): return filename return None diff --git a/portage_with_autodep/pym/portage/process.pyo b/portage_with_autodep/pym/portage/process.pyo Binary files differindex c53af30..323d956 100644 --- a/portage_with_autodep/pym/portage/process.pyo +++ b/portage_with_autodep/pym/portage/process.pyo diff --git a/portage_with_autodep/pym/portage/proxy/__init__.pyo b/portage_with_autodep/pym/portage/proxy/__init__.pyo Binary files differindex b3d096b..69a2a54 100644 --- a/portage_with_autodep/pym/portage/proxy/__init__.pyo +++ b/portage_with_autodep/pym/portage/proxy/__init__.pyo diff --git a/portage_with_autodep/pym/portage/proxy/lazyimport.py b/portage_with_autodep/pym/portage/proxy/lazyimport.py index ad4a542..3057c05 100644 --- a/portage_with_autodep/pym/portage/proxy/lazyimport.py +++ b/portage_with_autodep/pym/portage/proxy/lazyimport.py @@ -1,4 +1,4 @@ -# Copyright 2009 Gentoo Foundation +# Copyright 2009-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ['lazyimport'] @@ -32,7 +32,7 @@ def _preload_portage_submodules(): while True: remaining = False for name in list(_module_proxies): - if name.startswith('portage.'): + if name.startswith('portage.') or name.startswith('_emerge.'): if name in imported: continue imported.add(name) diff --git a/portage_with_autodep/pym/portage/proxy/lazyimport.pyo b/portage_with_autodep/pym/portage/proxy/lazyimport.pyo Binary files differindex 9da8089..cf4e80f 100644 --- a/portage_with_autodep/pym/portage/proxy/lazyimport.pyo +++ b/portage_with_autodep/pym/portage/proxy/lazyimport.pyo diff --git a/portage_with_autodep/pym/portage/proxy/objectproxy.py b/portage_with_autodep/pym/portage/proxy/objectproxy.py index 92b36d1..a755774 100644 --- a/portage_with_autodep/pym/portage/proxy/objectproxy.py +++ b/portage_with_autodep/pym/portage/proxy/objectproxy.py @@ -1,4 +1,4 @@ -# Copyright 2008-2009 Gentoo Foundation +# Copyright 2008-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import sys @@ -30,6 +30,13 @@ class ObjectProxy(object): result = object.__getattribute__(self, '_get_target')() return result(*args, **kwargs) + def __enter__(self): + return object.__getattribute__(self, '_get_target')().__enter__() + + def __exit__(self, exc_type, exc_value, traceback): + return object.__getattribute__(self, '_get_target')().__exit__( + exc_type, exc_value, traceback) + def __setitem__(self, key, value): object.__getattribute__(self, '_get_target')()[key] = value diff --git a/portage_with_autodep/pym/portage/proxy/objectproxy.pyo b/portage_with_autodep/pym/portage/proxy/objectproxy.pyo Binary files differindex f0919ff..a7c100a 100644 --- a/portage_with_autodep/pym/portage/proxy/objectproxy.pyo +++ b/portage_with_autodep/pym/portage/proxy/objectproxy.pyo diff --git a/portage_with_autodep/pym/portage/repository/__init__.pyo b/portage_with_autodep/pym/portage/repository/__init__.pyo Binary files differindex ab526c0..d98d5dc 100644 --- a/portage_with_autodep/pym/portage/repository/__init__.pyo +++ b/portage_with_autodep/pym/portage/repository/__init__.pyo diff --git a/portage_with_autodep/pym/portage/repository/config.py b/portage_with_autodep/pym/portage/repository/config.py index 20f1919..0d6edf4 100644 --- a/portage_with_autodep/pym/portage/repository/config.py +++ b/portage_with_autodep/pym/portage/repository/config.py @@ -1,6 +1,8 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + import io import logging import warnings @@ -8,25 +10,34 @@ import sys import re try: - from configparser import ParsingError + from configparser import Error as ConfigParserError if sys.hexversion >= 0x3020000: from configparser import ConfigParser as SafeConfigParser else: from configparser import SafeConfigParser except ImportError: - from ConfigParser import SafeConfigParser, ParsingError + from ConfigParser import SafeConfigParser, Error as ConfigParserError +import portage from portage import eclass_cache, os from portage.const import (MANIFEST2_HASH_FUNCTIONS, MANIFEST2_REQUIRED_HASH, - REPO_NAME_LOC, USER_CONFIG_PATH) + PORTAGE_BASE_PATH, REPO_NAME_LOC, USER_CONFIG_PATH) +from portage.eapi import eapi_allows_directories_on_profile_level_and_repository_level from portage.env.loaders import KeyValuePairFileLoader from portage.util import (normalize_path, read_corresponding_eapi_file, shlex_split, - stack_lists, writemsg, writemsg_level) + stack_lists, writemsg, writemsg_level, _recursive_file_list) +from portage.util._path import exists_raise_eaccess, isdir_raise_eaccess from portage.localization import _ from portage import _unicode_decode from portage import _unicode_encode from portage import _encodings from portage import manifest +if sys.hexversion >= 0x3000000: + basestring = str + +# Characters prohibited by repoman's file.name check. +_invalid_path_char_re = re.compile(r'[^a-zA-Z0-9._\-+:/]') + _valid_profile_formats = frozenset( ['pms', 'portage-1', 'portage-2']) @@ -48,38 +59,76 @@ def _gen_valid_repo(name): name = None return name +def _find_invalid_path_char(path, pos=0, endpos=None): + """ + Returns the position of the first invalid character found in basename, + or -1 if no invalid characters are found. + """ + if endpos is None: + endpos = len(path) + + m = _invalid_path_char_re.search(path, pos=pos, endpos=endpos) + if m is not None: + return m.start() + + return -1 + class RepoConfig(object): """Stores config of one repository""" __slots__ = ('aliases', 'allow_missing_manifest', 'allow_provide_virtual', 'cache_formats', 'create_manifest', 'disable_manifest', 'eapi', - 'eclass_db', 'eclass_locations', 'eclass_overrides', 'format', 'location', + 'eclass_db', 'eclass_locations', 'eclass_overrides', + 'find_invalid_path_char', 'force', 'format', 'local_config', 'location', 'main_repo', 'manifest_hashes', 'masters', 'missing_repo_name', 'name', 'portage1_profiles', 'portage1_profiles_compat', 'priority', - 'profile_formats', 'sign_commit', 'sign_manifest', 'sync', - 'thin_manifest', 'update_changelog', 'user_location') + 'profile_formats', 'sign_commit', 'sign_manifest', 'sync_cvs_repo', + 'sync_type', 'sync_uri', 'thin_manifest', 'update_changelog', + 'user_location', '_eapis_banned', '_eapis_deprecated', '_masters_orig') - def __init__(self, name, repo_opts): + def __init__(self, name, repo_opts, local_config=True): """Build a RepoConfig with options in repo_opts Try to read repo_name in repository location, but if it is not found use variable name as repository name""" - aliases = repo_opts.get('aliases') - if aliases is not None: - aliases = tuple(aliases.split()) + + force = repo_opts.get('force') + if force is not None: + force = tuple(force.split()) + self.force = force + if force is None: + force = () + + self.local_config = local_config + + if local_config or 'aliases' in force: + aliases = repo_opts.get('aliases') + if aliases is not None: + aliases = tuple(aliases.split()) + else: + aliases = None + self.aliases = aliases - eclass_overrides = repo_opts.get('eclass-overrides') - if eclass_overrides is not None: - eclass_overrides = tuple(eclass_overrides.split()) + if local_config or 'eclass-overrides' in force: + eclass_overrides = repo_opts.get('eclass-overrides') + if eclass_overrides is not None: + eclass_overrides = tuple(eclass_overrides.split()) + else: + eclass_overrides = None + self.eclass_overrides = eclass_overrides # Eclass databases and locations are computed later. self.eclass_db = None self.eclass_locations = None - # Masters from repos.conf override layout.conf. - masters = repo_opts.get('masters') - if masters is not None: - masters = tuple(masters.split()) + if local_config or 'masters' in force: + # Masters from repos.conf override layout.conf. + masters = repo_opts.get('masters') + if masters is not None: + masters = tuple(masters.split()) + else: + masters = None + self.masters = masters #The main-repo key makes only sense for the 'DEFAULT' section. @@ -93,11 +142,22 @@ class RepoConfig(object): priority = None self.priority = priority - sync = repo_opts.get('sync') - if sync is not None: - sync = sync.strip() - self.sync = sync + sync_cvs_repo = repo_opts.get('sync-cvs-repo') + if sync_cvs_repo is not None: + sync_cvs_repo = sync_cvs_repo.strip() + self.sync_cvs_repo = sync_cvs_repo or None + + sync_type = repo_opts.get('sync-type') + if sync_type is not None: + sync_type = sync_type.strip() + self.sync_type = sync_type or None + sync_uri = repo_opts.get('sync-uri') + if sync_uri is not None: + sync_uri = sync_uri.strip() + self.sync_uri = sync_uri or None + + # Not implemented. format = repo_opts.get('format') if format is not None: format = format.strip() @@ -106,7 +166,7 @@ class RepoConfig(object): location = repo_opts.get('location') self.user_location = location if location is not None and location.strip(): - if os.path.isdir(location): + if os.path.isdir(location) or portage._sync_disabled_warnings: location = os.path.realpath(location) else: location = None @@ -114,14 +174,23 @@ class RepoConfig(object): eapi = None missing = True + self.name = name if self.location is not None: eapi = read_corresponding_eapi_file(os.path.join(self.location, REPO_NAME_LOC)) - name, missing = self._read_valid_repo_name(self.location) - elif name == "DEFAULT": + self.name, missing = self._read_valid_repo_name(self.location) + if missing: + # The name from repos.conf has to be used here for + # things like emerge-webrsync to work when the repo + # is empty (bug #484950). + if name is not None: + self.name = name + if portage._sync_disabled_warnings: + missing = False + + elif name == "DEFAULT": missing = False self.eapi = eapi - self.name = name self.missing_repo_name = missing # sign_commit is disabled by default, since it requires Git >=1.7.9, # and key_id configured by `git config user.signingkey key_id` @@ -137,18 +206,20 @@ class RepoConfig(object): self.cache_formats = None self.portage1_profiles = True self.portage1_profiles_compat = False + self.find_invalid_path_char = _find_invalid_path_char + self._masters_orig = None # Parse layout.conf. if self.location: - layout_filename = os.path.join(self.location, "metadata", "layout.conf") layout_data = parse_layout_conf(self.location, self.name)[0] + self._masters_orig = layout_data['masters'] # layout.conf masters may be overridden here if we have a masters # setting from the user's repos.conf if self.masters is None: self.masters = layout_data['masters'] - if layout_data['aliases']: + if (local_config or 'aliases' in force) and layout_data['aliases']: aliases = self.aliases if aliases is None: aliases = () @@ -156,6 +227,12 @@ class RepoConfig(object): # them the ability to do incremental overrides self.aliases = layout_data['aliases'] + tuple(aliases) + if layout_data['repo-name']: + # allow layout.conf to override repository name + # useful when having two copies of the same repo enabled + # to avoid modifying profiles/repo_name in one of them + self.name = layout_data['repo-name'] + for value in ('allow-missing-manifest', 'allow-provide-virtual', 'cache-formats', 'create-manifest', 'disable-manifest', 'manifest-hashes', @@ -163,9 +240,19 @@ class RepoConfig(object): 'sign-commit', 'sign-manifest', 'thin-manifest', 'update-changelog'): setattr(self, value.lower().replace("-", "_"), layout_data[value]) - self.portage1_profiles = any(x in _portage1_profiles_allow_directories - for x in layout_data['profile-formats']) - self.portage1_profiles_compat = layout_data['profile-formats'] == ('portage-1-compat',) + self.portage1_profiles = eapi_allows_directories_on_profile_level_and_repository_level(eapi) or \ + any(x in _portage1_profiles_allow_directories for x in layout_data['profile-formats']) + self.portage1_profiles_compat = not eapi_allows_directories_on_profile_level_and_repository_level(eapi) and \ + layout_data['profile-formats'] == ('portage-1-compat',) + + self._eapis_banned = frozenset(layout_data['eapis-banned']) + self._eapis_deprecated = frozenset(layout_data['eapis-deprecated']) + + def eapi_is_banned(self, eapi): + return eapi in self._eapis_banned + + def eapi_is_deprecated(self, eapi): + return eapi in self._eapis_deprecated def iter_pregenerated_caches(self, auxdbkeys, readonly=True, force=False): """ @@ -178,7 +265,11 @@ class RepoConfig(object): if not formats: if not force: return - formats = ('pms',) + # The default egencache format was 'pms' prior to portage-2.1.11.32 + # (portage versions prior to portage-2.1.11.14 will NOT + # recognize md5-dict format unless it is explicitly listed in + # layout.conf). + formats = ('md5-dict',) for fmt in formats: name = None @@ -209,7 +300,8 @@ class RepoConfig(object): kwds['hashes'] = self.manifest_hashes if self.disable_manifest: kwds['from_scratch'] = True - return manifest.Manifest(*args, **kwds) + kwds['find_invalid_path_char'] = self.find_invalid_path_char + return manifest.Manifest(*args, **portage._native_kwargs(kwds)) def update(self, new_repo): """Update repository with options in another RepoConfig""" @@ -272,8 +364,12 @@ class RepoConfig(object): repo_msg.append(indent + "format: " + self.format) if self.user_location: repo_msg.append(indent + "location: " + self.user_location) - if self.sync: - repo_msg.append(indent + "sync: " + self.sync) + if self.sync_cvs_repo: + repo_msg.append(indent + "sync-cvs-repo: " + self.sync_cvs_repo) + if self.sync_type: + repo_msg.append(indent + "sync-type: " + self.sync_type) + if self.sync_uri: + repo_msg.append(indent + "sync-uri: " + self.sync_uri) if self.masters: repo_msg.append(indent + "masters: " + " ".join(master.name for master in self.masters)) if self.priority is not None: @@ -281,19 +377,19 @@ class RepoConfig(object): if self.aliases: repo_msg.append(indent + "aliases: " + " ".join(self.aliases)) if self.eclass_overrides: - repo_msg.append(indent + "eclass_overrides: " + \ + repo_msg.append(indent + "eclass-overrides: " + \ " ".join(self.eclass_overrides)) repo_msg.append("") return "\n".join(repo_msg) def __repr__(self): - return "<portage.repository.config.RepoConfig(name='%s', location='%s')>" % (self.name, _unicode_decode(self.location)) + return "<portage.repository.config.RepoConfig(name=%r, location=%r)>" % (self.name, _unicode_decode(self.location)) def __str__(self): d = {} for k in self.__slots__: d[k] = getattr(self, k, None) - return _unicode_decode("%s") % (d,) + return "%s" % (d,) if sys.hexversion < 0x3000000: @@ -306,11 +402,14 @@ class RepoConfigLoader(object): """Loads and store config of several repositories, loaded from PORTDIR_OVERLAY or repos.conf""" @staticmethod - def _add_repositories(portdir, portdir_overlay, prepos, ignored_map, ignored_location_map): + def _add_repositories(portdir, portdir_overlay, prepos, + ignored_map, ignored_location_map, local_config, default_portdir): """Add overlays in PORTDIR_OVERLAY as repositories""" overlays = [] + portdir_orig = None if portdir: portdir = normalize_path(portdir) + portdir_orig = portdir overlays.append(portdir) try: port_ov = [normalize_path(i) for i in shlex_split(portdir_overlay)] @@ -344,43 +443,57 @@ class RepoConfigLoader(object): #overlay priority is negative because we want them to be looked before any other repo base_priority = 0 for ov in overlays: - if os.path.isdir(ov): + # Ignore missing directory for 'gentoo' so that + # first sync with emerge-webrsync is possible. + if isdir_raise_eaccess(ov) or \ + (base_priority == 0 and ov is portdir): repo_opts = default_repo_opts.copy() repo_opts['location'] = ov - repo = RepoConfig(None, repo_opts) + repo = RepoConfig(None, repo_opts, local_config=local_config) # repos_conf_opts contains options from repos.conf repos_conf_opts = repos_conf.get(repo.name) if repos_conf_opts is not None: # Selectively copy only the attributes which # repos.conf is allowed to override. - for k in ('aliases', 'eclass_overrides', 'masters', 'priority'): + for k in ('aliases', 'eclass_overrides', 'force', 'masters', + 'priority', 'sync_cvs_repo', 'sync_type', 'sync_uri'): v = getattr(repos_conf_opts, k, None) if v is not None: setattr(repo, k, v) if repo.name in prepos: + # Silently ignore when PORTDIR overrides the location + # setting from the default repos.conf (bug #478544). old_location = prepos[repo.name].location - if old_location is not None and old_location != repo.location: + if old_location is not None and \ + old_location != repo.location and \ + not (base_priority == 0 and + old_location == default_portdir): ignored_map.setdefault(repo.name, []).append(old_location) ignored_location_map[old_location] = repo.name if old_location == portdir: portdir = repo.user_location - if ov == portdir and portdir not in port_ov: - repo.priority = -1000 - elif repo.priority is None: - repo.priority = base_priority - base_priority += 1 + if repo.priority is None: + if base_priority == 0 and ov == portdir_orig: + # If it's the original PORTDIR setting and it's not + # in PORTDIR_OVERLAY, then it will be assigned a + # special priority setting later. + pass + else: + repo.priority = base_priority + base_priority += 1 prepos[repo.name] = repo else: - writemsg(_("!!! Invalid PORTDIR_OVERLAY" - " (not a dir): '%s'\n") % ov, noiselevel=-1) + + if not portage._sync_disabled_warnings: + writemsg(_("!!! Invalid PORTDIR_OVERLAY (not a dir): '%s'\n") % ov, noiselevel=-1) return portdir @staticmethod - def _parse(paths, prepos, ignored_map, ignored_location_map): + def _parse(paths, prepos, ignored_map, ignored_location_map, local_config, portdir): """Parse files in paths to load config""" parser = SafeConfigParser() @@ -388,49 +501,78 @@ class RepoConfigLoader(object): try: # Python >=3.2 read_file = parser.read_file + source_kwarg = 'source' except AttributeError: read_file = parser.readfp + source_kwarg = 'filename' + recursive_paths = [] for p in paths: - f = None - try: - f = io.open(_unicode_encode(p, - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content'], - errors='replace') - except EnvironmentError: - pass + if isinstance(p, basestring): + recursive_paths.extend(_recursive_file_list(p)) else: + recursive_paths.append(p) + + for p in recursive_paths: + if isinstance(p, basestring): + f = None try: - read_file(f) - except ParsingError as e: - writemsg(_unicode_decode( - _("!!! Error while reading repo config file: %s\n") - ) % e, noiselevel=-1) - finally: - if f is not None: - f.close() - - prepos['DEFAULT'] = RepoConfig("DEFAULT", parser.defaults()) + f = io.open(_unicode_encode(p, + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') + except EnvironmentError: + pass + else: + # The 'source' keyword argument is needed since otherwise + # ConfigParser in Python <3.3.3 may throw a TypeError + # because it assumes that f.name is a native string rather + # than binary when constructing error messages. + kwargs = {source_kwarg: p} + read_file(f, **portage._native_kwargs(kwargs)) + finally: + if f is not None: + f.close() + elif isinstance(p, io.StringIO): + kwargs = {source_kwarg: "<io.StringIO>"} + read_file(p, **portage._native_kwargs(kwargs)) + else: + raise TypeError("Unsupported type %r of element %r of 'paths' argument" % (type(p), p)) + + prepos['DEFAULT'] = RepoConfig("DEFAULT", + parser.defaults(), local_config=local_config) + for sname in parser.sections(): optdict = {} for oname in parser.options(sname): optdict[oname] = parser.get(sname, oname) - repo = RepoConfig(sname, optdict) - if repo.location and not os.path.exists(repo.location): - writemsg(_("!!! Invalid repos.conf entry '%s'" - " (not a dir): '%s'\n") % (sname, repo.location), noiselevel=-1) + repo = RepoConfig(sname, optdict, local_config=local_config) + + if repo.sync_type is not None and repo.sync_uri is None: + writemsg_level("!!! %s\n" % _("Repository '%s' has sync-type attribute, but is missing sync-uri attribute") % + sname, level=logging.ERROR, noiselevel=-1) continue - if repo.name in prepos: - old_location = prepos[repo.name].location - if old_location is not None and repo.location is not None and old_location != repo.location: - ignored_map.setdefault(repo.name, []).append(old_location) - ignored_location_map[old_location] = repo.name - prepos[repo.name].update(repo) - else: - prepos[repo.name] = repo + if repo.sync_uri is not None and repo.sync_type is None: + writemsg_level("!!! %s\n" % _("Repository '%s' has sync-uri attribute, but is missing sync-type attribute") % + sname, level=logging.ERROR, noiselevel=-1) + continue + + if repo.sync_type not in (None, "cvs", "git", "rsync"): + writemsg_level("!!! %s\n" % _("Repository '%s' has sync-type attribute set to unsupported value: '%s'") % + (sname, repo.sync_type), level=logging.ERROR, noiselevel=-1) + continue + + if repo.sync_type == "cvs" and repo.sync_cvs_repo is None: + writemsg_level("!!! %s\n" % _("Repository '%s' has sync-type=cvs, but is missing sync-cvs-repo attribute") % + sname, level=logging.ERROR, noiselevel=-1) + continue + + # For backward compatibility with locations set via PORTDIR and + # PORTDIR_OVERLAY, delay validation of the location and repo.name + # until after PORTDIR and PORTDIR_OVERLAY have been processed. + prepos[sname] = repo def __init__(self, paths, settings): """Load config from files in paths""" @@ -441,15 +583,42 @@ class RepoConfigLoader(object): ignored_map = {} ignored_location_map = {} - portdir = settings.get('PORTDIR', '') - portdir_overlay = settings.get('PORTDIR_OVERLAY', '') + if "PORTAGE_REPOSITORIES" in settings: + portdir = "" + portdir_overlay = "" + portdir_sync = "" + else: + portdir = settings.get("PORTDIR", "") + portdir_overlay = settings.get("PORTDIR_OVERLAY", "") + portdir_sync = settings.get("SYNC", "") - self._parse(paths, prepos, ignored_map, ignored_location_map) + try: + self._parse(paths, prepos, ignored_map, + ignored_location_map, settings.local_config, + portdir) + except ConfigParserError as e: + writemsg( + _("!!! Error while reading repo config file: %s\n") % e, + noiselevel=-1) + # The configparser state is unreliable (prone to quirky + # exceptions) after it has thrown an error, so use empty + # config and try to fall back to PORTDIR{,_OVERLAY}. + prepos.clear() + prepos['DEFAULT'] = RepoConfig('DEFAULT', + {}, local_config=settings.local_config) + location_map.clear() + treemap.clear() + ignored_map.clear() + ignored_location_map.clear() + + default_portdir = os.path.join(os.sep, + settings['EPREFIX'].lstrip(os.sep), 'usr', 'portage') # If PORTDIR_OVERLAY contains a repo with the same repo_name as # PORTDIR, then PORTDIR is overridden. portdir = self._add_repositories(portdir, portdir_overlay, prepos, - ignored_map, ignored_location_map) + ignored_map, ignored_location_map, settings.local_config, + default_portdir) if portdir and portdir.strip(): portdir = os.path.realpath(portdir) @@ -460,9 +629,51 @@ class RepoConfigLoader(object): for repo in prepos.values() if repo.location is not None and repo.missing_repo_name) - #Take aliases into account. - new_prepos = {} - for repo_name, repo in prepos.items(): + # Do this before expanding aliases, so that location_map and + # treemap consistently map unaliased names whenever available. + for repo_name, repo in list(prepos.items()): + if repo.location is None: + if repo_name != 'DEFAULT': + # Skip this warning for repoman (bug #474578). + if settings.local_config and paths: + writemsg_level("!!! %s\n" % _("Section '%s' in repos.conf is missing location attribute") % + repo.name, level=logging.ERROR, noiselevel=-1) + del prepos[repo_name] + continue + else: + if not portage._sync_disabled_warnings: + if not isdir_raise_eaccess(repo.location): + writemsg_level("!!! %s\n" % _("Section '%s' in repos.conf has location attribute set " + "to nonexistent directory: '%s'") % + (repo_name, repo.location), level=logging.ERROR, noiselevel=-1) + + # Ignore missing directory for 'gentoo' so that + # first sync with emerge-webrsync is possible. + if repo.name != 'gentoo': + del prepos[repo_name] + continue + + # After removing support for PORTDIR_OVERLAY, the following check can be: + # if repo.missing_repo_name: + if repo.missing_repo_name and repo.name != repo_name: + writemsg_level("!!! %s\n" % _("Section '%s' in repos.conf refers to repository " + "without repository name set in '%s'") % + (repo_name, os.path.join(repo.location, REPO_NAME_LOC)), level=logging.ERROR, noiselevel=-1) + del prepos[repo_name] + continue + + if repo.name != repo_name: + writemsg_level("!!! %s\n" % _("Section '%s' in repos.conf has name different " + "from repository name '%s' set inside repository") % + (repo_name, repo.name), level=logging.ERROR, noiselevel=-1) + del prepos[repo_name] + continue + + location_map[repo.location] = repo_name + treemap[repo_name] = repo.location + + # Add alias mappings, but never replace unaliased mappings. + for repo_name, repo in list(prepos.items()): names = set() names.add(repo_name) if repo.aliases: @@ -470,36 +681,55 @@ class RepoConfigLoader(object): names.update(aliases) for name in names: - if name in new_prepos: + if name in prepos and prepos[name].location is not None: + if name == repo_name: + # unaliased names already handled earlier + continue writemsg_level(_("!!! Repository name or alias '%s', " + \ "defined for repository '%s', overrides " + \ "existing alias or repository.\n") % (name, repo_name), level=logging.WARNING, noiselevel=-1) - new_prepos[name] = repo - prepos = new_prepos + # Never replace an unaliased mapping with + # an aliased mapping. + continue + prepos[name] = repo + if repo.location is not None: + if repo.location not in location_map: + # Never replace an unaliased mapping with + # an aliased mapping. + location_map[repo.location] = name + treemap[name] = repo.location + + main_repo = prepos['DEFAULT'].main_repo + if main_repo is None or main_repo not in prepos: + #setting main_repo if it was not set in repos.conf + main_repo = location_map.get(portdir) + if main_repo is not None: + prepos['DEFAULT'].main_repo = main_repo + else: + prepos['DEFAULT'].main_repo = None + if portdir and not portage._sync_disabled_warnings: + writemsg(_("!!! main-repo not set in DEFAULT and PORTDIR is empty.\n"), noiselevel=-1) - for (name, r) in prepos.items(): - if r.location is not None: - location_map[r.location] = name - treemap[name] = r.location + if main_repo is not None and prepos[main_repo].priority is None: + # This happens if main-repo has been set in repos.conf. + prepos[main_repo].priority = -1000 - # filter duplicates from aliases, by only including - # items where repo.name == key + # Backward compatible SYNC support for mirrorselect. + if portdir_sync and main_repo is not None: + if portdir_sync.startswith("rsync://"): + prepos[main_repo].sync_uri = portdir_sync + prepos[main_repo].sync_type = "rsync" - prepos_order = sorted(prepos.items(), key=lambda r:r[1].priority or 0) + # Include repo.name in sort key, for predictable sorting + # even when priorities are equal. + prepos_order = sorted(prepos.items(), + key=lambda r:(r[1].priority or 0, r[1].name)) + # filter duplicates from aliases, by only including + # items where repo.name == key prepos_order = [repo.name for (key, repo) in prepos_order - if repo.name == key and repo.location is not None] - - if prepos['DEFAULT'].main_repo is None or \ - prepos['DEFAULT'].main_repo not in prepos: - #setting main_repo if it was not set in repos.conf - if portdir in location_map: - prepos['DEFAULT'].main_repo = location_map[portdir] - elif portdir in ignored_location_map: - prepos['DEFAULT'].main_repo = ignored_location_map[portdir] - else: - prepos['DEFAULT'].main_repo = None - writemsg(_("!!! main-repo not set in DEFAULT and PORTDIR is empty. \n"), noiselevel=-1) + if repo.name == key and key != 'DEFAULT' and + repo.location is not None] self.prepos = prepos self.prepos_order = prepos_order @@ -578,6 +808,17 @@ class RepoConfigLoader(object): eclass_db.append(tree_db) repo.eclass_db = eclass_db + for repo_name, repo in prepos.items(): + if repo_name == "DEFAULT": + continue + + if repo._masters_orig is None and self.mainRepo() and \ + repo.name != self.mainRepo().name and not portage._sync_disabled_warnings: + writemsg_level("!!! %s\n" % _("Repository '%s' is missing masters attribute in '%s'") % + (repo.name, os.path.join(repo.location, "metadata", "layout.conf")) + + "!!! %s\n" % _("Set 'masters = %s' in this file for future compatibility") % + self.mainRepo().name, level=logging.WARNING, noiselevel=-1) + self._prepos_changed = True self._repo_location_list = [] @@ -613,10 +854,10 @@ class RepoConfigLoader(object): def mainRepo(self): """Returns the main repo""" - maid_repo = self.prepos['DEFAULT'].main_repo - if maid_repo is None: + main_repo = self.prepos['DEFAULT'].main_repo + if main_repo is None: return None - return self.prepos[maid_repo] + return self.prepos[main_repo] def _check_locations(self): """Check if repositories location are correct and show a warning message if not""" @@ -625,7 +866,7 @@ class RepoConfigLoader(object): if r.location is None: writemsg(_("!!! Location not set for repository %s\n") % name, noiselevel=-1) else: - if not os.path.isdir(r.location): + if not isdir_raise_eaccess(r.location) and not portage._sync_disabled_warnings: self.prepos_order.remove(name) writemsg(_("!!! Invalid Repository Location" " (not a dir): '%s'\n") % r.location, noiselevel=-1) @@ -650,19 +891,66 @@ class RepoConfigLoader(object): def get_repo_for_location(self, location): return self.prepos[self.get_name_for_location(location)] + def __setitem__(self, repo_name, repo): + # self.prepos[repo_name] = repo + raise NotImplementedError + def __getitem__(self, repo_name): return self.prepos[repo_name] + def __delitem__(self, repo_name): + if repo_name == self.prepos['DEFAULT'].main_repo: + self.prepos['DEFAULT'].main_repo = None + location = self.prepos[repo_name].location + del self.prepos[repo_name] + if repo_name in self.prepos_order: + self.prepos_order.remove(repo_name) + for k, v in self.location_map.copy().items(): + if v == repo_name: + del self.location_map[k] + if repo_name in self.treemap: + del self.treemap[repo_name] + self._repo_location_list = tuple(x for x in self._repo_location_list if x != location) + def __iter__(self): for repo_name in self.prepos_order: yield self.prepos[repo_name] -def load_repository_config(settings): - #~ repoconfigpaths = [os.path.join(settings.global_config_path, "repos.conf")] + def __contains__(self, repo_name): + return repo_name in self.prepos + + def config_string(self): + str_or_int_keys = ("format", "location", "main_repo", "priority", "sync_cvs_repo", "sync_type", "sync_uri") + str_tuple_keys = ("aliases", "eclass_overrides", "force") + repo_config_tuple_keys = ("masters",) + keys = str_or_int_keys + str_tuple_keys + repo_config_tuple_keys + config_string = "" + for repo_name, repo in sorted(self.prepos.items()): + config_string += "\n[%s]\n" % repo_name + for key in sorted(keys): + if key == "main_repo" and repo_name != "DEFAULT": + continue + if getattr(repo, key) is not None: + if key in str_or_int_keys: + config_string += "%s = %s\n" % (key.replace("_", "-"), getattr(repo, key)) + elif key in str_tuple_keys: + config_string += "%s = %s\n" % (key.replace("_", "-"), " ".join(getattr(repo, key))) + elif key in repo_config_tuple_keys: + config_string += "%s = %s\n" % (key.replace("_", "-"), " ".join(x.name for x in getattr(repo, key))) + return config_string.lstrip("\n") + +def load_repository_config(settings, extra_files=None): repoconfigpaths = [] - if settings.local_config: - repoconfigpaths.append(os.path.join(settings["PORTAGE_CONFIGROOT"], - USER_CONFIG_PATH, "repos.conf")) + if "PORTAGE_REPOSITORIES" in settings: + repoconfigpaths.append(io.StringIO(settings["PORTAGE_REPOSITORIES"])) + else: + if portage._working_copy: + repoconfigpaths.append(os.path.join(PORTAGE_BASE_PATH, "cnf", "repos.conf")) + else: + repoconfigpaths.append(os.path.join(settings.global_config_path, "repos.conf")) + repoconfigpaths.append(os.path.join(settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH, "repos.conf")) + if extra_files: + repoconfigpaths.extend(extra_files) return RepoConfigLoader(repoconfigpaths, settings) def _get_repo_name(repo_location, cached=None): @@ -696,6 +984,9 @@ def parse_layout_conf(repo_location, repo_name=None): data['allow-provide-virtual'] = \ layout_data.get('allow-provide-virtuals', 'false').lower() == 'true' + data['eapis-banned'] = tuple(layout_data.get('eapis-banned', '').split()) + data['eapis-deprecated'] = tuple(layout_data.get('eapis-deprecated', '').split()) + data['sign-commit'] = layout_data.get('sign-commits', 'false').lower() \ == 'true' @@ -705,6 +996,8 @@ def parse_layout_conf(repo_location, repo_name=None): data['thin-manifest'] = layout_data.get('thin-manifests', 'false').lower() \ == 'true' + data['repo-name'] = _gen_valid_repo(layout_data.get('repo-name', '')) + manifest_policy = layout_data.get('use-manifests', 'strict').lower() data['allow-missing-manifest'] = manifest_policy != 'strict' data['create-manifest'] = manifest_policy != 'false' @@ -712,10 +1005,19 @@ def parse_layout_conf(repo_location, repo_name=None): # for compatibility w/ PMS, fallback to pms; but also check if the # cache exists or not. - cache_formats = layout_data.get('cache-formats', 'pms').lower().split() - if 'pms' in cache_formats and not os.path.isdir( - os.path.join(repo_location, 'metadata', 'cache')): - cache_formats.remove('pms') + cache_formats = layout_data.get('cache-formats', '').lower().split() + if not cache_formats: + # Auto-detect cache formats, and prefer md5-cache if available. + # This behavior was deployed in portage-2.1.11.14, so that the + # default egencache format could eventually be changed to md5-dict + # in portage-2.1.11.32. WARNING: Versions prior to portage-2.1.11.14 + # will NOT recognize md5-dict format unless it is explicitly + # listed in layout.conf. + cache_formats = [] + if os.path.isdir(os.path.join(repo_location, 'metadata', 'md5-cache')): + cache_formats.append('md5-dict') + if os.path.isdir(os.path.join(repo_location, 'metadata', 'cache')): + cache_formats.append('pms') data['cache-formats'] = tuple(cache_formats) manifest_hashes = layout_data.get('manifest-hashes') @@ -754,7 +1056,7 @@ def parse_layout_conf(repo_location, repo_name=None): raw_formats = layout_data.get('profile-formats') if raw_formats is None: - if eapi in ('4-python',): + if eapi_allows_directories_on_profile_level_and_repository_level(eapi): raw_formats = ('portage-1',) else: raw_formats = ('portage-1-compat',) diff --git a/portage_with_autodep/pym/portage/repository/config.pyo b/portage_with_autodep/pym/portage/repository/config.pyo Binary files differindex f9ee26d..b289c2c 100644 --- a/portage_with_autodep/pym/portage/repository/config.pyo +++ b/portage_with_autodep/pym/portage/repository/config.pyo diff --git a/portage_with_autodep/pym/portage/tests/__init__.py b/portage_with_autodep/pym/portage/tests/__init__.py index 492ece4..84e732a 100644 --- a/portage_with_autodep/pym/portage/tests/__init__.py +++ b/portage_with_autodep/pym/portage/tests/__init__.py @@ -1,5 +1,5 @@ # tests/__init__.py -- Portage Unit Test functionality -# Copyright 2006-2011 Gentoo Foundation +# Copyright 2006-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -7,26 +7,40 @@ from __future__ import print_function import sys import time import unittest -from optparse import OptionParser, OptionValueError try: from unittest.runner import _TextTestResult # new in python-2.7 except ImportError: from unittest import _TextTestResult +try: + # They added the skip framework to python-2.7. + # Drop this once we drop python-2.6 support. + unittest_skip_shims = False + import unittest.SkipTest as SkipTest # new in python-2.7 +except ImportError: + unittest_skip_shims = True + +import portage from portage import os from portage import _encodings from portage import _unicode_decode +from portage.util._argparse import ArgumentParser def main(): suite = unittest.TestSuite() basedir = os.path.dirname(os.path.realpath(__file__)) usage = "usage: %s [options] [tests to run]" % os.path.basename(sys.argv[0]) - parser = OptionParser(usage=usage) - parser.add_option("-l", "--list", help="list all tests", + parser = ArgumentParser(usage=usage) + parser.add_argument("-l", "--list", help="list all tests", action="store_true", dest="list_tests") - (options, args) = parser.parse_args(args=sys.argv) + options, args = parser.parse_known_args(args=sys.argv) + + if (os.environ.get('NOCOLOR') in ('yes', 'true') or + os.environ.get('TERM') == 'dumb' or + not sys.stdout.isatty()): + portage.output.nocolor() if options.list_tests: testdir = os.path.dirname(sys.argv[0]) @@ -70,15 +84,12 @@ def getTestFromCommandLine(args, base_path): def getTestDirs(base_path): TEST_FILE = b'__test__' - svn_dirname = b'.svn' testDirs = [] # the os.walk help mentions relative paths as being quirky # I was tired of adding dirs to the list, so now we add __test__ # to each dir we want tested. for root, dirs, files in os.walk(base_path): - if svn_dirname in dirs: - dirs.remove(svn_dirname) try: root = _unicode_decode(root, encoding=_encodings['fs'], errors='strict') @@ -93,7 +104,7 @@ def getTestDirs(base_path): def getTestNames(path): files = os.listdir(path) - files = [ f[:-3] for f in files if f.startswith("test") and f.endswith(".py") ] + files = [f[:-3] for f in files if f.startswith("test") and f.endswith(".py")] files.sort() return files @@ -134,14 +145,14 @@ class TextTestResult(_TextTestResult): self.portage_skipped = [] def addTodo(self, test, info): - self.todoed.append((test,info)) + self.todoed.append((test, info)) if self.showAll: self.stream.writeln("TODO") elif self.dots: self.stream.write(".") def addPortageSkip(self, test, info): - self.portage_skipped.append((test,info)) + self.portage_skipped.append((test, info)) if self.showAll: self.stream.writeln("SKIP") elif self.dots: @@ -185,10 +196,14 @@ class TestCase(unittest.TestCase): except: result.addError(self, sys.exc_info()) return + ok = False try: testMethod() ok = True + except SkipTest as e: + result.addPortageSkip(self, "%s: SKIP: %s" % + (testMethod, str(e))) except self.failureException: if self.portage_skip is not None: if self.portage_skip is True: @@ -197,13 +212,14 @@ class TestCase(unittest.TestCase): result.addPortageSkip(self, "%s: SKIP: %s" % (testMethod, self.portage_skip)) elif self.todo: - result.addTodo(self,"%s: TODO" % testMethod) + result.addTodo(self, "%s: TODO" % testMethod) else: result.addFailure(self, sys.exc_info()) except (KeyboardInterrupt, SystemExit): raise except: result.addError(self, sys.exc_info()) + try: self.tearDown() except SystemExit: @@ -213,7 +229,8 @@ class TestCase(unittest.TestCase): except: result.addError(self, sys.exc_info()) ok = False - if ok: result.addSuccess(self) + if ok: + result.addSuccess(self) finally: result.stopTest(self) @@ -230,10 +247,48 @@ class TestCase(unittest.TestCase): except excClass: return else: - if hasattr(excClass,'__name__'): excName = excClass.__name__ + if hasattr(excClass, '__name__'): excName = excClass.__name__ else: excName = str(excClass) raise self.failureException("%s not raised: %s" % (excName, msg)) + def assertExists(self, path): + """Make sure |path| exists""" + if not os.path.exists(path): + msg = ['path is missing: %s' % (path,)] + while path != '/': + path = os.path.dirname(path) + if not path: + # If we're given something like "foo", abort once we get to "". + break + result = os.path.exists(path) + msg.append('\tos.path.exists(%s): %s' % (path, result)) + if result: + msg.append('\tcontents: %r' % os.listdir(path)) + break + raise self.failureException('\n'.join(msg)) + + def assertNotExists(self, path): + """Make sure |path| does not exist""" + if os.path.exists(path): + raise self.failureException('path exists when it should not: %s' % path) + +if unittest_skip_shims: + # Shim code for <python-2.7. + class SkipTest(Exception): + """unittest.SkipTest shim for <python-2.7""" + + def skipTest(self, reason): + raise SkipTest(reason) + setattr(TestCase, 'skipTest', skipTest) + + def assertIn(self, member, container, msg=None): + self.assertTrue(member in container, msg=msg) + setattr(TestCase, 'assertIn', assertIn) + + def assertNotIn(self, member, container, msg=None): + self.assertFalse(member in container, msg=msg) + setattr(TestCase, 'assertNotIn', assertNotIn) + class TextTestRunner(unittest.TextTestRunner): """ We subclass unittest.TextTestRunner to output SKIP for tests that fail but are skippable @@ -271,8 +326,8 @@ class TextTestRunner(unittest.TextTestRunner): self.stream.writeln("OK") return result -test_cps = ['sys-apps/portage','virtual/portage'] -test_versions = ['1.0', '1.0-r1','2.3_p4','1.0_alpha57'] -test_slots = [ None, '1','gentoo-sources-2.6.17','spankywashere'] -test_usedeps = ['foo','-bar', ('foo','bar'), - ('foo','-bar'), ('foo?', '!bar?') ] +test_cps = ['sys-apps/portage', 'virtual/portage'] +test_versions = ['1.0', '1.0-r1', '2.3_p4', '1.0_alpha57'] +test_slots = [None, '1', 'gentoo-sources-2.6.17', 'spankywashere'] +test_usedeps = ['foo', '-bar', ('foo', 'bar'), + ('foo', '-bar'), ('foo?', '!bar?')] diff --git a/portage_with_autodep/pym/portage/tests/__init__.pyo b/portage_with_autodep/pym/portage/tests/__init__.pyo Binary files differindex 0e961b8..aa0215b 100644 --- a/portage_with_autodep/pym/portage/tests/__init__.pyo +++ b/portage_with_autodep/pym/portage/tests/__init__.pyo diff --git a/portage_with_autodep/pym/portage/tests/lint/__init__.pyo b/portage_with_autodep/pym/portage/tests/lint/__init__.pyo Binary files differindex a1241e5..2a1215a 100644 --- a/portage_with_autodep/pym/portage/tests/lint/__init__.pyo +++ b/portage_with_autodep/pym/portage/tests/lint/__init__.pyo diff --git a/portage_with_autodep/pym/portage/tests/lint/test_bash_syntax.py b/portage_with_autodep/pym/portage/tests/lint/test_bash_syntax.py index aef8d74..fdbb6fe 100644 --- a/portage_with_autodep/pym/portage/tests/lint/test_bash_syntax.py +++ b/portage_with_autodep/pym/portage/tests/lint/test_bash_syntax.py @@ -1,20 +1,26 @@ -# Copyright 2010 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from itertools import chain import stat +import subprocess +import sys -from portage.const import BASH_BINARY, PORTAGE_BIN_PATH +from portage.const import BASH_BINARY, PORTAGE_BASE_PATH, PORTAGE_BIN_PATH from portage.tests import TestCase from portage import os -from portage import subprocess_getstatusoutput from portage import _encodings -from portage import _shell_quote from portage import _unicode_decode, _unicode_encode class BashSyntaxTestCase(TestCase): def testBashSyntax(self): - for parent, dirs, files in os.walk(PORTAGE_BIN_PATH): + locations = [PORTAGE_BIN_PATH] + misc_dir = os.path.join(PORTAGE_BASE_PATH, "misc") + if os.path.isdir(misc_dir): + locations.append(misc_dir) + for parent, dirs, files in \ + chain.from_iterable(os.walk(x) for x in locations): parent = _unicode_decode(parent, encoding=_encodings['fs'], errors='strict') for x in files: @@ -36,7 +42,13 @@ class BashSyntaxTestCase(TestCase): f.close() if line[:2] == '#!' and \ 'bash' in line: - cmd = "%s -n %s" % (_shell_quote(BASH_BINARY), _shell_quote(x)) - status, output = subprocess_getstatusoutput(cmd) + cmd = [BASH_BINARY, "-n", x] + cmd = [_unicode_encode(x, + encoding=_encodings['fs'], errors='strict') for x in cmd] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + output = _unicode_decode(proc.communicate()[0], + encoding=_encodings['fs']) + status = proc.wait() self.assertEqual(os.WIFEXITED(status) and \ os.WEXITSTATUS(status) == os.EX_OK, True, msg=output) diff --git a/portage_with_autodep/pym/portage/tests/lint/test_bash_syntax.pyo b/portage_with_autodep/pym/portage/tests/lint/test_bash_syntax.pyo Binary files differindex a7ddc80..b066527 100644 --- a/portage_with_autodep/pym/portage/tests/lint/test_bash_syntax.pyo +++ b/portage_with_autodep/pym/portage/tests/lint/test_bash_syntax.pyo diff --git a/portage_with_autodep/pym/portage/tests/lint/test_compile_modules.py b/portage_with_autodep/pym/portage/tests/lint/test_compile_modules.py index f90a666..1d44e68 100644 --- a/portage_with_autodep/pym/portage/tests/lint/test_compile_modules.py +++ b/portage_with_autodep/pym/portage/tests/lint/test_compile_modules.py @@ -1,4 +1,4 @@ -# Copyright 2009-2010 Gentoo Foundation +# Copyright 2009-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import itertools @@ -10,8 +10,6 @@ from portage import os from portage import _encodings from portage import _unicode_decode, _unicode_encode -import py_compile - class CompileModulesTestCase(TestCase): def testCompileModules(self): @@ -34,13 +32,13 @@ class CompileModulesTestCase(TestCase): do_compile = True else: # Check for python shebang - f = open(_unicode_encode(x, - encoding=_encodings['fs'], errors='strict'), 'rb') - line = _unicode_decode(f.readline(), - encoding=_encodings['content'], errors='replace') - f.close() - if line[:2] == '#!' and \ - 'python' in line: + with open(_unicode_encode(x, + encoding=_encodings['fs'], errors='strict'), 'rb') as f: + line = _unicode_decode(f.readline(), + encoding=_encodings['content'], errors='replace') + if line[:2] == '#!' and 'python' in line: do_compile = True if do_compile: - py_compile.compile(x, cfile='/dev/null', doraise=True) + with open(_unicode_encode(x, + encoding=_encodings['fs'], errors='strict'), 'rb') as f: + compile(f.read(), x, 'exec') diff --git a/portage_with_autodep/pym/portage/tests/lint/test_compile_modules.pyo b/portage_with_autodep/pym/portage/tests/lint/test_compile_modules.pyo Binary files differindex 7b1460d..8e32e1f 100644 --- a/portage_with_autodep/pym/portage/tests/lint/test_compile_modules.pyo +++ b/portage_with_autodep/pym/portage/tests/lint/test_compile_modules.pyo diff --git a/portage_with_autodep/pym/portage/tests/lint/test_import_modules.py b/portage_with_autodep/pym/portage/tests/lint/test_import_modules.py index 8d257c5..34261f4 100644 --- a/portage_with_autodep/pym/portage/tests/lint/test_import_modules.py +++ b/portage_with_autodep/pym/portage/tests/lint/test_import_modules.py @@ -1,4 +1,4 @@ -# Copyright 2011 Gentoo Foundation +# Copyright 2011-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from portage.const import PORTAGE_PYM_PATH diff --git a/portage_with_autodep/pym/portage/tests/lint/test_import_modules.pyo b/portage_with_autodep/pym/portage/tests/lint/test_import_modules.pyo Binary files differindex b3a1d61..b566748 100644 --- a/portage_with_autodep/pym/portage/tests/lint/test_import_modules.pyo +++ b/portage_with_autodep/pym/portage/tests/lint/test_import_modules.pyo diff --git a/portage_with_autodep/pym/portage/tests/runTests b/portage_with_autodep/pym/portage/tests/runTests index 4c10087..60bcf31 100755 --- a/portage_with_autodep/pym/portage/tests/runTests +++ b/portage_with_autodep/pym/portage/tests/runTests @@ -1,18 +1,25 @@ #!/usr/bin/python -Wd # runTests.py -- Portage Unit Test Functionality -# Copyright 2006 Gentoo Foundation +# Copyright 2006-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import os, sys import os.path as osp import grp +import platform import pwd import signal def debug_signal(signum, frame): import pdb pdb.set_trace() -signal.signal(signal.SIGUSR1, debug_signal) + +if platform.python_implementation() == 'Jython': + debug_signum = signal.SIGUSR2 # bug #424259 +else: + debug_signum = signal.SIGUSR1 + +signal.signal(debug_signum, debug_signal) # Pretend that the current user's uid/gid are the 'portage' uid/gid, # so things go smoothly regardless of the current user and global @@ -22,23 +29,33 @@ os.environ["PORTAGE_GRPNAME"] = grp.getgrgid(os.getgid()).gr_name # Insert our parent dir so we can do shiny import "tests" # This line courtesy of Marienz and Pkgcore ;) -sys.path.insert(0, osp.dirname(osp.dirname(osp.dirname(osp.abspath(__file__))))) +sys.path.insert(0, osp.dirname(osp.dirname(osp.dirname(osp.realpath(__file__))))) import portage +portage._internal_caller = True # Ensure that we don't instantiate portage.settings, so that tests should # work the same regardless of global configuration file state/existence. portage._disable_legacy_globals() +if os.environ.get('NOCOLOR') in ('yes', 'true'): + portage.output.nocolor() + import portage.tests as tests from portage.const import PORTAGE_BIN_PATH path = os.environ.get("PATH", "").split(":") path = [x for x in path if x] -if not path or not os.path.samefile(path[0], PORTAGE_BIN_PATH): + +insert_bin_path = True +try: + insert_bin_path = not path or \ + not os.path.samefile(path[0], PORTAGE_BIN_PATH) +except OSError: + pass + +if insert_bin_path: path.insert(0, PORTAGE_BIN_PATH) os.environ["PATH"] = ":".join(path) -del path - if __name__ == "__main__": sys.exit(tests.main()) diff --git a/portage_with_autodep/pym/portage/update.py b/portage_with_autodep/pym/portage/update.py index 34e4663..92aba9a 100644 --- a/portage_with_autodep/pym/portage/update.py +++ b/portage_with_autodep/pym/portage/update.py @@ -1,11 +1,14 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + import errno import io import re import stat import sys +import warnings from portage import os from portage import _encodings @@ -13,50 +16,107 @@ from portage import _unicode_decode from portage import _unicode_encode import portage portage.proxy.lazyimport.lazyimport(globals(), - 'portage.dep:Atom,dep_getkey,isvalidatom,' + \ - 'remove_slot', + 'portage.dep:Atom,dep_getkey,isvalidatom,match_from_list', 'portage.util:ConfigProtect,new_protect_filename,' + \ 'normalize_path,write_atomic,writemsg', - 'portage.util.listdir:_ignorecvs_dirs', - 'portage.versions:ververify' + 'portage.versions:_get_slot_re', ) -from portage.const import USER_CONFIG_PATH +from portage.const import USER_CONFIG_PATH, VCS_DIRS +from portage.eapi import _get_eapi_attrs from portage.exception import DirectoryNotFound, InvalidAtom, PortageException from portage.localization import _ if sys.hexversion >= 0x3000000: long = int + _unicode = str +else: + _unicode = unicode ignored_dbentries = ("CONTENTS", "environment.bz2") -def update_dbentry(update_cmd, mycontent): +def update_dbentry(update_cmd, mycontent, eapi=None, parent=None): + + if parent is not None: + eapi = parent.eapi + if update_cmd[0] == "move": - old_value = str(update_cmd[1]) - if old_value in mycontent: - new_value = str(update_cmd[2]) - old_value = re.escape(old_value); - mycontent = re.sub(old_value+"(:|$|\\s)", new_value+"\\1", mycontent) - def myreplace(matchobj): - # Strip slot and * operator if necessary - # so that ververify works. - ver = remove_slot(matchobj.group(2)) - ver = ver.rstrip("*") - if ververify(ver): - return "%s-%s" % (new_value, matchobj.group(2)) - else: - return "".join(matchobj.groups()) - mycontent = re.sub("(%s-)(\\S*)" % old_value, myreplace, mycontent) + old_value = _unicode(update_cmd[1]) + new_value = _unicode(update_cmd[2]) + + # Use isvalidatom() to check if this move is valid for the + # EAPI (characters allowed in package names may vary). + if old_value in mycontent and isvalidatom(new_value, eapi=eapi): + # this split preserves existing whitespace + split_content = re.split(r'(\s+)', mycontent) + modified = False + for i, token in enumerate(split_content): + if old_value not in token: + continue + try: + atom = Atom(token, eapi=eapi) + except InvalidAtom: + continue + if atom.cp != old_value: + continue + + new_atom = Atom(token.replace(old_value, new_value, 1), + eapi=eapi) + + # Avoid creating self-blockers for bug #367215. + if new_atom.blocker and parent is not None and \ + parent.cp == new_atom.cp and \ + match_from_list(new_atom, [parent]): + continue + + split_content[i] = _unicode(new_atom) + modified = True + + if modified: + mycontent = "".join(split_content) + elif update_cmd[0] == "slotmove" and update_cmd[1].operator is None: - pkg, origslot, newslot = update_cmd[1:] - old_value = "%s:%s" % (pkg, origslot) - if old_value in mycontent: - old_value = re.escape(old_value) - new_value = "%s:%s" % (pkg, newslot) - mycontent = re.sub(old_value+"($|\\s)", new_value+"\\1", mycontent) + orig_atom, origslot, newslot = update_cmd[1:] + orig_cp = orig_atom.cp + + # We don't support versioned slotmove atoms here, since it can be + # difficult to determine if the version constraints really match + # the atoms that we're trying to update. + if orig_atom.version is None and orig_cp in mycontent: + # this split preserves existing whitespace + split_content = re.split(r'(\s+)', mycontent) + modified = False + for i, token in enumerate(split_content): + if orig_cp not in token: + continue + try: + atom = Atom(token, eapi=eapi) + except InvalidAtom: + continue + if atom.cp != orig_cp: + continue + if atom.slot is None or atom.slot != origslot: + continue + + slot_part = newslot + if atom.sub_slot is not None: + if atom.sub_slot == origslot: + sub_slot = newslot + else: + sub_slot = atom.sub_slot + slot_part += "/" + sub_slot + if atom.slot_operator is not None: + slot_part += atom.slot_operator + + split_content[i] = atom.with_slot(slot_part) + modified = True + + if modified: + mycontent = "".join(split_content) + return mycontent -def update_dbentries(update_iter, mydata): +def update_dbentries(update_iter, mydata, eapi=None, parent=None): """Performs update commands and returns a dict containing only the updated items.""" updated_items = {} @@ -70,7 +130,8 @@ def update_dbentries(update_iter, mydata): is_encoded = mycontent is not orig_content orig_content = mycontent for update_cmd in update_iter: - mycontent = update_dbentry(update_cmd, mycontent) + mycontent = update_dbentry(update_cmd, mycontent, + eapi=eapi, parent=parent) if mycontent != orig_content: if is_encoded: mycontent = _unicode_encode(mycontent, @@ -79,10 +140,14 @@ def update_dbentries(update_iter, mydata): updated_items[k] = mycontent return updated_items -def fixdbentries(update_iter, dbdir): |