diff options
| author |   Alexander Bersenev <bay@hackerdom.ru> | 2014-02-17 17:57:05 +0600 |
|---|---|---|
| committer | Alexander Bersenev <bay@hackerdom.ru> | 2014-02-17 17:57:05 +0600 |
| commit | 6563293d18daed502ccdb663f3c72b4bae5fe23a (patch) | |
| tree | d0a7d53a7c137feb4073c963408829f88ea75c92 /portage_with_autodep/pym/portage | |
| parent | updated portage to 2.2.8-r1 (diff) | |
| download | autodep-master.tar.gz autodep-master.tar.bz2 autodep-master.zip | |
Diffstat (limited to 'portage_with_autodep/pym/portage')
224 files changed, 6945 insertions, 3073 deletions
diff --git a/portage_with_autodep/pym/portage/__init__.py b/portage_with_autodep/pym/portage/__init__.py index 431dc26..2bce5d7 100644 --- a/portage_with_autodep/pym/portage/__init__.py +++ b/portage_with_autodep/pym/portage/__init__.py @@ -1,8 +1,10 @@ # portage.py -- core Portage functionality -# Copyright 1998-2011 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -VERSION="2.2.0_alpha108" +from __future__ import unicode_literals + +VERSION="2.2.8-r1" # =========================================================================== # START OF IMPORTS -- START OF IMPORTS -- START OF IMPORTS -- START OF IMPORT @@ -16,14 +18,6 @@ try: errno.ESTALE = -1 import re import types - - # Try the commands module first, since this allows us to eliminate - # the subprocess module from the baseline imports under python2. - try: - from commands import getstatusoutput as subprocess_getstatusoutput - except ImportError: - from subprocess import getstatusoutput as subprocess_getstatusoutput - import platform # Temporarily delete these imports, to ensure that only the @@ -70,6 +64,7 @@ try: 'match_from_list,match_to_list', 'portage.dep.dep_check:dep_check,dep_eval,dep_wordreduce,dep_zapdeps', 'portage.eclass_cache', + 'portage.elog', 'portage.exception', 'portage.getbinpkg', 'portage.locks', @@ -114,6 +109,7 @@ try: 'cpv_getkey@getCPFromCPV,endversion_keys,' + \ 'suffix_value@endversion,pkgcmp,pkgsplit,vercmp,ververify', 'portage.xpak', + 'subprocess', 'time', ) @@ -178,6 +174,15 @@ _encodings = { } if sys.hexversion >= 0x3000000: + + def _decode_argv(argv): + # With Python 3, the surrogateescape encoding error handler makes it + # possible to access the original argv bytes, which can be useful + # if their actual encoding does no match the filesystem encoding. + fs_encoding = sys.getfilesystemencoding() + return [_unicode_decode(x.encode(fs_encoding, 'surrogateescape')) + for x in argv] + def _unicode_encode(s, encoding=_encodings['content'], errors='backslashreplace'): if isinstance(s, str): s = s.encode(encoding, errors) @@ -187,7 +192,13 @@ if sys.hexversion >= 0x3000000: if isinstance(s, bytes): s = str(s, encoding=encoding, errors=errors) return s + + _native_string = _unicode_decode else: + + def _decode_argv(argv): + return [_unicode_decode(x) for x in argv] + def _unicode_encode(s, encoding=_encodings['content'], errors='backslashreplace'): if isinstance(s, unicode): s = s.encode(encoding, errors) @@ -198,6 +209,17 @@ else: s = unicode(s, encoding=encoding, errors=errors) return s + _native_string = _unicode_encode + +if sys.hexversion >= 0x20605f0: + def _native_kwargs(kwargs): + return kwargs +else: + # Avoid "TypeError: keywords must be strings" issue triggered + # by unicode_literals: http://bugs.python.org/issue4978 + def _native_kwargs(kwargs): + return dict((_native_string(k), v) for k, v in kwargs.iteritems()) + class _unicode_func_wrapper(object): """ Wraps a function, converts arguments from unicode to bytes, @@ -215,7 +237,7 @@ class _unicode_func_wrapper(object): self._func = func self._encoding = encoding - def __call__(self, *args, **kwargs): + def _process_args(self, args, kwargs): encoding = self._encoding wrapped_args = [_unicode_encode(x, encoding=encoding, errors='strict') @@ -227,6 +249,13 @@ class _unicode_func_wrapper(object): else: wrapped_kwargs = {} + return (wrapped_args, wrapped_kwargs) + + def __call__(self, *args, **kwargs): + + encoding = self._encoding + wrapped_args, wrapped_kwargs = self._process_args(args, kwargs) + rval = self._func(*wrapped_args, **wrapped_kwargs) # Don't use isinstance() since we don't want to convert subclasses @@ -294,12 +323,17 @@ class _unicode_module_wrapper(object): import os as _os _os_overrides = { id(_os.fdopen) : _os.fdopen, - id(_os.mkfifo) : _os.mkfifo, id(_os.popen) : _os.popen, id(_os.read) : _os.read, id(_os.system) : _os.system, } + +try: + _os_overrides[id(_os.mkfifo)] = _os.mkfifo +except AttributeError: + pass # Jython + if hasattr(_os, 'statvfs'): _os_overrides[id(_os.statvfs)] = _os.statvfs @@ -334,16 +368,25 @@ except (ImportError, OSError) as e: _python_interpreter = os.path.realpath(sys.executable) _bin_path = PORTAGE_BIN_PATH _pym_path = PORTAGE_PYM_PATH +_working_copy = VERSION == "HEAD" + +# Api consumers included in portage should set this to True. +_internal_caller = False -if sys.hexversion >= 0x3030000: - # Workaround for http://bugs.python.org/issue14007 - def _test_xml_etree_ElementTree_TreeBuilder_type(): - import subprocess - p = subprocess.Popen([_python_interpreter, "-c", - "import sys, xml.etree.ElementTree; sys.exit(not isinstance(xml.etree.ElementTree.TreeBuilder, type))"]) - if p.wait() != 0: - sys.modules["_elementtree"] = None - _test_xml_etree_ElementTree_TreeBuilder_type() +_sync_disabled_warnings = False + +def _get_stdin(): + """ + Buggy code in python's multiprocessing/process.py closes sys.stdin + and reassigns it to open(os.devnull), but fails to update the + corresponding __stdin__ reference. So, detect that case and handle + it appropriately. + """ + if not sys.__stdin__.closed: + return sys.__stdin__ + return sys.stdin + +_shell_quote_re = re.compile(r"[\s><=*\\\"'$`]") def _shell_quote(s): """ @@ -351,6 +394,8 @@ def _shell_quote(s): escape any backslashes, double-quotes, dollar signs, or backquotes in the string. """ + if _shell_quote_re.search(s) is None: + return s for letter in "\\\"$`": if letter in s: s = s.replace(letter, "\\" + letter) @@ -364,8 +409,27 @@ if platform.system() in ('FreeBSD',): @classmethod def chflags(cls, path, flags, opts=""): - cmd = 'chflags %s %o %s' % (opts, flags, _shell_quote(path)) - status, output = subprocess_getstatusoutput(cmd) + cmd = ['chflags'] + if opts: + cmd.append(opts) + cmd.append('%o' % (flags,)) + cmd.append(path) + + if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000: + # Python 3.1 _execvp throws TypeError for non-absolute executable + # path passed as bytes (see http://bugs.python.org/issue8513). + fullname = process.find_binary(cmd[0]) + if fullname is None: + raise exception.CommandNotFound(cmd[0]) + cmd[0] = fullname + + encoding = _encodings['fs'] + cmd = [_unicode_encode(x, encoding=encoding, errors='strict') + for x in cmd] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + output = proc.communicate()[0] + status = proc.wait() if os.WIFEXITED(status) and os.WEXITSTATUS(status) == os.EX_OK: return # Try to generate an ENOENT error if appropriate. @@ -378,6 +442,7 @@ if platform.system() in ('FreeBSD',): raise portage.exception.CommandNotFound('chflags') # Now we're not sure exactly why it failed or what # the real errno was, so just report EPERM. + output = _unicode_decode(output, encoding=encoding) e = OSError(errno.EPERM, output) e.errno = errno.EPERM e.filename = path @@ -406,7 +471,15 @@ def getcwd(): getcwd() def abssymlink(symlink, target=None): - "This reads symlinks, resolving the relative symlinks, and returning the absolute." + """ + This reads symlinks, resolving the relative symlinks, + and returning the absolute. + @param symlink: path of symlink (must be absolute) + @param target: the target of the symlink (as returned + by readlink) + @rtype: str + @return: the absolute path of the symlink target + """ if target is not None: mylink = target else: @@ -418,8 +491,9 @@ def abssymlink(symlink, target=None): _doebuild_manifest_exempt_depend = 0 -_testing_eapis = frozenset(["4-python"]) -_deprecated_eapis = frozenset(["4_pre1", "3_pre2", "3_pre1"]) +_testing_eapis = frozenset(["4-python", "4-slot-abi", "5-progress", "5-hdepend"]) +_deprecated_eapis = frozenset(["4_pre1", "3_pre2", "3_pre1", "5_pre1", "5_pre2"]) +_supported_eapis = frozenset([str(x) for x in range(portage.const.EAPI)] + list(_testing_eapis) + list(_deprecated_eapis)) def _eapi_is_deprecated(eapi): return eapi in _deprecated_eapis @@ -476,14 +550,13 @@ auxdbkeys = ( 'RESTRICT', 'HOMEPAGE', 'LICENSE', 'DESCRIPTION', 'KEYWORDS', 'INHERITED', 'IUSE', 'REQUIRED_USE', 'PDEPEND', 'PROVIDE', 'EAPI', - 'PROPERTIES', 'DEFINED_PHASES', 'UNUSED_05', 'UNUSED_04', + 'PROPERTIES', 'DEFINED_PHASES', 'HDEPEND', 'UNUSED_04', 'UNUSED_03', 'UNUSED_02', 'UNUSED_01', ) auxdbkeylen=len(auxdbkeys) def portageexit(): - if data.secpass > 1 and os.environ.get("SANDBOX_ON") != "1": - close_portdbapi_caches() + pass class _trees_dict(dict): __slots__ = ('_running_eroot', '_target_eroot',) @@ -494,13 +567,6 @@ class _trees_dict(dict): def create_trees(config_root=None, target_root=None, trees=None, env=None, eprefix=None): - if trees is not None: - # clean up any existing portdbapi instances - for myroot in trees: - portdb = trees[myroot]["porttree"].dbapi - portdb.close_caches() - portdbapi.portdbapi_instances.remove(portdb) - del trees[myroot]["porttree"], myroot, portdb if trees is None: trees = _trees_dict() @@ -518,7 +584,7 @@ def create_trees(config_root=None, target_root=None, trees=None, env=None, trees._target_eroot = settings['EROOT'] myroots = [(settings['EROOT'], settings)] - if settings["ROOT"] == "/": + if settings["ROOT"] == "/" and settings["EPREFIX"] == const.EPREFIX: trees._running_eroot = trees._target_eroot else: @@ -526,15 +592,15 @@ def create_trees(config_root=None, target_root=None, trees=None, env=None, # environment to apply to the config that's associated # with ROOT != "/", so pass a nearly empty dict for the env parameter. clean_env = {} - for k in ('PATH', 'PORTAGE_GRPNAME', 'PORTAGE_USERNAME', - 'SSH_AGENT_PID', 'SSH_AUTH_SOCK', 'TERM', + for k in ('PATH', 'PORTAGE_GRPNAME', 'PORTAGE_REPOSITORIES', 'PORTAGE_USERNAME', + 'PYTHONPATH', 'SSH_AGENT_PID', 'SSH_AUTH_SOCK', 'TERM', 'ftp_proxy', 'http_proxy', 'no_proxy', '__PORTAGE_TEST_HARDLINK_LOCKS'): v = settings.get(k) if v is not None: clean_env[k] = v settings = config(config_root=None, target_root="/", - env=clean_env, eprefix=eprefix) + env=clean_env, eprefix=None) settings.lock() trees._running_eroot = settings['EROOT'] myroots.append((settings['EROOT'], settings)) @@ -558,11 +624,17 @@ if VERSION == 'HEAD': if VERSION is not self: return VERSION if os.path.isdir(os.path.join(PORTAGE_BASE_PATH, '.git')): - status, output = subprocess_getstatusoutput(( - "cd %s ; git describe --tags || exit $? ; " + \ + encoding = _encodings['fs'] + cmd = [BASH_BINARY, "-c", ("cd %s ; git describe --tags || exit $? ; " + \ "if [ -n \"`git diff-index --name-only --diff-filter=M HEAD`\" ] ; " + \ "then echo modified ; git rev-list --format=%%ct -n 1 HEAD ; fi ; " + \ - "exit 0") % _shell_quote(PORTAGE_BASE_PATH)) + "exit 0") % _shell_quote(PORTAGE_BASE_PATH)] + cmd = [_unicode_encode(x, encoding=encoding, errors='strict') + for x in cmd] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + output = _unicode_decode(proc.communicate()[0], encoding=encoding) + status = proc.wait() if os.WIFEXITED(status) and os.WEXITSTATUS(status) == os.EX_OK: output_lines = output.splitlines() if output_lines: @@ -591,34 +663,17 @@ if VERSION == 'HEAD': return VERSION VERSION = _LazyVersion() -if "_legacy_globals_constructed" in globals(): - # The module has been reloaded, so perform any relevant cleanup - # and prevent memory leaks. - if "db" in _legacy_globals_constructed: - try: - db - except NameError: - pass - else: - if isinstance(db, dict) and db: - for _x in db.values(): - try: - if "porttree" in _x.lazy_items: - continue - except (AttributeError, TypeError): - continue - try: - _x = _x["porttree"].dbapi - except (AttributeError, KeyError): - continue - if not isinstance(_x, portdbapi): - continue - _x.close_caches() - try: - portdbapi.portdbapi_instances.remove(_x) - except ValueError: - pass - del _x +_legacy_global_var_names = ("archlist", "db", "features", + "groups", "mtimedb", "mtimedbfile", "pkglines", + "portdb", "profiledir", "root", "selinux_enabled", + "settings", "thirdpartymirrors") + +def _reset_legacy_globals(): + + global _legacy_globals_constructed + _legacy_globals_constructed = set() + for k in _legacy_global_var_names: + globals()[k] = _LegacyGlobalProxy(k) class _LegacyGlobalProxy(proxy.objectproxy.ObjectProxy): @@ -633,16 +688,7 @@ class _LegacyGlobalProxy(proxy.objectproxy.ObjectProxy): from portage._legacy_globals import _get_legacy_global return _get_legacy_global(name) -_legacy_global_var_names = ("archlist", "db", "features", - "groups", "mtimedb", "mtimedbfile", "pkglines", - "portdb", "profiledir", "root", "selinux_enabled", - "settings", "thirdpartymirrors") - -for k in _legacy_global_var_names: - globals()[k] = _LegacyGlobalProxy(k) -del k - -_legacy_globals_constructed = set() +_reset_legacy_globals() def _disable_legacy_globals(): """ diff --git a/portage_with_autodep/pym/portage/__init__.pyo b/portage_with_autodep/pym/portage/__init__.pyo Binary files differindex 9fc449e..ef60b98 100644 --- a/portage_with_autodep/pym/portage/__init__.pyo +++ b/portage_with_autodep/pym/portage/__init__.pyo diff --git a/portage_with_autodep/pym/portage/_global_updates.py b/portage_with_autodep/pym/portage/_global_updates.py index 5175043..9ae734b 100644 --- a/portage_with_autodep/pym/portage/_global_updates.py +++ b/portage_with_autodep/pym/portage/_global_updates.py @@ -1,4 +1,4 @@ -# Copyright 2010 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -32,22 +32,20 @@ def _global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True): """ # only do this if we're root and not running repoman/ebuild digest - retupd = False if secpass < 2 or \ "SANDBOX_ACTIVE" in os.environ or \ len(trees) != 1: - return retupd + return False + + return _do_global_updates(trees, prev_mtimes, + quiet=quiet, if_mtime_changed=if_mtime_changed) + +def _do_global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True): root = trees._running_eroot mysettings = trees[root]["vartree"].settings portdb = trees[root]["porttree"].dbapi vardb = trees[root]["vartree"].dbapi bindb = trees[root]["bintree"].dbapi - if not os.access(bindb.bintree.pkgdir, os.W_OK): - bindb = None - else: - # Call binarytree.populate(), since we want to make sure it's - # only populated with local packages here (getbinpkgs=0). - bindb.bintree.populate() world_file = os.path.join(mysettings['EROOT'], WORLD_FILE) world_list = grabfile(world_file) @@ -61,6 +59,7 @@ def _global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True): repo_map = {} timestamps = {} + retupd = False update_notice_printed = False for repo_name in portdb.getRepositories(): repo = portdb.getRepositoryPath(repo_name) @@ -115,6 +114,14 @@ def _global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True): if myupd: retupd = True + if retupd: + if os.access(bindb.bintree.pkgdir, os.W_OK): + # Call binarytree.populate(), since we want to make sure it's + # only populated with local packages here (getbinpkgs=0). + bindb.bintree.populate() + else: + bindb = None + master_repo = portdb.getRepositoryName(portdb.porttree_root) if master_repo in repo_map: repo_map['DEFAULT'] = repo_map[master_repo] @@ -237,8 +244,7 @@ def _global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True): # Update progress above is indicated by characters written to stdout so # we print a couple new lines here to separate the progress output from # what follows. - print() - print() + writemsg_stdout("\n\n") if do_upgrade_packagesmessage and bindb and \ bindb.cpv_all(): diff --git a/portage_with_autodep/pym/portage/_global_updates.pyo b/portage_with_autodep/pym/portage/_global_updates.pyo Binary files differindex 3e2e8de..fe5684e 100644 --- a/portage_with_autodep/pym/portage/_global_updates.pyo +++ b/portage_with_autodep/pym/portage/_global_updates.pyo diff --git a/portage_with_autodep/pym/portage/_legacy_globals.py b/portage_with_autodep/pym/portage/_legacy_globals.py index abffa0e..bb9691a 100644 --- a/portage_with_autodep/pym/portage/_legacy_globals.py +++ b/portage_with_autodep/pym/portage/_legacy_globals.py @@ -27,7 +27,8 @@ def _get_legacy_global(name): os.umask(0o22) kwargs = {} - for k, envvar in (("config_root", "PORTAGE_CONFIGROOT"), ("target_root", "ROOT")): + for k, envvar in (("config_root", "PORTAGE_CONFIGROOT"), + ("target_root", "ROOT"), ("eprefix", "EPREFIX")): kwargs[k] = os.environ.get(envvar) portage._initializing_globals = True diff --git a/portage_with_autodep/pym/portage/_legacy_globals.pyo b/portage_with_autodep/pym/portage/_legacy_globals.pyo Binary files differindex 2e50cbe..2a5d08a 100644 --- a/portage_with_autodep/pym/portage/_legacy_globals.pyo +++ b/portage_with_autodep/pym/portage/_legacy_globals.pyo diff --git a/portage_with_autodep/pym/portage/_selinux.py b/portage_with_autodep/pym/portage/_selinux.py index 9470978..e4621b1 100644 --- a/portage_with_autodep/pym/portage/_selinux.py +++ b/portage_with_autodep/pym/portage/_selinux.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # Don't use the unicode-wrapped os and shutil modules here since @@ -8,18 +8,18 @@ import shutil import portage from portage import _encodings -from portage import _unicode_decode -from portage import _unicode_encode +from portage import _native_string, _unicode_decode from portage.localization import _ portage.proxy.lazyimport.lazyimport(globals(), 'selinux') def copyfile(src, dest): - src = _unicode_encode(src, encoding=_encodings['fs'], errors='strict') - dest = _unicode_encode(dest, encoding=_encodings['fs'], errors='strict') + src = _native_string(src, encoding=_encodings['fs'], errors='strict') + dest = _native_string(dest, encoding=_encodings['fs'], errors='strict') (rc, ctx) = selinux.lgetfilecon(src) if rc < 0: - src = _unicode_decode(src, encoding=_encodings['fs'], errors='replace') + if sys.hexversion < 0x3000000: + src = _unicode_decode(src, encoding=_encodings['fs'], errors='replace') raise OSError(_("copyfile: Failed getting context of \"%s\".") % src) setfscreate(ctx) @@ -39,12 +39,12 @@ def is_selinux_enabled(): return selinux.is_selinux_enabled() def mkdir(target, refdir): - target = _unicode_encode(target, encoding=_encodings['fs'], errors='strict') - refdir = _unicode_encode(refdir, encoding=_encodings['fs'], errors='strict') + target = _native_string(target, encoding=_encodings['fs'], errors='strict') + refdir = _native_string(refdir, encoding=_encodings['fs'], errors='strict') (rc, ctx) = selinux.getfilecon(refdir) if rc < 0: - refdir = _unicode_decode(refdir, encoding=_encodings['fs'], - errors='replace') + if sys.hexversion < 0x3000000: + refdir = _unicode_decode(refdir, encoding=_encodings['fs'], errors='replace') raise OSError( _("mkdir: Failed getting context of reference directory \"%s\".") \ % refdir) @@ -56,11 +56,12 @@ def mkdir(target, refdir): setfscreate() def rename(src, dest): - src = _unicode_encode(src, encoding=_encodings['fs'], errors='strict') - dest = _unicode_encode(dest, encoding=_encodings['fs'], errors='strict') + src = _native_string(src, encoding=_encodings['fs'], errors='strict') + dest = _native_string(dest, encoding=_encodings['fs'], errors='strict') (rc, ctx) = selinux.lgetfilecon(src) if rc < 0: - src = _unicode_decode(src, encoding=_encodings['fs'], errors='replace') + if sys.hexversion < 0x3000000: + src = _unicode_decode(src, encoding=_encodings['fs'], errors='replace') raise OSError(_("rename: Failed getting context of \"%s\".") % src) setfscreate(ctx) @@ -75,10 +76,10 @@ def settype(newtype): return ":".join(ret) def setexec(ctx="\n"): - ctx = _unicode_encode(ctx, encoding=_encodings['content'], errors='strict') + ctx = _native_string(ctx, encoding=_encodings['content'], errors='strict') if selinux.setexeccon(ctx) < 0: - ctx = _unicode_decode(ctx, encoding=_encodings['content'], - errors='replace') + if sys.hexversion < 0x3000000: + ctx = _unicode_decode(ctx, encoding=_encodings['content'], errors='replace') if selinux.security_getenforce() == 1: raise OSError(_("Failed setting exec() context \"%s\".") % ctx) else: @@ -87,37 +88,47 @@ def setexec(ctx="\n"): noiselevel=-1) def setfscreate(ctx="\n"): - ctx = _unicode_encode(ctx, - encoding=_encodings['content'], errors='strict') + ctx = _native_string(ctx, encoding=_encodings['content'], errors='strict') if selinux.setfscreatecon(ctx) < 0: - ctx = _unicode_decode(ctx, - encoding=_encodings['content'], errors='replace') + if sys.hexversion < 0x3000000: + ctx = _unicode_decode(ctx, encoding=_encodings['content'], errors='replace') raise OSError( _("setfscreate: Failed setting fs create context \"%s\".") % ctx) -def spawn_wrapper(spawn_func, selinux_type): +class spawn_wrapper(object): + """ + Create a wrapper function for the given spawn function. When the wrapper + is called, it will adjust the arguments such that setexec() to be called + *after* the fork (thereby avoiding any interference with concurrent + threads in the calling process). + """ + __slots__ = ("_con", "_spawn_func") - selinux_type = _unicode_encode(selinux_type, - encoding=_encodings['content'], errors='strict') + def __init__(self, spawn_func, selinux_type): + self._spawn_func = spawn_func + selinux_type = _native_string(selinux_type, encoding=_encodings['content'], errors='strict') + self._con = settype(selinux_type) - def wrapper_func(*args, **kwargs): - con = settype(selinux_type) - setexec(con) - try: - return spawn_func(*args, **kwargs) - finally: - setexec() + def __call__(self, *args, **kwargs): - return wrapper_func + pre_exec = kwargs.get("pre_exec") + + def _pre_exec(): + if pre_exec is not None: + pre_exec() + setexec(self._con) + + kwargs["pre_exec"] = _pre_exec + return self._spawn_func(*args, **kwargs) def symlink(target, link, reflnk): - target = _unicode_encode(target, encoding=_encodings['fs'], errors='strict') - link = _unicode_encode(link, encoding=_encodings['fs'], errors='strict') - reflnk = _unicode_encode(reflnk, encoding=_encodings['fs'], errors='strict') + target = _native_string(target, encoding=_encodings['fs'], errors='strict') + link = _native_string(link, encoding=_encodings['fs'], errors='strict') + reflnk = _native_string(reflnk, encoding=_encodings['fs'], errors='strict') (rc, ctx) = selinux.lgetfilecon(reflnk) if rc < 0: - reflnk = _unicode_decode(reflnk, encoding=_encodings['fs'], - errors='replace') + if sys.hexversion < 0x3000000: + reflnk = _unicode_decode(reflnk, encoding=_encodings['fs'], errors='replace') raise OSError( _("symlink: Failed getting context of reference symlink \"%s\".") \ % reflnk) diff --git a/portage_with_autodep/pym/portage/_selinux.pyo b/portage_with_autodep/pym/portage/_selinux.pyo Binary files differindex 7a413e0..080613d 100644 --- a/portage_with_autodep/pym/portage/_selinux.pyo +++ b/portage_with_autodep/pym/portage/_selinux.pyo diff --git a/portage_with_autodep/pym/portage/_sets/__init__.py b/portage_with_autodep/pym/portage/_sets/__init__.py index 88a4b3b..75d1df7 100644 --- a/portage_with_autodep/pym/portage/_sets/__init__.py +++ b/portage_with_autodep/pym/portage/_sets/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2007-2011 Gentoo Foundation +# Copyright 2007-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -17,6 +17,7 @@ try: from configparser import SafeConfigParser except ImportError: from ConfigParser import SafeConfigParser, NoOptionError, ParsingError +import portage from portage import os from portage import load_mod from portage import _unicode_decode @@ -111,16 +112,51 @@ class SetConfig(object): """ parser = self._parser + parser.remove_section("world") parser.add_section("world") parser.set("world", "class", "portage.sets.base.DummyPackageSet") parser.set("world", "packages", "@selected @system") + parser.remove_section("selected") parser.add_section("selected") parser.set("selected", "class", "portage.sets.files.WorldSelectedSet") + parser.remove_section("system") parser.add_section("system") parser.set("system", "class", "portage.sets.profiles.PackagesSystemSet") + parser.remove_section("security") + parser.add_section("security") + parser.set("security", "class", "portage.sets.security.NewAffectedSet") + + parser.remove_section("usersets") + parser.add_section("usersets") + parser.set("usersets", "class", "portage.sets.files.StaticFileSet") + parser.set("usersets", "multiset", "true") + parser.set("usersets", "directory", "%(PORTAGE_CONFIGROOT)setc/portage/sets") + parser.set("usersets", "world-candidate", "true") + + parser.remove_section("live-rebuild") + parser.add_section("live-rebuild") + parser.set("live-rebuild", "class", "portage.sets.dbapi.VariableSet") + parser.set("live-rebuild", "variable", "INHERITED") + parser.set("live-rebuild", "includes", " ".join(sorted(portage.const.LIVE_ECLASSES))) + + parser.remove_section("module-rebuild") + parser.add_section("module-rebuild") + parser.set("module-rebuild", "class", "portage.sets.dbapi.OwnerSet") + parser.set("module-rebuild", "files", "/lib/modules") + + parser.remove_section("preserved-rebuild") + parser.add_section("preserved-rebuild") + parser.set("preserved-rebuild", "class", "portage.sets.libs.PreservedLibraryConsumerSet") + + parser.remove_section("x11-module-rebuild") + parser.add_section("x11-module-rebuild") + parser.set("x11-module-rebuild", "class", "portage.sets.dbapi.OwnerSet") + parser.set("x11-module-rebuild", "files", "/usr/lib/xorg/modules") + parser.set("x11-module-rebuild", "exclude-files", "/usr/bin/Xorg") + def update(self, setname, options): parser = self._parser self.errors = [] @@ -260,8 +296,8 @@ def load_default_config(settings, trees): return SetConfig(None, settings, trees) global_config_path = GLOBAL_CONFIG_PATH - if settings['EPREFIX']: - global_config_path = os.path.join(settings['EPREFIX'], + if portage.const.EPREFIX: + global_config_path = os.path.join(portage.const.EPREFIX, GLOBAL_CONFIG_PATH.lstrip(os.sep)) def _getfiles(): for path, dirs, files in os.walk(os.path.join(global_config_path, "sets")): diff --git a/portage_with_autodep/pym/portage/_sets/__init__.pyo b/portage_with_autodep/pym/portage/_sets/__init__.pyo Binary files differindex 5318dbe..500a5d3 100644 --- a/portage_with_autodep/pym/portage/_sets/__init__.pyo +++ b/portage_with_autodep/pym/portage/_sets/__init__.pyo diff --git a/portage_with_autodep/pym/portage/_sets/base.py b/portage_with_autodep/pym/portage/_sets/base.py index c8d3ae4..d368e00 100644 --- a/portage_with_autodep/pym/portage/_sets/base.py +++ b/portage_with_autodep/pym/portage/_sets/base.py @@ -126,7 +126,7 @@ class PackageSet(object): if modified_use is not None and modified_use is not pkg.use.enabled: pkg = pkg.copy() - pkg.metadata["USE"] = " ".join(modified_use) + pkg._metadata["USE"] = " ".join(modified_use) # Atoms matched via PROVIDE must be temporarily transformed since # match_from_list() only works correctly when atom.cp == pkg.cp. @@ -156,7 +156,7 @@ class PackageSet(object): for atom in atoms: if match_from_list(atom, cpv_slot_list): yield atom - provides = pkg.metadata['PROVIDE'] + provides = pkg._metadata['PROVIDE'] if not provides: return provides = provides.split() diff --git a/portage_with_autodep/pym/portage/_sets/base.pyo b/portage_with_autodep/pym/portage/_sets/base.pyo Binary files differindex 89e53be..09ade2f 100644 --- a/portage_with_autodep/pym/portage/_sets/base.pyo +++ b/portage_with_autodep/pym/portage/_sets/base.pyo diff --git a/portage_with_autodep/pym/portage/_sets/dbapi.py b/portage_with_autodep/pym/portage/_sets/dbapi.py index 4982a92..384fb3a 100644 --- a/portage_with_autodep/pym/portage/_sets/dbapi.py +++ b/portage_with_autodep/pym/portage/_sets/dbapi.py @@ -26,8 +26,7 @@ class EverythingSet(PackageSet): def load(self): myatoms = [] - db_keys = ["SLOT"] - aux_get = self._db.aux_get + pkg_str = self._db._pkg_str cp_list = self._db.cp_list for cp in self._db.cp_all(): @@ -35,8 +34,8 @@ class EverythingSet(PackageSet): # NOTE: Create SLOT atoms even when there is only one # SLOT installed, in order to avoid the possibility # of unwanted upgrades as reported in bug #338959. - slot, = aux_get(cpv, db_keys) - atom = Atom("%s:%s" % (cp, slot)) + pkg = pkg_str(cpv, None) + atom = Atom("%s:%s" % (pkg.cp, pkg.slot)) if self._filter: if self._filter(atom): myatoms.append(atom) @@ -68,20 +67,19 @@ class OwnerSet(PackageSet): """ rValue = set() vardb = self._db - aux_get = vardb.aux_get - aux_keys = ["SLOT"] + pkg_str = vardb._pkg_str if exclude_paths is None: for link, p in vardb._owners.iter_owners(paths): - slot, = aux_get(link.mycpv, aux_keys) - rValue.add("%s:%s" % (link.mycpv.cp, slot)) + pkg = pkg_str(link.mycpv, None) + rValue.add("%s:%s" % (pkg.cp, pkg.slot)) else: all_paths = set() all_paths.update(paths) all_paths.update(exclude_paths) exclude_atoms = set() for link, p in vardb._owners.iter_owners(all_paths): - slot, = aux_get(link.mycpv, aux_keys) - atom = "%s:%s" % (link.mycpv.cp, slot) + pkg = pkg_str(link.mycpv, None) + atom = "%s:%s" % (pkg.cp, pkg.slot) rValue.add(atom) if p in exclude_paths: exclude_atoms.add(atom) @@ -173,12 +171,11 @@ class DowngradeSet(PackageSet): xmatch = self._portdb.xmatch xmatch_level = "bestmatch-visible" cp_list = self._vardb.cp_list - aux_get = self._vardb.aux_get - aux_keys = ["SLOT"] + pkg_str = self._vardb._pkg_str for cp in self._vardb.cp_all(): for cpv in cp_list(cp): - slot, = aux_get(cpv, aux_keys) - slot_atom = "%s:%s" % (cp, slot) + pkg = pkg_str(cpv, None) + slot_atom = "%s:%s" % (pkg.cp, pkg.slot) ebuild = xmatch(xmatch_level, slot_atom) if not ebuild: continue @@ -326,6 +323,7 @@ class CategorySet(PackageSet): class AgeSet(EverythingSet): _operations = ["merge", "unmerge"] + _aux_keys = ('BUILD_TIME',) def __init__(self, vardb, mode="older", age=7): super(AgeSet, self).__init__(vardb) @@ -335,8 +333,12 @@ class AgeSet(EverythingSet): def _filter(self, atom): cpv = self._db.match(atom)[0] - path = self._db.getpath(cpv, filename="COUNTER") - age = (time.time() - os.stat(path).st_mtime) / (3600 * 24) + try: + date, = self._db.aux_get(cpv, self._aux_keys) + date = int(date) + except (KeyError, ValueError): + return bool(self._mode == "older") + age = (time.time() - date) / (3600 * 24) if ((self._mode == "older" and age <= self._age) \ or (self._mode == "newer" and age >= self._age)): return False @@ -355,6 +357,83 @@ class AgeSet(EverythingSet): singleBuilder = classmethod(singleBuilder) +class DateSet(EverythingSet): + _operations = ["merge", "unmerge"] + _aux_keys = ('BUILD_TIME',) + + def __init__(self, vardb, date, mode="older"): + super(DateSet, self).__init__(vardb) + self._mode = mode + self._date = date + + def _filter(self, atom): + + cpv = self._db.match(atom)[0] + try: + date, = self._db.aux_get(cpv, self._aux_keys) + date = int(date) + except (KeyError, ValueError): + return bool(self._mode == "older") + # Make sure inequality is _strict_ to exclude tested package + if ((self._mode == "older" and date < self._date) \ + or (self._mode == "newer" and date > self._date)): + return True + else: + return False + + def singleBuilder(cls, options, settings, trees): + vardbapi = trees["vartree"].dbapi + mode = options.get("mode", "older") + if str(mode).lower() not in ["newer", "older"]: + raise SetConfigError(_("invalid 'mode' value %s (use either 'newer' or 'older')") % mode) + + formats = [] + if options.get("package") is not None: + formats.append("package") + if options.get("filestamp") is not None: + formats.append("filestamp") + if options.get("seconds") is not None: + formats.append("seconds") + if options.get("date") is not None: + formats.append("date") + + if not formats: + raise SetConfigError(_("none of these options specified: 'package', 'filestamp', 'seconds', 'date'")) + elif len(formats) > 1: + raise SetConfigError(_("no more than one of these options is allowed: 'package', 'filestamp', 'seconds', 'date'")) + + format = formats[0] + + if (format == "package"): + package = options.get("package") + try: + cpv = vardbapi.match(package)[0] + date, = vardbapi.aux_get(cpv, ('BUILD_TIME',)) + date = int(date) + except (KeyError, ValueError): + raise SetConfigError(_("cannot determine installation date of package %s") % package) + elif (format == "filestamp"): + filestamp = options.get("filestamp") + try: + date = int(os.stat(filestamp).st_mtime) + except (OSError, ValueError): + raise SetConfigError(_("cannot determine 'filestamp' of '%s'") % filestamp) + elif (format == "seconds"): + try: + date = int(options.get("seconds")) + except ValueError: + raise SetConfigError(_("option 'seconds' must be an integer")) + else: + dateopt = options.get("date") + try: + dateformat = options.get("dateformat", "%x %X") + date = int(time.mktime(time.strptime(dateopt, dateformat))) + except ValueError: + raise SetConfigError(_("'date=%s' does not match 'dateformat=%s'") % (dateopt, dateformat)) + return DateSet(vardb=vardbapi, date=date, mode=mode) + + singleBuilder = classmethod(singleBuilder) + class RebuiltBinaries(EverythingSet): _operations = ('merge',) _aux_keys = ('BUILD_TIME',) diff --git a/portage_with_autodep/pym/portage/_sets/dbapi.pyo b/portage_with_autodep/pym/portage/_sets/dbapi.pyo Binary files differindex 20bf848..584cce4 100644 --- a/portage_with_autodep/pym/portage/_sets/dbapi.pyo +++ b/portage_with_autodep/pym/portage/_sets/dbapi.pyo diff --git a/portage_with_autodep/pym/portage/_sets/files.py b/portage_with_autodep/pym/portage/_sets/files.py index f19ecf6..2fb64de 100644 --- a/portage_with_autodep/pym/portage/_sets/files.py +++ b/portage_with_autodep/pym/portage/_sets/files.py @@ -1,4 +1,4 @@ -# Copyright 2007-2011 Gentoo Foundation +# Copyright 2007-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import errno @@ -11,7 +11,6 @@ from portage import _unicode_decode from portage import _unicode_encode from portage.util import grabfile, write_atomic, ensure_dirs, normalize_path from portage.const import USER_CONFIG_PATH, WORLD_FILE, WORLD_SETS_FILE -from portage.const import _ENABLE_SET_CONFIG from portage.localization import _ from portage.locks import lockfile, unlockfile from portage import portage_gid @@ -87,8 +86,8 @@ class StaticFileSet(EditablePackageSet): for a in data: matches = self.dbapi.match(a) for cpv in matches: - atoms.append("%s:%s" % (cpv_getkey(cpv), - self.dbapi.aux_get(cpv, ["SLOT"])[0])) + pkg = self.dbapi._pkg_str(cpv, None) + atoms.append("%s:%s" % (pkg.cp, pkg.slot)) # In addition to any installed slots, also try to pull # in the latest new slot that may be available. atoms.append(a) @@ -231,9 +230,8 @@ class WorldSelectedSet(EditablePackageSet): write_atomic(self._filename, "".join(sorted("%s\n" % x for x in self._atoms))) - if _ENABLE_SET_CONFIG: - write_atomic(self._filename2, - "".join(sorted("%s\n" % x for x in self._nonatoms))) + write_atomic(self._filename2, + "".join(sorted("%s\n" % x for x in self._nonatoms))) def load(self): atoms = [] @@ -263,9 +261,8 @@ class WorldSelectedSet(EditablePackageSet): else: atoms.extend(self._atoms) - if _ENABLE_SET_CONFIG: - changed2, nonatoms = self._load2() - atoms_changed |= changed2 + changed2, nonatoms = self._load2() + atoms_changed |= changed2 if atoms_changed: self._setAtoms(atoms+nonatoms) @@ -299,10 +296,14 @@ class WorldSelectedSet(EditablePackageSet): ensure_dirs(os.path.dirname(self._filename), gid=portage_gid, mode=0o2750, mask=0o2) def lock(self): + if self._lock is not None: + raise AssertionError("already locked") self._ensure_dirs() self._lock = lockfile(self._filename, wantnewlockfile=1) def unlock(self): + if self._lock is None: + raise AssertionError("not locked") unlockfile(self._lock) self._lock = None diff --git a/portage_with_autodep/pym/portage/_sets/files.pyo b/portage_with_autodep/pym/portage/_sets/files.pyo Binary files differindex eb03c00..f79345b 100644 --- a/portage_with_autodep/pym/portage/_sets/files.pyo +++ b/portage_with_autodep/pym/portage/_sets/files.pyo diff --git a/portage_with_autodep/pym/portage/_sets/libs.py b/portage_with_autodep/pym/portage/_sets/libs.py index 6c5babc..022e076 100644 --- a/portage_with_autodep/pym/portage/_sets/libs.py +++ b/portage_with_autodep/pym/portage/_sets/libs.py @@ -1,12 +1,12 @@ -# Copyright 2007-2011 Gentoo Foundation +# Copyright 2007-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function +from portage.exception import InvalidData from portage.localization import _ from portage._sets.base import PackageSet from portage._sets import get_boolean, SetConfigError -from portage.versions import cpv_getkey import portage class LibraryConsumerSet(PackageSet): @@ -22,14 +22,14 @@ class LibraryConsumerSet(PackageSet): for p in paths: for cpv in self.dbapi._linkmap.getOwners(p): try: - slot, = self.dbapi.aux_get(cpv, ["SLOT"]) - except KeyError: + pkg = self.dbapi._pkg_str(cpv, None) + except (KeyError, InvalidData): # This is expected for preserved libraries # of packages that have been uninstalled # without replacement. pass else: - rValue.add("%s:%s" % (cpv_getkey(cpv), slot)) + rValue.add("%s:%s" % (pkg.cp, pkg.slot)) return rValue class LibraryFileConsumerSet(LibraryConsumerSet): @@ -49,7 +49,8 @@ class LibraryFileConsumerSet(LibraryConsumerSet): def load(self): consumers = set() for lib in self.files: - consumers.update(self.dbapi._linkmap.findConsumers(lib)) + consumers.update( + self.dbapi._linkmap.findConsumers(lib, greedy=False)) if not consumers: return @@ -77,10 +78,10 @@ class PreservedLibraryConsumerSet(LibraryConsumerSet): for lib in libs: if self.debug: print(lib) - for x in sorted(self.dbapi._linkmap.findConsumers(lib)): + for x in sorted(self.dbapi._linkmap.findConsumers(lib, greedy=False)): print(" ", x) print("-"*40) - consumers.update(self.dbapi._linkmap.findConsumers(lib)) + consumers.update(self.dbapi._linkmap.findConsumers(lib, greedy=False)) # Don't rebuild packages just because they contain preserved # libs that happen to be consumers of other preserved libs. for libs in plib_dict.values(): diff --git a/portage_with_autodep/pym/portage/_sets/libs.pyo b/portage_with_autodep/pym/portage/_sets/libs.pyo Binary files differindex 72fc1bb..7a9f6d1 100644 --- a/portage_with_autodep/pym/portage/_sets/libs.pyo +++ b/portage_with_autodep/pym/portage/_sets/libs.pyo diff --git a/portage_with_autodep/pym/portage/_sets/profiles.pyo b/portage_with_autodep/pym/portage/_sets/profiles.pyo Binary files differindex 9502044..9ce72f1 100644 --- a/portage_with_autodep/pym/portage/_sets/profiles.pyo +++ b/portage_with_autodep/pym/portage/_sets/profiles.pyo diff --git a/portage_with_autodep/pym/portage/_sets/security.py b/portage_with_autodep/pym/portage/_sets/security.py index 7e856bc..f8dbef2 100644 --- a/portage_with_autodep/pym/portage/_sets/security.py +++ b/portage_with_autodep/pym/portage/_sets/security.py @@ -44,8 +44,8 @@ class SecuritySet(PackageSet): mydict = {} for atom in atomlist[:]: cpv = self._portdbapi.xmatch("match-all", atom)[0] - slot = self._portdbapi.aux_get(cpv, ["SLOT"])[0] - cps = "%s:%s" % (cpv.cp, slot) + pkg = self._portdbapi._pkg_str(cpv, None) + cps = "%s:%s" % (pkg.cp, pkg.slot) if not cps in mydict: mydict[cps] = (atom, cpv) else: diff --git a/portage_with_autodep/pym/portage/_sets/security.pyo b/portage_with_autodep/pym/portage/_sets/security.pyo Binary files differindex ea67514..9603bc8 100644 --- a/portage_with_autodep/pym/portage/_sets/security.pyo +++ b/portage_with_autodep/pym/portage/_sets/security.pyo diff --git a/portage_with_autodep/pym/portage/_sets/shell.pyo b/portage_with_autodep/pym/portage/_sets/shell.pyo Binary files differindex e5e4561..f153c51 100644 --- a/portage_with_autodep/pym/portage/_sets/shell.pyo +++ b/portage_with_autodep/pym/portage/_sets/shell.pyo diff --git a/portage_with_autodep/pym/portage/cache/__init__.pyo b/portage_with_autodep/pym/portage/cache/__init__.pyo Binary files differindex eb5a90e..735ab0b 100644 --- a/portage_with_autodep/pym/portage/cache/__init__.pyo +++ b/portage_with_autodep/pym/portage/cache/__init__.pyo diff --git a/portage_with_autodep/pym/portage/cache/anydbm.pyo b/portage_with_autodep/pym/portage/cache/anydbm.pyo Binary files differindex 5946da9..8422437 100644 --- a/portage_with_autodep/pym/portage/cache/anydbm.pyo +++ b/portage_with_autodep/pym/portage/cache/anydbm.pyo diff --git a/portage_with_autodep/pym/portage/cache/cache_errors.pyo b/portage_with_autodep/pym/portage/cache/cache_errors.pyo Binary files differindex 866088e..e323de7 100644 --- a/portage_with_autodep/pym/portage/cache/cache_errors.pyo +++ b/portage_with_autodep/pym/portage/cache/cache_errors.pyo diff --git a/portage_with_autodep/pym/portage/cache/ebuild_xattr.py b/portage_with_autodep/pym/portage/cache/ebuild_xattr.py index 0086e40..db6e177 100644 --- a/portage_with_autodep/pym/portage/cache/ebuild_xattr.py +++ b/portage_with_autodep/pym/portage/cache/ebuild_xattr.py @@ -1,4 +1,4 @@ -# -*- coding: UTF8 -*- +# -*- coding: utf-8 -*- # Copyright: 2009-2011 Gentoo Foundation # Author(s): Petteri Räty (betelgeuse@gentoo.org) # License: GPL2 diff --git a/portage_with_autodep/pym/portage/cache/ebuild_xattr.pyo b/portage_with_autodep/pym/portage/cache/ebuild_xattr.pyo Binary files differindex fe32dcc..405f372 100644 --- a/portage_with_autodep/pym/portage/cache/ebuild_xattr.pyo +++ b/portage_with_autodep/pym/portage/cache/ebuild_xattr.pyo diff --git a/portage_with_autodep/pym/portage/cache/flat_hash.py b/portage_with_autodep/pym/portage/cache/flat_hash.py index 2eae9f6..08dcbe8 100644 --- a/portage_with_autodep/pym/portage/cache/flat_hash.py +++ b/portage_with_autodep/pym/portage/cache/flat_hash.py @@ -1,7 +1,9 @@ -# Copyright: 2005-2011 Gentoo Foundation +# Copyright: 2005-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # Author(s): Brian Harring (ferringb@gentoo.org) +from __future__ import unicode_literals + from portage.cache import fs_template from portage.cache import cache_errors import errno @@ -11,16 +13,13 @@ import sys import os as _os from portage import os from portage import _encodings -from portage import _unicode_decode from portage import _unicode_encode +from portage.exception import InvalidData +from portage.versions import _pkg_str if sys.hexversion >= 0x3000000: long = int -# Coerce to unicode, in order to prevent TypeError when writing -# raw bytes to TextIOWrapper with python2. -_setitem_fmt = _unicode_decode("%s=%s\n") - class database(fs_template.FsBased): autocommits = True @@ -40,11 +39,10 @@ class database(fs_template.FsBased): # Don't use os.path.join, for better performance. fp = self.location + _os.sep + cpv try: - myf = io.open(_unicode_encode(fp, + with io.open(_unicode_encode(fp, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], - errors='replace') - try: + errors='replace') as myf: lines = myf.read().split("\n") if not lines[-1]: lines.pop() @@ -54,8 +52,6 @@ class database(fs_template.FsBased): # that uses mtime mangling. d['_mtime_'] = _os.fstat(myf.fileno())[stat.ST_MTIME] return d - finally: - myf.close() except (IOError, OSError) as e: if e.errno != errno.ENOENT: raise cache_errors.CacheCorruption(cpv, e) @@ -94,7 +90,10 @@ class database(fs_template.FsBased): v = values.get(k) if not v: continue - myf.write(_setitem_fmt % (k, v)) + # NOTE: This format string requires unicode_literals, so that + # k and v are coerced to unicode, in order to prevent TypeError + # when writing raw bytes to TextIOWrapper with Python 2. + myf.write("%s=%s\n" % (k, v)) finally: myf.close() self._ensure_access(fp) @@ -135,8 +134,6 @@ class database(fs_template.FsBased): del e continue for l in dir_list: - if l.endswith(".cpickle"): - continue p = os.path.join(dir_path, l) try: st = os.lstat(p) @@ -151,7 +148,11 @@ class database(fs_template.FsBased): if depth < 1: dirs.append((depth+1, p)) continue - yield p[len_base+1:] + + try: + yield _pkg_str(p[len_base+1:]) + except InvalidData: + continue class md5_database(database): diff --git a/portage_with_autodep/pym/portage/cache/flat_hash.pyo b/portage_with_autodep/pym/portage/cache/flat_hash.pyo Binary files differindex 4f568a8..90a1b36 100644 --- a/portage_with_autodep/pym/portage/cache/flat_hash.pyo +++ b/portage_with_autodep/pym/portage/cache/flat_hash.pyo diff --git a/portage_with_autodep/pym/portage/cache/flat_list.py b/portage_with_autodep/pym/portage/cache/flat_list.py deleted file mode 100644 index 7288307..0000000 --- a/portage_with_autodep/pym/portage/cache/flat_list.py +++ /dev/null @@ -1,134 +0,0 @@ -# Copyright 2005-2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.cache import fs_template -from portage.cache import cache_errors -from portage import os -from portage import _encodings -from portage import _unicode_decode -from portage import _unicode_encode -import errno -import io -import stat -import sys - -if sys.hexversion >= 0x3000000: - long = int - -# Coerce to unicode, in order to prevent TypeError when writing -# raw bytes to TextIOWrapper with python2. -_setitem_fmt = _unicode_decode("%s\n") - -# store the current key order *here*. -class database(fs_template.FsBased): - - autocommits = True - - # do not screw with this ordering. _eclasses_ needs to be last - auxdbkey_order=('DEPEND', 'RDEPEND', 'SLOT', 'SRC_URI', - 'RESTRICT', 'HOMEPAGE', 'LICENSE', 'DESCRIPTION', - 'KEYWORDS', 'IUSE', 'REQUIRED_USE', - 'PDEPEND', 'PROVIDE', 'EAPI', 'PROPERTIES', 'DEFINED_PHASES') - - def __init__(self, *args, **config): - super(database,self).__init__(*args, **config) - self.location = os.path.join(self.location, - self.label.lstrip(os.path.sep).rstrip(os.path.sep)) - - if len(self._known_keys) > len(self.auxdbkey_order) + 2: - raise Exception("less ordered keys then auxdbkeys") - if not os.path.exists(self.location): - self._ensure_dirs() - - - def _getitem(self, cpv): - d = {} - try: - myf = io.open(_unicode_encode(os.path.join(self.location, cpv), - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content'], - errors='replace') - for k,v in zip(self.auxdbkey_order, myf): - d[k] = v.rstrip("\n") - except (OSError, IOError) as e: - if errno.ENOENT == e.errno: - raise KeyError(cpv) - raise cache_errors.CacheCorruption(cpv, e) - - try: - d["_mtime_"] = os.fstat(myf.fileno())[stat.ST_MTIME] - except OSError as e: - myf.close() - raise cache_errors.CacheCorruption(cpv, e) - myf.close() - return d - - - def _setitem(self, cpv, values): - s = cpv.rfind("/") - fp=os.path.join(self.location,cpv[:s],".update.%i.%s" % (os.getpid(), cpv[s+1:])) - try: - myf = io.open(_unicode_encode(fp, - encoding=_encodings['fs'], errors='strict'), - mode='w', encoding=_encodings['repo.content'], - errors='backslashreplace') - except (OSError, IOError) as e: - if errno.ENOENT == e.errno: - try: - self._ensure_dirs(cpv) - myf = io.open(_unicode_encode(fp, - encoding=_encodings['fs'], errors='strict'), - mode='w', encoding=_encodings['repo.content'], - errors='backslashreplace') - except (OSError, IOError) as e: - raise cache_errors.CacheCorruption(cpv, e) - else: - raise cache_errors.CacheCorruption(cpv, e) - - - for x in self.auxdbkey_order: - myf.write(_setitem_fmt % (values.get(x, ""),)) - - myf.close() - self._ensure_access(fp, mtime=values["_mtime_"]) - #update written. now we move it. - new_fp = os.path.join(self.location,cpv) - try: - os.rename(fp, new_fp) - except (OSError, IOError) as e: - os.remove(fp) - raise cache_errors.CacheCorruption(cpv, e) - - - def _delitem(self, cpv): - try: - os.remove(os.path.join(self.location,cpv)) - except OSError as e: - if errno.ENOENT == e.errno: - raise KeyError(cpv) - else: - raise cache_errors.CacheCorruption(cpv, e) - - - def __contains__(self, cpv): - return os.path.exists(os.path.join(self.location, cpv)) - - - def __iter__(self): - """generator for walking the dir struct""" - dirs = [self.location] - len_base = len(self.location) - while len(dirs): - for l in os.listdir(dirs[0]): - if l.endswith(".cpickle"): - continue - p = os.path.join(dirs[0],l) - st = os.lstat(p) - if stat.S_ISDIR(st.st_mode): - dirs.append(p) - continue - yield p[len_base+1:] - dirs.pop(0) - - - def commit(self): pass diff --git a/portage_with_autodep/pym/portage/cache/flat_list.pyo b/portage_with_autodep/pym/portage/cache/flat_list.pyo Binary files differdeleted file mode 100644 index ab7dc82..0000000 --- a/portage_with_autodep/pym/portage/cache/flat_list.pyo +++ /dev/null diff --git a/portage_with_autodep/pym/portage/cache/fs_template.py b/portage_with_autodep/pym/portage/cache/fs_template.py index a82e862..0567c72 100644 --- a/portage_with_autodep/pym/portage/cache/fs_template.py +++ b/portage_with_autodep/pym/portage/cache/fs_template.py @@ -1,14 +1,14 @@ -# Copyright: 2005 Gentoo Foundation +# Copyright 2005-2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 # Author(s): Brian Harring (ferringb@gentoo.org) -# License: GPL2 +import os as _os import sys from portage.cache import template from portage import os from portage.proxy.lazyimport import lazyimport lazyimport(globals(), - 'portage.data:portage_gid', 'portage.exception:PortageException', 'portage.util:apply_permissions', ) @@ -22,14 +22,11 @@ class FsBased(template.database): attempt to ensure files have the specified owners/perms""" def __init__(self, *args, **config): - """throws InitializationError if needs args aren't specified - gid and perms aren't listed do to an oddity python currying mechanism - gid=portage_gid - perms=0665""" for x, y in (("gid", -1), ("perms", -1)): if x in config: - setattr(self, "_"+x, config[x]) + # Since Python 3.4, chown requires int type (no proxies). + setattr(self, "_" + x, int(config[x])) del config[x] else: setattr(self, "_"+x, y) @@ -78,7 +75,17 @@ class FsBased(template.database): if self._perms != -1: os.umask(um) - + def _prune_empty_dirs(self): + all_dirs = [] + for parent, dirs, files in os.walk(self.location): + for x in dirs: + all_dirs.append(_os.path.join(parent, x)) + while all_dirs: + try: + _os.rmdir(all_dirs.pop()) + except OSError: + pass + def gen_label(base, label): """if supplied label is a path, generate a unique label based upon label, and supplied base path""" if label.find(os.path.sep) == -1: diff --git a/portage_with_autodep/pym/portage/cache/fs_template.pyo b/portage_with_autodep/pym/portage/cache/fs_template.pyo Binary files differindex 6cbbc2f..de71ee6 100644 --- a/portage_with_autodep/pym/portage/cache/fs_template.pyo +++ b/portage_with_autodep/pym/portage/cache/fs_template.pyo diff --git a/portage_with_autodep/pym/portage/cache/mappings.py b/portage_with_autodep/pym/portage/cache/mappings.py index bc8ce9a..cd39a6e 100644 --- a/portage_with_autodep/pym/portage/cache/mappings.py +++ b/portage_with_autodep/pym/portage/cache/mappings.py @@ -199,10 +199,10 @@ class OrderedDict(UserDict): return iter(self._order) def __setitem__(self, key, item): - if key in self: - self._order.remove(key) + new_key = key not in self UserDict.__setitem__(self, key, item) - self._order.append(key) + if new_key: + self._order.append(key) def __delitem__(self, key): UserDict.__delitem__(self, key) diff --git a/portage_with_autodep/pym/portage/cache/mappings.pyo b/portage_with_autodep/pym/portage/cache/mappings.pyo Binary files differindex 1eb3f4f..c183246 100644 --- a/portage_with_autodep/pym/portage/cache/mappings.pyo +++ b/portage_with_autodep/pym/portage/cache/mappings.pyo diff --git a/portage_with_autodep/pym/portage/cache/metadata.py b/portage_with_autodep/pym/portage/cache/metadata.py index 9d2c3a5..6612e73 100644 --- a/portage_with_autodep/pym/portage/cache/metadata.py +++ b/portage_with_autodep/pym/portage/cache/metadata.py @@ -28,7 +28,8 @@ class database(flat_hash.database): auxdbkey_order=('DEPEND', 'RDEPEND', 'SLOT', 'SRC_URI', 'RESTRICT', 'HOMEPAGE', 'LICENSE', 'DESCRIPTION', 'KEYWORDS', 'INHERITED', 'IUSE', 'REQUIRED_USE', - 'PDEPEND', 'PROVIDE', 'EAPI', 'PROPERTIES', 'DEFINED_PHASES') + 'PDEPEND', 'PROVIDE', 'EAPI', 'PROPERTIES', + 'DEFINED_PHASES', 'HDEPEND') autocommits = True serialize_eclasses = False diff --git a/portage_with_autodep/pym/portage/cache/metadata.pyo b/portage_with_autodep/pym/portage/cache/metadata.pyo Binary files differindex c98445b..649ec5b 100644 --- a/portage_with_autodep/pym/portage/cache/metadata.pyo +++ b/portage_with_autodep/pym/portage/cache/metadata.pyo diff --git a/portage_with_autodep/pym/portage/cache/sql_template.pyo b/portage_with_autodep/pym/portage/cache/sql_template.pyo Binary files differindex e2c5974..54636ed 100644 --- a/portage_with_autodep/pym/portage/cache/sql_template.pyo +++ b/portage_with_autodep/pym/portage/cache/sql_template.pyo diff --git a/portage_with_autodep/pym/portage/cache/sqlite.py b/portage_with_autodep/pym/portage/cache/sqlite.py index fcc62ff..40db070 100644 --- a/portage_with_autodep/pym/portage/cache/sqlite.py +++ b/portage_with_autodep/pym/portage/cache/sqlite.py @@ -1,6 +1,9 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + +import re import sys from portage.cache import fs_template from portage.cache import cache_errors @@ -20,7 +23,6 @@ class database(fs_template.FsBased): # to calculate the number of pages requested, according to the following # equation: cache_bytes = page_bytes * page_count cache_bytes = 1024 * 1024 * 10 - _db_table = None def __init__(self, *args, **config): super(database, self).__init__(*args, **config) @@ -28,6 +30,7 @@ class database(fs_template.FsBased): self._allowed_keys = ["_mtime_", "_eclasses_"] self._allowed_keys.extend(self._known_keys) self._allowed_keys.sort() + self._allowed_keys_set = frozenset(self._allowed_keys) self.location = os.path.join(self.location, self.label.lstrip(os.path.sep).rstrip(os.path.sep)) @@ -37,8 +40,8 @@ class database(fs_template.FsBased): config.setdefault("autocommit", self.autocommits) config.setdefault("cache_bytes", self.cache_bytes) config.setdefault("synchronous", self.synchronous) - # Timeout for throwing a "database is locked" exception (pysqlite - # default is 5.0 seconds). + # Set longer timeout for throwing a "database is locked" exception. + # Default timeout in sqlite3 module is 5.0 seconds. config.setdefault("timeout", 15) self._db_init_connection(config) self._db_init_structures() @@ -47,11 +50,8 @@ class database(fs_template.FsBased): # sqlite3 is optional with >=python-2.5 try: import sqlite3 as db_module - except ImportError: - try: - from pysqlite2 import dbapi2 as db_module - except ImportError as e: - raise cache_errors.InitializationError(self.__class__, e) + except ImportError as e: + raise cache_errors.InitializationError(self.__class__, e) self._db_module = db_module self._db_error = db_module.Error @@ -62,7 +62,6 @@ class database(fs_template.FsBased): # Avoid potential UnicodeEncodeError in python-2.x by # only calling str() when it's absolutely necessary. s = str(s) - # This is equivalent to the _quote function from pysqlite 1.1. return "'%s'" % s.replace("'", "''") def _db_init_connection(self, config): @@ -92,9 +91,6 @@ class database(fs_template.FsBased): self._db_table["packages"]["table_name"] = mytable self._db_table["packages"]["package_id"] = "internal_db_package_id" self._db_table["packages"]["package_key"] = "portage_package_key" - self._db_table["packages"]["internal_columns"] = \ - [self._db_table["packages"]["package_id"], - self._db_table["packages"]["package_key"]] create_statement = [] create_statement.append("CREATE TABLE") create_statement.append(mytable) @@ -109,15 +105,18 @@ class database(fs_template.FsBased): create_statement.append(")") self._db_table["packages"]["create"] = " ".join(create_statement) - self._db_table["packages"]["columns"] = \ - self._db_table["packages"]["internal_columns"] + \ - self._allowed_keys cursor = self._db_cursor for k, v in self._db_table.items(): if self._db_table_exists(v["table_name"]): create_statement = self._db_table_get_create(v["table_name"]) - if create_statement != v["create"]: + table_ok, missing_keys = self._db_validate_create_statement(create_statement) + if table_ok: + if missing_keys: + for k in sorted(missing_keys): + cursor.execute("ALTER TABLE %s ADD COLUMN %s TEXT" % + (self._db_table["packages"]["table_name"], k)) + else: writemsg(_("sqlite: dropping old table: %s\n") % v["table_name"]) cursor.execute("DROP TABLE %s" % v["table_name"]) cursor.execute(v["create"]) @@ -138,6 +137,37 @@ class database(fs_template.FsBased): self._db_escape_string(table_name)) return cursor.fetchall()[0][0] + def _db_validate_create_statement(self, statement): + missing_keys = None + if statement == self._db_table["packages"]["create"]: + return True, missing_keys + + m = re.match(r'^\s*CREATE\s*TABLE\s*%s\s*\(\s*%s\s*INTEGER\s*PRIMARY\s*KEY\s*AUTOINCREMENT\s*,(.*)\)\s*$' % + (self._db_table["packages"]["table_name"], + self._db_table["packages"]["package_id"]), + statement) + if m is None: + return False, missing_keys + + unique_constraints = set([self._db_table["packages"]["package_key"]]) + missing_keys = set(self._allowed_keys) + unique_re = re.compile(r'^\s*UNIQUE\s*\(\s*(\w*)\s*\)\s*$') + column_re = re.compile(r'^\s*(\w*)\s*TEXT\s*$') + for x in m.group(1).split(","): + m = column_re.match(x) + if m is not None: + missing_keys.discard(m.group(1)) + continue + m = unique_re.match(x) + if m is not None: + unique_constraints.discard(m.group(1)) + continue + + if unique_constraints: + return False, missing_keys + + return True, missing_keys + def _db_init_cache_size(self, cache_bytes): cursor = self._db_cursor cursor.execute("PRAGMA page_size") @@ -173,13 +203,17 @@ class database(fs_template.FsBased): raise KeyError(cpv) else: raise cache_errors.CacheCorruption(cpv, "key is not unique") + result = result[0] d = {} - internal_columns = self._db_table["packages"]["internal_columns"] - column_index = -1 - for k in self._db_table["packages"]["columns"]: - column_index +=1 - if k not in internal_columns: - d[k] = result[0][column_index] + allowed_keys_set = self._allowed_keys_set + for column_index, column_info in enumerate(cursor.description): + k = column_info[0] + if k in allowed_keys_set: + v = result[column_index] + if v is None: + # This happens after a new empty column has been added. + v = "" + d[k] = v return d diff --git a/portage_with_autodep/pym/portage/cache/sqlite.pyo b/portage_with_autodep/pym/portage/cache/sqlite.pyo Binary files differindex a82d25f..aba2f6c 100644 --- a/portage_with_autodep/pym/portage/cache/sqlite.pyo +++ b/portage_with_autodep/pym/portage/cache/sqlite.pyo diff --git a/portage_with_autodep/pym/portage/cache/template.py b/portage_with_autodep/pym/portage/cache/template.py index cf1e8ae..9b8a4d3 100644 --- a/portage_with_autodep/pym/portage/cache/template.py +++ b/portage_with_autodep/pym/portage/cache/template.py @@ -1,6 +1,6 @@ -# Copyright: 2005-2012 Gentoo Foundation +# Copyright 2005-2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 # Author(s): Brian Harring (ferringb@gentoo.org) -# License: GPL2 from portage.cache import cache_errors from portage.cache.cache_errors import InvalidRestriction @@ -164,7 +164,14 @@ class database(object): def commit(self): if not self.autocommits: - raise NotImplementedError + raise NotImplementedError(self) + + def __del__(self): + # This used to be handled by an atexit hook that called + # close_portdbapi_caches() for all portdbapi instances, but that was + # prone to memory leaks for API consumers that needed to create/destroy + # many portdbapi instances. So, instead we rely on __del__. + self.sync() def __contains__(self, cpv): """This method should always be overridden. It is provided only for diff --git a/portage_with_autodep/pym/portage/cache/template.pyo b/portage_with_autodep/pym/portage/cache/template.pyo Binary files differindex 45da015..3315771 100644 --- a/portage_with_autodep/pym/portage/cache/template.pyo +++ b/portage_with_autodep/pym/portage/cache/template.pyo diff --git a/portage_with_autodep/pym/portage/cache/volatile.pyo b/portage_with_autodep/pym/portage/cache/volatile.pyo Binary files differindex fac5d55..9ebb5b8 100644 --- a/portage_with_autodep/pym/portage/cache/volatile.pyo +++ b/portage_with_autodep/pym/portage/cache/volatile.pyo diff --git a/portage_with_autodep/pym/portage/checksum.py b/portage_with_autodep/pym/portage/checksum.py index bd416ac..cd1572e 100644 --- a/portage_with_autodep/pym/portage/checksum.py +++ b/portage_with_autodep/pym/portage/checksum.py @@ -1,5 +1,5 @@ # checksum.py -- core Portage functionality -# Copyright 1998-2012 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import portage @@ -10,6 +10,8 @@ from portage import _encodings from portage import _unicode_encode import errno import stat +import sys +import subprocess import tempfile #dict of all available hash functions @@ -48,16 +50,15 @@ class _generate_hash_function(object): @type filename: String @return: The hash and size of the data """ - f = _open_file(filename) - blocksize = HASHING_BLOCKSIZE - data = f.read(blocksize) - size = 0 - checksum = self._hashobject() - while data: - checksum.update(data) - size = size + len(data) + with _open_file(filename) as f: + blocksize = HASHING_BLOCKSIZE + size = 0 + checksum = self._hashobject() data = f.read(blocksize) - f.close() + while data: + checksum.update(data) + size = size + len(data) + data = f.read(blocksize) return (checksum.hexdigest(), size) @@ -137,8 +138,10 @@ try: except ImportError: pass +_whirlpool_unaccelerated = False if "WHIRLPOOL" not in hashfunc_map: # Bundled WHIRLPOOL implementation + _whirlpool_unaccelerated = True from portage.util.whirlpool import new as _new_whirlpool whirlpoolhash = _generate_hash_function("WHIRLPOOL", _new_whirlpool, origin="bundled") @@ -161,11 +164,16 @@ hashfunc_map["size"] = getsize prelink_capable = False if os.path.exists(PRELINK_BINARY): - results = portage.subprocess_getstatusoutput( - "%s --version > /dev/null 2>&1" % (PRELINK_BINARY,)) - if (results[0] >> 8) == 0: + cmd = [PRELINK_BINARY, "--version"] + cmd = [_unicode_encode(x, encoding=_encodings['fs'], errors='strict') + for x in cmd] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + proc.communicate() + status = proc.wait() + if os.WIFEXITED(status) and os.WEXITSTATUS(status) == os.EX_OK: prelink_capable=1 - del results + del cmd, proc, status def is_prelinkable_elf(filename): f = _open_file(filename) @@ -197,6 +205,82 @@ def get_hash_origin(hashtype): raise KeyError(hashtype) return hashorigin_map.get(hashtype, "unknown") +def _filter_unaccelarated_hashes(digests): + """ + If multiple digests are available and some are unaccelerated, + then return a new dict that omits the unaccelerated ones. This + allows extreme performance problems like bug #425046 to be + avoided whenever practical, especially for cases like stage + builds where acceleration may not be available for some hashes + due to minimization of dependencies. + """ + if _whirlpool_unaccelerated and "WHIRLPOOL" in digests: + verifiable_hash_types = set(digests).intersection(hashfunc_map) + verifiable_hash_types.discard("size") + if len(verifiable_hash_types) > 1: + digests = dict(digests) + digests.pop("WHIRLPOOL") + + return digests + +class _hash_filter(object): + """ + Implements filtering for PORTAGE_CHECKSUM_FILTER. + """ + + __slots__ = ('transparent', '_tokens',) + + def __init__(self, filter_str): + tokens = filter_str.upper().split() + if not tokens or tokens[-1] == "*": + del tokens[:] + self.transparent = not tokens + tokens.reverse() + self._tokens = tuple(tokens) + + def __call__(self, hash_name): + if self.transparent: + return True + matches = ("*", hash_name) + for token in self._tokens: + if token in matches: + return True + elif token[:1] == "-": + if token[1:] in matches: + return False + return False + +def _apply_hash_filter(digests, hash_filter): + """ + Return a new dict containing the filtered digests, or the same + dict if no changes are necessary. This will always preserve at + at least one digest, in order to ensure that they are not all + discarded. + @param digests: dictionary of digests + @type digests: dict + @param hash_filter: A callable that takes a single hash name + argument, and returns True if the hash is to be used or + False otherwise + @type hash_filter: callable + """ + + verifiable_hash_types = set(digests).intersection(hashfunc_map) + verifiable_hash_types.discard("size") + modified = False + if len(verifiable_hash_types) > 1: + for k in list(verifiable_hash_types): + if not hash_filter(k): + modified = True + verifiable_hash_types.remove(k) + if len(verifiable_hash_types) == 1: + break + + if modified: + digests = dict((k, v) for (k, v) in digests.items() + if k == "size" or k in verifiable_hash_types) + + return digests + def verify_all(filename, mydict, calc_prelink=0, strict=0): """ Verify all checksums against a file. diff --git a/portage_with_autodep/pym/portage/checksum.pyo b/portage_with_autodep/pym/portage/checksum.pyo Binary files differindex 00231af..46441a3 100644 --- a/portage_with_autodep/pym/portage/checksum.pyo +++ b/portage_with_autodep/pym/portage/checksum.pyo diff --git a/portage_with_autodep/pym/portage/const.py b/portage_with_autodep/pym/portage/const.py index 614dcdb..0ac503f 100644 --- a/portage_with_autodep/pym/portage/const.py +++ b/portage_with_autodep/pym/portage/const.py @@ -1,7 +1,9 @@ # portage: Constants -# Copyright 1998-2011 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + import os # =========================================================================== @@ -27,8 +29,8 @@ import os # The variables in this file are grouped by config_root, target_root. # variables used with config_root (these need to be relative) -MAKE_CONF_FILE = "etc/make.conf" USER_CONFIG_PATH = "etc/portage" +MAKE_CONF_FILE = USER_CONFIG_PATH + "/make.conf" MODULES_FILE_PATH = USER_CONFIG_PATH + "/modules" CUSTOM_PROFILE_PATH = USER_CONFIG_PATH + "/profile" USER_VIRTUALS_FILE = USER_CONFIG_PATH + "/virtuals" @@ -36,7 +38,7 @@ EBUILD_SH_ENV_FILE = USER_CONFIG_PATH + "/bashrc" EBUILD_SH_ENV_DIR = USER_CONFIG_PATH + "/env" CUSTOM_MIRRORS_FILE = USER_CONFIG_PATH + "/mirrors" COLOR_MAP_FILE = USER_CONFIG_PATH + "/color.map" -PROFILE_PATH = "etc/make.profile" +PROFILE_PATH = USER_CONFIG_PATH + "/make.profile" MAKE_DEFAULTS_FILE = PROFILE_PATH + "/make.defaults" # FIXME: not used DEPRECATED_PROFILE_FILE = PROFILE_PATH + "/deprecated" @@ -56,7 +58,10 @@ DEPCACHE_PATH = "/var/cache/edb/dep" GLOBAL_CONFIG_PATH = "/usr/share/portage/config" # these variables are not used with target_root or config_root -PORTAGE_BASE_PATH = os.path.join(os.sep, os.sep.join(__file__.split(os.sep)[:-3])) +# NOTE: Use realpath(__file__) so that python module symlinks in site-packages +# are followed back to the real location of the whole portage installation. +PORTAGE_BASE_PATH = os.path.join(os.sep, os.sep.join(os.path.realpath( + __file__.rstrip("co")).split(os.sep)[:-3])) PORTAGE_BIN_PATH = PORTAGE_BASE_PATH + "/bin" PORTAGE_PYM_PATH = PORTAGE_BASE_PATH + "/pym" LOCALE_DATA_PATH = PORTAGE_BASE_PATH + "/locale" # FIXME: not used @@ -69,7 +74,6 @@ MOVE_BINARY = "/bin/mv" PRELINK_BINARY = "/usr/sbin/prelink" AUTODEP_LIBRARY = "/usr/lib/file_hook.so" - INVALID_ENV_FILE = "/etc/spork/is/not/valid/profile.env" REPO_NAME_FILE = "repo_name" REPO_NAME_LOC = "profiles" + "/" + REPO_NAME_FILE @@ -77,50 +81,139 @@ REPO_NAME_LOC = "profiles" + "/" + REPO_NAME_FILE PORTAGE_PACKAGE_ATOM = "sys-apps/portage" LIBC_PACKAGE_ATOM = "virtual/libc" OS_HEADERS_PACKAGE_ATOM = "virtual/os-headers" +CVS_PACKAGE_ATOM = "dev-vcs/cvs" +GIT_PACKAGE_ATOM = "dev-vcs/git" +RSYNC_PACKAGE_ATOM = "net-misc/rsync" -INCREMENTALS = ("USE", "USE_EXPAND", "USE_EXPAND_HIDDEN", - "FEATURES", "ACCEPT_KEYWORDS", - "CONFIG_PROTECT_MASK", "CONFIG_PROTECT", - "PRELINK_PATH", "PRELINK_PATH_MASK", - "PROFILE_ONLY_VARIABLES") -EBUILD_PHASES = ("pretend", "setup", "unpack", "prepare", "configure", - "compile", "test", "install", - "package", "preinst", "postinst","prerm", "postrm", - "nofetch", "config", "info", "other") +INCREMENTALS = ( + "ACCEPT_KEYWORDS", + "CONFIG_PROTECT", + "CONFIG_PROTECT_MASK", + "FEATURES", + "IUSE_IMPLICIT", + "PRELINK_PATH", + "PRELINK_PATH_MASK", + "PROFILE_ONLY_VARIABLES", + "USE", + "USE_EXPAND", + "USE_EXPAND_HIDDEN", + "USE_EXPAND_IMPLICIT", + "USE_EXPAND_UNPREFIXED", +) +EBUILD_PHASES = ( + "pretend", + "setup", + "unpack", + "prepare", + "configure", + "compile", + "test", + "install", + "package", + "preinst", + "postinst", + "prerm", + "postrm", + "nofetch", + "config", + "info", + "other", +) SUPPORTED_FEATURES = frozenset([ - "assume-digests", "binpkg-logs", "buildpkg", "buildsyspkg", "candy", - "ccache", "chflags", "clean-logs", - "collision-protect", "compress-build-logs", "compressdebug", - "config-protect-if-modified", "depcheck", "depcheckstrict", - "digest", "distcc", "distcc-pump", "distlocks", "ebuild-locks", "fakeroot", - "fail-clean", "force-mirror", "force-prefix", "getbinpkg", - "installsources", "keeptemp", "keepwork", "fixlafiles", "lmirror", - "metadata-transfer", "mirror", "multilib-strict", "news", - "noauto", "noclean", "nodoc", "noinfo", "noman", - "nostrip", "notitles", "parallel-fetch", "parallel-install", - "parse-eapi-ebuild-head", - "prelink-checksums", "preserve-libs", - "protect-owned", "python-trace", "sandbox", - "selinux", "sesandbox", "sfperms", - "sign", "skiprocheck", "split-elog", "split-log", "splitdebug", - "strict", "stricter", "suidctl", "test", "test-fail-continue", - "unknown-features-filter", "unknown-features-warn", - "unmerge-logs", "unmerge-orphans", "userfetch", "userpriv", - "usersandbox", "usersync", "webrsync-gpg", "xattr"]) - -EAPI = 4 + "assume-digests", + "binpkg-logs", + "buildpkg", + "buildsyspkg", + "candy", + "ccache", + "cgroup", + "chflags", + "clean-logs", + "collision-protect", + "compress-build-logs", + "compressdebug", + "compress-index", + "config-protect-if-modified", + "depcheck", + "depcheckstrict", + "digest", + "distcc", + "distcc-pump", + "distlocks", + "downgrade-backup", + "ebuild-locks", + "fail-clean", + "fakeroot", + "fixlafiles", + "force-mirror", + "force-prefix", + "getbinpkg", + "installsources", + "ipc-sandbox", + "keeptemp", + "keepwork", + "lmirror", + "merge-sync", + "metadata-transfer", + "mirror", + "multilib-strict", + "network-sandbox", + "news", + "noauto", + "noclean", + "nodoc", + "noinfo", + "noman", + "nostrip", + "notitles", + "parallel-fetch", + "parallel-install", + "prelink-checksums", + "preserve-libs", + "protect-owned", + "python-trace", + "sandbox", + "selinux", + "sesandbox", + "sfperms", + "sign", + "skiprocheck", + "splitdebug", + "split-elog", + "split-log", + "strict", + "stricter", + "suidctl", + "test", + "test-fail-continue", + "unknown-features-filter", + "unknown-features-warn", + "unmerge-backup", + "unmerge-logs", + "unmerge-orphans", + "userfetch", + "userpriv", + "usersandbox", + "usersync", + "webrsync-gpg", + "xattr", +]) + +EAPI = 5 HASHING_BLOCKSIZE = 32768 MANIFEST1_HASH_FUNCTIONS = ("MD5", "SHA256", "RMD160") MANIFEST1_REQUIRED_HASH = "MD5" -# Future events: +# Past events: # -# After WHIRLPOOL is supported in stable portage: -# - Add SHA256 and WHIRLPOOL to MANIFEST2_HASH_DEFAULTS. -# - Remove SHA1 and RMD160 from MANIFEST2_HASH_*. +# 20120704 - After WHIRLPOOL is supported in stable portage: # - Set manifest-hashes in gentoo-x86/metadata/layout.conf as follows: # manifest-hashes = SHA256 SHA512 WHIRLPOOL +# - Add SHA512 and WHIRLPOOL to MANIFEST2_HASH_DEFAULTS. +# - Remove SHA1 and RMD160 from MANIFEST2_HASH_*. +# +# Future events: # # After WHIRLPOOL is supported in stable portage for at least 1 year: # - Change MANIFEST2_REQUIRED_HASH to WHIRLPOOL. @@ -138,8 +231,8 @@ MANIFEST1_REQUIRED_HASH = "MD5" # After layout.conf settings correspond to defaults in stable portage: # - Remove redundant settings from gentoo-x86/metadata/layout.conf. -MANIFEST2_HASH_FUNCTIONS = ("RMD160", "SHA1", "SHA256", "SHA512", "WHIRLPOOL") -MANIFEST2_HASH_DEFAULTS = frozenset(["SHA1", "SHA256", "RMD160"]) +MANIFEST2_HASH_FUNCTIONS = ("SHA256", "SHA512", "WHIRLPOOL") +MANIFEST2_HASH_DEFAULTS = frozenset(["SHA256", "SHA512", "WHIRLPOOL"]) MANIFEST2_REQUIRED_HASH = "SHA256" MANIFEST2_IDENTIFIERS = ("AUX", "MISC", "DIST", "EBUILD") @@ -149,13 +242,35 @@ MANIFEST2_IDENTIFIERS = ("AUX", "MISC", "DIST", "EBUILD") # a config instance (since it's possible to contruct a config instance with # a different EPREFIX). Therefore, the EPREFIX constant should *NOT* be used # in the definition of any other constants within this file. -EPREFIX="" +EPREFIX = "" # pick up EPREFIX from the environment if set if "PORTAGE_OVERRIDE_EPREFIX" in os.environ: EPREFIX = os.environ["PORTAGE_OVERRIDE_EPREFIX"] if EPREFIX: EPREFIX = os.path.normpath(EPREFIX) + if EPREFIX == os.sep: + EPREFIX = "" + +VCS_DIRS = ("CVS", "RCS", "SCCS", ".bzr", ".git", ".hg", ".svn") + +# List of known live eclasses. Keep it in sync with cnf/sets/portage.conf +LIVE_ECLASSES = frozenset([ + "bzr", + "cvs", + "darcs", + "git", + "git-2", + "git-r3", + "mercurial", + "subversion", + "tla", +]) + +SUPPORTED_BINPKG_FORMATS = ("tar", "rpm") + +# Time formats used in various places like metadata.chk. +TIMESTAMP_FORMAT = "%a, %d %b %Y %H:%M:%S +0000" # to be used with time.gmtime() # =========================================================================== # END OF CONSTANTS -- END OF CONSTANTS -- END OF CONSTANTS -- END OF CONSTANT @@ -163,19 +278,5 @@ if "PORTAGE_OVERRIDE_EPREFIX" in os.environ: # Private constants for use in conditional code in order to minimize the diff # between branches. -_ENABLE_DYN_LINK_MAP = True -_ENABLE_PRESERVE_LIBS = True -_ENABLE_REPO_NAME_WARN = True +_DEPCLEAN_LIB_CHECK_DEFAULT = True _ENABLE_SET_CONFIG = True - - -# The definitions above will differ between branches, so it's useful to have -# common lines of diff context here in order to avoid merge conflicts. - -if not _ENABLE_PRESERVE_LIBS: - SUPPORTED_FEATURES = set(SUPPORTED_FEATURES) - SUPPORTED_FEATURES.remove("preserve-libs") - SUPPORTED_FEATURES = frozenset(SUPPORTED_FEATURES) - -if not _ENABLE_SET_CONFIG: - WORLD_SETS_FILE = '/dev/null' diff --git a/portage_with_autodep/pym/portage/const.py.rej b/portage_with_autodep/pym/portage/const.py.rej deleted file mode 100644 index 9fe70f8..0000000 --- a/portage_with_autodep/pym/portage/const.py.rej +++ /dev/null @@ -1,12 +0,0 @@ ---- pym/portage/const.py -+++ pym/portage/const.py -@@ -90,7 +92,8 @@ - SUPPORTED_FEATURES = frozenset([ - "allow-missing-manifests", - "assume-digests", "binpkg-logs", "buildpkg", "buildsyspkg", "candy", -- "ccache", "chflags", "collision-protect", "compress-build-logs", -+ "ccache", "chflags", "collision-protect", "compress-build-logs", -+ "depcheck", "depcheckstrict", - "digest", "distcc", "distcc-pump", "distlocks", "ebuild-locks", "fakeroot", - "fail-clean", "fixpackages", "force-mirror", "getbinpkg", - "installsources", "keeptemp", "keepwork", "fixlafiles", "lmirror", diff --git a/portage_with_autodep/pym/portage/const.pyo b/portage_with_autodep/pym/portage/const.pyo Binary files differindex 804420f..3b9812b 100644 --- a/portage_with_autodep/pym/portage/const.pyo +++ b/portage_with_autodep/pym/portage/const.pyo diff --git a/portage_with_autodep/pym/portage/cvstree.pyo b/portage_with_autodep/pym/portage/cvstree.pyo Binary files differindex 4719daf..1c18d23 100644 --- a/portage_with_autodep/pym/portage/cvstree.pyo +++ b/portage_with_autodep/pym/portage/cvstree.pyo diff --git a/portage_with_autodep/pym/portage/data.py b/portage_with_autodep/pym/portage/data.py index c4d967a..44104c2 100644 --- a/portage_with_autodep/pym/portage/data.py +++ b/portage_with_autodep/pym/portage/data.py @@ -1,13 +1,14 @@ # data.py -- Calculated/Discovered Data Values -# Copyright 1998-2011 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -import os, pwd, grp, platform +import os, pwd, grp, platform, sys import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.output:colorize', 'portage.util:writemsg', + 'subprocess' ) from portage.localization import _ @@ -85,19 +86,27 @@ def _get_global(k): elif portage.const.EPREFIX: secpass = 2 #Discover the uid and gid of the portage user/group + keyerror = False try: portage_uid = pwd.getpwnam(_get_global('_portage_username')).pw_uid - _portage_grpname = _get_global('_portage_grpname') - if platform.python_implementation() == 'PyPy': - # Somehow this prevents "TypeError: expected string" errors - # from grp.getgrnam() with PyPy 1.7 - _portage_grpname = str(_portage_grpname) - portage_gid = grp.getgrnam(_portage_grpname).gr_gid - if secpass < 1 and portage_gid in os.getgroups(): - secpass = 1 except KeyError: + keyerror = True portage_uid = 0 + + try: + portage_gid = grp.getgrnam(_get_global('_portage_grpname')).gr_gid + except KeyError: + keyerror = True portage_gid = 0 + + if secpass < 1 and portage_gid in os.getgroups(): + secpass = 1 + + # Suppress this error message if both PORTAGE_GRPNAME and + # PORTAGE_USERNAME are set to "root", for things like + # Android (see bug #454060). + if keyerror and not (_get_global('_portage_username') == "root" and + _get_global('_portage_grpname') == "root"): writemsg(colorize("BAD", _("portage: 'portage' user or group missing.")) + "\n", noiselevel=-1) writemsg(_( @@ -129,10 +138,28 @@ def _get_global(k): # Get a list of group IDs for the portage user. Do not use # grp.getgrall() since it is known to trigger spurious # SIGPIPE problems with nss_ldap. - mystatus, myoutput = \ - portage.subprocess_getstatusoutput("id -G %s" % _portage_username) - if mystatus == os.EX_OK: - for x in myoutput.split(): + cmd = ["id", "-G", _portage_username] + + if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000: + # Python 3.1 _execvp throws TypeError for non-absolute executable + # path passed as bytes (see http://bugs.python.org/issue8513). + fullname = portage.process.find_binary(cmd[0]) + if fullname is None: + globals()[k] = v + _initialized_globals.add(k) + return v + cmd[0] = fullname + + encoding = portage._encodings['content'] + cmd = [portage._unicode_encode(x, + encoding=encoding, errors='strict') for x in cmd] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + myoutput = proc.communicate()[0] + status = proc.wait() + if os.WIFEXITED(status) and os.WEXITSTATUS(status) == os.EX_OK: + for x in portage._unicode_decode(myoutput, + encoding=encoding, errors='strict').split(): try: v.append(int(x)) except ValueError: @@ -213,10 +240,18 @@ def _init(settings): if '_portage_grpname' not in _initialized_globals and \ '_portage_username' not in _initialized_globals: + # Prevents "TypeError: expected string" errors + # from grp.getgrnam() with PyPy + native_string = platform.python_implementation() == 'PyPy' + v = settings.get('PORTAGE_GRPNAME', 'portage') + if native_string: + v = portage._native_string(v) globals()['_portage_grpname'] = v _initialized_globals.add('_portage_grpname') v = settings.get('PORTAGE_USERNAME', 'portage') + if native_string: + v = portage._native_string(v) globals()['_portage_username'] = v _initialized_globals.add('_portage_username') diff --git a/portage_with_autodep/pym/portage/data.pyo b/portage_with_autodep/pym/portage/data.pyo Binary files differindex 7f749e0..8115c7a 100644 --- a/portage_with_autodep/pym/portage/data.pyo +++ b/portage_with_autodep/pym/portage/data.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/_MergeProcess.py b/portage_with_autodep/pym/portage/dbapi/_MergeProcess.py index b5f6a0b..956dbb9 100644 --- a/portage_with_autodep/pym/portage/dbapi/_MergeProcess.py +++ b/portage_with_autodep/pym/portage/dbapi/_MergeProcess.py @@ -1,7 +1,8 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import io +import platform import signal import sys import traceback @@ -10,10 +11,11 @@ import errno import fcntl import portage from portage import os, _unicode_decode +from portage.util._ctypes import find_library import portage.elog.messages -from _emerge.SpawnProcess import SpawnProcess +from portage.util._async.ForkProcess import ForkProcess -class MergeProcess(SpawnProcess): +class MergeProcess(ForkProcess): """ Merge packages in a subprocess, so the Scheduler can run in the main thread while files are moved or copied asynchronously. @@ -40,11 +42,20 @@ class MergeProcess(SpawnProcess): settings.reset() settings.setcpv(cpv, mydb=self.mydbapi) + # This caches the libc library lookup in the current + # process, so that it's only done once rather than + # for each child process. + if platform.system() == "Linux" and \ + "merge-sync" in settings.features: + find_library("c") + # Inherit stdin by default, so that the pdb SIGUSR1 # handler is usable for the subprocess. if self.fd_pipes is None: self.fd_pipes = {} - self.fd_pipes.setdefault(0, sys.stdin.fileno()) + else: + self.fd_pipes = self.fd_pipes.copy() + self.fd_pipes.setdefault(0, portage._get_stdin().fileno()) super(MergeProcess, self)._start() @@ -90,7 +101,7 @@ class MergeProcess(SpawnProcess): reporter(msg, phase=phase, key=key, out=out) if event & self.scheduler.IO_HUP: - self.scheduler.unregister(self._elog_reg_id) + self.scheduler.source_remove(self._elog_reg_id) self._elog_reg_id = None os.close(self._elog_reader_fd) self._elog_reader_fd = None @@ -101,12 +112,24 @@ class MergeProcess(SpawnProcess): def _spawn(self, args, fd_pipes, **kwargs): """ Fork a subprocess, apply local settings, and call - dblink.merge(). + dblink.merge(). TODO: Share code with ForkProcess. """ elog_reader_fd, elog_writer_fd = os.pipe() + fcntl.fcntl(elog_reader_fd, fcntl.F_SETFL, fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK) + + # FD_CLOEXEC is enabled by default in Python >=3.4. + if sys.hexversion < 0x3040000: + try: + fcntl.FD_CLOEXEC + except AttributeError: + pass + else: + fcntl.fcntl(elog_reader_fd, fcntl.F_SETFD, + fcntl.fcntl(elog_reader_fd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC) + blockers = None if self.blockers is not None: # Query blockers in the main process, since closing @@ -116,10 +139,9 @@ class MergeProcess(SpawnProcess): blockers = self.blockers() mylink = portage.dblink(self.mycat, self.mypkg, settings=self.settings, treetype=self.treetype, vartree=self.vartree, - blockers=blockers, scheduler=self.scheduler, - pipe=elog_writer_fd) + blockers=blockers, pipe=elog_writer_fd) fd_pipes[elog_writer_fd] = elog_writer_fd - self._elog_reg_id = self.scheduler.register(elog_reader_fd, + self._elog_reg_id = self.scheduler.io_add_watch(elog_reader_fd, self._registered_events, self._elog_output_handler) # If a concurrent emerge process tries to install a package @@ -133,88 +155,100 @@ class MergeProcess(SpawnProcess): if not self.unmerge: counter = self.vartree.dbapi.counter_tick() - pid = os.fork() - if pid != 0: - if not isinstance(pid, int): - raise AssertionError( - "fork returned non-integer: %s" % (repr(pid),)) - - os.close(elog_writer_fd) - self._elog_reader_fd = elog_reader_fd - self._buf = "" - self._elog_keys = set() - - # invalidate relevant vardbapi caches - if self.vartree.dbapi._categories is not None: - self.vartree.dbapi._categories = None - self.vartree.dbapi._pkgs_changed = True - self.vartree.dbapi._clear_pkg_cache(mylink) - - portage.process.spawned_pids.append(pid) - return [pid] - - os.close(elog_reader_fd) - portage.locks._close_fds() - # Disable close_fds since we don't exec (see _setup_pipes docstring). - portage.process._setup_pipes(fd_pipes, close_fds=False) - - # Use default signal handlers since the ones inherited - # from the parent process are irrelevant here. - signal.signal(signal.SIGINT, signal.SIG_DFL) - signal.signal(signal.SIGTERM, signal.SIG_DFL) - - portage.output.havecolor = self.settings.get('NOCOLOR') \ - not in ('yes', 'true') - - # In this subprocess we want mylink._display_merge() to use - # stdout/stderr directly since they are pipes. This behavior - # is triggered when mylink._scheduler is None. - mylink._scheduler = None - - # Avoid wastful updates of the vdb cache. - self.vartree.dbapi._flush_cache_enabled = False - - # In this subprocess we don't want PORTAGE_BACKGROUND to - # suppress stdout/stderr output since they are pipes. We - # also don't want to open PORTAGE_LOG_FILE, since it will - # already be opened by the parent process, so we set the - # "subprocess" value for use in conditional logging code - # involving PORTAGE_LOG_FILE. - if not self.unmerge: - # unmerge phases have separate logs - if self.settings.get("PORTAGE_BACKGROUND") == "1": - self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "1" - else: - self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "0" - self.settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE") - self.settings["PORTAGE_BACKGROUND"] = "subprocess" - self.settings.backup_changes("PORTAGE_BACKGROUND") - - rval = 1 + parent_pid = os.getpid() + pid = None try: - if self.unmerge: - if not mylink.exists(): - rval = os.EX_OK - elif mylink.unmerge( - ldpath_mtimes=self.prev_mtimes) == os.EX_OK: - mylink.lockdb() - try: - mylink.delete() - finally: - mylink.unlockdb() - rval = os.EX_OK - else: - rval = mylink.merge(self.pkgloc, self.infloc, - myebuild=self.myebuild, mydbapi=self.mydbapi, - prev_mtimes=self.prev_mtimes, counter=counter) - except SystemExit: - raise - except: - traceback.print_exc() + pid = os.fork() + + if pid != 0: + if not isinstance(pid, int): + raise AssertionError( + "fork returned non-integer: %s" % (repr(pid),)) + + os.close(elog_writer_fd) + self._elog_reader_fd = elog_reader_fd + self._buf = "" + self._elog_keys = set() + # Discard messages which will be collected by the subprocess, + # in order to avoid duplicates (bug #446136). + portage.elog.messages.collect_messages(key=mylink.mycpv) + + # invalidate relevant vardbapi caches + if self.vartree.dbapi._categories is not None: + self.vartree.dbapi._categories = None + self.vartree.dbapi._pkgs_changed = True + self.vartree.dbapi._clear_pkg_cache(mylink) + + return [pid] + + os.close(elog_reader_fd) + + # Use default signal handlers in order to avoid problems + # killing subprocesses as reported in bug #353239. + signal.signal(signal.SIGINT, signal.SIG_DFL) + signal.signal(signal.SIGTERM, signal.SIG_DFL) + + portage.locks._close_fds() + # We don't exec, so use close_fds=False + # (see _setup_pipes docstring). + portage.process._setup_pipes(fd_pipes, close_fds=False) + + portage.output.havecolor = self.settings.get('NOCOLOR') \ + not in ('yes', 'true') + + # Avoid wastful updates of the vdb cache. + self.vartree.dbapi._flush_cache_enabled = False + + # In this subprocess we don't want PORTAGE_BACKGROUND to + # suppress stdout/stderr output since they are pipes. We + # also don't want to open PORTAGE_LOG_FILE, since it will + # already be opened by the parent process, so we set the + # "subprocess" value for use in conditional logging code + # involving PORTAGE_LOG_FILE. + if not self.unmerge: + # unmerge phases have separate logs + if self.settings.get("PORTAGE_BACKGROUND") == "1": + self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "1" + else: + self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "0" + self.settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE") + self.settings["PORTAGE_BACKGROUND"] = "subprocess" + self.settings.backup_changes("PORTAGE_BACKGROUND") + + rval = 1 + try: + if self.unmerge: + if not mylink.exists(): + rval = os.EX_OK + elif mylink.unmerge( + ldpath_mtimes=self.prev_mtimes) == os.EX_OK: + mylink.lockdb() + try: + mylink.delete() + finally: + mylink.unlockdb() + rval = os.EX_OK + else: + rval = mylink.merge(self.pkgloc, self.infloc, + myebuild=self.myebuild, mydbapi=self.mydbapi, + prev_mtimes=self.prev_mtimes, counter=counter) + except SystemExit: + raise + except: + traceback.print_exc() + # os._exit() skips stderr flush! + sys.stderr.flush() + finally: + os._exit(rval) + finally: - # Call os._exit() from finally block, in order to suppress any - # finally blocks from earlier in the call stack. See bug #345289. - os._exit(rval) + if pid == 0 or (pid is None and os.getpid() != parent_pid): + # Call os._exit() from a finally block in order + # to suppress any finally blocks from earlier + # in the call stack (see bug #345289). This + # finally block has to be setup before the fork + # in order to avoid a race condition. + os._exit(1) def _unregister(self): """ @@ -231,7 +265,7 @@ class MergeProcess(SpawnProcess): self._unlock_vdb() if self._elog_reg_id is not None: - self.scheduler.unregister(self._elog_reg_id) + self.scheduler.source_remove(self._elog_reg_id) self._elog_reg_id = None if self._elog_reader_fd is not None: os.close(self._elog_reader_fd) diff --git a/portage_with_autodep/pym/portage/dbapi/_MergeProcess.pyo b/portage_with_autodep/pym/portage/dbapi/_MergeProcess.pyo Binary files differindex 5839ad8..abee4be 100644 --- a/portage_with_autodep/pym/portage/dbapi/_MergeProcess.pyo +++ b/portage_with_autodep/pym/portage/dbapi/_MergeProcess.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/__init__.py b/portage_with_autodep/pym/portage/dbapi/__init__.py index a1c5c56..a20a1e8 100644 --- a/portage_with_autodep/pym/portage/dbapi/__init__.py +++ b/portage_with_autodep/pym/portage/dbapi/__init__.py @@ -1,6 +1,8 @@ -# Copyright 1998-2012 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ["dbapi"] import re @@ -8,7 +10,7 @@ import re import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.dbapi.dep_expand:dep_expand@_dep_expand', - 'portage.dep:match_from_list', + 'portage.dep:Atom,match_from_list,_match_slot', 'portage.output:colorize', 'portage.util:cmp_sort_key,writemsg', 'portage.versions:catsplit,catpkgsplit,vercmp,_pkg_str', @@ -16,14 +18,19 @@ portage.proxy.lazyimport.lazyimport(globals(), from portage import os from portage import auxdbkeys +from portage.eapi import _get_eapi_attrs +from portage.exception import InvalidData from portage.localization import _ +from _emerge.Package import Package class dbapi(object): - _category_re = re.compile(r'^\w[-.+\w]*$') + _category_re = re.compile(r'^\w[-.+\w]*$', re.UNICODE) _categories = None _use_mutable = False _known_keys = frozenset(x for x in auxdbkeys if not x.startswith("UNUSED_0")) + _pkg_str_aux_keys = ("EAPI", "KEYWORDS", "SLOT", "repository") + def __init__(self): pass @@ -125,29 +132,52 @@ class dbapi(object): def _iter_match(self, atom, cpv_iter): cpv_iter = iter(match_from_list(atom, cpv_iter)) + if atom.repo: + cpv_iter = self._iter_match_repo(atom, cpv_iter) if atom.slot: cpv_iter = self._iter_match_slot(atom, cpv_iter) if atom.unevaluated_atom.use: cpv_iter = self._iter_match_use(atom, cpv_iter) - if atom.repo: - cpv_iter = self._iter_match_repo(atom, cpv_iter) return cpv_iter + def _pkg_str(self, cpv, repo): + """ + This is used to contruct _pkg_str instances on-demand during + matching. If cpv is a _pkg_str instance with slot attribute, + then simply return it. Otherwise, fetch metadata and construct + a _pkg_str instance. This may raise KeyError or InvalidData. + """ + try: + cpv.slot + except AttributeError: + pass + else: + return cpv + + metadata = dict(zip(self._pkg_str_aux_keys, + self.aux_get(cpv, self._pkg_str_aux_keys, myrepo=repo))) + + return _pkg_str(cpv, metadata=metadata, settings=self.settings) + def _iter_match_repo(self, atom, cpv_iter): for cpv in cpv_iter: try: - if self.aux_get(cpv, ["repository"], myrepo=atom.repo)[0] == atom.repo: - yield cpv - except KeyError: - continue + pkg_str = self._pkg_str(cpv, atom.repo) + except (KeyError, InvalidData): + pass + else: + if pkg_str.repo == atom.repo: + yield pkg_str def _iter_match_slot(self, atom, cpv_iter): for cpv in cpv_iter: try: - if self.aux_get(cpv, ["SLOT"], myrepo=atom.repo)[0] == atom.slot: - yield cpv - except KeyError: - continue + pkg_str = self._pkg_str(cpv, atom.repo) + except (KeyError, InvalidData): + pass + else: + if _match_slot(atom, pkg_str): + yield pkg_str def _iter_match_use(self, atom, cpv_iter): """ @@ -155,7 +185,7 @@ class dbapi(object): 2) Check enabled/disabled flag states. """ - aux_keys = ["IUSE", "SLOT", "USE", "repository"] + aux_keys = ["EAPI", "IUSE", "KEYWORDS", "SLOT", "USE", "repository"] for cpv in cpv_iter: try: metadata = dict(zip(aux_keys, @@ -163,17 +193,31 @@ class dbapi(object): except KeyError: continue + try: + cpv.slot + except AttributeError: + try: + cpv = _pkg_str(cpv, metadata=metadata, + settings=self.settings) + except InvalidData: + continue + if not self._match_use(atom, cpv, metadata): continue yield cpv - def _match_use(self, atom, cpv, metadata): - iuse_implicit_match = self.settings._iuse_implicit_match - iuse = frozenset(x.lstrip('+-') for x in metadata["IUSE"].split()) + def _match_use(self, atom, pkg, metadata): + eapi_attrs = _get_eapi_attrs(metadata["EAPI"]) + if eapi_attrs.iuse_effective: + iuse_implicit_match = self.settings._iuse_effective_match + else: + iuse_implicit_match = self.settings._iuse_implicit_match + usealiases = self.settings._use_manager.getUseAliases(pkg) + iuse = Package._iuse(None, metadata["IUSE"].split(), iuse_implicit_match, usealiases, metadata["EAPI"]) for x in atom.unevaluated_atom.use.required: - if x not in iuse and not iuse_implicit_match(x): + if iuse.get_real_flag(x) is None: return False if atom.use is None: @@ -183,45 +227,54 @@ class dbapi(object): # Use IUSE to validate USE settings for built packages, # in case the package manager that built this package # failed to do that for some reason (or in case of - # data corruption). - use = frozenset(x for x in metadata["USE"].split() - if x in iuse or iuse_implicit_match(x)) - missing_enabled = atom.use.missing_enabled.difference(iuse) - missing_disabled = atom.use.missing_disabled.difference(iuse) - - if atom.use.enabled: - if atom.use.enabled.intersection(missing_disabled): + # data corruption). The enabled flags must be consistent + # with implicit IUSE, in order to avoid potential + # inconsistencies in USE dep matching (see bug #453400). + use = frozenset(x for x in metadata["USE"].split() if iuse.get_real_flag(x) is not None) + missing_enabled = frozenset(x for x in atom.use.missing_enabled if iuse.get_real_flag(x) is None) + missing_disabled = frozenset(x for x in atom.use.missing_disabled if iuse.get_real_flag(x) is None) + enabled = frozenset((iuse.get_real_flag(x) or x) for x in atom.use.enabled) + disabled = frozenset((iuse.get_real_flag(x) or x) for x in atom.use.disabled) + + if enabled: + if any(x in enabled for x in missing_disabled): return False - need_enabled = atom.use.enabled.difference(use) + need_enabled = enabled.difference(use) if need_enabled: - need_enabled = need_enabled.difference(missing_enabled) - if need_enabled: + if any(x not in missing_enabled for x in need_enabled): return False - if atom.use.disabled: - if atom.use.disabled.intersection(missing_enabled): + if disabled: + if any(x in disabled for x in missing_enabled): return False - need_disabled = atom.use.disabled.intersection(use) + need_disabled = disabled.intersection(use) if need_disabled: - need_disabled = need_disabled.difference(missing_disabled) - if need_disabled: + if any(x not in missing_disabled for x in need_disabled): return False elif not self.settings.local_config: # Check masked and forced flags for repoman. - if hasattr(cpv, 'slot'): - pkg = cpv - else: - pkg = _pkg_str(cpv, slot=metadata["SLOT"], - repo=metadata.get("repository")) - usemask = self.settings._getUseMask(pkg) - if usemask.intersection(atom.use.enabled): + usemask = self.settings._getUseMask(pkg, + stable=self.settings._parent_stable) + if any(x in usemask for x in atom.use.enabled): return False - useforce = self.settings._getUseForce(pkg).difference(usemask) - if useforce.intersection(atom.use.disabled): + useforce = self.settings._getUseForce(pkg, + stable=self.settings._parent_stable) + if any(x in useforce and x not in usemask + for x in atom.use.disabled): return False + # Check unsatisfied use-default deps + if atom.use.enabled: + missing_disabled = frozenset(x for x in atom.use.missing_disabled if iuse.get_real_flag(x) is None) + if any(x in atom.use.enabled for x in missing_disabled): + return False + if atom.use.disabled: + missing_enabled = frozenset(x for x in atom.use.missing_enabled if iuse.get_real_flag(x) is None) + if any(x in atom.use.disabled for x in missing_enabled): + return False + return True def invalidentry(self, mypath): @@ -249,23 +302,30 @@ class dbapi(object): maxval = len(cpv_all) aux_get = self.aux_get aux_update = self.aux_update - meta_keys = ["DEPEND", "RDEPEND", "PDEPEND", "PROVIDE", 'repository'] + update_keys = Package._dep_keys + ("PROVIDE",) + meta_keys = update_keys + self._pkg_str_aux_keys repo_dict = None if isinstance(updates, dict): repo_dict = updates - from portage.update import update_dbentries if onUpdate: onUpdate(maxval, 0) if onProgress: onProgress(maxval, 0) for i, cpv in enumerate(cpv_all): - metadata = dict(zip(meta_keys, aux_get(cpv, meta_keys))) - repo = metadata.pop('repository') + try: + metadata = dict(zip(meta_keys, aux_get(cpv, meta_keys))) + except KeyError: + continue + try: + pkg = _pkg_str(cpv, metadata=metadata, settings=self.settings) + except InvalidData: + continue + metadata = dict((k, metadata[k]) for k in update_keys) if repo_dict is None: updates_list = updates else: try: - updates_list = repo_dict[repo] + updates_list = repo_dict[pkg.repo] except KeyError: try: updates_list = repo_dict['DEFAULT'] @@ -275,7 +335,8 @@ class dbapi(object): if not updates_list: continue - metadata_updates = update_dbentries(updates_list, metadata) + metadata_updates = \ + portage.update_dbentries(updates_list, metadata, parent=pkg) if metadata_updates: aux_update(cpv, metadata_updates) if onUpdate: @@ -286,27 +347,39 @@ class dbapi(object): def move_slot_ent(self, mylist, repo_match=None): """This function takes a sequence: Args: - mylist: a sequence of (package, originalslot, newslot) + mylist: a sequence of (atom, originalslot, newslot) repo_match: callable that takes single repo_name argument and returns True if the update should be applied Returns: The number of slotmoves this function did """ - pkg = mylist[1] + atom = mylist[1] origslot = mylist[2] newslot = mylist[3] - origmatches = self.match(pkg) + + try: + atom.with_slot + except AttributeError: + atom = Atom(atom).with_slot(origslot) + else: + atom = atom.with_slot(origslot) + + origmatches = self.match(atom) moves = 0 if not origmatches: return moves for mycpv in origmatches: - slot = self.aux_get(mycpv, ["SLOT"])[0] - if slot != origslot: + try: + mycpv = self._pkg_str(mycpv, atom.repo) + except (KeyError, InvalidData): continue - if repo_match is not None \ - and not repo_match(self.aux_get(mycpv, ['repository'])[0]): + if repo_match is not None and not repo_match(mycpv.repo): continue moves += 1 + if "/" not in newslot and \ + mycpv.sub_slot and \ + mycpv.sub_slot not in (mycpv.slot, newslot): + newslot = "%s/%s" % (newslot, mycpv.sub_slot) mydata = {"SLOT": newslot+"\n"} self.aux_update(mycpv, mydata) return moves diff --git a/portage_with_autodep/pym/portage/dbapi/__init__.pyo b/portage_with_autodep/pym/portage/dbapi/__init__.pyo Binary files differindex e7b494d..d4d47b2 100644 --- a/portage_with_autodep/pym/portage/dbapi/__init__.pyo +++ b/portage_with_autodep/pym/portage/dbapi/__init__.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/_expand_new_virt.py b/portage_with_autodep/pym/portage/dbapi/_expand_new_virt.py index d379b4c..9aa603d 100644 --- a/portage_with_autodep/pym/portage/dbapi/_expand_new_virt.py +++ b/portage_with_autodep/pym/portage/dbapi/_expand_new_virt.py @@ -1,8 +1,11 @@ -# Copyright 2011 Gentoo Foundation +# Copyright 2011-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + import portage from portage.dep import Atom, _get_useflag_re +from portage.eapi import _get_eapi_attrs def expand_new_virt(vardb, atom): """ @@ -44,6 +47,7 @@ def expand_new_virt(vardb, atom): yield atom continue + eapi_attrs = _get_eapi_attrs(eapi) # Validate IUSE and IUSE, for early detection of vardb corruption. useflag_re = _get_useflag_re(eapi) valid_iuse = [] @@ -54,7 +58,11 @@ def expand_new_virt(vardb, atom): valid_iuse.append(x) valid_iuse = frozenset(valid_iuse) - iuse_implicit_match = vardb.settings._iuse_implicit_match + if eapi_attrs.iuse_effective: + iuse_implicit_match = vardb.settings._iuse_effective_match + else: + iuse_implicit_match = vardb.settings._iuse_implicit_match + valid_use = [] for x in use.split(): if x in valid_iuse or iuse_implicit_match(x): diff --git a/portage_with_autodep/pym/portage/dbapi/_expand_new_virt.pyo b/portage_with_autodep/pym/portage/dbapi/_expand_new_virt.pyo Binary files differindex 6c23a7e..7884393 100644 --- a/portage_with_autodep/pym/portage/dbapi/_expand_new_virt.pyo +++ b/portage_with_autodep/pym/portage/dbapi/_expand_new_virt.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/bintree.py b/portage_with_autodep/pym/portage/dbapi/bintree.py index a8027ee..b1f67ae 100644 --- a/portage_with_autodep/pym/portage/dbapi/bintree.py +++ b/portage_with_autodep/pym/portage/dbapi/bintree.py @@ -1,16 +1,18 @@ -# Copyright 1998-2012 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ["bindbapi", "binarytree"] import portage portage.proxy.lazyimport.lazyimport(globals(), - 'portage.checksum:hashfunc_map,perform_multiple_checksums,verify_all', + 'portage.checksum:hashfunc_map,perform_multiple_checksums,' + \ + 'verify_all,_apply_hash_filter,_hash_filter', 'portage.dbapi.dep_expand:dep_expand', - 'portage.dep:dep_getkey,isjustname,match_from_list', + 'portage.dep:dep_getkey,isjustname,isvalidatom,match_from_list', 'portage.output:EOutput,colorize', 'portage.locks:lockfile,unlockfile', - 'portage.package.ebuild.doebuild:_vdb_use_conditional_atoms', 'portage.package.ebuild.fetch:_check_distfile,_hide_url_passwd', 'portage.update:update_dbentries', 'portage.util:atomic_ofstream,ensure_dirs,normalize_path,' + \ @@ -41,7 +43,9 @@ import subprocess import sys import tempfile import textwrap +import traceback import warnings +from gzip import GzipFile from itertools import chain try: from urllib.parse import urlparse @@ -49,8 +53,16 @@ except ImportError: from urlparse import urlparse if sys.hexversion >= 0x3000000: + _unicode = str basestring = str long = int +else: + _unicode = unicode + +class UseCachedCopyOfRemoteIndex(Exception): + # If the local copy is recent enough + # then fetching the remote index can be skipped. + pass class bindbapi(fakedbapi): _known_keys = frozenset(list(fakedbapi._known_keys) + \ @@ -63,9 +75,10 @@ class bindbapi(fakedbapi): self.cpdict={} # Selectively cache metadata in order to optimize dep matching. self._aux_cache_keys = set( - ["BUILD_TIME", "CHOST", "DEPEND", "EAPI", "IUSE", "KEYWORDS", + ["BUILD_TIME", "CHOST", "DEPEND", "EAPI", + "HDEPEND", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROPERTIES", "PROVIDE", - "RDEPEND", "repository", "RESTRICT", "SLOT", "USE", "DEFINED_PHASES", + "RDEPEND", "repository", "RESTRICT", "SLOT", "USE", "DEFINED_PHASES" ]) self._aux_cache_slot_dict = slot_dict_class(self._aux_cache_keys) self._aux_cache = {} @@ -128,15 +141,15 @@ class bindbapi(fakedbapi): if myval: mydata[x] = " ".join(myval.split()) - if not mydata.setdefault('EAPI', _unicode_decode('0')): - mydata['EAPI'] = _unicode_decode('0') + if not mydata.setdefault('EAPI', '0'): + mydata['EAPI'] = '0' if cache_me: aux_cache = self._aux_cache_slot_dict() for x in self._aux_cache_keys: - aux_cache[x] = mydata.get(x, _unicode_decode('')) + aux_cache[x] = mydata.get(x, '') self._aux_cache[mycpv] = aux_cache - return [mydata.get(x, _unicode_decode('')) for x in wants] + return [mydata.get(x, '') for x in wants] def aux_update(self, cpv, values): if not self.bintree.populated: @@ -248,7 +261,7 @@ def _pkgindex_cpv_map_latest_build(pkgindex): class binarytree(object): "this tree scans for a list of all packages available in PKGDIR" - def __init__(self, _unused=None, pkgdir=None, + def __init__(self, _unused=DeprecationWarning, pkgdir=None, virtual=DeprecationWarning, settings=None): if pkgdir is None: @@ -257,11 +270,11 @@ class binarytree(object): if settings is None: raise TypeError("settings parameter is required") - if _unused is not None and _unused != settings['ROOT']: - warnings.warn("The root parameter of the " + if _unused is not DeprecationWarning: + warnings.warn("The first parameter of the " "portage.dbapi.bintree.binarytree" - " constructor is now unused. Use " - "settings['ROOT'] instead.", + " constructor is now unused. Instead " + "settings['ROOT'] is used.", DeprecationWarning, stacklevel=2) if virtual is not DeprecationWarning: @@ -293,22 +306,26 @@ class binarytree(object): self._pkgindex_keys.update(["CPV", "MTIME", "SIZE"]) self._pkgindex_aux_keys = \ ["BUILD_TIME", "CHOST", "DEPEND", "DESCRIPTION", "EAPI", - "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROPERTIES", - "PROVIDE", "RDEPEND", "repository", "SLOT", "USE", "DEFINED_PHASES", + "HDEPEND", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROPERTIES", + "PROVIDE", "RESTRICT", "RDEPEND", "repository", "SLOT", "USE", "DEFINED_PHASES", "BASE_URI"] self._pkgindex_aux_keys = list(self._pkgindex_aux_keys) self._pkgindex_use_evaluated_keys = \ - ("LICENSE", "RDEPEND", "DEPEND", - "PDEPEND", "PROPERTIES", "PROVIDE") + ("DEPEND", "HDEPEND", "LICENSE", "RDEPEND", + "PDEPEND", "PROPERTIES", "PROVIDE", "RESTRICT") self._pkgindex_header_keys = set([ "ACCEPT_KEYWORDS", "ACCEPT_LICENSE", - "ACCEPT_PROPERTIES", "CBUILD", + "ACCEPT_PROPERTIES", "ACCEPT_RESTRICT", "CBUILD", "CONFIG_PROTECT", "CONFIG_PROTECT_MASK", "FEATURES", - "GENTOO_MIRRORS", "INSTALL_MASK", "SYNC", "USE"]) + "GENTOO_MIRRORS", "INSTALL_MASK", "IUSE_IMPLICIT", "USE", + "USE_EXPAND", "USE_EXPAND_HIDDEN", "USE_EXPAND_IMPLICIT", + "USE_EXPAND_UNPREFIXED"]) self._pkgindex_default_pkg_data = { "BUILD_TIME" : "", + "DEFINED_PHASES" : "", "DEPEND" : "", "EAPI" : "0", + "HDEPEND" : "", "IUSE" : "", "KEYWORDS": "", "LICENSE" : "", @@ -320,7 +337,6 @@ class binarytree(object): "RESTRICT": "", "SLOT" : "0", "USE" : "", - "DEFINED_PHASES" : "", } self._pkgindex_inherited_keys = ["CHOST", "repository"] @@ -378,15 +394,24 @@ class binarytree(object): if not origmatches: return moves for mycpv in origmatches: + try: + mycpv = self.dbapi._pkg_str(mycpv, None) + except (KeyError, InvalidData): + continue mycpv_cp = portage.cpv_getkey(mycpv) if mycpv_cp != origcp: # Ignore PROVIDE virtual match. continue if repo_match is not None \ - and not repo_match(self.dbapi.aux_get(mycpv, - ['repository'])[0]): + and not repo_match(mycpv.repo): + continue + + # Use isvalidatom() to check if this move is valid for the + # EAPI (characters allowed in package names may vary). + if not isvalidatom(newcp, eapi=mycpv.eapi): continue - mynewcpv = mycpv.replace(mycpv_cp, str(newcp), 1) + + mynewcpv = mycpv.replace(mycpv_cp, _unicode(newcp), 1) myoldpkg = catsplit(mycpv)[1] mynewpkg = catsplit(mynewcpv)[1] @@ -405,7 +430,7 @@ class binarytree(object): moves += 1 mytbz2 = portage.xpak.tbz2(tbz2path) mydata = mytbz2.get_data() - updated_items = update_dbentries([mylist], mydata) + updated_items = update_dbentries([mylist], mydata, parent=mycpv) mydata.update(updated_items) mydata[b'PF'] = \ _unicode_encode(mynewpkg + "\n", @@ -541,6 +566,20 @@ class binarytree(object): if not os.path.isdir(path): raise + def _file_permissions(self, path): + try: + pkgdir_st = os.stat(self.pkgdir) + except OSError: + pass + else: + pkgdir_gid = pkgdir_st.st_gid + pkgdir_grp_mode = 0o0060 & pkgdir_st.st_mode + try: + portage.util.apply_permissions(path, gid=pkgdir_gid, + mode=pkgdir_grp_mode, mask=0) + except PortageException: + pass + def _move_to_all(self, cpv): """If the file exists, move it. Whether or not it exists, update state for future getname() calls.""" @@ -796,9 +835,7 @@ class binarytree(object): del pkgindex.packages[:] pkgindex.packages.extend(iter(metadata.values())) self._update_pkgindex_header(pkgindex.header) - f = atomic_ofstream(self._pkgindex_file) - pkgindex.write(f) - f.close() + self._pkgindex_write(pkgindex) if getbinpkgs and not self.settings["PORTAGE_BINHOST"]: writemsg(_("!!! PORTAGE_BINHOST unset, but use is requested.\n"), @@ -841,6 +878,7 @@ class binarytree(object): if e.errno != errno.ENOENT: raise local_timestamp = pkgindex.header.get("TIMESTAMP", None) + remote_timestamp = None rmt_idx = self._new_pkgindex() proc = None tmp_filename = None @@ -849,41 +887,76 @@ class binarytree(object): # protocols and requires the base url to have a trailing # slash, so join manually... url = base_url.rstrip("/") + "/Packages" - try: - f = _urlopen(url) - except IOError: - path = parsed_url.path.rstrip("/") + "/Packages" + f = None + + # Don't use urlopen for https, since it doesn't support + # certificate/hostname verification (bug #469888). + if parsed_url.scheme not in ('https',): + try: + f = _urlopen(url, if_modified_since=local_timestamp) + if hasattr(f, 'headers') and f.headers.get('timestamp', ''): + remote_timestamp = f.headers.get('timestamp') + except IOError as err: + if hasattr(err, 'code') and err.code == 304: # not modified (since local_timestamp) + raise UseCachedCopyOfRemoteIndex() + + if parsed_url.scheme in ('ftp', 'http', 'https'): + # This protocol is supposedly supported by urlopen, + # so apparently there's a problem with the url + # or a bug in urlopen. + if self.settings.get("PORTAGE_DEBUG", "0") != "0": + traceback.print_exc() - if parsed_url.scheme == 'sftp': - # The sftp command complains about 'Illegal seek' if - # we try to make it write to /dev/stdout, so use a - # temp file instead. - fd, tmp_filename = tempfile.mkstemp() - os.close(fd) - if port is not None: - port_args = ['-P', "%s" % (port,)] - proc = subprocess.Popen(['sftp'] + port_args + \ - [user_passwd + host + ":" + path, tmp_filename]) - if proc.wait() != os.EX_OK: raise - f = open(tmp_filename, 'rb') - elif parsed_url.scheme == 'ssh': + + if f is None: + + path = parsed_url.path.rstrip("/") + "/Packages" + + if parsed_url.scheme == 'ssh': + # Use a pipe so that we can terminate the download + # early if we detect that the TIMESTAMP header + # matches that of the cached Packages file. + ssh_args = ['ssh'] if port is not None: - port_args = ['-p', "%s" % (port,)] - proc = subprocess.Popen(['ssh'] + port_args + \ - [user_passwd + host, '--', 'cat', path], + ssh_args.append("-p%s" % (port,)) + # NOTE: shlex evaluates embedded quotes + ssh_args.extend(portage.util.shlex_split( + self.settings.get("PORTAGE_SSH_OPTS", ""))) + ssh_args.append(user_passwd + host) + ssh_args.append('--') + ssh_args.append('cat') + ssh_args.append(path) + + proc = subprocess.Popen(ssh_args, stdout=subprocess.PIPE) f = proc.stdout else: setting = 'FETCHCOMMAND_' + parsed_url.scheme.upper() fcmd = self.settings.get(setting) if not fcmd: - raise + fcmd = self.settings.get('FETCHCOMMAND') + if not fcmd: + raise EnvironmentError("FETCHCOMMAND is unset") + fd, tmp_filename = tempfile.mkstemp() tmp_dirname, tmp_basename = os.path.split(tmp_filename) os.close(fd) - success = portage.getbinpkg.file_get(url, - tmp_dirname, fcmd=fcmd, filename=tmp_basename) + + fcmd_vars = { + "DISTDIR": tmp_dirname, + "FILE": tmp_basename, + "URI": url + } + + for k in ("PORTAGE_SSH_OPTS",): + try: + fcmd_vars[k] = self.settings[k] + except KeyError: + pass + + success = portage.getbinpkg.file_get( + fcmd=fcmd, fcmd_vars=fcmd_vars) if not success: raise EnvironmentError("%s failed" % (setting,)) f = open(tmp_filename, 'rb') @@ -892,7 +965,8 @@ class binarytree(object): _encodings['repo.content'], errors='replace') try: rmt_idx.readHeader(f_dec) - remote_timestamp = rmt_idx.header.get("TIMESTAMP", None) + if not remote_timestamp: # in case it had not been read from HTTP header + remote_timestamp = rmt_idx.header.get("TIMESTAMP", None) if not remote_timestamp: # no timestamp in the header, something's wrong pkgindex = None @@ -920,6 +994,12 @@ class binarytree(object): writemsg("\n\n!!! %s\n" % \ _("Timed out while closing connection to binhost"), noiselevel=-1) + except UseCachedCopyOfRemoteIndex: + writemsg_stdout("\n") + writemsg_stdout( + colorize("GOOD", _("Local copy of remote index is up-to-date and will be used.")) + \ + "\n") + rmt_idx = pkgindex except EnvironmentError as e: writemsg(_("\n\n!!! Error fetching binhost package" \ " info from '%s'\n") % _hide_url_passwd(base_url)) @@ -988,75 +1068,7 @@ class binarytree(object): # Local package instances override remote instances. for cpv in metadata: self._remotepkgs.pop(cpv, None) - continue - try: - chunk_size = long(self.settings["PORTAGE_BINHOST_CHUNKSIZE"]) - if chunk_size < 8: - chunk_size = 8 - except (ValueError, KeyError): - chunk_size = 3000 - writemsg_stdout("\n") - writemsg_stdout( - colorize("GOOD", _("Fetching bininfo from ")) + \ - _hide_url_passwd(base_url) + "\n") - remotepkgs = portage.getbinpkg.dir_get_metadata( - base_url, chunk_size=chunk_size) - - for mypkg, remote_metadata in remotepkgs.items(): - mycat = remote_metadata.get("CATEGORY") - if mycat is None: - #old-style or corrupt package - writemsg(_("!!! Invalid remote binary package: %s\n") % mypkg, - noiselevel=-1) - continue - mycat = mycat.strip() - try: - fullpkg = _pkg_str(mycat+"/"+mypkg[:-5]) - except InvalidData: - writemsg(_("!!! Invalid remote binary package: %s\n") % mypkg, - noiselevel=-1) - continue - - if fullpkg in metadata: - # When using this old protocol, comparison with the remote - # package isn't supported, so the local package is always - # preferred even if getbinpkgsonly is enabled. - continue - - if not self.dbapi._category_re.match(mycat): - writemsg(_("!!! Remote binary package has an " \ - "unrecognized category: '%s'\n") % fullpkg, - noiselevel=-1) - writemsg(_("!!! '%s' has a category that is not" \ - " listed in %setc/portage/categories\n") % \ - (fullpkg, self.settings["PORTAGE_CONFIGROOT"]), - noiselevel=-1) - continue - mykey = portage.cpv_getkey(fullpkg) - try: - # invalid tbz2's can hurt things. - self.dbapi.cpv_inject(fullpkg) - for k, v in remote_metadata.items(): - remote_metadata[k] = v.strip() - remote_metadata["BASE_URI"] = base_url - - # Eliminate metadata values with names that digestCheck - # uses, since they are not valid when using the old - # protocol. Typically this is needed for SIZE metadata - # which corresponds to the size of the unpacked files - # rather than the binpkg file size, triggering digest - # verification failures as reported in bug #303211. - remote_metadata.pop('SIZE', None) - for k in portage.checksum.hashfunc_map: - remote_metadata.pop(k, None) - - self._remotepkgs[fullpkg] = remote_metadata - except SystemExit as e: - raise - except: - writemsg(_("!!! Failed to inject remote binary package: %s\n") % fullpkg, - noiselevel=-1) - continue + self.populated=1 def inject(self, cpv, filename=None): @@ -1110,6 +1122,10 @@ class binarytree(object): if not samefile: self._ensure_dir(os.path.dirname(new_filename)) _movefile(filename, new_filename, mysettings=self.settings) + full_path = new_filename + + self._file_permissions(full_path) + if self._all_directory and \ self.getname(cpv).split(os.path.sep)[-2] == "All": self._create_symlink(cpv) @@ -1157,13 +1173,35 @@ class binarytree(object): pkgindex.packages.append(d) self._update_pkgindex_header(pkgindex.header) - f = atomic_ofstream(os.path.join(self.pkgdir, "Packages")) - pkgindex.write(f) - f.close() + self._pkgindex_write(pkgindex) + finally: if pkgindex_lock: unlockfile(pkgindex_lock) + def _pkgindex_write(self, pkgindex): + contents = codecs.getwriter(_encodings['repo.content'])(io.BytesIO()) + pkgindex.write(contents) + contents = contents.getvalue() + atime = mtime = long(pkgindex.header["TIMESTAMP"]) + output_files = [(atomic_ofstream(self._pkgindex_file, mode="wb"), + self._pkgindex_file, None)] + + if "compress-index" in self.settings.features: + gz_fname = self._pkgindex_file + ".gz" + fileobj = atomic_ofstream(gz_fname, mode="wb") + output_files.append((GzipFile(filename='', mode="wb", + fileobj=fileobj, mtime=mtime), gz_fname, fileobj)) + + for f, fname, f_close in output_files: + f.write(contents) + f.close() + if f_close is not None: + f_close.close() + self._file_permissions(fname) + # some seconds might have elapsed since TIMESTAMP + os.utime(fname, (atime, mtime)) + def _pkgindex_entry(self, cpv): """ Performs checksums and evaluates USE flag conditionals. @@ -1223,6 +1261,16 @@ class binarytree(object): else: header.pop(k, None) + # These values may be useful for using a binhost without + # having a local copy of the profile (bug #470006). + for k in self.settings.get("USE_EXPAND_IMPLICIT", "").split(): + k = "USE_EXPAND_VALUES_" + k + v = self.settings.get(k) + if v: + header[k] = v + else: + header.pop(k, None) + def _pkgindex_version_supported(self, pkgindex): version = pkgindex.header.get("VERSION") if version: @@ -1235,11 +1283,6 @@ class binarytree(object): def _eval_use_flags(self, cpv, metadata): use = frozenset(metadata["USE"].split()) - raw_use = use - iuse = set(f.lstrip("-+") for f in metadata["IUSE"].split()) - use = [f for f in use if f in iuse] - use.sort() - metadata["USE"] = " ".join(use) for k in self._pkgindex_use_evaluated_keys: if k.endswith('DEPEND'): token_class = Atom @@ -1248,7 +1291,7 @@ class binarytree(object): try: deps = metadata[k] - deps = use_reduce(deps, uselist=raw_use, token_class=token_class) + deps = use_reduce(deps, uselist=use, token_class=token_class) deps = paren_enclose(deps) except portage.exception.InvalidDependString as e: writemsg("%s: %s\n" % (k, str(e)), @@ -1313,6 +1356,8 @@ class binarytree(object): """Returns the URI to the Packages file for a given package.""" return self._pkgindex_uri.get(pkgname) + + def gettbz2(self, pkgname): """Fetches the package from a remote site, if necessary. Attempts to resume if the file appears to be partially downloaded.""" @@ -1320,7 +1365,7 @@ class binarytree(object): tbz2name = os.path.basename(tbz2_path) resume = False if os.path.exists(tbz2_path): - if (tbz2name not in self.invalids): + if tbz2name[:-5] not in self.invalids: return else: resume = True @@ -1370,19 +1415,14 @@ class binarytree(object): f.close() return pkgindex - def digestCheck(self, pkg): - """ - Verify digests for the given package and raise DigestException - if verification fails. - @rtype: bool - @return: True if digests could be located, False otherwise. - """ - cpv = pkg - if not isinstance(cpv, basestring): + def _get_digests(self, pkg): + + try: cpv = pkg.cpv - pkg = None + except AttributeError: + cpv = pkg - pkg_path = self.getname(cpv) + digests = {} metadata = None if self._remotepkgs is None or cpv not in self._remotepkgs: for d in self._load_pkgindex().packages: @@ -1392,9 +1432,8 @@ class binarytree(object): else: metadata = self._remotepkgs[cpv] if metadata is None: - return False + return digests - digests = {} for k in hashfunc_map: v = metadata.get(k) if not v: @@ -1408,9 +1447,31 @@ class binarytree(object): writemsg(_("!!! Malformed SIZE attribute in remote " \ "metadata for '%s'\n") % cpv) + return digests + + def digestCheck(self, pkg): + """ + Verify digests for the given package and raise DigestException + if verification fails. + @rtype: bool + @return: True if digests could be located, False otherwise. + """ + + digests = self._get_digests(pkg) + if not digests: return False + try: + cpv = pkg.cpv + except AttributeError: + cpv = pkg + + pkg_path = self.getname(cpv) + hash_filter = _hash_filter( + self.settings.get("PORTAGE_CHECKSUM_FILTER", "")) + if not hash_filter.transparent: + digests = _apply_hash_filter(digests, hash_filter) eout = EOutput() eout.quiet = self.settings.get("PORTAGE_QUIET") == "1" ok, st = _check_distfile(pkg_path, digests, eout, show_errors=0) @@ -1426,9 +1487,7 @@ class binarytree(object): "Get a slot for a catpkg; assume it exists." myslot = "" try: - myslot = self.dbapi.aux_get(mycatpkg,["SLOT"])[0] - except SystemExit as e: - raise - except Exception as e: + myslot = self.dbapi._pkg_str(mycatpkg, None).slot + except KeyError: pass return myslot diff --git a/portage_with_autodep/pym/portage/dbapi/bintree.pyo b/portage_with_autodep/pym/portage/dbapi/bintree.pyo Binary files differindex f99f377..90d0c6a 100644 --- a/portage_with_autodep/pym/portage/dbapi/bintree.pyo +++ b/portage_with_autodep/pym/portage/dbapi/bintree.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/cpv_expand.py b/portage_with_autodep/pym/portage/dbapi/cpv_expand.py index 947194c..70ee782 100644 --- a/portage_with_autodep/pym/portage/dbapi/cpv_expand.py +++ b/portage_with_autodep/pym/portage/dbapi/cpv_expand.py @@ -1,6 +1,8 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ["cpv_expand"] import portage diff --git a/portage_with_autodep/pym/portage/dbapi/cpv_expand.pyo b/portage_with_autodep/pym/portage/dbapi/cpv_expand.pyo Binary files differindex cf1a428..7c38720 100644 --- a/portage_with_autodep/pym/portage/dbapi/cpv_expand.pyo +++ b/portage_with_autodep/pym/portage/dbapi/cpv_expand.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/dep_expand.py b/portage_with_autodep/pym/portage/dbapi/dep_expand.py index ac8ccf4..3de5d8f 100644 --- a/portage_with_autodep/pym/portage/dbapi/dep_expand.py +++ b/portage_with_autodep/pym/portage/dbapi/dep_expand.py @@ -1,6 +1,8 @@ -# Copyright 2010 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ["dep_expand"] import re @@ -23,7 +25,7 @@ def dep_expand(mydep, mydb=None, use_cache=1, settings=None): if mydep[0] == "*": mydep = mydep[1:] orig_dep = mydep - has_cat = '/' in orig_dep + has_cat = '/' in orig_dep.split(':')[0] if not has_cat: alphanum = re.search(r'\w', orig_dep) if alphanum: diff --git a/portage_with_autodep/pym/portage/dbapi/dep_expand.pyo b/portage_with_autodep/pym/portage/dbapi/dep_expand.pyo Binary files differindex b323f5b..bcaf8e3 100644 --- a/portage_with_autodep/pym/portage/dbapi/dep_expand.pyo +++ b/portage_with_autodep/pym/portage/dbapi/dep_expand.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/porttree.py b/portage_with_autodep/pym/portage/dbapi/porttree.py index c5ee770..fc3fc03 100644 --- a/portage_with_autodep/pym/portage/dbapi/porttree.py +++ b/portage_with_autodep/pym/portage/dbapi/porttree.py @@ -1,6 +1,8 @@ -# Copyright 1998-2012 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = [ "close_portdbapi_caches", "FetchlistDict", "portagetree", "portdbapi" ] @@ -10,7 +12,7 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.checksum', 'portage.data:portage_gid,secpass', 'portage.dbapi.dep_expand:dep_expand', - 'portage.dep:Atom,dep_getkey,match_from_list,use_reduce', + 'portage.dep:Atom,dep_getkey,match_from_list,use_reduce,_match_slot', 'portage.package.ebuild.doebuild:doebuild', 'portage.util:ensure_dirs,shlex_split,writemsg,writemsg_level', 'portage.util.listdir:listdir', @@ -22,7 +24,8 @@ from portage.cache.cache_errors import CacheError from portage.cache.mappings import Mapping from portage.dbapi import dbapi from portage.exception import PortageException, \ - FileNotFound, InvalidAtom, InvalidDependString, InvalidPackageName + FileNotFound, InvalidAtom, InvalidData, \ + InvalidDependString, InvalidPackageName from portage.localization import _ from portage import eclass_cache, \ @@ -32,21 +35,74 @@ from portage import os from portage import _encodings from portage import _unicode_encode from portage import OrderedDict +from portage.util._eventloop.EventLoop import EventLoop +from portage.util._eventloop.global_event_loop import global_event_loop from _emerge.EbuildMetadataPhase import EbuildMetadataPhase -from _emerge.PollScheduler import PollScheduler import os as _os import sys import traceback import warnings +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse + if sys.hexversion >= 0x3000000: basestring = str long = int +def close_portdbapi_caches(): + # The python interpreter does _not_ guarantee that destructors are + # called for objects that remain when the interpreter exits, so we + # use an atexit hook to call destructors for any global portdbapi + # instances that may have been constructed. + try: + portage._legacy_globals_constructed + except AttributeError: + pass + else: + if "db" in portage._legacy_globals_constructed: + try: + db = portage.db + except AttributeError: + pass + else: + if isinstance(db, dict): + for x in db.values(): + try: + if "porttree" in x.lazy_items: + continue + except (AttributeError, TypeError): + continue + try: + x = x.pop("porttree").dbapi + except (AttributeError, KeyError): + continue + if not isinstance(x, portdbapi): + continue + x.close_caches() + +portage.process.atexit_register(close_portdbapi_caches) + +# It used to be necessary for API consumers to remove portdbapi instances +# from portdbapi_instances, in order to avoid having accumulated instances +# consume memory. Now, portdbapi_instances is just an empty dummy list, so +# for backward compatibility, ignore ValueError for removal on non-existent +# items. +class _dummy_list(list): + def remove(self, item): + # TODO: Trigger a DeprecationWarning here, after stable portage + # has dummy portdbapi_instances. + try: + list.remove(self, item) + except ValueError: + pass + class portdbapi(dbapi): """this tree will scan a portage directory located at root (passed to init)""" - portdbapi_instances = [] + portdbapi_instances = _dummy_list() _use_mutable = True @property @@ -64,14 +120,13 @@ class portdbapi(dbapi): return None return main_repo.eclass_db - def __init__(self, _unused_param=None, mysettings=None): + def __init__(self, _unused_param=DeprecationWarning, mysettings=None): """ @param _unused_param: deprecated, use mysettings['PORTDIR'] instead @type _unused_param: None @param mysettings: an immutable config instance @type mysettings: portage.config """ - portdbapi.portdbapi_instances.append(self) from portage import config if mysettings: @@ -80,7 +135,7 @@ class portdbapi(dbapi): from portage import settings self.settings = config(clone=settings) - if _unused_param is not None: + if _unused_param is not DeprecationWarning: warnings.warn("The first parameter of the " + \ "portage.dbapi.porttree.portdbapi" + \ " constructor is unused since portage-2.1.8. " + \ @@ -95,7 +150,6 @@ class portdbapi(dbapi): # this purpose because doebuild makes many changes to the config # instance that is passed in. self.doebuild_settings = config(clone=self.settings) - self._scheduler = PollScheduler().sched_iface self.depcachedir = os.path.realpath(self.settings.depcachedir) if os.environ.get("SANDBOX_ON") == "1": @@ -152,10 +206,10 @@ class portdbapi(dbapi): # portage group. depcachedir_unshared = True else: - cache_kwargs.update({ + cache_kwargs.update(portage._native_kwargs({ 'gid' : portage_gid, 'perms' : 0o664 - }) + })) # If secpass < 1, we don't want to write to the cache # since then we won't be able to apply group permissions @@ -186,13 +240,25 @@ class portdbapi(dbapi): self._pregen_auxdb[x] = cache # Selectively cache metadata in order to optimize dep matching. self._aux_cache_keys = set( - ["DEPEND", "EAPI", "INHERITED", "IUSE", "KEYWORDS", "LICENSE", + ["DEPEND", "EAPI", "HDEPEND", + "INHERITED", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROPERTIES", "PROVIDE", "RDEPEND", "repository", "RESTRICT", "SLOT", "DEFINED_PHASES", "REQUIRED_USE"]) self._aux_cache = {} self._broken_ebuilds = set() + @property + def _event_loop(self): + if portage._internal_caller: + # For internal portage usage, the global_event_loop is safe. + return global_event_loop() + else: + # For external API consumers, use a local EventLoop, since + # we don't want to assume that it's safe to override the + # global SIGCHLD handler. + return EventLoop(main=False) + def _create_pregen_cache(self, tree): conf = self.repositories.get_repo_for_location(tree) cache = conf.get_pregenerated_cache( @@ -202,6 +268,13 @@ class portdbapi(dbapi): cache.ec = self.repositories.get_repo_for_location(tree).eclass_db except AttributeError: pass + + if not cache.complete_eclass_entries: + warnings.warn( + ("Repository '%s' used deprecated 'pms' cache format. " + "Please migrate to 'md5-dict' format.") % (conf.name,), + DeprecationWarning) + return cache def _init_cache_dirs(self): @@ -446,7 +519,7 @@ class portdbapi(dbapi): proc = EbuildMetadataPhase(cpv=mycpv, ebuild_hash=ebuild_hash, portdb=self, - repo_path=mylocation, scheduler=self._scheduler, + repo_path=mylocation, scheduler=self._event_loop, settings=self.doebuild_settings) proc.start() @@ -626,13 +699,14 @@ class portdbapi(dbapi): else: return 0 - def cp_all(self, categories=None, trees=None): + def cp_all(self, categories=None, trees=None, reverse=False): """ This returns a list of all keys in our tree or trees @param categories: optional list of categories to search or defaults to self.settings.categories @param trees: optional list of trees to search the categories in or defaults to self.porttrees + @param reverse: reverse sort order (default is False) @rtype list of [cat/pkg,...] """ d = {} @@ -651,7 +725,7 @@ class portdbapi(dbapi): continue d[atom.cp] = None l = list(d) - l.sort() + l.sort(reverse=reverse) return l def cp_list(self, mycp, use_cache=1, mytree=None): @@ -825,18 +899,24 @@ class portdbapi(dbapi): # ebuild not in this repo, or masked by corruption continue - if visibility_filter and not self._visible(cpv, metadata): + try: + pkg_str = _pkg_str(cpv, metadata=metadata, + settings=self.settings) + except InvalidData: + continue + + if visibility_filter and not self._visible(pkg_str, metadata): continue if mydep.slot is not None and \ - mydep.slot != metadata["SLOT"]: + not _match_slot(mydep, pkg_str): continue if mydep.unevaluated_atom.use is not None and \ - not self._match_use(mydep, cpv, metadata): + not self._match_use(mydep, pkg_str, metadata): continue - myval.append(cpv) + myval.append(pkg_str) # only yield a given cpv once break @@ -959,19 +1039,16 @@ class portdbapi(dbapi): return False if settings._getMissingProperties(cpv, metadata): return False + if settings._getMissingRestrict(cpv, metadata): + return False except InvalidDependString: return False return True -def close_portdbapi_caches(): - for i in portdbapi.portdbapi_instances: - i.close_caches() - -portage.process.atexit_register(portage.portageexit) - class portagetree(object): - def __init__(self, root=None, virtual=DeprecationWarning, settings=None): + def __init__(self, root=DeprecationWarning, virtual=DeprecationWarning, + settings=None): """ Constructor for a PortageTree @@ -987,7 +1064,7 @@ class portagetree(object): settings = portage.settings self.settings = settings - if root is not None and root != settings['ROOT']: + if root is not DeprecationWarning: warnings.warn("The root parameter of the " + \ "portage.dbapi.porttree.portagetree" + \ " constructor is now unused. Use " + \ @@ -1055,10 +1132,8 @@ class portagetree(object): "Get a slot for a catpkg; assume it exists." myslot = "" try: - myslot = self.dbapi.aux_get(mycatpkg, ["SLOT"])[0] - except SystemExit: - raise - except Exception: + myslot = self.dbapi._pkg_str(mycatpkg, None).slot + except KeyError: pass return myslot @@ -1130,9 +1205,18 @@ def _parse_uri_map(cpv, metadata, use=None): uri_set = uri_map.get(distfile) if uri_set is None: - uri_set = set() + # Use OrderedDict to preserve order from SRC_URI + # while ensuring uniqueness. + uri_set = OrderedDict() uri_map[distfile] = uri_set - uri_set.add(uri) - uri = None + + # SRC_URI may contain a file name with no scheme, and in + # this case it does not belong in uri_set. + if urlparse(uri).scheme: + uri_set[uri] = True + + # Convert OrderedDicts to tuples. + for k, v in uri_map.items(): + uri_map[k] = tuple(v) return uri_map diff --git a/portage_with_autodep/pym/portage/dbapi/porttree.pyo b/portage_with_autodep/pym/portage/dbapi/porttree.pyo Binary files differindex fb57919..43ce4a8 100644 --- a/portage_with_autodep/pym/portage/dbapi/porttree.pyo +++ b/portage_with_autodep/pym/portage/dbapi/porttree.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/vartree.py b/portage_with_autodep/pym/portage/dbapi/vartree.py index 517c873..ed62323 100644 --- a/portage_with_autodep/pym/portage/dbapi/vartree.py +++ b/portage_with_autodep/pym/portage/dbapi/vartree.py @@ -1,6 +1,8 @@ -# Copyright 1998-2012 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = [ "vardbapi", "vartree", "dblink"] + \ ["write_contents", "tar_contents"] @@ -11,8 +13,10 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.data:portage_gid,portage_uid,secpass', 'portage.dbapi.dep_expand:dep_expand', 'portage.dbapi._MergeProcess:MergeProcess', - 'portage.dep:dep_getkey,isjustname,match_from_list,' + \ - 'use_reduce,_slot_re', + 'portage.dbapi._SyncfsProcess:SyncfsProcess', + 'portage.dep:dep_getkey,isjustname,isvalidatom,match_from_list,' + \ + 'use_reduce,_slot_separator,_repo_separator', + 'portage.eapi:_get_eapi_attrs', 'portage.elog:collect_ebuild_messages,collect_messages,' + \ 'elog_process,_merge_logentries', 'portage.locks:lockdir,unlockdir,lockfile,unlockfile', @@ -20,7 +24,7 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.package.ebuild.doebuild:doebuild_environment,' + \ '_merge_unicode_error', '_spawn_phase', 'portage.package.ebuild.prepare_build_dirs:prepare_build_dirs', - 'portage.update:fixdbentries', + 'portage.package.ebuild._ipc.QueryCommand:QueryCommand', 'portage.util:apply_secpass_permissions,ConfigProtect,ensure_dirs,' + \ 'writemsg,writemsg_level,write_atomic,atomic_ofstream,writedict,' + \ 'grabdict,normalize_path,new_protect_filename', @@ -30,15 +34,17 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.util.movefile:movefile', 'portage.util._dyn_libs.PreservedLibsRegistry:PreservedLibsRegistry', 'portage.util._dyn_libs.LinkageMapELF:LinkageMapELF@LinkageMap', + 'portage.util._async.SchedulerInterface:SchedulerInterface', + 'portage.util._eventloop.EventLoop:EventLoop', + 'portage.util._eventloop.global_event_loop:global_event_loop', 'portage.versions:best,catpkgsplit,catsplit,cpv_getkey,vercmp,' + \ - '_pkgsplit@pkgsplit,_pkg_str', + '_get_slot_re,_pkgsplit@pkgsplit,_pkg_str,_unknown_repo', 'subprocess', 'tarfile', ) from portage.const import CACHE_PATH, CONFIG_MEMORY_FILE, \ PORTAGE_PACKAGE_ATOM, PRIVATE_PATH, VDB_PATH -from portage.const import _ENABLE_DYN_LINK_MAP, _ENABLE_PRESERVE_LIBS from portage.dbapi import dbapi from portage.exception import CommandNotFound, \ InvalidData, InvalidLocation, InvalidPackageName, \ @@ -59,8 +65,8 @@ from portage import _unicode_encode from _emerge.EbuildBuildDir import EbuildBuildDir from _emerge.EbuildPhase import EbuildPhase from _emerge.emergelog import emergelog -from _emerge.PollScheduler import PollScheduler from _emerge.MiscFunctionsProcess import MiscFunctionsProcess +from _emerge.SpawnProcess import SpawnProcess import errno import fnmatch @@ -70,6 +76,7 @@ import io from itertools import chain import logging import os as _os +import platform import pwd import re import stat @@ -108,7 +115,8 @@ class vardbapi(dbapi): _aux_cache_keys_re = re.compile(r'^NEEDED\..*$') _aux_multi_line_re = re.compile(r'^(CONTENTS|NEEDED\..*)$') - def __init__(self, _unused_param=None, categories=None, settings=None, vartree=None): + def __init__(self, _unused_param=DeprecationWarning, + categories=None, settings=None, vartree=None): """ The categories parameter is unused since the dbapi class now has a categories property that is generated from the @@ -138,11 +146,11 @@ class vardbapi(dbapi): settings = portage.settings self.settings = settings - if _unused_param is not None and _unused_param != settings['ROOT']: + if _unused_param is not DeprecationWarning: warnings.warn("The first parameter of the " "portage.dbapi.vartree.vardbapi" - " constructor is now unused. Use " - "settings['ROOT'] instead.", + " constructor is now unused. Instead " + "settings['ROOT'] is used.", DeprecationWarning, stacklevel=2) self._eroot = settings['EROOT'] @@ -159,7 +167,7 @@ class vardbapi(dbapi): self.vartree = vartree self._aux_cache_keys = set( ["BUILD_TIME", "CHOST", "COUNTER", "DEPEND", "DESCRIPTION", - "EAPI", "HOMEPAGE", "IUSE", "KEYWORDS", + "EAPI", "HDEPEND", "HOMEPAGE", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROPERTIES", "PROVIDE", "RDEPEND", "repository", "RESTRICT" , "SLOT", "USE", "DEFINED_PHASES", ]) @@ -169,15 +177,9 @@ class vardbapi(dbapi): self._counter_path = os.path.join(self._eroot, CACHE_PATH, "counter") - self._plib_registry = None - if _ENABLE_PRESERVE_LIBS: - self._plib_registry = PreservedLibsRegistry(settings["ROOT"], - os.path.join(self._eroot, PRIVATE_PATH, - "preserved_libs_registry")) - - self._linkmap = None - if _ENABLE_DYN_LINK_MAP: - self._linkmap = LinkageMap(self) + self._plib_registry = PreservedLibsRegistry(settings["ROOT"], + os.path.join(self._eroot, PRIVATE_PATH, "preserved_libs_registry")) + self._linkmap = LinkageMap(self) self._owners = self._owners_db(self) self._cached_counter = None @@ -318,14 +320,24 @@ class vardbapi(dbapi): if not origmatches: return moves for mycpv in origmatches: + try: + mycpv = self._pkg_str(mycpv, None) + except (KeyError, InvalidData): + continue mycpv_cp = cpv_getkey(mycpv) if mycpv_cp != origcp: # Ignore PROVIDE virtual match. continue if repo_match is not None \ - and not repo_match(self.aux_get(mycpv, ['repository'])[0]): + and not repo_match(mycpv.repo): continue - mynewcpv = mycpv.replace(mycpv_cp, str(newcp), 1) + + # Use isvalidatom() to check if this move is valid for the + # EAPI (characters allowed in package names may vary). + if not isvalidatom(newcp, eapi=mycpv.eapi): + continue + + mynewcpv = mycpv.replace(mycpv_cp, _unicode(newcp), 1) mynewcat = catsplit(newcp)[0] origpath = self.getpath(mycpv) if not os.path.exists(origpath): @@ -355,7 +367,7 @@ class vardbapi(dbapi): del e write_atomic(os.path.join(newpath, "PF"), new_pf+"\n") write_atomic(os.path.join(newpath, "CATEGORY"), mynewcat+"\n") - fixdbentries([mylist], newpath) + return moves def cp_list(self, mycp, use_cache=1): @@ -363,7 +375,10 @@ class vardbapi(dbapi): if mysplit[0] == '*': mysplit[0] = mysplit[0][1:] try: - mystat = os.stat(self.getpath(mysplit[0])).st_mtime + if sys.hexversion >= 0x3030000: + mystat = os.stat(self.getpath(mysplit[0])).st_mtime_ns + else: + mystat = os.stat(self.getpath(mysplit[0])).st_mtime except OSError: mystat = 0 if use_cache and mycp in self.cpcache: @@ -498,7 +513,10 @@ class vardbapi(dbapi): return list(self._iter_match(mydep, self.cp_list(mydep.cp, use_cache=use_cache))) try: - curmtime = os.stat(os.path.join(self._eroot, VDB_PATH, mycat)).st_mtime + if sys.hexversion >= 0x3030000: + curmtime = os.stat(os.path.join(self._eroot, VDB_PATH, mycat)).st_mtime_ns + else: + curmtime = os.stat(os.path.join(self._eroot, VDB_PATH, mycat)).st_mtime except (IOError, OSError): curmtime=0 @@ -553,31 +571,32 @@ class vardbapi(dbapi): def _aux_cache_init(self): aux_cache = None open_kwargs = {} - if sys.hexversion >= 0x3000000: + if sys.hexversion >= 0x3000000 and sys.hexversion < 0x3020000: # Buffered io triggers extreme performance issues in # Unpickler.load() (problem observed with python-3.0.1). # Unfortunately, performance is still poor relative to - # python-2.x, but buffering makes it much worse. + # python-2.x, but buffering makes it much worse (problem + # appears to be solved in Python >=3.2 at least). open_kwargs["buffering"] = 0 try: - f = open(_unicode_encode(self._aux_cache_filename, + with open(_unicode_encode(self._aux_cache_filename, encoding=_encodings['fs'], errors='strict'), - mode='rb', **open_kwargs) - mypickle = pickle.Unpickler(f) - try: - mypickle.find_global = None - except AttributeError: - # TODO: If py3k, override Unpickler.find_class(). - pass - aux_cache = mypickle.load() - f.close() - del f - except (AttributeError, EOFError, EnvironmentError, ValueError, pickle.UnpicklingError) as e: + mode='rb', **open_kwargs) as f: + mypickle = pickle.Unpickler(f) + try: + mypickle.find_global = None + except AttributeError: + # TODO: If py3k, override Unpickler.find_class(). + pass + aux_cache = mypickle.load() + except (SystemExit, KeyboardInterrupt): + raise + except Exception as e: if isinstance(e, EnvironmentError) and \ getattr(e, 'errno', None) in (errno.ENOENT, errno.EACCES): pass else: - writemsg(_unicode_decode(_("!!! Error loading '%s': %s\n")) % \ + writemsg(_("!!! Error loading '%s': %s\n") % \ (self._aux_cache_filename, e), noiselevel=-1) del e @@ -644,7 +663,8 @@ class vardbapi(dbapi): if e.errno != errno.ENOENT: raise raise KeyError(mycpv) - mydir_mtime = mydir_stat[stat.ST_MTIME] + # Use float mtime when available. + mydir_mtime = mydir_stat.st_mtime pkg_data = self._aux_cache["packages"].get(mycpv) pull_me = cache_these.union(wants) mydata = {"_mtime_" : mydir_mtime} @@ -657,13 +677,18 @@ class vardbapi(dbapi): pkg_data = None else: cache_mtime, metadata = pkg_data - if not isinstance(cache_mtime, (long, int)) or \ + if not isinstance(cache_mtime, (float, long, int)) or \ not isinstance(metadata, dict): pkg_data = None if pkg_data: cache_mtime, metadata = pkg_data - cache_valid = cache_mtime == mydir_mtime + if isinstance(cache_mtime, float): + cache_valid = cache_mtime == mydir_stat.st_mtime + else: + # Cache may contain integer mtime. + cache_valid = cache_mtime == mydir_stat[stat.ST_MTIME] + if cache_valid: # Migrate old metadata to unicode. for k, v in metadata.items(): @@ -687,10 +712,11 @@ class vardbapi(dbapi): (mydir_mtime, cache_data) self._aux_cache["modified"].add(mycpv) - if _slot_re.match(mydata['SLOT']) is None: + eapi_attrs = _get_eapi_attrs(mydata['EAPI']) + if _get_slot_re(eapi_attrs).match(mydata['SLOT']) is None: # Empty or invalid slot triggers InvalidAtom exceptions when # generating slot atoms for packages, so translate it to '0' here. - mydata['SLOT'] = _unicode_decode('0') + mydata['SLOT'] = '0' return [mydata[x] for x in wants] @@ -715,21 +741,18 @@ class vardbapi(dbapi): results[x] = st[stat.ST_MTIME] continue try: - myf = io.open( + with io.open( _unicode_encode(os.path.join(mydir, x), encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], - errors='replace') - try: - myd = myf.read() - finally: - myf.close() + errors='replace') as f: + myd = f.read() except IOError: if x not in self._aux_cache_keys and \ self._aux_cache_keys_re.match(x) is None: env_keys.append(x) continue - myd = _unicode_decode('') + myd = '' # Preserve \n for metadata that is known to # contain multiple lines. @@ -743,13 +766,13 @@ class vardbapi(dbapi): for k in env_keys: v = env_results.get(k) if v is None: - v = _unicode_decode('') + v = '' if self._aux_multi_line_re.match(k) is None: v = " ".join(v.split()) results[k] = v if results.get("EAPI") == "": - results[_unicode_decode("EAPI")] = _unicode_decode('0') + results["EAPI"] = '0' return results @@ -869,11 +892,17 @@ class vardbapi(dbapi): del myroot counter = -1 try: - cfile = io.open( + with io.open( _unicode_encode(self._counter_path, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], - errors='replace') + errors='replace') as f: + try: + counter = long(f.readline().strip()) + except (OverflowError, ValueError) as e: + writemsg(_("!!! COUNTER file is corrupt: '%s'\n") % + self._counter_path, noiselevel=-1) + writemsg("!!! %s\n" % (e,), noiselevel=-1) except EnvironmentError as e: # Silently allow ENOENT since files under # /var/cache/ are allowed to disappear. @@ -882,17 +911,6 @@ class vardbapi(dbapi): self._counter_path, noiselevel=-1) writemsg("!!! %s\n" % str(e), noiselevel=-1) del e - else: - try: - try: - counter = long(cfile.readline().strip()) - finally: - cfile.close() - except (OverflowError, ValueError) as e: - writemsg(_("!!! COUNTER file is corrupt: '%s'\n") % \ - self._counter_path, noiselevel=-1) - writemsg("!!! %s\n" % str(e), noiselevel=-1) - del e if self._cached_counter == counter: max_counter = counter @@ -984,16 +1002,31 @@ class vardbapi(dbapi): relative_filename = filename[root_len:] contents_key = pkg._match_contents(relative_filename) if contents_key: - del new_contents[contents_key] + # It's possible for two different paths to refer to the same + # contents_key, due to directory symlinks. Therefore, pass a + # default value to pop, in order to avoid a KeyError which + # could otherwise be triggered (see bug #454400). + new_contents.pop(contents_key, None) removed += 1 if removed: - self._bump_mtime(pkg.mycpv) - f = atomic_ofstream(os.path.join(pkg.dbdir, "CONTENTS")) - write_contents(new_contents, root, f) - f.close() - self._bump_mtime(pkg.mycpv) - pkg._clear_contents_cache() + self.writeContentsToContentsFile(pkg, new_contents) + + def writeContentsToContentsFile(self, pkg, new_contents): + """ + @param pkg: package to write contents file for + @type pkg: dblink + @param new_contents: contents to write to CONTENTS file + @type new_contents: contents dictionary of the form + {u'/path/to/file' : (contents_attribute 1, ...), ...} + """ + root = self.settings['ROOT'] + self._bump_mtime(pkg.mycpv) + f = atomic_ofstream(os.path.join(pkg.dbdir, "CONTENTS")) + write_contents(new_contents, root, f) + f.close() + self._bump_mtime(pkg.mycpv) + pkg._clear_contents_cache() class _owners_cache(object): """ @@ -1238,18 +1271,35 @@ class vardbapi(dbapi): name = os.path.basename(path.rstrip(os.path.sep)) path_info_list.append((path, name, is_basename)) + # Do work via the global event loop, so that it can be used + # for indication of progress during the search (bug #461412). + event_loop = (portage._internal_caller and + global_event_loop() or EventLoop(main=False)) root = self._vardb._eroot - for cpv in self._vardb.cpv_all(): - dblnk = self._vardb._dblink(cpv) + def search_pkg(cpv): + dblnk = self._vardb._dblink(cpv) for path, name, is_basename in path_info_list: if is_basename: for p in dblnk.getcontents(): if os.path.basename(p) == name: - yield dblnk, p[len(root):] + search_pkg.results.append((dblnk, p[len(root):])) else: if dblnk.isowner(path): - yield dblnk, path + search_pkg.results.append((dblnk, path)) + search_pkg.complete = True + return False + + search_pkg.results = [] + + for cpv in self._vardb.cpv_all(): + del search_pkg.results[:] + search_pkg.complete = False + event_loop.idle_add(search_pkg, cpv) + while not search_pkg.complete: + event_loop.iteration() + for result in search_pkg.results: + yield result class vartree(object): "this tree will scan a var/db/pkg database located at root (passed to init)" @@ -1370,7 +1420,7 @@ class vartree(object): def getslot(self, mycatpkg): "Get a slot for a catpkg; assume it exists." try: - return self.dbapi.aux_get(mycatpkg, ["SLOT"])[0] + return self.dbapi._pkg_str(mycatpkg, None).slot except KeyError: return "" @@ -1463,11 +1513,16 @@ class dblink(object): self._contents_inodes = None self._contents_basenames = None self._linkmap_broken = False + self._device_path_map = {} self._hardlink_merge_map = {} self._hash_key = (self._eroot, self.mycpv) self._protect_obj = None self._pipe = pipe + # When necessary, this attribute is modified for + # compliance with RESTRICT=preserve-libs. + self._preserve_libs = "preserve-libs" in mysettings.features + def __hash__(self): return hash(self._hash_key) @@ -1510,7 +1565,11 @@ class dblink(object): """ Remove this entry from the database """ - if not os.path.exists(self.dbdir): + try: + os.lstat(self.dbdir) + except OSError as e: + if e.errno not in (errno.ENOENT, errno.ENOTDIR, errno.ESTALE): + raise return # Check validity of self.dbdir before attempting to remove it. @@ -1527,6 +1586,14 @@ class dblink(object): pass self.vartree.dbapi._remove(self) + # Use self.dbroot since we need an existing path for syncfs. + try: + self._merged_path(self.dbroot, os.lstat(self.dbroot)) + except OSError: + pass + + self._post_merge_sync() + def clearcontents(self): """ For a given db entry (self), erase the CONTENTS values. @@ -1552,18 +1619,18 @@ class dblink(object): return self.contentscache pkgfiles = {} try: - myc = io.open(_unicode_encode(contents_file, + with io.open(_unicode_encode(contents_file, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], - errors='replace') + errors='replace') as f: + mylines = f.readlines() except EnvironmentError as e: if e.errno != errno.ENOENT: raise del e self.contentscache = pkgfiles return pkgfiles - mylines = myc.readlines() - myc.close() + null_byte = "\0" normalize_needed = self._normalize_needed contents_re = self._contents_re @@ -1578,7 +1645,7 @@ class dblink(object): if myroot == os.path.sep: myroot = None # used to generate parent dir entries - dir_entry = (_unicode_decode("dir"),) + dir_entry = ("dir",) eroot_split_len = len(self.settings["EROOT"].split(os.sep)) - 1 pos = 0 errors = [] @@ -1678,8 +1745,11 @@ class dblink(object): unmerge_preserve = \ self._find_libs_to_preserve(unmerge=True) counter = self.vartree.dbapi.cpv_counter(self.mycpv) - plib_registry.unregister(self.mycpv, - self.settings["SLOT"], counter) + try: + slot = self.mycpv.slot + except AttributeError: + slot = _pkg_str(self.mycpv, slot=self.settings["SLOT"]).slot + plib_registry.unregister(self.mycpv, slot, counter) if unmerge_preserve: for path in sorted(unmerge_preserve): contents_key = self._match_contents(path) @@ -1689,7 +1759,7 @@ class dblink(object): self._display_merge(_(">>> needed %s %s\n") % \ (obj_type, contents_key), noiselevel=-1) plib_registry.register(self.mycpv, - self.settings["SLOT"], counter, unmerge_preserve) + slot, counter, unmerge_preserve) # Remove the preserved files from our contents # so that they won't be unmerged. self.vartree.dbapi.removeFromContents(self, @@ -1759,7 +1829,8 @@ class dblink(object): if self._scheduler is None: # We create a scheduler instance and use it to # log unmerge output separately from merge output. - self._scheduler = PollScheduler().sched_iface + self._scheduler = SchedulerInterface(portage._internal_caller and + global_event_loop() or EventLoop(main=False)) if self.settings.get("PORTAGE_BACKGROUND") == "subprocess": if self.settings.get("PORTAGE_BACKGROUND_UNMERGE") == "1": self.settings["PORTAGE_BACKGROUND"] = "1" @@ -1775,11 +1846,16 @@ class dblink(object): showMessage = self._display_merge if self.vartree.dbapi._categories is not None: self.vartree.dbapi._categories = None + + # When others_in_slot is not None, the backup has already been + # handled by the caller. + caller_handles_backup = others_in_slot is not None + # When others_in_slot is supplied, the security check has already been # done for this slot, so it shouldn't be repeated until the next # replacement or unmerge operation. if others_in_slot is None: - slot = self.vartree.dbapi.aux_get(self.mycpv, ["SLOT"])[0] + slot = self.vartree.dbapi._pkg_str(self.mycpv, None).slot slot_matches = self.vartree.dbapi.match( "%s:%s" % (portage.cpv_getkey(self.mycpv), slot)) others_in_slot = [] @@ -1823,8 +1899,9 @@ class dblink(object): except UnsupportedAPIException as e: eapi_unsupported = e - self._prune_plib_registry(unmerge=True, needed=needed, - preserve_paths=preserve_paths) + if self._preserve_libs and "preserve-libs" in \ + self.settings["PORTAGE_RESTRICT"].split(): + self._preserve_libs = False builddir_lock = None scheduler = self._scheduler @@ -1832,7 +1909,7 @@ class dblink(object): try: # Only create builddir_lock if the caller # has not already acquired the lock. - if "PORTAGE_BUILDIR_LOCKED" not in self.settings: + if "PORTAGE_BUILDDIR_LOCKED" not in self.settings: builddir_lock = EbuildBuildDir( scheduler=scheduler, settings=self.settings) @@ -1840,6 +1917,19 @@ class dblink(object): prepare_build_dirs(settings=self.settings, cleanup=True) log_path = self.settings.get("PORTAGE_LOG_FILE") + # Do this before the following _prune_plib_registry call, since + # that removes preserved libraries from our CONTENTS, and we + # may want to backup those libraries first. + if not caller_handles_backup: + retval = self._pre_unmerge_backup(background) + if retval != os.EX_OK: + showMessage(_("!!! FAILED prerm: quickpkg: %s\n") % retval, + level=logging.ERROR, noiselevel=-1) + return retval + + self._prune_plib_registry(unmerge=True, needed=needed, + preserve_paths=preserve_paths) + # Log the error after PORTAGE_LOG_FILE is initialized # by prepare_build_dirs above. if eapi_unsupported: @@ -1848,7 +1938,7 @@ class dblink(object): showMessage(_("!!! FAILED prerm: %s\n") % \ os.path.join(self.dbdir, "EAPI"), level=logging.ERROR, noiselevel=-1) - showMessage(_unicode_decode("%s\n") % (eapi_unsupported,), + showMessage("%s\n" % (eapi_unsupported,), level=logging.ERROR, noiselevel=-1) elif os.path.isfile(myebuildpath): phase = EbuildPhase(background=background, @@ -2037,7 +2127,7 @@ class dblink(object): if others_in_slot is None: others_in_slot = [] - slot = self.vartree.dbapi.aux_get(self.mycpv, ["SLOT"])[0] + slot = self.vartree.dbapi._pkg_str(self.mycpv, None).slot slot_matches = self.vartree.dbapi.match( "%s:%s" % (portage.cpv_getkey(self.mycpv), slot)) for cur_cpv in slot_matches: @@ -2062,7 +2152,9 @@ class dblink(object): #process symlinks second-to-last, directories last. mydirs = set() - modprotect = os.path.join(self._eroot, "lib/modules/") + + uninstall_ignore = portage.util.shlex_split( + self.settings.get("UNINSTALL_IGNORE", "")) def unlink(file_name, lstatobj): if bsd_chflags: @@ -2092,6 +2184,14 @@ class dblink(object): self._eerror("postrm", ["Could not chmod or unlink '%s': %s" % \ (file_name, ose)]) + else: + + # Even though the file no longer exists, we log it + # here so that _unmerge_dirs can see that we've + # removed a file from this device, and will record + # the parent directory for a syncfs call. + self._merged_path(file_name, lstatobj, exists=False) + finally: if bsd_chflags and pflags != 0: # Restore the parent flags we saved before unlinking @@ -2169,6 +2269,24 @@ class dblink(object): if lstatobj is None: show_unmerge("---", unmerge_desc["!found"], file_type, obj) continue + + f_match = obj[len(eroot)-1:] + ignore = False + for pattern in uninstall_ignore: + if fnmatch.fnmatch(f_match, pattern): + ignore = True + break + + if not ignore: + if islink and f_match in \ + ("/lib", "/usr/lib", "/usr/local/lib"): + # Ignore libdir symlinks for bug #423127. + ignore = True + + if ignore: + show_unmerge("---", unmerge_desc["cfgpro"], file_type, obj) + continue + # don't use EROOT, CONTENTS entries already contain EPREFIX if obj.startswith(real_root): relative_path = obj[real_root_len:] @@ -2178,8 +2296,9 @@ class dblink(object): is_owned = True break - if file_type == "sym" and is_owned and \ - (islink and statobj and stat.S_ISDIR(statobj.st_mode)): + if is_owned and islink and \ + file_type in ("sym", "dir") and \ + statobj and stat.S_ISDIR(statobj.st_mode): # A new instance of this package claims the file, so # don't unmerge it. If the file is symlink to a # directory and the unmerging package installed it as @@ -2211,18 +2330,6 @@ class dblink(object): continue elif relative_path in cfgfiledict: stale_confmem.append(relative_path) - # next line includes a tweak to protect modules from being unmerged, - # but we don't protect modules from being overwritten if they are - # upgraded. We effectively only want one half of the config protection - # functionality for /lib/modules. For portage-ng both capabilities - # should be able to be independently specified. - # TODO: For rebuilds, re-parent previous modules to the new - # installed instance (so they are not orphans). For normal - # uninstall (not rebuild/reinstall), remove the modules along - # with all other files (leave no orphans). - if obj.startswith(modprotect): - show_unmerge("---", unmerge_desc["cfgpro"], file_type, obj) - continue # Don't unlink symlinks to directories here since that can # remove /lib and /usr/lib symlinks. @@ -2244,12 +2351,12 @@ class dblink(object): show_unmerge("---", unmerge_desc["!mtime"], file_type, obj) continue - if pkgfiles[objkey][0] == "dir": + if file_type == "dir" and not islink: if lstatobj is None or not stat.S_ISDIR(lstatobj.st_mode): show_unmerge("---", unmerge_desc["!dir"], file_type, obj) continue mydirs.add((obj, (lstatobj.st_dev, lstatobj.st_ino))) - elif pkgfiles[objkey][0] == "sym": + elif file_type == "sym" or (file_type == "dir" and islink): if not islink: show_unmerge("---", unmerge_desc["!sym"], file_type, obj) continue @@ -2359,7 +2466,11 @@ class dblink(object): if protected_symlinks: msg = "One or more symlinks to directories have been " + \ "preserved in order to ensure that files installed " + \ - "via these symlinks remain accessible:" + "via these symlinks remain accessible. " + \ + "This indicates that the mentioned symlink(s) may " + \ + "be obsolete remnants of an old install, and it " + \ + "may be appropriate to replace a given symlink " + \ + "with the directory that it points to." lines = textwrap.wrap(msg, 72) lines.append("") flat_list = set() @@ -2369,7 +2480,7 @@ class dblink(object): lines.append("\t%s" % (os.path.join(real_root, f.lstrip(os.sep)))) lines.append("") - self._elog("eerror", "postrm", lines) + self._elog("elog", "postrm", lines) # Remove stale entries from config memory. if stale_confmem: @@ -2501,15 +2612,19 @@ class dblink(object): raise del e show_unmerge("!!!", "", "obj", child) + try: + parent_name = os.path.dirname(obj) + parent_stat = os.stat(parent_name) + if bsd_chflags: lstatobj = os.lstat(obj) if lstatobj.st_flags != 0: bsd_chflags.lchflags(obj, 0) - parent_name = os.path.dirname(obj) + # Use normal stat/chflags for the parent since we want to # follow any symlinks to the real parent directory. - pflags = os.stat(parent_name).st_flags + pflags = parent_stat.st_flags if pflags != 0: bsd_chflags.chflags(parent_name, 0) try: @@ -2518,13 +2633,34 @@ class dblink(object): if bsd_chflags and pflags != 0: # Restore the parent flags we saved before unlinking bsd_chflags.chflags(parent_name, pflags) + + # Record the parent directory for use in syncfs calls. + # Note that we use a realpath and a regular stat here, since + # we want to follow any symlinks back to the real device where + # the real parent directory resides. + self._merged_path(os.path.realpath(parent_name), parent_stat) + show_unmerge("<<<", "", "dir", obj) except EnvironmentError as e: if e.errno not in ignored_rmdir_errnos: raise if e.errno != errno.ENOENT: show_unmerge("---", unmerge_desc["!empty"], "dir", obj) - del e + + # Since we didn't remove this directory, record the directory + # itself for use in syncfs calls, if we have removed another + # file from the same device. + # Note that we use a realpath and a regular stat here, since + # we want to follow any symlinks back to the real device where + # the real directory resides. + try: + dir_stat = os.stat(obj) + except OSError: + pass + else: + if dir_stat.st_dev in self._device_path_map: + self._merged_path(os.path.realpath(obj), dir_stat) + else: # When a directory is successfully removed, there's # no need to protect symlinks that point to it. @@ -2751,7 +2887,7 @@ class dblink(object): self.vartree.dbapi._linkmap is None or \ self.vartree.dbapi._plib_registry is None or \ (not unmerge and self._installed_instance is None) or \ - "preserve-libs" not in self.settings.features: + not self._preserve_libs: return set() os = _os_merge @@ -3335,7 +3471,10 @@ class dblink(object): else: logdir = os.path.join(self.settings["T"], "logging") ebuild_logentries = collect_ebuild_messages(logdir) - py_logentries = collect_messages(key=cpv).get(cpv, {}) + # phasefilter is irrelevant for the above collect_ebuild_messages + # call, since this package instance has a private logdir. However, + # it may be relevant for the following collect_messages call. + py_logentries = collect_messages(key=cpv, phasefilter=phasefilter).get(cpv, {}) logentries = _merge_logentries(py_logentries, ebuild_logentries) funcnames = { "INFO": "einfo", @@ -3356,7 +3495,9 @@ class dblink(object): str_buffer.append(' '.join(fields)) str_buffer.append('\n') if str_buffer: - os.write(self._pipe, _unicode_encode(''.join(str_buffer))) + str_buffer = _unicode_encode(''.join(str_buffer)) + while str_buffer: + str_buffer = str_buffer[os.write(self._pipe, str_buffer):] def _emerge_log(self, msg): emergelog(False, msg) @@ -3414,6 +3555,7 @@ class dblink(object): level=logging.ERROR, noiselevel=-1) return 1 + is_binpkg = self.settings.get("EMERGE_FROM") == "binary" slot = '' for var_name in ('CHOST', 'SLOT'): if var_name == 'CHOST' and self.cat == 'virtual': @@ -3423,22 +3565,18 @@ class dblink(object): pass continue - f = None try: - f = io.open(_unicode_encode( + with io.open(_unicode_encode( os.path.join(inforoot, var_name), encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], - errors='replace') - val = f.readline().strip() + errors='replace') as f: + val = f.readline().strip() except EnvironmentError as e: if e.errno != errno.ENOENT: raise del e val = '' - finally: - if f is not None: - f.close() if var_name == 'SLOT': slot = val @@ -3451,7 +3589,9 @@ class dblink(object): return 1 write_atomic(os.path.join(inforoot, var_name), slot + '\n') - if val != self.settings.get(var_name, ''): + # This check only applies when built from source, since + # inforoot values are written just after src_install. + if not is_binpkg and val != self.settings.get(var_name, ''): self._eqawarn('preinst', [_("QA Notice: Expected %(var_name)s='%(expected_value)s', got '%(actual_value)s'\n") % \ {"var_name":var_name, "expected_value":self.settings.get(var_name, ''), "actual_value":val}]) @@ -3462,30 +3602,47 @@ class dblink(object): if not os.path.exists(self.dbcatdir): ensure_dirs(self.dbcatdir) + # NOTE: We use SLOT obtained from the inforoot + # directory, in order to support USE=multislot. + # Use _pkg_str discard the sub-slot part if necessary. + slot = _pkg_str(self.mycpv, slot=slot).slot cp = self.mysplit[0] slot_atom = "%s:%s" % (cp, slot) - # filter any old-style virtual matches - slot_matches = [cpv for cpv in self.vartree.dbapi.match(slot_atom) \ - if cpv_getkey(cpv) == cp] - - if self.mycpv not in slot_matches and \ - self.vartree.dbapi.cpv_exists(self.mycpv): - # handle multislot or unapplied slotmove - slot_matches.append(self.mycpv) - - others_in_slot = [] - from portage import config - for cur_cpv in slot_matches: - # Clone the config in case one of these has to be unmerged since - # we need it to have private ${T} etc... for things like elog. - settings_clone = config(clone=self.settings) - settings_clone.pop("PORTAGE_BUILDIR_LOCKED", None) - settings_clone.reset() - others_in_slot.append(dblink(self.cat, catsplit(cur_cpv)[1], - settings=settings_clone, - vartree=self.vartree, treetype="vartree", - scheduler=self._scheduler, pipe=self._pipe)) + self.lockdb() + try: + # filter any old-style virtual matches + slot_matches = [cpv for cpv in self.vartree.dbapi.match(slot_atom) + if cpv_getkey(cpv) == cp] + + if self.mycpv not in slot_matches and \ + self.vartree.dbapi.cpv_exists(self.mycpv): + # handle multislot or unapplied slotmove + slot_matches.append(self.mycpv) + + others_in_slot = [] + for cur_cpv in slot_matches: + # Clone the config in case one of these has to be unmerged, + # since we need it to have private ${T} etc... for things + # like elog. + settings_clone = portage.config(clone=self.settings) + settings_clone.pop("PORTAGE_BUILDDIR_LOCKED", None) + settings_clone.setcpv(cur_cpv, mydb=self.vartree.dbapi) + if self._preserve_libs and "preserve-libs" in \ + settings_clone["PORTAGE_RESTRICT"].split(): + self._preserve_libs = False + others_in_slot.append(dblink(self.cat, catsplit(cur_cpv)[1], + settings=settings_clone, + vartree=self.vartree, treetype="vartree", + scheduler=self._scheduler, pipe=self._pipe)) + finally: + self.unlockdb() + + # If any instance has RESTRICT=preserve-libs, then + # restrict it for all instances. + if not self._preserve_libs: + for dblnk in others_in_slot: + dblnk._preserve_libs = False retval = self._security_check(others_in_slot) if retval: @@ -3596,6 +3753,13 @@ class dblink(object): # to an infinite recursion loop. mylinklist.append(relative_path) + myto = _unicode_decode( + _os.readlink(_unicode_encode(fpath, + encoding=_encodings['merge'], errors='strict')), + encoding=_encodings['merge'], errors='replace') + if line_ending_re.search(myto) is not None: + paths_with_newlines.append(relative_path) + if unicode_error: break @@ -3647,7 +3811,7 @@ class dblink(object): _("Manually run `emerge --unmerge =%s` if you " "really want to remove the above files. Set " "PORTAGE_PACKAGE_EMPTY_ABORT=\"0\" in " - "/etc/make.conf if you do not want to " + "/etc/portage/make.conf if you do not want to " "abort in cases like this.") % other_dblink.mycpv, wrap_width)) eerror(msg) @@ -3713,7 +3877,9 @@ class dblink(object): " enough information to determine if a real problem" " exists. Please do NOT file a bug report at" " http://bugs.gentoo.org unless you report exactly which" - " two packages install the same file(s). Once again," + " two packages install the same file(s). See" + " http://wiki.gentoo.org/wiki/Knowledge_Base:Blockers" + " for tips on how to solve the problem. And once again," " please do NOT file a bug report unless you have" " completely understood the above message.") @@ -3748,17 +3914,28 @@ class dblink(object): # get_owners is slow for large numbers of files, so # don't look them all up. collisions = collisions[:20] + + pkg_info_strs = {} self.lockdb() try: owners = self.vartree.dbapi._owners.get_owners(collisions) self.vartree.dbapi.flush_cache() + + for pkg in owners: + pkg = self.vartree.dbapi._pkg_str(pkg.mycpv, None) + pkg_info_str = "%s%s%s" % (pkg, + _slot_separator, pkg.slot) + if pkg.repo != _unknown_repo: + pkg_info_str += "%s%s" % (_repo_separator, + pkg.repo) + pkg_info_strs[pkg] = pkg_info_str + finally: self.unlockdb() for pkg, owned_files in owners.items(): - cpv = pkg.mycpv msg = [] - msg.append("%s" % cpv) + msg.append(pkg_info_strs[pkg.mycpv]) for f in sorted(owned_files): msg.append("\t%s" % os.path.join(destroot, f.lstrip(os.path.sep))) @@ -3814,6 +3991,20 @@ class dblink(object): self.delete() ensure_dirs(self.dbtmpdir) + downgrade = False + if self._installed_instance is not None and \ + vercmp(self.mycpv.version, + self._installed_instance.mycpv.version) < 0: + downgrade = True + + if self._installed_instance is not None: + rval = self._pre_merge_backup(self._installed_instance, downgrade) + if rval != os.EX_OK: + showMessage(_("!!! FAILED preinst: ") + + "quickpkg: %s\n" % rval, + level=logging.ERROR, noiselevel=-1) + return rval + # run preinst script showMessage(_(">>> Merging %(cpv)s to %(destroot)s\n") % \ {"cpv":self.mycpv, "destroot":destroot}) @@ -3835,32 +4026,26 @@ class dblink(object): # write local package counter for recording if counter is None: counter = self.vartree.dbapi.counter_tick(mycpv=self.mycpv) - f = io.open(_unicode_encode(os.path.join(self.dbtmpdir, 'COUNTER'), + with io.open(_unicode_encode(os.path.join(self.dbtmpdir, 'COUNTER'), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['repo.content'], - errors='backslashreplace') - f.write(_unicode_decode(str(counter))) - f.close() + errors='backslashreplace') as f: + f.write("%s" % counter) self.updateprotect() #if we have a file containing previously-merged config file md5sums, grab it. self.vartree.dbapi._fs_lock() try: + # Always behave like --noconfmem is enabled for downgrades + # so that people who don't know about this option are less + # likely to get confused when doing upgrade/downgrade cycles. cfgfiledict = grabdict(self.vartree.dbapi._conf_mem_file) - if "NOCONFMEM" in self.settings: + if "NOCONFMEM" in self.settings or downgrade: cfgfiledict["IGNORE"]=1 else: cfgfiledict["IGNORE"]=0 - # Always behave like --noconfmem is enabled for downgrades - # so that people who don't know about this option are less - # likely to get confused when doing upgrade/downgrade cycles. - for other in others_in_slot: - if vercmp(self.mycpv.version, other.mycpv.version) < 0: - cfgfiledict["IGNORE"] = 1 - break - rval = self._merge_contents(srcroot, destroot, cfgfiledict) if rval != os.EX_OK: return rval @@ -3970,6 +4155,7 @@ class dblink(object): try: self.delete() _movefile(self.dbtmpdir, self.dbpkgdir, mysettings=self.settings) + self._merged_path(self.dbpkgdir, os.lstat(self.dbpkgdir)) finally: self.unlockdb() @@ -4014,9 +4200,9 @@ class dblink(object): self.vartree.dbapi.lock() try: try: - slot, counter = self.vartree.dbapi.aux_get( - cpv, ["SLOT", "COUNTER"]) - except KeyError: + slot = self.vartree.dbapi._pkg_str(cpv, None).slot + counter = self.vartree.dbapi.cpv_counter(cpv) + except (KeyError, InvalidData): pass else: has_vdb_entry = True @@ -4085,6 +4271,7 @@ class dblink(object): # For gcc upgrades, preserved libs have to be removed after the # the library path has been updated. self._prune_plib_registry() + self._post_merge_sync() return os.EX_OK @@ -4100,7 +4287,7 @@ class dblink(object): x = -1 while True: x += 1 - backup_p = p + '.backup.' + str(x).rjust(4, '0') + backup_p = '%s.backup.%04d' % (p, x) try: os.lstat(backup_p) except OSError: @@ -4201,8 +4388,9 @@ class dblink(object): @type stufftomerge: String or List @param cfgfiledict: { File:mtime } mapping for config_protected files @type cfgfiledict: Dictionary - @param thismtime: The current time (typically long(time.time()) - @type thismtime: Long + @param thismtime: None or new mtime for merged files (expressed in seconds + in Python <3.3 and nanoseconds in Python >=3.3) + @type thismtime: None or Int @rtype: None or Boolean @return: 1. True on failure @@ -4227,18 +4415,18 @@ class dblink(object): # this is supposed to merge a list of files. There will be 2 forms of argument passing. if isinstance(stufftomerge, basestring): #A directory is specified. Figure out protection paths, listdir() it and process it. - mergelist = os.listdir(join(srcroot, stufftomerge)) - offset = stufftomerge + mergelist = [join(stufftomerge, child) for child in \ + os.listdir(join(srcroot, stufftomerge))] else: - mergelist = stufftomerge - offset = "" + mergelist = stufftomerge[:] - for i, x in enumerate(mergelist): + while mergelist: - mysrc = join(srcroot, offset, x) - mydest = join(destroot, offset, x) + relative_path = mergelist.pop() + mysrc = join(srcroot, relative_path) + mydest = join(destroot, relative_path) # myrealdest is mydest without the $ROOT prefix (makes a difference if ROOT!="/") - myrealdest = join(sep, offset, x) + myrealdest = join(sep, relative_path) # stat file once, test using S_* macros many times (faster that way) mystat = os.lstat(mysrc) mymode = mystat[stat.ST_MODE] @@ -4333,9 +4521,26 @@ class dblink(object): mymtime = movefile(mysrc, mydest, newmtime=thismtime, sstat=mystat, mysettings=self.settings, encoding=_encodings['merge']) + + try: + self._merged_path(mydest, os.lstat(mydest)) + except OSError: + pass + if mymtime != None: + # Use lexists, since if the target happens to be a broken + # symlink then that should trigger an independent warning. + if not (os.path.lexists(myrealto) or + os.path.lexists(join(srcroot, myabsto))): + self._eqawarn('preinst', + [_("QA Notice: Symbolic link /%s points to /%s which does not exist.") + % (relative_path, myabsto)]) + showMessage(">>> %s -> %s\n" % (mydest, myto)) - outfile.write("sym "+myrealdest+" -> "+myto+" "+str(mymtime)+"\n") + if sys.hexversion >= 0x3030000: + outfile.write("sym "+myrealdest+" -> "+myto+" "+str(mymtime // 1000000000)+"\n") + else: + outfile.write("sym "+myrealdest+" -> "+myto+" "+str(mymtime)+"\n") else: showMessage(_("!!! Failed to move file.\n"), level=logging.ERROR, noiselevel=-1) @@ -4429,11 +4634,17 @@ class dblink(object): os.chmod(mydest, mystat[0]) os.chown(mydest, mystat[4], mystat[5]) showMessage(">>> %s/\n" % mydest) + + try: + self._merged_path(mydest, os.lstat(mydest)) + except OSError: + pass + outfile.write("dir "+myrealdest+"\n") # recurse and merge this directory - if self.mergeme(srcroot, destroot, outfile, secondhand, - join(offset, x), cfgfiledict, thismtime): - return 1 + mergelist.extend(join(relative_path, child) for child in + os.listdir(join(srcroot, relative_path))) + elif stat.S_ISREG(mymode): # we are merging a regular file mymd5 = perform_md5(mysrc, calc_prelink=calc_prelink) @@ -4489,7 +4700,10 @@ class dblink(object): cfgprot = cfgfiledict["IGNORE"] if not moveme: zing = "---" - mymtime = mystat[stat.ST_MTIME] + if sys.hexversion >= 0x3030000: + mymtime = mystat.st_mtime_ns + else: + mymtime = mystat[stat.ST_MTIME] else: moveme = 1 cfgprot = 1 @@ -4525,8 +4739,16 @@ class dblink(object): hardlink_candidates.append(mydest) zing = ">>>" + try: + self._merged_path(mydest, os.lstat(mydest)) + except OSError: + pass + if mymtime != None: - outfile.write("obj "+myrealdest+" "+mymd5+" "+str(mymtime)+"\n") + if sys.hexversion >= 0x3030000: + outfile.write("obj "+myrealdest+" "+mymd5+" "+str(mymtime // 1000000000)+"\n") + else: + outfile.write("obj "+myrealdest+" "+mymd5+" "+str(mymtime)+"\n") showMessage("%s %s\n" % (zing,mydest)) else: # we are merging a fifo or device node @@ -4537,6 +4759,12 @@ class dblink(object): sstat=mystat, mysettings=self.settings, encoding=_encodings['merge']) is not None: zing = ">>>" + + try: + self._merged_path(mydest, os.lstat(mydest)) + except OSError: + pass + else: return 1 if stat.S_ISFIFO(mymode): @@ -4545,6 +4773,52 @@ class dblink(object): outfile.write("dev %s\n" % myrealdest) showMessage(zing + " " + mydest + "\n") + def _merged_path(self, path, lstatobj, exists=True): + previous_path = self._device_path_map.get(lstatobj.st_dev) + if previous_path is None or previous_path is False or \ + (exists and len(path) < len(previous_path)): + if exists: + self._device_path_map[lstatobj.st_dev] = path + else: + # This entry is used to indicate that we've unmerged + # a file from this device, and later, this entry is + # replaced by a parent directory. + self._device_path_map[lstatobj.st_dev] = False + + def _post_merge_sync(self): + """ + Call this after merge or unmerge, in order to sync relevant files to + disk and avoid data-loss in the event of a power failure. This method + does nothing if FEATURES=merge-sync is disabled. + """ + if not self._device_path_map or \ + "merge-sync" not in self.settings.features: + return + + returncode = None + if platform.system() == "Linux": + + paths = [] + for path in self._device_path_map.values(): + if path is not False: + paths.append(path) + paths = tuple(paths) + + proc = SyncfsProcess(paths=paths, + scheduler=(self._scheduler or + portage._internal_caller and global_event_loop() or + EventLoop(main=False))) + proc.start() + returncode = proc.wait() + + if returncode is None or returncode != os.EX_OK: + try: + proc = subprocess.Popen(["sync"]) + except EnvironmentError: + pass + else: + proc.wait() + def merge(self, mergeroot, inforoot, myroot=None, myebuild=None, cleanup=0, mydbapi=None, prev_mtimes=None, counter=None): """ @@ -4557,7 +4831,8 @@ class dblink(object): self.lockdb() self.vartree.dbapi._bump_mtime(self.mycpv) if self._scheduler is None: - self._scheduler = PollScheduler().sched_iface + self._scheduler = SchedulerInterface(portage._internal_caller and + global_event_loop() or EventLoop(main=False)) try: retval = self.treewalk(mergeroot, myroot, inforoot, myebuild, cleanup=cleanup, mydbapi=mydbapi, prev_mtimes=prev_mtimes, @@ -4608,11 +4883,12 @@ class dblink(object): "returns contents of a file with whitespace converted to spaces" if not os.path.exists(self.dbdir+"/"+name): return "" - mydata = io.open( + with io.open( _unicode_encode(os.path.join(self.dbdir, name), encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace' - ).read().split() + ) as f: + mydata = f.read().split() return " ".join(mydata) def copyfile(self,fname): @@ -4621,10 +4897,11 @@ class dblink(object): def getfile(self,fname): if not os.path.exists(self.dbdir+"/"+fname): return "" - return io.open(_unicode_encode(os.path.join(self.dbdir, fname), + with io.open(_unicode_encode(os.path.join(self.dbdir, fname), encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace' - ).read() + ) as f: + return f.read() def setfile(self,fname,data): kwargs = {} @@ -4633,16 +4910,18 @@ class dblink(object): else: kwargs['mode'] = 'w' kwargs['encoding'] = _encodings['repo.content'] - write_atomic(os.path.join(self.dbdir, fname), data, **kwargs) + write_atomic(os.path.join(self.dbdir, fname), data, + **portage._native_kwargs(kwargs)) def getelements(self,ename): if not os.path.exists(self.dbdir+"/"+ename): return [] - mylines = io.open(_unicode_encode( + with io.open(_unicode_encode( os.path.join(self.dbdir, ename), encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace' - ).readlines() + ) as f: + mylines = f.readlines() myreturn = [] for x in mylines: for y in x[:-1].split(): @@ -4650,23 +4929,82 @@ class dblink(object): return myreturn def setelements(self,mylist,ename): - myelement = io.open(_unicode_encode( + with io.open(_unicode_encode( os.path.join(self.dbdir, ename), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['repo.content'], - errors='backslashreplace') - for x in mylist: - myelement.write(_unicode_decode(x+"\n")) - myelement.close() + errors='backslashreplace') as f: + for x in mylist: + f.write("%s\n" % x) def isregular(self): "Is this a regular package (does it have a CATEGORY file? A dblink can be virtual *and* regular)" return os.path.exists(os.path.join(self.dbdir, "CATEGORY")) + def _pre_merge_backup(self, backup_dblink, downgrade): + + if ("unmerge-backup" in self.settings.features or + (downgrade and "downgrade-backup" in self.settings.features)): + return self._quickpkg_dblink(backup_dblink, False, None) + + return os.EX_OK + + def _pre_unmerge_backup(self, background): + + if "unmerge-backup" in self.settings.features : + logfile = None + if self.settings.get("PORTAGE_BACKGROUND") != "subprocess": + logfile = self.settings.get("PORTAGE_LOG_FILE") + return self._quickpkg_dblink(self, background, logfile) + + return os.EX_OK + + def _quickpkg_dblink(self, backup_dblink, background, logfile): + + trees = QueryCommand.get_db()[self.settings["EROOT"]] + bintree = trees["bintree"] + binpkg_path = bintree.getname(backup_dblink.mycpv) + if os.path.exists(binpkg_path) and \ + catsplit(backup_dblink.mycpv)[1] not in bintree.invalids: + return os.EX_OK + + self.lockdb() + try: + + if not backup_dblink.exists(): + # It got unmerged by a concurrent process. + return os.EX_OK + + # Call quickpkg for support of QUICKPKG_DEFAULT_OPTS and stuff. + quickpkg_binary = os.path.join(self.settings["PORTAGE_BIN_PATH"], + "quickpkg") + + # Let quickpkg inherit the global vartree config's env. + env = dict(self.vartree.settings.items()) + env["__PORTAGE_INHERIT_VARDB_LOCK"] = "1" + + pythonpath = [x for x in env.get('PYTHONPATH', '').split(":") if x] + if not pythonpath or \ + not os.path.samefile(pythonpath[0], portage._pym_path): + pythonpath.insert(0, portage._pym_path) + env['PYTHONPATH'] = ":".join(pythonpath) + + quickpkg_proc = SpawnProcess( + args=[portage._python_interpreter, quickpkg_binary, + "=%s" % (backup_dblink.mycpv,)], + background=background, env=env, + scheduler=self._scheduler, logfile=logfile) + quickpkg_proc.start() + + return quickpkg_proc.wait() + + finally: + self.unlockdb() + def merge(mycat, mypkg, pkgloc, infloc, myroot=None, settings=None, myebuild=None, mytree=None, mydbapi=None, vartree=None, prev_mtimes=None, blockers=None, - scheduler=None): + scheduler=None, fd_pipes=None): """ @param myroot: ignored, settings['EROOT'] is used instead """ @@ -4681,10 +5019,12 @@ def merge(mycat, mypkg, pkgloc, infloc, merge_task = MergeProcess( mycat=mycat, mypkg=mypkg, settings=settings, treetype=mytree, vartree=vartree, - scheduler=(scheduler or PollScheduler().sched_iface), + scheduler=(scheduler or portage._internal_caller and + global_event_loop() or EventLoop(main=False)), background=background, blockers=blockers, pkgloc=pkgloc, infloc=infloc, myebuild=myebuild, mydbapi=mydbapi, - prev_mtimes=prev_mtimes, logfile=settings.get('PORTAGE_LOG_FILE')) + prev_mtimes=prev_mtimes, logfile=settings.get('PORTAGE_LOG_FILE'), + fd_pipes=fd_pipes) merge_task.start() retcode = merge_task.wait() return retcode @@ -4864,13 +5204,11 @@ def tar_contents(contents, root, tar, protect=None, onProgress=None): tar.addfile(tarinfo, f) f.close() else: - f = open(_unicode_encode(path, + with open(_unicode_encode(path, encoding=encoding, - errors='strict'), 'rb') - try: + errors='strict'), 'rb') as f: tar.addfile(tarinfo, f) - finally: - f.close() + else: tar.addfile(tarinfo) if onProgress: diff --git a/portage_with_autodep/pym/portage/dbapi/vartree.pyo b/portage_with_autodep/pym/portage/dbapi/vartree.pyo Binary files differindex 7c186cf..745d15f 100644 --- a/portage_with_autodep/pym/portage/dbapi/vartree.pyo +++ b/portage_with_autodep/pym/portage/dbapi/vartree.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/virtual.py b/portage_with_autodep/pym/portage/dbapi/virtual.py index da15983..ba9745c 100644 --- a/portage_with_autodep/pym/portage/dbapi/virtual.py +++ b/portage_with_autodep/pym/portage/dbapi/virtual.py @@ -1,6 +1,7 @@ -# Copyright 1998-2012 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals from portage.dbapi import dbapi from portage.dbapi.dep_expand import dep_expand @@ -74,30 +75,55 @@ class fakedbapi(dbapi): @param metadata: dict """ self._clear_cache() - if not hasattr(mycpv, 'cp'): + + try: + mycp = mycpv.cp + except AttributeError: + mycp = None + try: + myslot = mycpv.slot + except AttributeError: + myslot = None + + if mycp is None or \ + (myslot is None and metadata is not None and metadata.get('SLOT')): if metadata is None: mycpv = _pkg_str(mycpv) else: - mycpv = _pkg_str(mycpv, slot=metadata.get('SLOT'), - repo=metadata.get('repository')) - mycp = mycpv.cp + mycpv = _pkg_str(mycpv, metadata=metadata, + settings=self.settings) + + mycp = mycpv.cp + try: + myslot = mycpv.slot + except AttributeError: + pass + self.cpvdict[mycpv] = metadata - myslot = None - if self._exclusive_slots and metadata: - myslot = metadata.get("SLOT", None) + if not self._exclusive_slots: + myslot = None if myslot and mycp in self.cpdict: # If necessary, remove another package in the same SLOT. for cpv in self.cpdict[mycp]: if mycpv != cpv: - other_metadata = self.cpvdict[cpv] - if other_metadata: - if myslot == other_metadata.get("SLOT", None): + try: + other_slot = cpv.slot + except AttributeError: + pass + else: + if myslot == other_slot: self.cpv_remove(cpv) break - if mycp not in self.cpdict: - self.cpdict[mycp] = [] - if not mycpv in self.cpdict[mycp]: - self.cpdict[mycp].append(mycpv) + + cp_list = self.cpdict.get(mycp) + if cp_list is None: + cp_list = [] + self.cpdict[mycp] = cp_list + try: + cp_list.remove(mycpv) + except ValueError: + pass + cp_list.append(mycpv) def cpv_remove(self,mycpv): """Removes a cpv from the list of available packages.""" diff --git a/portage_with_autodep/pym/portage/dbapi/virtual.pyo b/portage_with_autodep/pym/portage/dbapi/virtual.pyo Binary files differindex 9f7c667..ca52263 100644 --- a/portage_with_autodep/pym/portage/dbapi/virtual.pyo +++ b/portage_with_autodep/pym/portage/dbapi/virtual.pyo diff --git a/portage_with_autodep/pym/portage/debug.pyo b/portage_with_autodep/pym/portage/debug.pyo Binary files differindex 82a5e8f..a3c437d 100644 --- a/portage_with_autodep/pym/portage/debug.pyo +++ b/portage_with_autodep/pym/portage/debug.pyo diff --git a/portage_with_autodep/pym/portage/dep/__init__.py b/portage_with_autodep/pym/portage/dep/__init__.py index 152af0a..798903f 100644 --- a/portage_with_autodep/pym/portage/dep/__init__.py +++ b/portage_with_autodep/pym/portage/dep/__init__.py @@ -1,7 +1,9 @@ # deps.py -- Portage dependency resolution functions -# Copyright 2003-2012 Gentoo Foundation +# Copyright 2003-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = [ 'Atom', 'best_match_to_list', 'cpvequal', 'dep_getcpv', 'dep_getkey', 'dep_getslot', @@ -13,20 +15,6 @@ __all__ = [ '_repo_separator', '_slot_separator', ] -# DEPEND SYNTAX: -# -# 'use?' only affects the immediately following word! -# Nesting is the only legal way to form multiple '[!]use?' requirements. -# -# Where: 'a' and 'b' are use flags, and 'z' is a depend atom. -# -# "a? z" -- If 'a' in [use], then b is valid. -# "a? ( z )" -- Syntax with parenthesis. -# "a? b? z" -- Deprecated. -# "a? ( b? z )" -- Valid -# "a? ( b? ( z ) ) -- Valid -# - import re, sys import warnings from itertools import chain @@ -36,23 +24,164 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.util:cmp_sort_key,writemsg', ) -from portage import _unicode_decode -from portage.eapi import eapi_has_slot_deps, eapi_has_src_uri_arrows, \ - eapi_has_use_deps, eapi_has_strong_blocks, eapi_has_use_dep_defaults, \ - eapi_has_repo_deps, eapi_allows_dots_in_PN, eapi_allows_dots_in_use_flags +from portage import _encodings, _unicode_decode, _unicode_encode +from portage.eapi import _get_eapi_attrs from portage.exception import InvalidAtom, InvalidData, InvalidDependString from portage.localization import _ from portage.versions import catpkgsplit, catsplit, \ - vercmp, ververify, _cp, _cpv, _pkg_str, _unknown_repo + vercmp, ververify, _cp, _cpv, _pkg_str, _slot, _unknown_repo, _vr import portage.cache.mappings if sys.hexversion >= 0x3000000: basestring = str + _unicode = str +else: + _unicode = unicode + +# \w is [a-zA-Z0-9_] + +# PMS 3.1.3: A slot name may contain any of the characters [A-Za-z0-9+_.-]. +# It must not begin with a hyphen or a dot. +_slot_separator = ":" +# loosly match SLOT, which may have an optional ABI part +_slot_loose = r'([\w+./*=-]+)' + +_use = r'\[.*\]' +_op = r'([=~]|[><]=?)' + +_repo_separator = "::" +_repo_name = r'[\w][\w-]*' +_repo_name_re = re.compile('^' + _repo_name + '$', re.UNICODE) +_repo = r'(?:' + _repo_separator + '(' + _repo_name + ')' + ')?' + +_extended_cat = r'[\w+*][\w+.*-]*' + +_slot_dep_re_cache = {} + +def _get_slot_dep_re(eapi_attrs): + cache_key = eapi_attrs.slot_operator + slot_re = _slot_dep_re_cache.get(cache_key) + if slot_re is not None: + return slot_re + + if eapi_attrs.slot_operator: + slot_re = _slot + r'?(\*|=|/' + _slot + r'=?)?' + else: + slot_re = _slot + + slot_re = re.compile('^' + slot_re + '$', re.VERBOSE | re.UNICODE) + + _slot_dep_re_cache[cache_key] = slot_re + return slot_re + +def _match_slot(atom, pkg): + if pkg.slot == atom.slot: + if not atom.sub_slot: + return True + elif atom.sub_slot == pkg.sub_slot: + return True + return False + +_atom_re_cache = {} + +def _get_atom_re(eapi_attrs): + cache_key = eapi_attrs.dots_in_PN + atom_re = _atom_re_cache.get(cache_key) + if atom_re is not None: + return atom_re + + if eapi_attrs.dots_in_PN: + cp_re = _cp['dots_allowed_in_PN'] + cpv_re = _cpv['dots_allowed_in_PN'] + else: + cp_re = _cp['dots_disallowed_in_PN'] + cpv_re = _cpv['dots_disallowed_in_PN'] + + atom_re = re.compile('^(?P<without_use>(?:' + + '(?P<op>' + _op + cpv_re + ')|' + + '(?P<star>=' + cpv_re + r'\*)|' + + '(?P<simple>' + cp_re + '))' + + '(' + _slot_separator + _slot_loose + ')?' + + _repo + ')(' + _use + ')?$', re.VERBOSE | re.UNICODE) + + _atom_re_cache[cache_key] = atom_re + return atom_re + +_atom_wildcard_re_cache = {} + +def _get_atom_wildcard_re(eapi_attrs): + cache_key = eapi_attrs.dots_in_PN + atom_re = _atom_wildcard_re_cache.get(cache_key) + if atom_re is not None: + return atom_re + + if eapi_attrs.dots_in_PN: + pkg_re = r'[\w+*][\w+.*-]*?' + else: + pkg_re = r'[\w+*][\w+*-]*?' + + atom_re = re.compile(r'((?P<simple>(' + + _extended_cat + r')/(' + pkg_re + r'(-' + _vr + ')?))' + \ + '|(?P<star>=((' + _extended_cat + r')/(' + pkg_re + r'))-(?P<version>\*\w+\*)))' + \ + '(:(?P<slot>' + _slot_loose + r'))?(' + + _repo_separator + r'(?P<repo>' + _repo_name + r'))?$', re.UNICODE) + + _atom_wildcard_re_cache[cache_key] = atom_re + return atom_re + +_usedep_re_cache = {} + +def _get_usedep_re(eapi_attrs): + """ + @param eapi_attrs: The EAPI attributes from _get_eapi_attrs + @type eapi_attrs: _eapi_attrs + @rtype: regular expression object + @return: A regular expression object that matches valid USE deps for the + given eapi. + """ + cache_key = eapi_attrs.dots_in_use_flags + usedep_re = _usedep_re_cache.get(cache_key) + if usedep_re is not None: + return usedep_re + + if eapi_attrs.dots_in_use_flags: + _flag_re = r'[A-Za-z0-9][A-Za-z0-9+_@.-]*' + else: + _flag_re = r'[A-Za-z0-9][A-Za-z0-9+_@-]*' + + usedep_re = re.compile(r'^(?P<prefix>[!-]?)(?P<flag>' + + _flag_re + r')(?P<default>(\(\+\)|\(\-\))?)(?P<suffix>[?=]?)$') + + _usedep_re_cache[cache_key] = usedep_re + return usedep_re -# Api consumers included in portage should set this to True. -# Once the relevant api changes are in a portage release with -# stable keywords, make these warnings unconditional. -_internal_warnings = False +_useflag_re_cache = {} + +def _get_useflag_re(eapi): + """ + When eapi is None then validation is not as strict, since we want the + same to work for multiple EAPIs that may have slightly different rules. + @param eapi: The EAPI + @type eapi: String or None + @rtype: regular expression object + @return: A regular expression object that matches valid USE flags for the + given eapi. + """ + eapi_attrs = _get_eapi_attrs(eapi) + cache_key = eapi_attrs.dots_in_use_flags + useflag_re = _useflag_re_cache.get(cache_key) + if useflag_re is not None: + return useflag_re + + if eapi_attrs.dots_in_use_flags: + flag_re = r'[A-Za-z0-9][A-Za-z0-9+_@.-]*' + else: + flag_re = r'[A-Za-z0-9][A-Za-z0-9+_@-]*' + + useflag_re = re.compile(r'^' + flag_re + r'$') + + _useflag_re_cache[cache_key] = useflag_re + return useflag_re def cpvequal(cpv1, cpv2): """ @@ -109,7 +238,7 @@ def strip_empty(myarr): ('portage.dep.strip_empty',), DeprecationWarning, stacklevel=2) return [x for x in myarr if x] -def paren_reduce(mystr): +def paren_reduce(mystr, _deprecation_warn=True): """ Take a string and convert all paren enclosed entities into sublists and split the list elements by spaces. All redundant brackets are removed. @@ -123,7 +252,7 @@ def paren_reduce(mystr): @rtype: Array @return: The reduced string in an array """ - if _internal_warnings: + if portage._internal_caller and _deprecation_warn: warnings.warn(_("%s is deprecated and will be removed without replacement.") % \ ('portage.dep.paren_reduce',), DeprecationWarning, stacklevel=2) mysplit = mystr.split() @@ -215,7 +344,7 @@ class paren_normalize(list): """Take a dependency structure as returned by paren_reduce or use_reduce and generate an equivalent structure that has no redundant lists.""" def __init__(self, src): - if _internal_warnings: + if portage._internal_caller: warnings.warn(_("%s is deprecated and will be removed without replacement.") % \ ('portage.dep.paren_normalize',), DeprecationWarning, stacklevel=2) list.__init__(self) @@ -250,7 +379,7 @@ class paren_normalize(list): self._zap_parens(x, dest) return dest -def paren_enclose(mylist, unevaluated_atom=False): +def paren_enclose(mylist, unevaluated_atom=False, opconvert=False): """ Convert a list to a string with sublists enclosed with parens. @@ -267,7 +396,10 @@ def paren_enclose(mylist, unevaluated_atom=False): mystrparts = [] for x in mylist: if isinstance(x, list): - mystrparts.append("( "+paren_enclose(x)+" )") + if opconvert and x and x[0] == "||": + mystrparts.append("%s ( %s )" % (x[0], paren_enclose(x[1:]))) + else: + mystrparts.append("( %s )" % paren_enclose(x)) else: if unevaluated_atom: x = getattr(x, 'unevaluated_atom', x) @@ -308,7 +440,7 @@ def use_reduce(depstr, uselist=[], masklist=[], matchall=False, excludeall=[], i @return: The use reduced depend array """ if isinstance(depstr, list): - if _internal_warnings: + if portage._internal_caller: warnings.warn(_("Passing paren_reduced dep arrays to %s is deprecated. " + \ "Pass the original dep string instead.") % \ ('portage.dep.use_reduce',), DeprecationWarning, stacklevel=2) @@ -320,6 +452,7 @@ def use_reduce(depstr, uselist=[], masklist=[], matchall=False, excludeall=[], i if matchall and matchnone: raise ValueError("portage.dep.use_reduce: 'matchall' and 'matchnone' are mutually exclusive") + eapi_attrs = _get_eapi_attrs(eapi) useflag_re = _get_useflag_re(eapi) def is_active(conditional): @@ -535,7 +668,7 @@ def use_reduce(depstr, uselist=[], masklist=[], matchall=False, excludeall=[], i if not is_src_uri: raise InvalidDependString( _("SRC_URI arrow are only allowed in SRC_URI: token %s") % (pos+1,)) - if eapi is None or not eapi_has_src_uri_arrows(eapi): + if not eapi_attrs.src_uri_arrows: raise InvalidDependString( _("SRC_URI arrow not allowed in EAPI %s: token %s") % (eapi, pos+1)) need_simple_token = True @@ -608,7 +741,7 @@ def dep_opconvert(deplist): @return: The new list with the new ordering """ - if _internal_warnings: + if portage._internal_caller: warnings.warn(_("%s is deprecated. Use %s with the opconvert parameter set to True instead.") % \ ('portage.dep.dep_opconvert', 'portage.dep.use_reduce'), DeprecationWarning, stacklevel=2) @@ -639,7 +772,7 @@ def flatten(mylist): @rtype: List @return: A single list containing only non-list elements. """ - if _internal_warnings: + if portage._internal_caller: warnings.warn(_("%s is deprecated and will be removed without replacement.") % \ ('portage.dep.flatten',), DeprecationWarning, stacklevel=2) @@ -651,30 +784,9 @@ def flatten(mylist): newlist.append(x) return newlist - -_usedep_re = { - "dots_disallowed_in_use_flags": re.compile("^(?P<prefix>[!-]?)(?P<flag>[A-Za-z0-9][A-Za-z0-9+_@-]*)(?P<default>(\(\+\)|\(\-\))?)(?P<suffix>[?=]?)$"), - "dots_allowed_in_use_flags": re.compile("^(?P<prefix>[!-]?)(?P<flag>[A-Za-z0-9][A-Za-z0-9+_@.-]*)(?P<default>(\(\+\)|\(\-\))?)(?P<suffix>[?=]?)$"), -} - -def _get_usedep_re(eapi): - """ - When eapi is None then validation is not as strict, since we want the - same to work for multiple EAPIs that may have slightly different rules. - @param eapi: The EAPI - @type eapi: String or None - @rtype: regular expression object - @return: A regular expression object that matches valid USE deps for the - given eapi. - """ - if eapi is None or eapi_allows_dots_in_use_flags(eapi): - return _usedep_re["dots_allowed_in_use_flags"] - else: - return _usedep_re["dots_disallowed_in_use_flags"] - class _use_dep(object): - __slots__ = ("__weakref__", "eapi", "conditional", "missing_enabled", "missing_disabled", + __slots__ = ("_eapi_attrs", "conditional", "missing_enabled", "missing_disabled", "disabled", "enabled", "tokens", "required") class _conditionals_class(object): @@ -700,10 +812,10 @@ class _use_dep(object): 'not_equal': '!%s=', } - def __init__(self, use, eapi, enabled_flags=None, disabled_flags=None, missing_enabled=None, \ + def __init__(self, use, eapi_attrs, enabled_flags=None, disabled_flags=None, missing_enabled=None, missing_disabled=None, conditional=None, required=None): - self.eapi = eapi + self._eapi_attrs = eapi_attrs if enabled_flags is not None: #A shortcut for the classe's own methods. @@ -732,7 +844,7 @@ class _use_dep(object): no_default = set() conditional = {} - usedep_re = _get_usedep_re(self.eapi) + usedep_re = _get_usedep_re(self._eapi_attrs) for x in use: m = usedep_re.match(x) @@ -800,6 +912,14 @@ class _use_dep(object): return "" return "[%s]" % (",".join(self.tokens),) + if sys.hexversion < 0x3000000: + + __unicode__ = __str__ + + def __str__(self): + return _unicode_encode(self.__unicode__(), + encoding=_encodings['content'], errors='backslashreplace') + def __repr__(self): return "portage.dep._use_dep(%s)" % repr(self.tokens) @@ -835,7 +955,7 @@ class _use_dep(object): disabled_flags = set(self.disabled) tokens = [] - usedep_re = _get_usedep_re(self.eapi) + usedep_re = _get_usedep_re(self._eapi_attrs) for x in self.tokens: m = usedep_re.match(x) @@ -871,7 +991,7 @@ class _use_dep(object): else: tokens.append(x) - return _use_dep(tokens, self.eapi, enabled_flags=enabled_flags, disabled_flags=disabled_flags, \ + return _use_dep(tokens, self._eapi_attrs, enabled_flags=enabled_flags, disabled_flags=disabled_flags, missing_enabled=self.missing_enabled, missing_disabled=self.missing_disabled, required=self.required) def violated_conditionals(self, other_use, is_valid_flag, parent_use=None): @@ -893,7 +1013,7 @@ class _use_dep(object): def validate_flag(flag): return is_valid_flag(flag) or flag in all_defaults - usedep_re = _get_usedep_re(self.eapi) + usedep_re = _get_usedep_re(self._eapi_attrs) for x in self.tokens: m = usedep_re.match(x) @@ -985,7 +1105,7 @@ class _use_dep(object): tokens.append(x) conditional.setdefault("disabled", set()).add(flag) - return _use_dep(tokens, self.eapi, enabled_flags=enabled_flags, disabled_flags=disabled_flags, \ + return _use_dep(tokens, self._eapi_attrs, enabled_flags=enabled_flags, disabled_flags=disabled_flags, missing_enabled=self.missing_enabled, missing_disabled=self.missing_disabled, \ conditional=conditional, required=self.required) @@ -1005,7 +1125,7 @@ class _use_dep(object): missing_disabled = self.missing_disabled tokens = [] - usedep_re = _get_usedep_re(self.eapi) + usedep_re = _get_usedep_re(self._eapi_attrs) for x in self.tokens: m = usedep_re.match(x) @@ -1041,15 +1161,10 @@ class _use_dep(object): else: tokens.append(x) - return _use_dep(tokens, self.eapi, enabled_flags=enabled_flags, disabled_flags=disabled_flags, \ + return _use_dep(tokens, self._eapi_attrs, enabled_flags=enabled_flags, disabled_flags=disabled_flags, missing_enabled=missing_enabled, missing_disabled=missing_disabled, required=self.required) -if sys.hexversion < 0x3000000: - _atom_base = unicode -else: - _atom_base = str - -class Atom(_atom_base): +class Atom(_unicode): """ For compatibility with existing atom string manipulation code, this @@ -1068,51 +1183,72 @@ class Atom(_atom_base): def __init__(self, forbid_overlap=False): self.overlap = self._overlap(forbid=forbid_overlap) - def __new__(cls, s, unevaluated_atom=None, allow_wildcard=False, allow_repo=False, + def __new__(cls, s, unevaluated_atom=None, allow_wildcard=False, allow_repo=None, _use=None, eapi=None, is_valid_flag=None): - return _atom_base.__new__(cls, s) + return _unicode.__new__(cls, s) - def __init__(self, s, unevaluated_atom=None, allow_wildcard=False, allow_repo=False, + def __init__(self, s, unevaluated_atom=None, allow_wildcard=False, allow_repo=None, _use=None, eapi=None, is_valid_flag=None): if isinstance(s, Atom): # This is an efficiency assertion, to ensure that the Atom # constructor is not called redundantly. raise TypeError(_("Expected %s, got %s") % \ - (_atom_base, type(s))) + (_unicode, type(s))) - if not isinstance(s, _atom_base): - # Avoid TypeError from _atom_base.__init__ with PyPy. + if not isinstance(s, _unicode): + # Avoid TypeError from _unicode.__init__ with PyPy. s = _unicode_decode(s) - _atom_base.__init__(s) + _unicode.__init__(s) + + eapi_attrs = _get_eapi_attrs(eapi) + atom_re = _get_atom_re(eapi_attrs) - atom_re = _get_atom_re(eapi) - if eapi_has_repo_deps(eapi): - allow_repo = True + self.__dict__['eapi'] = eapi + if eapi is not None: + # Ignore allow_repo when eapi is specified. + allow_repo = eapi_attrs.repo_deps + else: + if allow_repo is None: + allow_repo = True + blocker_prefix = "" if "!" == s[:1]: blocker = self._blocker(forbid_overlap=("!" == s[1:2])) if blocker.overlap.forbid: + blocker_prefix = s[:2] s = s[2:] else: + blocker_prefix = s[:1] s = s[1:] else: blocker = False self.__dict__['blocker'] = blocker m = atom_re.match(s) extended_syntax = False + extended_version = None if m is None: if allow_wildcard: - m = _get_atom_wildcard_re(eapi).match(s) + atom_re = _get_atom_wildcard_re(eapi_attrs) + m = atom_re.match(s) if m is None: raise InvalidAtom(self) - op = None gdict = m.groupdict() - cpv = cp = gdict['simple'] + if m.group('star') is not None: + op = '=*' + base = atom_re.groupindex['star'] + cp = m.group(base + 1) + cpv = m.group('star')[1:] + extended_version = m.group(base + 4) + else: + op = None + cpv = cp = m.group('simple') + if m.group(atom_re.groupindex['simple'] + 3) is not None: + raise InvalidAtom(self) if cpv.find("**") != -1: raise InvalidAtom(self) - slot = gdict['slot'] - repo = gdict['repo'] + slot = m.group('slot') + repo = m.group('repo') use_str = None extended_syntax = True else: @@ -1155,9 +1291,38 @@ class Atom(_atom_base): except InvalidData: # plain cp, wildcard, or something self.__dict__['cpv'] = cpv - self.__dict__['version'] = None + self.__dict__['version'] = extended_version self.__dict__['repo'] = repo - self.__dict__['slot'] = slot + if slot is None: + self.__dict__['slot'] = None + self.__dict__['sub_slot'] = None + self.__dict__['slot_operator'] = None + else: + slot_re = _get_slot_dep_re(eapi_attrs) + slot_match = slot_re.match(slot) + if slot_match is None: + raise InvalidAtom(self) + if eapi_attrs.slot_operator: + self.__dict__['slot'] = slot_match.group(1) + sub_slot = slot_match.group(2) + if sub_slot is not None: + sub_slot = sub_slot.lstrip("/") + if sub_slot in ("*", "="): + self.__dict__['sub_slot'] = None + self.__dict__['slot_operator'] = sub_slot + else: + slot_operator = None + if sub_slot is not None and sub_slot[-1:] == "=": + slot_operator = sub_slot[-1:] + sub_slot = sub_slot[:-1] + self.__dict__['sub_slot'] = sub_slot + self.__dict__['slot_operator'] = slot_operator + if self.slot is not None and self.slot_operator == "*": + raise InvalidAtom(self) + else: + self.__dict__['slot'] = slot + self.__dict__['sub_slot'] = None + self.__dict__['slot_operator'] = None self.__dict__['operator'] = op self.__dict__['extended_syntax'] = extended_syntax @@ -1168,16 +1333,19 @@ class Atom(_atom_base): if _use is not None: use = _use else: - use = _use_dep(use_str[1:-1].split(","), eapi) - without_use = Atom(m.group('without_use'), allow_repo=allow_repo) + use = _use_dep(use_str[1:-1].split(","), eapi_attrs) + without_use = Atom(blocker_prefix + m.group('without_use'), + allow_repo=allow_repo) else: use = None if unevaluated_atom is not None and \ unevaluated_atom.use is not None: # unevaluated_atom.use is used for IUSE checks when matching # packages, so it must not propagate to without_use - without_use = Atom(s, allow_wildcard=allow_wildcard, - allow_repo=allow_repo) + without_use = Atom(_unicode(self), + allow_wildcard=allow_wildcard, + allow_repo=allow_repo, + eapi=eapi) else: without_use = self @@ -1193,16 +1361,16 @@ class Atom(_atom_base): if not isinstance(eapi, basestring): raise TypeError('expected eapi argument of ' + \ '%s, got %s: %s' % (basestring, type(eapi), eapi,)) - if self.slot and not eapi_has_slot_deps(eapi): + if self.slot and not eapi_attrs.slot_deps: raise InvalidAtom( _("Slot deps are not allowed in EAPI %s: '%s'") \ % (eapi, self), category='EAPI.incompatible') if self.use: - if not eapi_has_use_deps(eapi): + if not eapi_attrs.use_deps: raise InvalidAtom( _("Use deps are not allowed in EAPI %s: '%s'") \ % (eapi, self), category='EAPI.incompatible') - elif not eapi_has_use_dep_defaults(eapi) and \ + elif not eapi_attrs.use_dep_defaults and \ (self.use.missing_enabled or self.use.missing_disabled): raise InvalidAtom( _("Use dep defaults are not allowed in EAPI %s: '%s'") \ @@ -1225,12 +1393,21 @@ class Atom(_atom_base): "conditional '%s' in atom '%s' is not in IUSE") \ % (flag, conditional_str % flag, self) raise InvalidAtom(msg, category='IUSE.missing') - if self.blocker and self.blocker.overlap.forbid and not eapi_has_strong_blocks(eapi): + if self.blocker and self.blocker.overlap.forbid and not eapi_attrs.strong_blocks: raise InvalidAtom( _("Strong blocks are not allowed in EAPI %s: '%s'") \ % (eapi, self), category='EAPI.incompatible') @property + def slot_operator_built(self): + """ + Returns True if slot_operator == "=" and sub_slot is not None. + NOTE: foo/bar:2= is unbuilt and returns False, whereas foo/bar:2/2= + is built and returns True. + """ + return self.slot_operator == "=" and self.sub_slot is not None + + @property def without_repo(self): if self.repo is None: return self @@ -1239,18 +1416,29 @@ class Atom(_atom_base): @property def without_slot(self): - if self.slot is None: + if self.slot is None and self.slot_operator is None: return self - return Atom(self.replace(_slot_separator + self.slot, '', 1), + atom = remove_slot(self) + if self.repo is not None: + atom += _repo_separator + self.repo + if self.use is not None: + atom += _unicode(self.use) + return Atom(atom, allow_repo=True, allow_wildcard=True) def with_repo(self, repo): atom = remove_slot(self) - if self.slot is not None: - atom += _slot_separator + self.slot + if self.slot is not None or self.slot_operator is not None: + atom += _slot_separator + if self.slot is not None: + atom += self.slot + if self.sub_slot is not None: + atom += "/%s" % self.sub_slot + if self.slot_operator is not None: + atom += self.slot_operator atom += _repo_separator + repo if self.use is not None: - atom += str(self.use) + atom += _unicode(self.use) return Atom(atom, allow_repo=True, allow_wildcard=True) def with_slot(self, slot): @@ -1258,7 +1446,7 @@ class Atom(_atom_base): if self.repo is not None: atom += _repo_separator + self.repo if self.use is not None: - atom += str(self.use) + atom += _unicode(self.use) return Atom(atom, allow_repo=True, allow_wildcard=True) def __setattr__(self, name, value): @@ -1307,10 +1495,16 @@ class Atom(_atom_base): if not (self.use and self.use.conditional): return self atom = remove_slot(self) - if self.slot: - atom += ":%s" % self.slot + if self.slot is not None or self.slot_operator is not None: + atom += _slot_separator + if self.slot is not None: + atom += self.slot + if self.sub_slot is not None: + atom += "/%s" % self.sub_slot + if self.slot_operator is not None: + atom += self.slot_operator use_dep = self.use.evaluate_conditionals(use) - atom += str(use_dep) + atom += _unicode(use_dep) return Atom(atom, unevaluated_atom=self, allow_repo=(self.repo is not None), _use=use_dep) def violated_conditionals(self, other_use, is_valid_flag, parent_use=None): @@ -1329,20 +1523,32 @@ class Atom(_atom_base): if not self.use: return self atom = remove_slot(self) - if self.slot: - atom += ":%s" % self.slot + if self.slot is not None or self.slot_operator is not None: + atom += _slot_separator + if self.slot is not None: + atom += self.slot + if self.sub_slot is not None: + atom += "/%s" % self.sub_slot + if self.slot_operator is not None: + atom += self.slot_operator use_dep = self.use.violated_conditionals(other_use, is_valid_flag, parent_use) - atom += str(use_dep) + atom += _unicode(use_dep) return Atom(atom, unevaluated_atom=self, allow_repo=(self.repo is not None), _use=use_dep) def _eval_qa_conditionals(self, use_mask, use_force): if not (self.use and self.use.conditional): return self atom = remove_slot(self) - if self.slot: - atom += ":%s" % self.slot + if self.slot is not None or self.slot_operator is not None: + atom += _slot_separator + if self.slot is not None: + atom += self.slot + if self.sub_slot is not None: + atom += "/%s" % self.sub_slot + if self.slot_operator is not None: + atom += self.slot_operator use_dep = self.use._eval_qa_conditionals(use_mask, use_force) - atom += str(use_dep) + atom += _unicode(use_dep) return Atom(atom, unevaluated_atom=self, allow_repo=(self.repo is not None), _use=use_dep) def __copy__(self): @@ -1366,7 +1572,7 @@ def extended_cp_match(extended_cp, other_cp): extended_cp_re = _extended_cp_re_cache.get(extended_cp) if extended_cp_re is None: extended_cp_re = re.compile("^" + re.escape(extended_cp).replace( - r'\*', '[^/]*') + "$") + r'\*', '[^/]*') + "$", re.UNICODE) _extended_cp_re_cache[extended_cp] = extended_cp_re return extended_cp_re.match(other_cp) is not None @@ -1644,77 +1850,8 @@ def dep_getusedeps( depend ): open_bracket = depend.find( '[', open_bracket+1 ) return tuple(use_list) -# \w is [a-zA-Z0-9_] - -# 2.1.3 A slot name may contain any of the characters [A-Za-z0-9+_.-]. -# It must not begin with a hyphen or a dot. -_slot_separator = ":" -_slot = r'([\w+][\w+.-]*)' -_slot_re = re.compile('^' + _slot + '$', re.VERBOSE) - -_use = r'\[.*\]' -_op = r'([=~]|[><]=?)' -_repo_separator = "::" -_repo_name = r'[\w][\w-]*' -_repo = r'(?:' + _repo_separator + '(' + _repo_name + ')' + ')?' - -_atom_re = { - "dots_disallowed_in_PN": re.compile('^(?P<without_use>(?:' + - '(?P<op>' + _op + _cpv['dots_disallowed_in_PN'] + ')|' + - '(?P<star>=' + _cpv['dots_disallowed_in_PN'] + r'\*)|' + - '(?P<simple>' + _cp['dots_disallowed_in_PN'] + '))' + - '(' + _slot_separator + _slot + ')?' + _repo + ')(' + _use + ')?$', re.VERBOSE), - "dots_allowed_in_PN": re.compile('^(?P<without_use>(?:' + - '(?P<op>' + _op + _cpv['dots_allowed_in_PN'] + ')|' + - '(?P<star>=' + _cpv['dots_allowed_in_PN'] + r'\*)|' + - '(?P<simple>' + _cp['dots_allowed_in_PN'] + '))' + - '(' + _slot_separator + _slot + ')?' + _repo + ')(' + _use + ')?$', re.VERBOSE), -} - -def _get_atom_re(eapi): - if eapi is None or eapi_allows_dots_in_PN(eapi): - return _atom_re["dots_allowed_in_PN"] - else: - return _atom_re["dots_disallowed_in_PN"] - -_extended_cat = r'[\w+*][\w+.*-]*' -_extended_pkg = { - "dots_disallowed_in_PN": r'[\w+*][\w+*-]*?', - "dots_allowed_in_PN": r'[\w+*][\w+.*-]*?', -} - -_atom_wildcard_re = { - "dots_disallowed_in_PN": re.compile('(?P<simple>(' + _extended_cat + ')/(' + _extended_pkg['dots_disallowed_in_PN'] + '))(:(?P<slot>' + _slot + '))?(' + _repo_separator + '(?P<repo>' + _repo_name + '))?$'), - "dots_allowed_in_PN": re.compile('(?P<simple>(' + _extended_cat + ')/(' + _extended_pkg['dots_allowed_in_PN'] + '))(:(?P<slot>' + _slot + '))?(' + _repo_separator + '(?P<repo>' + _repo_name + '))?$'), -} - -def _get_atom_wildcard_re(eapi): - if eapi is None or eapi_allows_dots_in_PN(eapi): - return _atom_wildcard_re["dots_allowed_in_PN"] - else: - return _atom_wildcard_re["dots_disallowed_in_PN"] - -_useflag_re = { - "dots_disallowed_in_use_flags": re.compile(r'^[A-Za-z0-9][A-Za-z0-9+_@-]*$'), - "dots_allowed_in_use_flags": re.compile(r'^[A-Za-z0-9][A-Za-z0-9+_@.-]*$'), -} - -def _get_useflag_re(eapi): - """ - When eapi is None then validation is not as strict, since we want the - same to work for multiple EAPIs that may have slightly different rules. - @param eapi: The EAPI - @type eapi: String or None - @rtype: regular expression object - @return: A regular expression object that matches valid USE flags for the - given eapi. - """ - if eapi is None or eapi_allows_dots_in_use_flags(eapi): - return _useflag_re["dots_allowed_in_use_flags"] - else: - return _useflag_re["dots_disallowed_in_use_flags"] - -def isvalidatom(atom, allow_blockers=False, allow_wildcard=False, allow_repo=False): +def isvalidatom(atom, allow_blockers=False, allow_wildcard=False, + allow_repo=False, eapi=None): """ Check to see if a depend atom is valid @@ -1731,9 +1868,15 @@ def isvalidatom(atom, allow_blockers=False, allow_wildcard=False, allow_repo=Fal 1) False if the atom is invalid 2) True if the atom is valid """ + + if eapi is not None and isinstance(atom, Atom) and atom.eapi != eapi: + # We'll construct a new atom with the given eapi. + atom = _unicode(atom) + try: if not isinstance(atom, Atom): - atom = Atom(atom, allow_wildcard=allow_wildcard, allow_repo=allow_repo) + atom = Atom(atom, allow_wildcard=allow_wildcard, + allow_repo=allow_repo, eapi=eapi) if not allow_blockers and atom.blocker: return False return True @@ -1859,19 +2002,23 @@ def best_match_to_list(mypkg, mylist): """ operator_values = {'=':6, '~':5, '=*':4, '>':2, '<':2, '>=':2, '<=':2, None:1} - maxvalue = -2 + maxvalue = -99 bestm = None mypkg_cpv = None for x in match_to_list(mypkg, mylist): if x.extended_syntax: - if dep_getslot(x) is not None: + if x.operator == '=*': if maxvalue < 0: maxvalue = 0 bestm = x - else: + elif x.slot is not None: if maxvalue < -1: maxvalue = -1 bestm = x + else: + if maxvalue < -2: + maxvalue = -2 + bestm = x continue if dep_getslot(x) is not None: if maxvalue < 3: @@ -1955,7 +2102,8 @@ def match_from_list(mydep, candidate_list): mylist = [] - if operator is None: + if mydep.extended_syntax: + for x in candidate_list: cp = getattr(x, "cp", None) if cp is None: @@ -1966,8 +2114,38 @@ def match_from_list(mydep, candidate_list): if cp is None: continue - if cp == mycpv or (mydep.extended_syntax and \ - extended_cp_match(mydep.cp, cp)): + if cp == mycpv or extended_cp_match(mydep.cp, cp): + mylist.append(x) + + if mylist and mydep.operator == "=*": + + candidate_list = mylist + mylist = [] + # Currently, only \*\w+\* is supported. + ver = mydep.version[1:-1] + + for x in candidate_list: + x_ver = getattr(x, "version", None) + if x_ver is None: + xs = catpkgsplit(remove_slot(x)) + if xs is None: + continue + x_ver = "-".join(xs[-2:]) + if ver in x_ver: + mylist.append(x) + + elif operator is None: + for x in candidate_list: + cp = getattr(x, "cp", None) + if cp is None: + mysplit = catpkgsplit(remove_slot(x)) + if mysplit is not None: + cp = mysplit[0] + '/' + mysplit[1] + + if cp is None: + continue + + if cp == mydep.cp: mylist.append(x) elif operator == "=": # Exact match @@ -1983,19 +2161,40 @@ def match_from_list(mydep, candidate_list): # XXX: Nasty special casing for leading zeros # Required as =* is a literal prefix match, so can't # use vercmp - mysplit = catpkgsplit(mycpv) - myver = mysplit[2].lstrip("0") + myver = mycpv_cps[2].lstrip("0") if not myver or not myver[0].isdigit(): myver = "0"+myver - mycpv_cmp = mysplit[0]+"/"+mysplit[1]+"-"+myver + if myver == mycpv_cps[2]: + mycpv_cmp = mycpv + else: + # Use replace to preserve the revision part if it exists + # (mycpv_cps[3] can't be trusted because in contains r0 + # even when the input has no revision part). + mycpv_cmp = mycpv.replace( + mydep.cp + "-" + mycpv_cps[2], + mydep.cp + "-" + myver, 1) for x in candidate_list: - xs = getattr(x, "cpv_split", None) - if xs is None: - xs = catpkgsplit(remove_slot(x)) + try: + x.cp + except AttributeError: + try: + pkg = _pkg_str(remove_slot(x)) + except InvalidData: + continue + else: + pkg = x + + xs = pkg.cpv_split myver = xs[2].lstrip("0") if not myver or not myver[0].isdigit(): myver = "0"+myver - xcpv = xs[0]+"/"+xs[1]+"-"+myver + if myver == xs[2]: + xcpv = pkg.cpv + else: + # Use replace to preserve the revision part if it exists. + xcpv = pkg.cpv.replace( + pkg.cp + "-" + xs[2], + pkg.cp + "-" + myver, 1) if xcpv.startswith(mycpv_cmp): mylist.append(x) @@ -2048,16 +2247,33 @@ def match_from_list(mydep, candidate_list): else: raise KeyError(_("Unknown operator: %s") % mydep) - if slot is not None and not mydep.extended_syntax: + if mydep.slot is not None: candidate_list = mylist mylist = [] for x in candidate_list: - xslot = getattr(x, "slot", False) - if xslot is False: + x_pkg = None + try: + x.cpv + except AttributeError: xslot = dep_getslot(x) - if xslot is not None and xslot != slot: - continue - mylist.append(x) + if xslot is not None: + try: + x_pkg = _pkg_str(remove_slot(x), slot=xslot) + except InvalidData: + continue + else: + x_pkg = x + + if x_pkg is None: + mylist.append(x) + else: + try: + x_pkg.slot + except AttributeError: + mylist.append(x) + else: + if _match_slot(mydep, x_pkg): + mylist.append(x) if mydep.unevaluated_atom.use: candidate_list = mylist @@ -2071,26 +2287,26 @@ def match_from_list(mydep, candidate_list): continue if mydep.use: - - missing_enabled = mydep.use.missing_enabled.difference(x.iuse.all) - missing_disabled = mydep.use.missing_disabled.difference(x.iuse.all) + is_valid_flag = x.iuse.is_valid_flag + missing_enabled = frozenset(flag for flag in + mydep.use.missing_enabled if not is_valid_flag(flag)) + missing_disabled = frozenset(flag for flag in + mydep.use.missing_disabled if not is_valid_flag(flag)) if mydep.use.enabled: - if mydep.use.enabled.intersection(missing_disabled): + if any(f in mydep.use.enabled for f in missing_disabled): continue need_enabled = mydep.use.enabled.difference(use.enabled) if need_enabled: - need_enabled = need_enabled.difference(missing_enabled) - if need_enabled: + if any(f not in missing_enabled for f in need_enabled): continue if mydep.use.disabled: - if mydep.use.disabled.intersection(missing_enabled): + if any(f in mydep.use.disabled for f in missing_enabled): continue need_disabled = mydep.use.disabled.intersection(use.enabled) if need_disabled: - need_disabled = need_disabled.difference(missing_disabled) - if need_disabled: + if any(f not in missing_disabled for f in need_disabled): continue mylist.append(x) @@ -2110,9 +2326,9 @@ def match_from_list(mydep, candidate_list): return mylist def human_readable_required_use(required_use): - return required_use.replace("^^", "exactly-one-of").replace("||", "any-of") + return required_use.replace("^^", "exactly-one-of").replace("||", "any-of").replace("??", "at-most-one-of") -def get_required_use_flags(required_use): +def get_required_use_flags(required_use, eapi=None): """ Returns a set of use flags that are used in the given REQUIRED_USE string @@ -2122,6 +2338,12 @@ def get_required_use_flags(required_use): @return: Set of use flags that are used in the given REQUIRED_USE string """ + eapi_attrs = _get_eapi_attrs(eapi) + if eapi_attrs.required_use_at_most_one_of: + valid_operators = ("||", "^^", "??") + else: + valid_operators = ("||", "^^") + mysplit = required_use.split() level = 0 stack = [[]] @@ -2150,7 +2372,7 @@ def get_required_use_flags(required_use): l = stack.pop() ignore = False if stack[level]: - if stack[level][-1] in ("||", "^^") or \ + if stack[level][-1] in valid_operators or \ (not isinstance(stack[level][-1], bool) and \ stack[level][-1][-1] == "?"): ignore = True @@ -2162,15 +2384,14 @@ def get_required_use_flags(required_use): else: raise InvalidDependString( _("malformed syntax: '%s'") % required_use) - elif token in ("||", "^^"): + elif token in valid_operators: if need_bracket: raise InvalidDependString( _("malformed syntax: '%s'") % required_use) need_bracket = True stack[level].append(token) else: - if need_bracket or "(" in token or ")" in token or \ - "|" in token or "^" in token: + if need_bracket: raise InvalidDependString( _("malformed syntax: '%s'") % required_use) @@ -2225,7 +2446,7 @@ class _RequiredUseBranch(object): complex_nesting = False node = self while node != None and not complex_nesting: - if node._operator in ("||", "^^"): + if node._operator in ("||", "^^", "??"): complex_nesting = True else: node = node._parent @@ -2246,7 +2467,7 @@ class _RequiredUseBranch(object): if sys.hexversion < 0x3000000: __nonzero__ = __bool__ -def check_required_use(required_use, use, iuse_match): +def check_required_use(required_use, use, iuse_match, eapi=None): """ Checks if the use flags listed in 'use' satisfy all constraints specified in 'constraints'. @@ -2262,6 +2483,12 @@ def check_required_use(required_use, use, iuse_match): @return: Indicates if REQUIRED_USE constraints are satisfied """ + eapi_attrs = _get_eapi_attrs(eapi) + if eapi_attrs.required_use_at_most_one_of: + valid_operators = ("||", "^^", "??") + else: + valid_operators = ("||", "^^") + def is_active(token): if token.startswith("!"): flag = token[1:] @@ -2271,6 +2498,11 @@ def check_required_use(required_use, use, iuse_match): is_negated = False if not flag or not iuse_match(flag): + if not eapi_attrs.required_use_at_most_one_of and flag == "?": + msg = _("Operator '??' is not supported with EAPI '%s'") \ + % (eapi,) + e = InvalidData(msg, category='EAPI.incompatible') + raise InvalidDependString(msg, errors=(e,)) msg = _("USE flag '%s' is not in IUSE") \ % (flag,) e = InvalidData(msg, category='IUSE.missing') @@ -2288,6 +2520,8 @@ def check_required_use(required_use, use, iuse_match): return (True in argument) elif operator == "^^": return (argument.count(True) == 1) + elif operator == "??": + return (argument.count(True) <= 1) elif operator[-1] == "?": return (False not in argument) @@ -2317,7 +2551,7 @@ def check_required_use(required_use, use, iuse_match): l = stack.pop() op = None if stack[level]: - if stack[level][-1] in ("||", "^^"): + if stack[level][-1] in valid_operators: op = stack[level].pop() satisfied = is_satisfied(op, l) stack[level].append(satisfied) @@ -2346,7 +2580,7 @@ def check_required_use(required_use, use, iuse_match): stack[level].append(satisfied) if len(node._children) <= 1 or \ - node._parent._operator not in ("||", "^^"): + node._parent._operator not in valid_operators: last_node = node._parent._children.pop() if last_node is not node: raise AssertionError( @@ -2362,7 +2596,7 @@ def check_required_use(required_use, use, iuse_match): raise AssertionError( "node is not last child of parent") - elif len(node._children) == 1 and op in ("||", "^^"): + elif len(node._children) == 1 and op in valid_operators: last_node = node._parent._children.pop() if last_node is not node: raise AssertionError( @@ -2372,7 +2606,7 @@ def check_required_use(required_use, use, iuse_match): node._children[0]._parent = node._parent node = node._children[0] if node._operator is None and \ - node._parent._operator not in ("||", "^^"): + node._parent._operator not in valid_operators: last_node = node._parent._children.pop() if last_node is not node: raise AssertionError( @@ -2386,7 +2620,7 @@ def check_required_use(required_use, use, iuse_match): else: raise InvalidDependString( _("malformed syntax: '%s'") % required_use) - elif token in ("||", "^^"): + elif token in valid_operators: if need_bracket: raise InvalidDependString( _("malformed syntax: '%s'") % required_use) @@ -2396,8 +2630,7 @@ def check_required_use(required_use, use, iuse_match): node._children.append(child) node = child else: - if need_bracket or "(" in token or ")" in token or \ - "|" in token or "^" in token: + if need_bracket: raise InvalidDependString( _("malformed syntax: '%s'") % required_use) @@ -2425,16 +2658,16 @@ def extract_affecting_use(mystr, atom, eapi=None): that decide if the given atom is in effect. Example usage: - >>> extract_use_cond('sasl? ( dev-libs/cyrus-sasl ) \ + >>> extract_affecting_use('sasl? ( dev-libs/cyrus-sasl ) \ !minimal? ( cxx? ( dev-libs/cyrus-sasl ) )', 'dev-libs/cyrus-sasl') - (['sasl', 'minimal', 'cxx']) + {'cxx', 'minimal', 'sasl'} - @param dep: The dependency string + @param mystr: The dependency string @type mystr: String @param atom: The atom to get into effect @type atom: String - @rtype: Tuple of two lists of strings - @return: List of use flags that need to be enabled, List of use flag that need to be disabled + @rtype: Set of strings + @return: Set of use flags affecting given atom """ useflag_re = _get_useflag_re(eapi) mysplit = mystr.split() @@ -2540,3 +2773,48 @@ def extract_affecting_use(mystr, atom, eapi=None): _("malformed syntax: '%s'") % mystr) return affecting_use + +def extract_unpack_dependencies(src_uri, unpackers): + """ + Return unpack dependencies string for given SRC_URI string. + + @param src_uri: SRC_URI string + @type src_uri: String + @param unpackers: Dictionary mapping archive suffixes to dependency strings + @type unpackers: Dictionary + @rtype: String + @return: Dependency string specifying packages required to unpack archives. + """ + src_uri = src_uri.split() + + depend = [] + for i in range(len(src_uri)): + if src_uri[i][-1] == "?" or src_uri[i] in ("(", ")"): + depend.append(src_uri[i]) + elif (i+1 < len(src_uri) and src_uri[i+1] == "->") or src_uri[i] == "->": + continue + else: + for suffix in sorted(unpackers, key=lambda x: len(x), reverse=True): + suffix = suffix.lower() + if src_uri[i].lower().endswith(suffix): + depend.append(unpackers[suffix]) + break + + while True: + cleaned_depend = depend[:] + for i in range(len(cleaned_depend)): + if cleaned_depend[i] is None: + continue + elif cleaned_depend[i] == "(" and cleaned_depend[i+1] == ")": + cleaned_depend[i] = None + cleaned_depend[i+1] = None + elif cleaned_depend[i][-1] == "?" and cleaned_depend[i+1] == "(" and cleaned_depend[i+2] == ")": + cleaned_depend[i] = None + cleaned_depend[i+1] = None + cleaned_depend[i+2] = None + if depend == cleaned_depend: + break + else: + depend = [x for x in cleaned_depend if x is not None] + + return " ".join(depend) diff --git a/portage_with_autodep/pym/portage/dep/__init__.pyo b/portage_with_autodep/pym/portage/dep/__init__.pyo Binary files differindex c78bb23..515ec01 100644 --- a/portage_with_autodep/pym/portage/dep/__init__.pyo +++ b/portage_with_autodep/pym/portage/dep/__init__.pyo diff --git a/portage_with_autodep/pym/portage/dep/dep_check.py b/portage_with_autodep/pym/portage/dep/dep_check.py index 99a5eb0..b5ace3d 100644 --- a/portage_with_autodep/pym/portage/dep/dep_check.py +++ b/portage_with_autodep/pym/portage/dep/dep_check.py @@ -1,16 +1,19 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ['dep_check', 'dep_eval', 'dep_wordreduce', 'dep_zapdeps'] import logging +import operator import portage -from portage import _unicode_decode from portage.dep import Atom, match_from_list, use_reduce from portage.exception import InvalidDependString, ParseError from portage.localization import _ from portage.util import writemsg, writemsg_level +from portage.util.SlotObject import SlotObject from portage.versions import vercmp, _pkg_str def _expand_new_virtuals(mysplit, edebug, mydbapi, mysettings, myroot="/", @@ -160,7 +163,7 @@ def _expand_new_virtuals(mysplit, edebug, mydbapi, mysettings, myroot="/", # According to GLEP 37, RDEPEND is the only dependency # type that is valid for new-style virtuals. Repoman # should enforce this. - depstring = pkg.metadata['RDEPEND'] + depstring = pkg._metadata['RDEPEND'] pkg_kwargs = kwargs.copy() pkg_kwargs["myuse"] = pkg_use_enabled(pkg) if edebug: @@ -183,7 +186,7 @@ def _expand_new_virtuals(mysplit, edebug, mydbapi, mysettings, myroot="/", del mytrees["virt_parent"] if not mycheck[0]: - raise ParseError(_unicode_decode("%s: %s '%s'") % \ + raise ParseError("%s: %s '%s'" % \ (pkg, mycheck[1], depstring)) # pull in the new-style virtual @@ -254,6 +257,10 @@ def dep_eval(deplist): return 0 return 1 +class _dep_choice(SlotObject): + __slots__ = ('atoms', 'slot_map', 'cp_map', 'all_available', + 'all_installed_slots') + def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): """ Takes an unreduced and reduced deplist and removes satisfied dependencies. @@ -316,6 +323,7 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): priority = trees[myroot].get("priority") graph_db = trees[myroot].get("graph_db") graph = trees[myroot].get("graph") + want_update_pkg = trees[myroot].get("want_update_pkg") vardb = None if "vartree" in trees[myroot]: vardb = trees[myroot]["vartree"].dbapi @@ -324,6 +332,13 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): else: mydbapi = trees[myroot]["porttree"].dbapi + try: + mydbapi_match_pkgs = mydbapi.match_pkgs + except AttributeError: + def mydbapi_match_pkgs(atom): + return [mydbapi._pkg_str(cpv, atom.repo) + for cpv in mydbapi.match(atom)] + # Sort the deps into installed, not installed but already # in the graph and other, not installed and not in the graph # and other, with values of [[required_atom], availablility] @@ -347,24 +362,17 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): continue # Ignore USE dependencies here since we don't want USE # settings to adversely affect || preference evaluation. - avail_pkg = mydbapi.match(atom.without_use) + avail_pkg = mydbapi_match_pkgs(atom.without_use) if avail_pkg: avail_pkg = avail_pkg[-1] # highest (ascending order) - try: - slot = avail_pkg.slot - except AttributeError: - eapi, slot, repo = mydbapi.aux_get(avail_pkg, - ["EAPI", "SLOT", "repository"]) - avail_pkg = _pkg_str(avail_pkg, eapi=eapi, - slot=slot, repo=repo) - avail_slot = Atom("%s:%s" % (atom.cp, slot)) + avail_slot = Atom("%s:%s" % (atom.cp, avail_pkg.slot)) if not avail_pkg: all_available = False all_use_satisfied = False break if atom.use: - avail_pkg_use = mydbapi.match(atom) + avail_pkg_use = mydbapi_match_pkgs(atom) if not avail_pkg_use: all_use_satisfied = False else: @@ -372,13 +380,7 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): avail_pkg_use = avail_pkg_use[-1] if avail_pkg_use != avail_pkg: avail_pkg = avail_pkg_use - try: - slot = avail_pkg.slot - except AttributeError: - eapi, slot, repo = mydbapi.aux_get(avail_pkg, - ["EAPI", "SLOT", "repository"]) - avail_pkg = _pkg_str(avail_pkg, - eapi=eapi, slot=slot, repo=repo) + avail_slot = Atom("%s:%s" % (atom.cp, avail_pkg.slot)) slot_map[avail_slot] = avail_pkg highest_cpv = cp_map.get(avail_pkg.cp) @@ -386,7 +388,9 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): vercmp(avail_pkg.version, highest_cpv.version) > 0: cp_map[avail_pkg.cp] = avail_pkg - this_choice = (atoms, slot_map, cp_map, all_available) + this_choice = _dep_choice(atoms=atoms, slot_map=slot_map, + cp_map=cp_map, all_available=all_available, + all_installed_slots=False) if all_available: # The "all installed" criterion is not version or slot specific. # If any version of a package is already in the graph then we @@ -407,6 +411,7 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): not slot_atom.startswith("virtual/"): all_installed_slots = False break + this_choice.all_installed_slots = all_installed_slots if graph_db is None: if all_use_satisfied: if all_installed: @@ -468,8 +473,27 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): elif all_installed: if all_installed_slots: preferred_installed.append(this_choice) - else: + elif parent is None or want_update_pkg is None: preferred_any_slot.append(this_choice) + else: + # When appropriate, prefer a slot that is not + # installed yet for bug #478188. + want_update = True + for slot_atom, avail_pkg in slot_map.items(): + if avail_pkg in graph: + continue + # New-style virtuals have zero cost to install. + if slot_atom.startswith("virtual/") or \ + vardb.match(slot_atom): + continue + if not want_update_pkg(parent, avail_pkg): + want_update = False + break + + if want_update: + preferred_installed.append(this_choice) + else: + preferred_any_slot.append(this_choice) else: preferred_non_installed.append(this_choice) else: @@ -490,6 +514,7 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): all_installed = False if all_installed: + this_choice.all_installed_slots = True other_installed.append(this_choice) elif some_installed: other_installed_some.append(this_choice) @@ -506,22 +531,23 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): for choices in choice_bins: if len(choices) < 2: continue + # Prefer choices with all_installed_slots for bug #480736. + choices.sort(key=operator.attrgetter('all_installed_slots'), + reverse=True) for choice_1 in choices[1:]: - atoms_1, slot_map_1, cp_map_1, all_available_1 = choice_1 - cps = set(cp_map_1) + cps = set(choice_1.cp_map) for choice_2 in choices: if choice_1 is choice_2: # choice_1 will not be promoted, so move on break - atoms_2, slot_map_2, cp_map_2, all_available_2 = choice_2 - intersecting_cps = cps.intersection(cp_map_2) + intersecting_cps = cps.intersection(choice_2.cp_map) if not intersecting_cps: continue has_upgrade = False has_downgrade = False for cp in intersecting_cps: - version_1 = cp_map_1[cp] - version_2 = cp_map_2[cp] + version_1 = choice_1.cp_map[cp] + version_2 = choice_2.cp_map[cp] difference = vercmp(version_1.version, version_2.version) if difference != 0: if difference > 0: @@ -538,9 +564,9 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): for allow_masked in (False, True): for choices in choice_bins: - for atoms, slot_map, cp_map, all_available in choices: - if all_available or allow_masked: - return atoms + for choice in choices: + if choice.all_available or allow_masked: + return choice.atoms assert(False) # This point should not be reachable @@ -575,18 +601,15 @@ def dep_check(depstring, mydbapi, mysettings, use="yes", mode=None, myuse=None, mymasks = set() useforce = set() - useforce.add(mysettings["ARCH"]) if use == "all": - # This masking/forcing is only for repoman. In other cases, relevant - # masking/forcing should have already been applied via - # config.regenerate(). Also, binary or installed packages may have - # been built with flags that are now masked, and it would be - # inconsistent to mask them now. Additionally, myuse may consist of - # flags from a parent package that is being merged to a $ROOT that is - # different from the one that mysettings represents. + # This is only for repoman, in order to constrain the use_reduce + # matchall behavior to account for profile use.mask/force. The + # ARCH/archlist code here may be redundant, since the profile + # really should be handling ARCH masking/forcing itself. mymasks.update(mysettings.usemask) mymasks.update(mysettings.archlist()) mymasks.discard(mysettings["ARCH"]) + useforce.add(mysettings["ARCH"]) useforce.update(mysettings.useforce) useforce.difference_update(mymasks) @@ -609,14 +632,17 @@ def dep_check(depstring, mydbapi, mysettings, use="yes", mode=None, myuse=None, # dependencies so that things like --depclean work as well as possible # in spite of partial invalidity. if not current_parent.installed: - eapi = current_parent.metadata['EAPI'] + eapi = current_parent.eapi - try: - mysplit = use_reduce(depstring, uselist=myusesplit, masklist=mymasks, \ - matchall=(use=="all"), excludeall=useforce, opconvert=True, \ - token_class=Atom, eapi=eapi) - except InvalidDependString as e: - return [0, _unicode_decode("%s") % (e,)] + if isinstance(depstring, list): + mysplit = depstring + else: + try: + mysplit = use_reduce(depstring, uselist=myusesplit, + masklist=mymasks, matchall=(use=="all"), excludeall=useforce, + opconvert=True, token_class=Atom, eapi=eapi) + except InvalidDependString as e: + return [0, "%s" % (e,)] if mysplit == []: #dependencies were reduced to nothing @@ -630,10 +656,10 @@ def dep_check(depstring, mydbapi, mysettings, use="yes", mode=None, myuse=None, use_force=useforce, use_mask=mymasks, use_cache=use_cache, use_binaries=use_binaries, myroot=myroot, trees=trees) except ParseError as e: - return [0, _unicode_decode("%s") % (e,)] + return [0, "%s" % (e,)] - mysplit2=mysplit[:] - mysplit2=dep_wordreduce(mysplit2,mysettings,mydbapi,mode,use_cache=use_cache) + mysplit2 = dep_wordreduce(mysplit, + mysettings, mydbapi, mode, use_cache=use_cache) if mysplit2 is None: return [0, _("Invalid token")] diff --git a/portage_with_autodep/pym/portage/dep/dep_check.pyo b/portage_with_autodep/pym/portage/dep/dep_check.pyo Binary files differindex 1b9e03f..feec00e 100644 --- a/portage_with_autodep/pym/portage/dep/dep_check.pyo +++ b/portage_with_autodep/pym/portage/dep/dep_check.pyo diff --git a/portage_with_autodep/pym/portage/dispatch_conf.py b/portage_with_autodep/pym/portage/dispatch_conf.py index 4c68dfc..79daa9f 100644 --- a/portage_with_autodep/pym/portage/dispatch_conf.py +++ b/portage_with_autodep/pym/portage/dispatch_conf.py @@ -1,5 +1,5 @@ # archive_conf.py -- functionality common to archive-conf and dispatch-conf -# Copyright 2003-2012 Gentoo Foundation +# Copyright 2003-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 @@ -31,9 +31,17 @@ def diffstatusoutput(cmd, file1, file2): # Use Popen to emulate getstatusoutput(), since getstatusoutput() may # raise a UnicodeDecodeError which makes the output inaccessible. args = shlex_split(cmd % (file1, file2)) - if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000: - # Python 3.1 does not support bytes in Popen args. - args = [portage._unicode_encode(x, errors='strict') for x in args] + + if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \ + not os.path.isabs(args[0]): + # Python 3.1 _execvp throws TypeError for non-absolute executable + # path passed as bytes (see http://bugs.python.org/issue8513). + fullname = portage.process.find_binary(args[0]) + if fullname is None: + raise portage.exception.CommandNotFound(args[0]) + args[0] = fullname + + args = [portage._unicode_encode(x, errors='strict') for x in args] proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output = portage._unicode_decode(proc.communicate()[0]) @@ -43,7 +51,7 @@ def diffstatusoutput(cmd, file1, file2): return (proc.wait(), output) def read_config(mandatory_opts): - eprefix = portage.const.EPREFIX + eprefix = portage.settings["EPREFIX"] config_path = os.path.join(eprefix or os.sep, "etc/dispatch-conf.conf") loader = KeyValuePairFileLoader(config_path, None) opts, errors = loader.load() diff --git a/portage_with_autodep/pym/portage/dispatch_conf.pyo b/portage_with_autodep/pym/portage/dispatch_conf.pyo Binary files differindex 6239859..2241d20 100644 --- a/portage_with_autodep/pym/portage/dispatch_conf.pyo +++ b/portage_with_autodep/pym/portage/dispatch_conf.pyo diff --git a/portage_with_autodep/pym/portage/eapi.py b/portage_with_autodep/pym/portage/eapi.py index 79cf891..4f77910 100644 --- a/portage_with_autodep/pym/portage/eapi.py +++ b/portage_with_autodep/pym/portage/eapi.py @@ -1,12 +1,22 @@ # Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +import collections + +from portage import eapi_is_supported + def eapi_has_iuse_defaults(eapi): return eapi != "0" +def eapi_has_iuse_effective(eapi): + return eapi not in ("0", "1", "2", "3", "4", "4-python", "4-slot-abi") + def eapi_has_slot_deps(eapi): return eapi != "0" +def eapi_has_slot_operator(eapi): + return eapi not in ("0", "1", "2", "3", "4", "4-python") + def eapi_has_src_uri_arrows(eapi): return eapi not in ("0", "1") @@ -34,8 +44,11 @@ def eapi_exports_merge_type(eapi): def eapi_exports_replace_vars(eapi): return eapi not in ("0", "1", "2", "3") +def eapi_exports_EBUILD_PHASE_FUNC(eapi): + return eapi not in ("0", "1", "2", "3", "4", "4-python", "4-slot-abi") + def eapi_exports_REPOSITORY(eapi): - return eapi in ("4-python",) + return eapi in ("4-python", "5-progress") def eapi_has_pkg_pretend(eapi): return eapi not in ("0", "1", "2", "3") @@ -49,14 +62,83 @@ def eapi_has_dosed_dohard(eapi): def eapi_has_required_use(eapi): return eapi not in ("0", "1", "2", "3") +def eapi_has_required_use_at_most_one_of(eapi): + return eapi not in ("0", "1", "2", "3", "4", "4-python", "4-slot-abi") + def eapi_has_use_dep_defaults(eapi): return eapi not in ("0", "1", "2", "3") def eapi_has_repo_deps(eapi): - return eapi in ("4-python",) + return eapi in ("4-python", "5-progress") def eapi_allows_dots_in_PN(eapi): - return eapi in ("4-python",) + return eapi in ("4-python", "5-progress") def eapi_allows_dots_in_use_flags(eapi): - return eapi in ("4-python",) + return eapi in ("4-python", "5-progress") + +def eapi_supports_stable_use_forcing_and_masking(eapi): + return eapi not in ("0", "1", "2", "3", "4", "4-python", "4-slot-abi") + +def eapi_allows_directories_on_profile_level_and_repository_level(eapi): + return eapi in ("4-python", "5-progress") + +def eapi_has_use_aliases(eapi): + return eapi in ("4-python", "5-progress") + +def eapi_has_automatic_unpack_dependencies(eapi): + return eapi in ("5-progress",) + +def eapi_has_hdepend(eapi): + return eapi in ("5-hdepend",) + +def eapi_has_targetroot(eapi): + return eapi in ("5-hdepend",) + +_eapi_attrs = collections.namedtuple('_eapi_attrs', + 'dots_in_PN dots_in_use_flags exports_EBUILD_PHASE_FUNC ' + 'feature_flag_test feature_flag_targetroot ' + 'hdepend iuse_defaults iuse_effective ' + 'repo_deps required_use required_use_at_most_one_of slot_operator slot_deps ' + 'src_uri_arrows strong_blocks use_deps use_dep_defaults') + +_eapi_attrs_cache = {} + +def _get_eapi_attrs(eapi): + """ + When eapi is None then validation is not as strict, since we want the + same to work for multiple EAPIs that may have slightly different rules. + An unsupported eapi is handled the same as when eapi is None, which may + be helpful for handling of corrupt EAPI metadata in essential functions + such as pkgsplit. + """ + eapi_attrs = _eapi_attrs_cache.get(eapi) + if eapi_attrs is not None: + return eapi_attrs + + orig_eapi = eapi + if eapi is not None and not eapi_is_supported(eapi): + eapi = None + + eapi_attrs = _eapi_attrs( + dots_in_PN = (eapi is None or eapi_allows_dots_in_PN(eapi)), + dots_in_use_flags = (eapi is None or eapi_allows_dots_in_use_flags(eapi)), + exports_EBUILD_PHASE_FUNC = (eapi is None or eapi_exports_EBUILD_PHASE_FUNC(eapi)), + feature_flag_test = True, + feature_flag_targetroot = (eapi is not None and eapi_has_targetroot(eapi)), + hdepend = (eapi is not None and eapi_has_hdepend(eapi)), + iuse_defaults = (eapi is None or eapi_has_iuse_defaults(eapi)), + iuse_effective = (eapi is not None and eapi_has_iuse_effective(eapi)), + repo_deps = (eapi is None or eapi_has_repo_deps(eapi)), + required_use = (eapi is None or eapi_has_required_use(eapi)), + required_use_at_most_one_of = (eapi is None or eapi_has_required_use_at_most_one_of(eapi)), + slot_deps = (eapi is None or eapi_has_slot_deps(eapi)), + slot_operator = (eapi is None or eapi_has_slot_operator(eapi)), + src_uri_arrows = (eapi is None or eapi_has_src_uri_arrows(eapi)), + strong_blocks = (eapi is None or eapi_has_strong_blocks(eapi)), + use_deps = (eapi is None or eapi_has_use_deps(eapi)), + use_dep_defaults = (eapi is None or eapi_has_use_dep_defaults(eapi)) + ) + + _eapi_attrs_cache[orig_eapi] = eapi_attrs + return eapi_attrs diff --git a/portage_with_autodep/pym/portage/eapi.pyo b/portage_with_autodep/pym/portage/eapi.pyo Binary files differindex ce67ab1..cfce6bd 100644 --- a/portage_with_autodep/pym/portage/eapi.pyo +++ b/portage_with_autodep/pym/portage/eapi.pyo diff --git a/portage_with_autodep/pym/portage/eclass_cache.py b/portage_with_autodep/pym/portage/eclass_cache.py index cb2cf8a..8c209c8 100644 --- a/portage_with_autodep/pym/portage/eclass_cache.py +++ b/portage_with_autodep/pym/portage/eclass_cache.py @@ -1,7 +1,9 @@ -# Copyright 2005-2011 Gentoo Foundation +# Copyright 2005-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # Author(s): Nicholas Carpaski (carpaski@gentoo.org), Brian Harring (ferringb@gentoo.org) +from __future__ import unicode_literals + __all__ = ["cache"] import stat @@ -12,6 +14,7 @@ import errno from portage.exception import FileNotFound, PermissionDenied from portage import os from portage import checksum +from portage import _shell_quote if sys.hexversion >= 0x3000000: long = int @@ -60,6 +63,7 @@ class cache(object): self.eclasses = {} # {"Name": hashed_path} self._eclass_locations = {} + self._eclass_locations_str = None # screw with the porttree ordering, w/out having bash inherit match it, and I'll hurt you. # ~harring @@ -98,6 +102,7 @@ class cache(object): self.porttrees = self.porttrees + other.porttrees self.eclasses.update(other.eclasses) self._eclass_locations.update(other._eclass_locations) + self._eclass_locations_str = None def update_eclasses(self): self.eclasses = {} @@ -169,3 +174,10 @@ class cache(object): ec_dict[x] = self.eclasses[x] return ec_dict + + @property + def eclass_locations_string(self): + if self._eclass_locations_str is None: + self._eclass_locations_str = " ".join(_shell_quote(x) + for x in reversed(self.porttrees)) + return self._eclass_locations_str diff --git a/portage_with_autodep/pym/portage/eclass_cache.pyo b/portage_with_autodep/pym/portage/eclass_cache.pyo Binary files differindex ebe3463..ffd56f7 100644 --- a/portage_with_autodep/pym/portage/eclass_cache.pyo +++ b/portage_with_autodep/pym/portage/eclass_cache.pyo diff --git a/portage_with_autodep/pym/portage/elog/__init__.pyo b/portage_with_autodep/pym/portage/elog/__init__.pyo Binary files differindex 39dc595..6bf9898 100644 --- a/portage_with_autodep/pym/portage/elog/__init__.pyo +++ b/portage_with_autodep/pym/portage/elog/__init__.pyo diff --git a/portage_with_autodep/pym/portage/elog/filtering.pyo b/portage_with_autodep/pym/portage/elog/filtering.pyo Binary files differindex 3a040cc..c67c7bb 100644 --- a/portage_with_autodep/pym/portage/elog/filtering.pyo +++ b/portage_with_autodep/pym/portage/elog/filtering.pyo diff --git a/portage_with_autodep/pym/portage/elog/messages.pyo b/portage_with_autodep/pym/portage/elog/messages.pyo Binary files differindex c033f55..0e3239f 100644 --- a/portage_with_autodep/pym/portage/elog/messages.pyo +++ b/portage_with_autodep/pym/portage/elog/messages.pyo diff --git a/portage_with_autodep/pym/portage/elog/mod_custom.pyo b/portage_with_autodep/pym/portage/elog/mod_custom.pyo Binary files differindex 317fab4..cc95c24 100644 --- a/portage_with_autodep/pym/portage/elog/mod_custom.pyo +++ b/portage_with_autodep/pym/portage/elog/mod_custom.pyo diff --git a/portage_with_autodep/pym/portage/elog/mod_echo.pyo b/portage_with_autodep/pym/portage/elog/mod_echo.pyo Binary files differindex 6a00d4c..be383e2 100644 --- a/portage_with_autodep/pym/portage/elog/mod_echo.pyo +++ b/portage_with_autodep/pym/portage/elog/mod_echo.pyo diff --git a/portage_with_autodep/pym/portage/elog/mod_mail.pyo b/portage_with_autodep/pym/portage/elog/mod_mail.pyo Binary files differindex 5d87aa6..bfed1d3 100644 --- a/portage_with_autodep/pym/portage/elog/mod_mail.pyo +++ b/portage_with_autodep/pym/portage/elog/mod_mail.pyo diff --git a/portage_with_autodep/pym/portage/elog/mod_mail_summary.pyo b/portage_with_autodep/pym/portage/elog/mod_mail_summary.pyo Binary files differindex d7306b5..17c2135 100644 --- a/portage_with_autodep/pym/portage/elog/mod_mail_summary.pyo +++ b/portage_with_autodep/pym/portage/elog/mod_mail_summary.pyo diff --git a/portage_with_autodep/pym/portage/elog/mod_save.py b/portage_with_autodep/pym/portage/elog/mod_save.py index c69f4a3..7b1cd46 100644 --- a/portage_with_autodep/pym/portage/elog/mod_save.py +++ b/portage_with_autodep/pym/portage/elog/mod_save.py @@ -1,7 +1,8 @@ # elog/mod_save.py - elog dispatch module -# Copyright 2006-2011 Gentoo Foundation +# Copyright 2006-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +import errno import io import time import portage @@ -47,11 +48,22 @@ def process(mysettings, key, logentries, fulltext): elogfilename = os.path.join(log_subdir, cat + ':' + elogfilename) _ensure_log_subdirs(logdir, log_subdir) - elogfile = io.open(_unicode_encode(elogfilename, - encoding=_encodings['fs'], errors='strict'), - mode='w', encoding=_encodings['content'], errors='backslashreplace') - elogfile.write(_unicode_decode(fulltext)) - elogfile.close() + try: + with io.open(_unicode_encode(elogfilename, + encoding=_encodings['fs'], errors='strict'), mode='w', + encoding=_encodings['content'], + errors='backslashreplace') as elogfile: + elogfile.write(_unicode_decode(fulltext)) + except IOError as e: + func_call = "open('%s', 'w')" % elogfilename + if e.errno == errno.EACCES: + raise portage.exception.PermissionDenied(func_call) + elif e.errno == errno.EPERM: + raise portage.exception.OperationNotPermitted(func_call) + elif e.errno == errno.EROFS: + raise portage.exception.ReadOnlyFileSystem(func_call) + else: + raise # Copy group permission bits from parent directory. elogdir_st = os.stat(log_subdir) diff --git a/portage_with_autodep/pym/portage/elog/mod_save.pyo b/portage_with_autodep/pym/portage/elog/mod_save.pyo Binary files differindex fb28b76..82d683d 100644 --- a/portage_with_autodep/pym/portage/elog/mod_save.pyo +++ b/portage_with_autodep/pym/portage/elog/mod_save.pyo diff --git a/portage_with_autodep/pym/portage/elog/mod_save_summary.py b/portage_with_autodep/pym/portage/elog/mod_save_summary.py index 347f66e..786f894 100644 --- a/portage_with_autodep/pym/portage/elog/mod_save_summary.py +++ b/portage_with_autodep/pym/portage/elog/mod_save_summary.py @@ -1,8 +1,12 @@ # elog/mod_save_summary.py - elog dispatch module -# Copyright 2006-2011 Gentoo Foundation +# Copyright 2006-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + +import errno import io +import sys import time import portage from portage import os @@ -37,9 +41,21 @@ def process(mysettings, key, logentries, fulltext): # TODO: Locking elogfilename = elogdir+"/summary.log" - elogfile = io.open(_unicode_encode(elogfilename, - encoding=_encodings['fs'], errors='strict'), - mode='a', encoding=_encodings['content'], errors='backslashreplace') + try: + elogfile = io.open(_unicode_encode(elogfilename, + encoding=_encodings['fs'], errors='strict'), + mode='a', encoding=_encodings['content'], + errors='backslashreplace') + except IOError as e: + func_call = "open('%s', 'a')" % elogfilename + if e.errno == errno.EACCES: + raise portage.exception.PermissionDenied(func_call) + elif e.errno == errno.EPERM: + raise portage.exception.OperationNotPermitted(func_call) + elif e.errno == errno.EROFS: + raise portage.exception.ReadOnlyFileSystem(func_call) + else: + raise # Copy group permission bits from parent directory. elogdir_st = os.stat(elogdir) @@ -58,17 +74,19 @@ def process(mysettings, key, logentries, fulltext): apply_permissions(elogfilename, uid=logfile_uid, gid=elogdir_gid, mode=elogdir_grp_mode, mask=0) - time_str = time.strftime("%Y-%m-%d %H:%M:%S %Z", - time.localtime(time.time())) - # Avoid potential UnicodeDecodeError later. + time_fmt = "%Y-%m-%d %H:%M:%S %Z" + if sys.hexversion < 0x3000000: + time_fmt = _unicode_encode(time_fmt) + time_str = time.strftime(time_fmt, time.localtime(time.time())) + # Avoid potential UnicodeDecodeError in Python 2, since strftime + # returns bytes in Python 2, and %Z may contain non-ascii chars. time_str = _unicode_decode(time_str, encoding=_encodings['content'], errors='replace') - elogfile.write(_unicode_decode( - _(">>> Messages generated by process " + + elogfile.write(_(">>> Messages generated by process " "%(pid)d on %(time)s for package %(pkg)s:\n\n") % - {"pid": os.getpid(), "time": time_str, "pkg": key})) + {"pid": os.getpid(), "time": time_str, "pkg": key}) elogfile.write(_unicode_decode(fulltext)) - elogfile.write(_unicode_decode("\n")) + elogfile.write("\n") elogfile.close() return elogfilename diff --git a/portage_with_autodep/pym/portage/elog/mod_save_summary.pyo b/portage_with_autodep/pym/portage/elog/mod_save_summary.pyo Binary files differindex 8f99c51..19e4422 100644 --- a/portage_with_autodep/pym/portage/elog/mod_save_summary.pyo +++ b/portage_with_autodep/pym/portage/elog/mod_save_summary.pyo diff --git a/portage_with_autodep/pym/portage/elog/mod_syslog.pyo b/portage_with_autodep/pym/portage/elog/mod_syslog.pyo Binary files differindex c7b4248..bb01208 100644 --- a/portage_with_autodep/pym/portage/elog/mod_syslog.pyo +++ b/portage_with_autodep/pym/portage/elog/mod_syslog.pyo diff --git a/portage_with_autodep/pym/portage/env/__init__.pyo b/portage_with_autodep/pym/portage/env/__init__.pyo Binary files differindex 846aea3..b45054e 100644 --- a/portage_with_autodep/pym/portage/env/__init__.pyo +++ b/portage_with_autodep/pym/portage/env/__init__.pyo diff --git a/portage_with_autodep/pym/portage/env/config.pyo b/portage_with_autodep/pym/portage/env/config.pyo Binary files differindex 13c2e86..0a7faa3 100644 --- a/portage_with_autodep/pym/portage/env/config.pyo +++ b/portage_with_autodep/pym/portage/env/config.pyo diff --git a/portage_with_autodep/pym/portage/env/loaders.py b/portage_with_autodep/pym/portage/env/loaders.py index 372bc12..f869884 100644 --- a/portage_with_autodep/pym/portage/env/loaders.py +++ b/portage_with_autodep/pym/portage/env/loaders.py @@ -1,10 +1,14 @@ # config.py -- Portage Config -# Copyright 2007-2011 Gentoo Foundation +# Copyright 2007-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import errno import io import stat +import portage +portage.proxy.lazyimport.lazyimport(globals(), + 'portage.util:writemsg', +) from portage import os from portage import _encodings from portage import _unicode_decode @@ -149,17 +153,21 @@ class FileLoader(DataLoader): func = self.lineParser for fn in RecursiveFileLoader(self.fname): try: - f = io.open(_unicode_encode(fn, + with io.open(_unicode_encode(fn, encoding=_encodings['fs'], errors='strict'), mode='r', - encoding=_encodings['content'], errors='replace') + encoding=_encodings['content'], errors='replace') as f: + lines = f.readlines() except EnvironmentError as e: - if e.errno not in (errno.ENOENT, errno.ESTALE): + if e.errno == errno.EACCES: + writemsg(_("Permission denied: '%s'\n") % fn, noiselevel=-1) + del e + elif e.errno in (errno.ENOENT, errno.ESTALE): + del e + else: raise - del e - continue - for line_num, line in enumerate(f): - func(line, line_num, data, errors) - f.close() + else: + for line_num, line in enumerate(lines): + func(line, line_num, data, errors) return (data, errors) def lineParser(self, line, line_num, data, errors): diff --git a/portage_with_autodep/pym/portage/env/loaders.pyo b/portage_with_autodep/pym/portage/env/loaders.pyo Binary files differindex 2622a9f..91c3efa 100644 --- a/portage_with_autodep/pym/portage/env/loaders.pyo +++ b/portage_with_autodep/pym/portage/env/loaders.pyo diff --git a/portage_with_autodep/pym/portage/env/validators.pyo b/portage_with_autodep/pym/portage/env/validators.pyo Binary files differindex cd18adb..24e1510 100644 --- a/portage_with_autodep/pym/portage/env/validators.pyo +++ b/portage_with_autodep/pym/portage/env/validators.pyo diff --git a/portage_with_autodep/pym/portage/exception.py b/portage_with_autodep/pym/portage/exception.py index 5ccd750..1388c49 100644 --- a/portage_with_autodep/pym/portage/exception.py +++ b/portage_with_autodep/pym/portage/exception.py @@ -1,4 +1,4 @@ -# Copyright 1998-2011 Gentoo Foundation +# Copyright 1998-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import signal @@ -11,26 +11,35 @@ if sys.hexversion >= 0x3000000: class PortageException(Exception): """General superclass for portage exceptions""" - def __init__(self,value): - self.value = value[:] - if isinstance(self.value, basestring): - self.value = _unicode_decode(self.value, - encoding=_encodings['content'], errors='replace') + if sys.hexversion >= 0x3000000: + def __init__(self,value): + self.value = value[:] - def __str__(self): - if isinstance(self.value, basestring): - return self.value - else: - return _unicode_decode(repr(self.value), - encoding=_encodings['content'], errors='replace') - - if sys.hexversion < 0x3000000: - - __unicode__ = __str__ + def __str__(self): + if isinstance(self.value, str): + return self.value + else: + return repr(self.value) + else: + def __init__(self,value): + self.value = value[:] + if isinstance(self.value, basestring): + self.value = _unicode_decode(self.value, + encoding=_encodings['content'], errors='replace') + + def __unicode__(self): + if isinstance(self.value, unicode): + return self.value + else: + return _unicode_decode(repr(self.value), + encoding=_encodings['content'], errors='replace') def __str__(self): - return _unicode_encode(self.__unicode__(), - encoding=_encodings['content'], errors='backslashreplace') + if isinstance(self.value, unicode): + return _unicode_encode(self.value, + encoding=_encodings['content'], errors='backslashreplace') + else: + return repr(self.value) class CorruptionError(PortageException): """Corruption indication""" diff --git a/portage_with_autodep/pym/portage/exception.pyo b/portage_with_autodep/pym/portage/exception.pyo Binary files differindex 3a60e7c..c60c3a0 100644 --- a/portage_with_autodep/pym/portage/exception.pyo +++ b/portage_with_autodep/pym/portage/exception.pyo diff --git a/portage_with_autodep/pym/portage/getbinpkg.py b/portage_with_autodep/pym/portage/getbinpkg.py index 212f788..ff656ba 100644 --- a/portage_with_autodep/pym/portage/getbinpkg.py +++ b/portage_with_autodep/pym/portage/getbinpkg.py @@ -1,7 +1,9 @@ # getbinpkg.py -- Portage binary-package helper functions -# Copyright 2003-2012 Gentoo Foundation +# Copyright 2003-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + from portage.output import colorize from portage.cache.mappings import slot_dict_class from portage.localization import _ @@ -18,6 +20,7 @@ import socket import time import tempfile import base64 +import warnings _all_errors = [NotImplementedError, ValueError, socket.error] @@ -65,6 +68,10 @@ if sys.hexversion >= 0x3000000: long = int def make_metadata_dict(data): + + warnings.warn("portage.getbinpkg.make_metadata_dict() is deprecated", + DeprecationWarning, stacklevel=2) + myid,myglob = data mydict = {} @@ -84,6 +91,10 @@ class ParseLinks(html_parser_HTMLParser): """Parser class that overrides HTMLParser to grab all anchors from an html page and provide suffix and prefix limitors""" def __init__(self): + + warnings.warn("portage.getbinpkg.ParseLinks is deprecated", + DeprecationWarning, stacklevel=2) + self.PL_anchors = [] html_parser_HTMLParser.__init__(self) @@ -122,6 +133,9 @@ def create_conn(baseurl,conn=None): optional connection. If connection is already active, it is passed on. baseurl is reduced to address and is returned in tuple (conn,address)""" + warnings.warn("portage.getbinpkg.create_conn() is deprecated", + DeprecationWarning, stacklevel=2) + parts = baseurl.split("://",1) if len(parts) != 2: raise ValueError(_("Provided URI does not " @@ -221,6 +235,10 @@ def create_conn(baseurl,conn=None): def make_ftp_request(conn, address, rest=None, dest=None): """(conn,address,rest) --- uses the conn object to request the data from address and issuing a rest if it is passed.""" + + warnings.warn("portage.getbinpkg.make_ftp_request() is deprecated", + DeprecationWarning, stacklevel=2) + try: if dest: @@ -270,6 +288,9 @@ def make_http_request(conn, address, params={}, headers={}, dest=None): the data from address, performing Location forwarding and using the optional params and headers.""" + warnings.warn("portage.getbinpkg.make_http_request() is deprecated", + DeprecationWarning, stacklevel=2) + rc = 0 response = None while (rc == 0) or (rc == 301) or (rc == 302): @@ -312,6 +333,10 @@ def make_http_request(conn, address, params={}, headers={}, dest=None): def match_in_array(array, prefix="", suffix="", match_both=1, allow_overlap=0): + + warnings.warn("portage.getbinpkg.match_in_array() is deprecated", + DeprecationWarning, stacklevel=2) + myarray = [] if not (prefix and suffix): @@ -352,6 +377,9 @@ def dir_get_list(baseurl,conn=None): URI should be in the form <proto>://<site>[:port]<path> Connection is used for persistent connection instances.""" + warnings.warn("portage.getbinpkg.dir_get_list() is deprecated", + DeprecationWarning, stacklevel=2) + if not conn: keepconnection = 0 else: @@ -400,6 +428,9 @@ def file_get_metadata(baseurl,conn=None, chunk_size=3000): URI should be in the form <proto>://<site>[:port]<path> Connection is used for persistent connection instances.""" + warnings.warn("portage.getbinpkg.file_get_metadata() is deprecated", + DeprecationWarning, stacklevel=2) + if not conn: keepconnection = 0 else: @@ -446,30 +477,53 @@ def file_get_metadata(baseurl,conn=None, chunk_size=3000): return myid -def file_get(baseurl,dest,conn=None,fcmd=None,filename=None): +def file_get(baseurl=None, dest=None, conn=None, fcmd=None, filename=None, + fcmd_vars=None): """(baseurl,dest,fcmd=) -- Takes a base url to connect to and read from. URI should be in the form <proto>://[user[:pass]@]<site>[:port]<path>""" if not fcmd: + + warnings.warn("Use of portage.getbinpkg.file_get() without the fcmd " + "parameter is deprecated", DeprecationWarning, stacklevel=2) + return file_get_lib(baseurl,dest,conn) - if not filename: - filename = os.path.basename(baseurl) - variables = { - "DISTDIR": dest, - "URI": baseurl, - "FILE": filename - } + variables = {} + + if fcmd_vars is not None: + variables.update(fcmd_vars) + + if "DISTDIR" not in variables: + if dest is None: + raise portage.exception.MissingParameter( + _("%s is missing required '%s' key") % + ("fcmd_vars", "DISTDIR")) + variables["DISTDIR"] = dest + + if "URI" not in variables: + if baseurl is None: + raise portage.exception.MissingParameter( + _("%s is missing required '%s' key") % + ("fcmd_vars", "URI")) + variables["URI"] = baseurl + + if "FILE" not in variables: + if filename is None: + filename = os.path.basename(variables["URI"]) + variables["FILE"] = filename from portage.util import varexpand from portage.process import spawn myfetch = portage.util.shlex_split(fcmd) myfetch = [varexpand(x, mydict=variables) for x in myfetch] fd_pipes= { - 0:sys.stdin.fileno(), - 1:sys.stdout.fileno(), - 2:sys.stdout.fileno() + 0:portage._get_stdin().fileno(), + 1:sys.__stdout__.fileno(), + 2:sys.__stdout__.fileno() } + sys.__stdout__.flush() + sys.__stderr__.flush() retval = spawn(myfetch, env=os.environ.copy(), fd_pipes=fd_pipes) if retval != os.EX_OK: sys.stderr.write(_("Fetcher exited with a failure condition.\n")) @@ -481,6 +535,9 @@ def file_get_lib(baseurl,dest,conn=None): URI should be in the form <proto>://<site>[:port]<path> Connection is used for persistent connection instances.""" + warnings.warn("portage.getbinpkg.file_get_lib() is deprecated", + DeprecationWarning, stacklevel=2) + if not conn: keepconnection = 0 else: @@ -524,6 +581,10 @@ def file_get_lib(baseurl,dest,conn=None): def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache=1, makepickle=None): """(baseurl,conn,chunk_size,verbose) -- """ + + warnings.warn("portage.getbinpkg.dir_get_metadata() is deprecated", + DeprecationWarning, stacklevel=2) + if not conn: keepconnection = 0 else: @@ -557,7 +618,9 @@ def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache= out.write(_("Loaded metadata pickle.\n")) out.flush() metadatafile.close() - except (AttributeError, EOFError, EnvironmentError, ValueError, pickle.UnpicklingError): + except (SystemExit, KeyboardInterrupt): + raise + except Exception: metadata = {} if baseurl not in metadata: metadata[baseurl]={} diff --git a/portage_with_autodep/pym/portage/getbinpkg.pyo b/portage_with_autodep/pym/portage/getbinpkg.pyo Binary files differindex 53ec2e9..578acd9 100644 --- a/portage_with_autodep/pym/portage/getbinpkg.pyo +++ b/portage_with_autodep/pym/portage/getbinpkg.pyo diff --git a/portage_with_autodep/pym/portage/glsa.py b/portage_with_autodep/pym/portage/glsa.py index 1857695..cac0f1a 100644 --- a/portage_with_autodep/pym/portage/glsa.py +++ b/portage_with_autodep/pym/portage/glsa.py @@ -1,7 +1,7 @@ # Copyright 2003-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals import io import sys @@ -9,8 +9,12 @@ try: from urllib.request import urlopen as urllib_request_urlopen except ImportError: from urllib import urlopen as urllib_request_urlopen +import codecs import re +import operator import xml.dom.minidom +from io import StringIO +from functools import reduce import portage from portage import os @@ -19,13 +23,13 @@ from portage import _unicode_decode from portage import _unicode_encode from portage.versions import pkgsplit, vercmp, best from portage.util import grabfile -from portage.const import CACHE_PATH +from portage.const import PRIVATE_PATH from portage.localization import _ from portage.dep import _slot_separator # Note: the space for rgt and rlt is important !! # FIXME: use slot deps instead, requires GLSA format versioning -opMapping = {"le": "<=", "lt": "<", "eq": "=", "gt": ">", "ge": ">=", +opMapping = {"le": "<=", "lt": "<", "eq": "=", "gt": ">", "ge": ">=", "rge": ">=~", "rle": "<=~", "rgt": " >~", "rlt": " <~"} NEWLINE_ESCAPE = "!;\\n" # some random string to mark newlines that should be preserved SPACE_ESCAPE = "!;_" # some random string to mark spaces that should be preserved @@ -39,22 +43,22 @@ def get_applied_glsas(settings): @rtype: list @return: list of glsa IDs """ - return grabfile(os.path.join(settings["EROOT"], CACHE_PATH, "glsa")) + return grabfile(os.path.join(settings["EROOT"], PRIVATE_PATH, "glsa_injected")) # TODO: use the textwrap module instead def wrap(text, width, caption=""): """ Wraps the given text at column I{width}, optionally indenting - it so that no text is under I{caption}. It's possible to encode + it so that no text is under I{caption}. It's possible to encode hard linebreaks in I{text} with L{NEWLINE_ESCAPE}. - + @type text: String @param text: the text to be wrapped @type width: Integer @param width: the column at which the text should be wrapped @type caption: String - @param caption: this string is inserted at the beginning of the + @param caption: this string is inserted at the beginning of the return value and the paragraph is indented up to C{len(caption)}. @rtype: String @@ -65,7 +69,7 @@ def wrap(text, width, caption=""): text = text.replace(2*NEWLINE_ESCAPE, NEWLINE_ESCAPE+" "+NEWLINE_ESCAPE) words = text.split() indentLevel = len(caption)+1 - + for w in words: if line != "" and line[-1] == "\n": rValue += line @@ -94,10 +98,10 @@ def get_glsa_list(myconfig): Returns a list of all available GLSAs in the given repository by comparing the filelist there with the pattern described in the config. - + @type myconfig: portage.config @param myconfig: Portage settings instance - + @rtype: List of Strings @return: a list of GLSA IDs in this repository """ @@ -113,10 +117,10 @@ def get_glsa_list(myconfig): dirlist = os.listdir(repository) prefix = "glsa-" suffix = ".xml" - + for f in dirlist: try: - if f[:len(prefix)] == prefix: + if f[:len(prefix)] == prefix and f[-1*len(suffix):] == suffix: rValue.append(f[len(prefix):-1*len(suffix)]) except IndexError: pass @@ -125,22 +129,20 @@ def get_glsa_list(myconfig): def getListElements(listnode): """ Get all <li> elements for a given <ol> or <ul> node. - + @type listnode: xml.dom.Node @param listnode: <ul> or <ol> list to get the elements for @rtype: List of Strings @return: a list that contains the value of the <li> elements """ - rValue = [] if not listnode.nodeName in ["ul", "ol"]: raise GlsaFormatException("Invalid function call: listnode is not <ul> or <ol>") - for li in listnode.childNodes: - if li.nodeType != xml.dom.Node.ELEMENT_NODE: - continue - rValue.append(getText(li, format="strip")) + rValue = [getText(li, format="strip") \ + for li in listnode.childNodes \ + if li.nodeType == xml.dom.Node.ELEMENT_NODE] return rValue -def getText(node, format): +def getText(node, format, textfd = None): """ This is the main parser function. It takes a node and traverses recursive over the subnodes, getting the text of each (and the @@ -148,7 +150,7 @@ def getText(node, format): parameter the text might be formatted by adding/removing newlines, tabs and spaces. This function is only useful for the GLSA DTD, it's not applicable for other DTDs. - + @type node: xml.dom.Node @param node: the root node to start with the parsing @type format: String @@ -158,45 +160,54 @@ def getText(node, format): replaces multiple spaces with one space. I{xml} does some more formatting, depending on the type of the encountered nodes. + @type textfd: writable file-like object + @param textfd: the file-like object to write the output to @rtype: String @return: the (formatted) content of the node and its subnodes + except if textfd was not none """ - rValue = "" + if not textfd: + textfd = StringIO() + returnNone = False + else: + returnNone = True if format in ["strip", "keep"]: if node.nodeName in ["uri", "mail"]: - rValue += node.childNodes[0].data+": "+node.getAttribute("link") + textfd.write(node.childNodes[0].data+": "+node.getAttribute("link")) else: for subnode in node.childNodes: if subnode.nodeName == "#text": - rValue += subnode.data + textfd.write(subnode.data) else: - rValue += getText(subnode, format) - else: + getText(subnode, format, textfd) + else: # format = "xml" for subnode in node.childNodes: if subnode.nodeName == "p": for p_subnode in subnode.childNodes: if p_subnode.nodeName == "#text": - rValue += p_subnode.data.strip() + textfd.write(p_subnode.data.strip()) elif p_subnode.nodeName in ["uri", "mail"]: - rValue += p_subnode.childNodes[0].data - rValue += " ( "+p_subnode.getAttribute("link")+" )" - rValue += NEWLINE_ESCAPE + textfd.write(p_subnode.childNodes[0].data) + textfd.write(" ( "+p_subnode.getAttribute("link")+" )") + textfd.write(NEWLINE_ESCAPE) elif subnode.nodeName == "ul": for li in getListElements(subnode): - rValue += "-"+SPACE_ESCAPE+li+NEWLINE_ESCAPE+" " + textfd.write("-"+SPACE_ESCAPE+li+NEWLINE_ESCAPE+" ") elif subnode.nodeName == "ol": i = 0 for li in getListElements(subnode): i = i+1 - rValue += str(i)+"."+SPACE_ESCAPE+li+NEWLINE_ESCAPE+" " + textfd.write(str(i)+"."+SPACE_ESCAPE+li+NEWLINE_ESCAPE+" ") elif subnode.nodeName == "code": - rValue += getText(subnode, format="keep").replace("\n", NEWLINE_ESCAPE) - if rValue[-1*len(NEWLINE_ESCAPE):] != NEWLINE_ESCAPE: - rValue += NEWLINE_ESCAPE + textfd.write(getText(subnode, format="keep").lstrip().replace("\n", NEWLINE_ESCAPE)) + textfd.write(NEWLINE_ESCAPE) elif subnode.nodeName == "#text": - rValue += subnode.data + textfd.write(subnode.data) else: raise GlsaFormatException(_("Invalid Tag found: "), subnode.nodeName) + if returnNone: + return None + rValue = textfd.getvalue() if format == "strip": rValue = rValue.strip(" \n\t") rValue = re.sub("[\s]{2,}", " ", rValue) @@ -206,7 +217,7 @@ def getMultiTagsText(rootnode, tagname, format): """ Returns a list with the text of all subnodes of type I{tagname} under I{rootnode} (which itself is not parsed) using the given I{format}. - + @type rootnode: xml.dom.Node @param rootnode: the node to search for I{tagname} @type tagname: String @@ -216,16 +227,15 @@ def getMultiTagsText(rootnode, tagname, format): @rtype: List of Strings @return: a list containing the text of all I{tagname} childnodes """ - rValue = [] - for e in rootnode.getElementsByTagName(tagname): - rValue.append(getText(e, format)) + rValue = [getText(e, format) \ + for e in rootnode.getElementsByTagName(tagname)] return rValue def makeAtom(pkgname, versionNode): """ - creates from the given package name and information in the + creates from the given package name and information in the I{versionNode} a (syntactical) valid portage atom. - + @type pkgname: String @param pkgname: the name of the package for this atom @type versionNode: xml.dom.Node @@ -248,9 +258,9 @@ def makeAtom(pkgname, versionNode): def makeVersion(versionNode): """ - creates from the information in the I{versionNode} a + creates from the information in the I{versionNode} a version string (format <op><version>). - + @type versionNode: xml.dom.Node @param versionNode: a <vulnerable> or <unaffected> Node that contains the version information for this atom @@ -270,17 +280,17 @@ def makeVersion(versionNode): def match(atom, dbapi, match_type="default"): """ - wrapper that calls revisionMatch() or portage.dbapi.dbapi.match() depending on + wrapper that calls revisionMatch() or portage.dbapi.dbapi.match() depending on the given atom. - + @type atom: string @param atom: a <~ or >~ atom or a normal portage atom that contains the atom to match against @type dbapi: portage.dbapi.dbapi @param dbapi: one of the portage databases to use as information source @type match_type: string - @param match_type: if != "default" passed as first argument to dbapi.xmatch + @param match_type: if != "default" passed as first argument to dbapi.xmatch to apply the wanted visibility filters - + @rtype: list of strings @return: a list with the matching versions """ @@ -296,15 +306,15 @@ def revisionMatch(revisionAtom, dbapi, match_type="default"): handler for the special >~, >=~, <=~ and <~ atoms that are supposed to behave as > and < except that they are limited to the same version, the range only applies to the revision part. - + @type revisionAtom: string @param revisionAtom: a <~ or >~ atom that contains the atom to match against @type dbapi: portage.dbapi.dbapi @param dbapi: one of the portage databases to use as information source @type match_type: string - @param match_type: if != "default" passed as first argument to portdb.xmatch + @param match_type: if != "default" passed as first argument to portdb.xmatch to apply the wanted visibility filters - + @rtype: list of strings @return: a list with the matching versions """ @@ -325,18 +335,19 @@ def revisionMatch(revisionAtom, dbapi, match_type="default"): if eval(r1+" "+revisionAtom[0:2]+" "+r2): rValue.append(v) return rValue - + def getMinUpgrade(vulnerableList, unaffectedList, portdbapi, vardbapi, minimize=True): """ Checks if the systemstate is matching an atom in I{vulnerableList} and returns string describing - the lowest version for the package that matches an atom in + the lowest version for the package that matches an atom in I{unaffectedList} and is greater than the currently installed - version or None if the system is not affected. Both - I{vulnerableList} and I{unaffectedList} should have the + version. It will return an empty list if the system is affected, + and no upgrade is possible or None if the system is not affected. + Both I{vulnerableList} and I{unaffectedList} should have the same base package. - + @type vulnerableList: List of Strings @param vulnerableList: atoms matching vulnerable package versions @type unaffectedList: List of Strings @@ -347,46 +358,51 @@ def getMinUpgrade(vulnerableList, unaffectedList, portdbapi, vardbapi, minimize= @param vardbapi: Installed package repository @type minimize: Boolean @param minimize: True for a least-change upgrade, False for emerge-like algorithm - + @rtype: String | None @return: the lowest unaffected version that is greater than the installed version. - """ - rValue = None - v_installed = [] - u_installed = [] - for v in vulnerableList: - v_installed += match(v, vardbapi) + """ + rValue = "" + v_installed = reduce(operator.add, [match(v, vardbapi) for v in vulnerableList], []) + u_installed = reduce(operator.add, [match(u, vardbapi) for u in unaffectedList], []) - for u in unaffectedList: - u_installed += match(u, vardbapi) - - install_unaffected = True - for i in v_installed: - if i not in u_installed: - install_unaffected = False + # remove all unaffected atoms from vulnerable list + v_installed = list(set(v_installed).difference(set(u_installed))) - if install_unaffected: - return rValue - + if not v_installed: + return None + + # this tuple holds all vulnerable atoms, and the related upgrade atom + vuln_update = [] + avail_updates = set() for u in unaffectedList: - mylist = match(u, portdbapi, match_type="match-all") - for c in mylist: - i = best(v_installed) - if vercmp(c.version, i.version) > 0 \ - and (rValue == None \ - or not match("="+rValue, portdbapi) \ - or (minimize ^ (vercmp(c.version, rValue.version) > 0)) \ - and match("="+c, portdbapi)) \ - and portdbapi.aux_get(c, ["SLOT"]) == vardbapi.aux_get(best(v_installed), ["SLOT"]): - rValue = c - return rValue + # TODO: This had match_type="match-all" before. I don't think it should + # since we disregarded masked items later anyway (match(=rValue, "porttree")) + avail_updates.update(match(u, portdbapi)) + # if an atom is already installed, we should not consider it for upgrades + avail_updates.difference_update(u_installed) + + for vuln in v_installed: + update = "" + for c in avail_updates: + c_pv = portage.catpkgsplit(c) + if vercmp(c.version, vuln.version) > 0 \ + and (update == "" \ + or (minimize ^ (vercmp(c.version, update.version) > 0))) \ + and portdbapi._pkg_str(c, None).slot == vardbapi._pkg_str(vuln, None).slot: + update = c_pv[0]+"/"+c_pv[1]+"-"+c_pv[2] + if c_pv[3] != "r0": # we don't like -r0 for display + update += "-"+c_pv[3] + vuln_update.append([vuln, update]) + + return vuln_update def format_date(datestr): """ Takes a date (announced, revised) date from a GLSA and formats it as readable text (i.e. "January 1, 2008"). - + @type date: String @param date: the date string to reformat @rtype: String @@ -396,16 +412,16 @@ def format_date(datestr): splitdate = datestr.split("-", 2) if len(splitdate) != 3: return datestr - + # This cannot raise an error as we use () instead of [] splitdate = (int(x) for x in splitdate) - + from datetime import date try: d = date(*splitdate) except ValueError: return datestr - + # TODO We could format to local date format '%x' here? return _unicode_decode(d.strftime("%B %d, %Y"), encoding=_encodings['content'], errors='replace') @@ -417,7 +433,7 @@ class GlsaTypeException(Exception): class GlsaFormatException(Exception): pass - + class GlsaArgumentException(Exception): pass @@ -429,9 +445,9 @@ class Glsa: """ def __init__(self, myid, myconfig, vardbapi, portdbapi): """ - Simple constructor to set the ID, store the config and gets the + Simple constructor to set the ID, store the config and gets the XML data by calling C{self.read()}. - + @type myid: String @param myid: String describing the id for the GLSA object (standard GLSAs have an ID of the form YYYYMM-nn) or an existing @@ -461,7 +477,7 @@ class Glsa: """ Here we build the filename from the config and the ID and pass it to urllib to fetch it from the filesystem or a remote server. - + @rtype: None @return: None """ @@ -473,15 +489,21 @@ class Glsa: myurl = "file://"+self.nr else: myurl = repository + "glsa-%s.xml" % str(self.nr) - self.parse(urllib_request_urlopen(myurl)) + + f = urllib_request_urlopen(myurl) + try: + self.parse(f) + finally: + f.close() + return None def parse(self, myfile): """ - This method parses the XML file and sets up the internal data + This method parses the XML file and sets up the internal data structures by calling the different helper functions in this module. - + @type myfile: String @param myfile: Filename to grab the XML data from @rtype: None @@ -504,27 +526,27 @@ class Glsa: self.title = getText(myroot.getElementsByTagName("title")[0], format="strip") self.synopsis = getText(myroot.getElementsByTagName("synopsis")[0], format="strip") self.announced = format_date(getText(myroot.getElementsByTagName("announced")[0], format="strip")) - - count = 1 + # Support both formats of revised: # <revised>December 30, 2007: 02</revised> # <revised count="2">2007-12-30</revised> revisedEl = myroot.getElementsByTagName("revised")[0] self.revised = getText(revisedEl, format="strip") - if ((sys.hexversion >= 0x3000000 and "count" in revisedEl.attributes) or - (sys.hexversion < 0x3000000 and revisedEl.attributes.has_key("count"))): - count = revisedEl.getAttribute("count") - elif (self.revised.find(":") >= 0): - (self.revised, count) = self.revised.split(":") - + count = revisedEl.attributes.get("count") + if count is None: + if self.revised.find(":") >= 0: + (self.revised, count) = self.revised.split(":") + else: + count = 1 + self.revised = format_date(self.revised) - + try: self.count = int(count) except ValueError: # TODO should this raise a GlsaFormatException? self.count = 1 - + # now the optional and 0-n toplevel, #PCDATA tags and references try: self.access = getText(myroot.getElementsByTagName("access")[0], format="strip") @@ -532,7 +554,7 @@ class Glsa: self.access = "" self.bugs = getMultiTagsText(myroot, "bug", format="strip") self.references = getMultiTagsText(myroot.getElementsByTagName("references")[0], "uri", format="keep") - + # and now the formatted text elements self.description = getText(myroot.getElementsByTagName("description")[0], format="xml") self.workaround = getText(myroot.getElementsByTagName("workaround")[0], format="xml") @@ -542,7 +564,7 @@ class Glsa: try: self.background = getText(myroot.getElementsByTagName("background")[0], format="xml") except IndexError: - self.background = "" + self.background = "" # finally the interesting tags (product, affected, package) self.glsatype = myroot.getElementsByTagName("product")[0].getAttribute("type") @@ -572,16 +594,18 @@ class Glsa: self.services = self.affected.getElementsByTagName("service") return None - def dump(self, outstream=sys.stdout): + def dump(self, outstream=sys.stdout, encoding="utf-8"): """ - Dumps a plaintext representation of this GLSA to I{outfile} or + Dumps a plaintext representation of this GLSA to I{outfile} or B{stdout} if it is ommitted. You can specify an alternate - I{encoding} if needed (default is latin1). - + I{encoding} if needed (default is utf-8). + @type outstream: File @param outfile: Stream that should be used for writing (defaults to sys.stdout) """ + outstream = getattr(outstream, "buffer", outstream) + outstream = codecs.getwriter(encoding)(outstream) width = 76 outstream.write(("GLSA %s: \n%s" % (self.nr, self.title)).center(width)+"\n") outstream.write((width*"=")+"\n") @@ -606,30 +630,24 @@ class Glsa: pass if len(self.bugs) > 0: outstream.write(_("\nRelated bugs: ")) - for i in range(0, len(self.bugs)): - outstream.write(self.bugs[i]) - if i < len(self.bugs)-1: - outstream.write(", ") - else: - outstream.write("\n") + outstream.write(", ".join(self.bugs)) + outstream.write("\n") if self.background: outstream.write("\n"+wrap(self.background, width, caption=_("Background: "))) outstream.write("\n"+wrap(self.description, width, caption=_("Description: "))) outstream.write("\n"+wrap(self.impact_text, width, caption=_("Impact: "))) outstream.write("\n"+wrap(self.workaround, width, caption=_("Workaround: "))) outstream.write("\n"+wrap(self.resolution, width, caption=_("Resolution: "))) - myreferences = "" - for r in self.references: - myreferences += (r.replace(" ", SPACE_ESCAPE)+NEWLINE_ESCAPE+" ") + myreferences = " ".join(r.replace(" ", SPACE_ESCAPE)+NEWLINE_ESCAPE for r in self.references) outstream.write("\n"+wrap(myreferences, width, caption=_("References: "))) outstream.write("\n") - + def isVulnerable(self): """ Tests if the system is affected by this GLSA by checking if any vulnerable package versions are installed. Also checks for affected architectures. - + @rtype: Boolean @return: True if the system is affected, False if not """ @@ -641,56 +659,67 @@ class Glsa: for v in path["vul_atoms"]: rValue = rValue \ or (len(match(v, self.vardbapi)) > 0 \ - and getMinUpgrade(path["vul_atoms"], path["unaff_atoms"], \ + and None != getMinUpgrade(path["vul_atoms"], path["unaff_atoms"], \ self.portdbapi, self.vardbapi)) return rValue - - def isApplied(self): + + def isInjected(self): """ - Looks if the GLSA IDis in the GLSA checkfile to check if this - GLSA was already applied. - + Looks if the GLSA ID is in the GLSA checkfile to check if this + GLSA should be marked as applied. + @rtype: Boolean - @return: True if the GLSA was applied, False if not + @returns: True if the GLSA is in the inject file, False if not """ + if not os.access(os.path.join(self.config["EROOT"], + PRIVATE_PATH, "glsa_injected"), os.R_OK): + return False return (self.nr in get_applied_glsas(self.config)) def inject(self): """ Puts the ID of this GLSA into the GLSA checkfile, so it won't - show up on future checks. Should be called after a GLSA is + show up on future checks. Should be called after a GLSA is applied or on explicit user request. @rtype: None @return: None """ - if not self.isApplied(): + if not self.isInjected(): checkfile = io.open( _unicode_encode(os.path.join(self.config["EROOT"], - CACHE_PATH, "glsa"), - encoding=_encodings['fs'], errors='strict'), + PRIVATE_PATH, "glsa_injected"), + encoding=_encodings['fs'], errors='strict'), mode='a+', encoding=_encodings['content'], errors='strict') checkfile.write(_unicode_decode(self.nr + "\n")) checkfile.close() return None - + def getMergeList(self, least_change=True): """ Returns the list of package-versions that have to be merged to - apply this GLSA properly. The versions are as low as possible + apply this GLSA properly. The versions are as low as possible while avoiding downgrades (see L{getMinUpgrade}). - + @type least_change: Boolean @param least_change: True if the smallest possible upgrade should be selected, False for an emerge-like algorithm @rtype: List of Strings @return: list of package-versions that have to be merged """ - rValue = [] - for pkg in self.packages: + return list(set(update for (vuln, update) in self.getAffectionTable(least_change) if update)) + + def getAffectionTable(self, least_change=True): + """ + Will initialize the self.systemAffection list of + atoms installed on the system that are affected + by this GLSA, and the atoms that are minimal upgrades. + """ + systemAffection = [] + for pkg in self.packages.keys(): for path in self.packages[pkg]: - update = getMinUpgrade(path["vul_atoms"], path["unaff_atoms"], \ + update = getMinUpgrade(path["vul_atoms"], path["unaff_atoms"], self.portdbapi, self.vardbapi, minimize=least_change) if update: - rValue.append(update) - return rValue + systemAffection.extend(update) + return systemAffection diff --git a/portage_with_autodep/pym/portage/glsa.pyo b/portage_with_autodep/pym/portage/glsa.pyo Binary files differindex 65162f1..dd97e49 100644 --- a/portage_with_autodep/pym/portage/glsa.pyo +++ b/portage_with_autodep/pym/portage/glsa.pyo diff --git a/portage_with_autodep/pym/portage/localization.py b/portage_with_autodep/pym/portage/localization.py index d16c4b1..2815ef5 100644 --- a/portage_with_autodep/pym/portage/localization.py +++ b/portage_with_autodep/pym/portage/localization.py @@ -1,12 +1,18 @@ # localization.py -- Code to manage/help portage localization. -# Copyright 2004 Gentoo Foundation +# Copyright 2004-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from portage import _unicode_decode # We define this to make the transition easier for us. def _(mystr): - return mystr - + """ + Always returns unicode, regardless of the input type. This is + helpful for avoiding UnicodeDecodeError from __str__() with + Python 2, by ensuring that string format operations invoke + __unicode__() instead of __str__(). + """ + return _unicode_decode(mystr) def localization_example(): # Dict references allow translators to rearrange word order. diff --git a/portage_with_autodep/pym/portage/localization.pyo b/portage_with_autodep/pym/portage/localization.pyo Binary files differindex e992e3a..50a2805 100644 --- a/portage_with_autodep/pym/portage/localization.pyo +++ b/portage_with_autodep/pym/portage/localization.pyo diff --git a/portage_with_autodep/pym/portage/locks.py b/portage_with_autodep/pym/portage/locks.py index 59fbc6e..8571d8c 100644 --- a/portage_with_autodep/pym/portage/locks.py +++ b/portage_with_autodep/pym/portage/locks.py @@ -1,5 +1,5 @@ # portage: Lock management code -# Copyright 2004-2012 Gentoo Foundation +# Copyright 2004-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ["lockdir", "unlockdir", "lockfile", "unlockfile", \ @@ -17,7 +17,6 @@ import portage from portage import os, _encodings, _unicode_decode from portage.exception import DirectoryNotFound, FileNotFound, \ InvalidData, TryAgain, OperationNotPermitted, PermissionDenied -from portage.data import portage_gid from portage.util import writemsg from portage.localization import _ @@ -64,6 +63,9 @@ def lockfile(mypath, wantnewlockfile=0, unlinkfile=0, if not mypath: raise InvalidData(_("Empty path given")) + # Since Python 3.4, chown requires int type (no proxies). + portage_gid = int(portage.data.portage_gid) + # Support for file object or integer file descriptor parameters is # deprecated due to ambiguity in whether or not it's safe to close # the file descriptor, making it prone to "Bad file descriptor" errors @@ -148,7 +150,7 @@ def lockfile(mypath, wantnewlockfile=0, unlinkfile=0, except IOError as e: if not hasattr(e, "errno"): raise - if e.errno in (errno.EACCES, errno.EAGAIN): + if e.errno in (errno.EACCES, errno.EAGAIN, errno.ENOLCK): # resource temp unavailable; eg, someone beat us to the lock. if flags & os.O_NONBLOCK: os.close(myfd) @@ -163,19 +165,43 @@ def lockfile(mypath, wantnewlockfile=0, unlinkfile=0, if isinstance(mypath, int): waiting_msg = _("waiting for lock on fd %i") % myfd else: - waiting_msg = _("waiting for lock on %s\n") % lockfilename + waiting_msg = _("waiting for lock on %s") % lockfilename if out is not None: out.ebegin(waiting_msg) # try for the exclusive lock now. - try: - locking_method(myfd, fcntl.LOCK_EX) - except EnvironmentError as e: - if out is not None: - out.eend(1, str(e)) - raise + enolock_msg_shown = False + while True: + try: + locking_method(myfd, fcntl.LOCK_EX) + except EnvironmentError as e: + if e.errno == errno.ENOLCK: + # This is known to occur on Solaris NFS (see + # bug #462694). Assume that the error is due + # to temporary exhaustion of record locks, + # and loop until one becomes available. + if not enolock_msg_shown: + enolock_msg_shown = True + if isinstance(mypath, int): + context_desc = _("Error while waiting " + "to lock fd %i") % myfd + else: + context_desc = _("Error while waiting " + "to lock '%s'") % lockfilename + writemsg("\n!!! %s: %s\n" % (context_desc, e), + noiselevel=-1) + + time.sleep(_HARDLINK_POLL_LATENCY) + continue + + if out is not None: + out.eend(1, str(e)) + raise + else: + break + if out is not None: out.eend(os.EX_OK) - elif e.errno in (errno.ENOSYS, errno.ENOLCK): + elif e.errno in (errno.ENOSYS,): # We're not allowed to lock on this FS. if not isinstance(lockfilename, int): # If a file object was passed in, it's not safe @@ -207,6 +233,17 @@ def lockfile(mypath, wantnewlockfile=0, unlinkfile=0, waiting_msg=waiting_msg, flags=flags) if myfd != HARDLINK_FD: + + # FD_CLOEXEC is enabled by default in Python >=3.4. + if sys.hexversion < 0x3040000: + try: + fcntl.FD_CLOEXEC + except AttributeError: + pass + else: + fcntl.fcntl(myfd, fcntl.F_SETFD, + fcntl.fcntl(myfd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC) + _open_fds.add(myfd) writemsg(str((lockfilename,myfd,unlinkfile))+"\n",1) @@ -339,6 +376,9 @@ def hardlink_lockfile(lockfilename, max_wait=DeprecationWarning, preexisting = os.path.exists(lockfilename) myhardlock = hardlock_name(lockfilename) + # Since Python 3.4, chown requires int type (no proxies). + portage_gid = int(portage.data.portage_gid) + # myhardlock must not exist prior to our link() call, and we can # safely unlink it since its file name is unique to our PID try: diff --git a/portage_with_autodep/pym/portage/locks.pyo b/portage_with_autodep/pym/portage/locks.pyo Binary files differindex 9c90a2f..c2e24fb 100644 --- a/portage_with_autodep/pym/portage/locks.pyo +++ b/portage_with_autodep/pym/portage/locks.pyo diff --git a/portage_with_autodep/pym/portage/mail.pyo b/portage_with_autodep/pym/portage/mail.pyo Binary files differindex bc3a76d..f0ef5df 100644 --- a/portage_with_autodep/pym/portage/mail.pyo +++ b/portage_with_autodep/pym/portage/mail.pyo diff --git a/portage_with_autodep/pym/portage/manifest.py b/portage_with_autodep/pym/portage/manifest.py index 90324ee..510e203 100644 --- a/portage_with_autodep/pym/portage/manifest.py +++ b/portage_with_autodep/pym/portage/manifest.py @@ -1,14 +1,19 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + import errno import io import re +import sys import warnings import portage portage.proxy.lazyimport.lazyimport(globals(), - 'portage.checksum:hashfunc_map,perform_multiple_checksums,verify_all', + 'portage.checksum:hashfunc_map,perform_multiple_checksums,' + \ + 'verify_all,_apply_hash_filter,_filter_unaccelarated_hashes', + 'portage.repository.config:_find_invalid_path_char', 'portage.util:write_atomic', ) @@ -23,8 +28,15 @@ from portage.const import (MANIFEST1_HASH_FUNCTIONS, MANIFEST2_HASH_DEFAULTS, MANIFEST2_HASH_FUNCTIONS, MANIFEST2_IDENTIFIERS, MANIFEST2_REQUIRED_HASH) from portage.localization import _ -# Characters prohibited by repoman's file.name check. -_prohibited_filename_chars_re = re.compile(r'[^a-zA-Z0-9._\-+:]') +_manifest_re = re.compile( + r'^(' + '|'.join(MANIFEST2_IDENTIFIERS) + r') (.*)( \d+( \S+ \S+)+)$', + re.UNICODE) + +if sys.hexversion >= 0x3000000: + _unicode = str + basestring = str +else: + _unicode = unicode class FileNotInManifestException(PortageException): pass @@ -37,15 +49,10 @@ def manifest2AuxfileFilter(filename): for x in mysplit: if x[:1] == '.': return False - if _prohibited_filename_chars_re.search(x) is not None: - return False return not filename[:7] == 'digest-' def manifest2MiscfileFilter(filename): - filename = filename.strip(os.sep) - if _prohibited_filename_chars_re.search(filename) is not None: - return False - return not (filename in ["CVS", ".svn", "files", "Manifest"] or filename.endswith(".ebuild")) + return not (filename == "Manifest" or filename.endswith(".ebuild")) def guessManifestFileType(filename): """ Perform a best effort guess of which type the given filename is, avoid using this if possible """ @@ -66,18 +73,17 @@ def guessThinManifestFileType(filename): return None return "DIST" -def parseManifest2(mysplit): +def parseManifest2(line): + if not isinstance(line, basestring): + line = ' '.join(line) myentry = None - if len(mysplit) > 4 and mysplit[0] in MANIFEST2_IDENTIFIERS: - mytype = mysplit[0] - myname = mysplit[1] - try: - mysize = int(mysplit[2]) - except ValueError: - return None - myhashes = dict(zip(mysplit[3::2], mysplit[4::2])) - myhashes["size"] = mysize - myentry = Manifest2Entry(type=mytype, name=myname, hashes=myhashes) + match = _manifest_re.match(line) + if match is not None: + tokens = match.group(3).split() + hashes = dict(zip(tokens[1::2], tokens[2::2])) + hashes["size"] = int(tokens[0]) + myentry = Manifest2Entry(type=match.group(1), + name=match.group(2), hashes=hashes) return myentry class ManifestEntry(object): @@ -107,11 +113,20 @@ class Manifest2Entry(ManifestEntry): def __ne__(self, other): return not self.__eq__(other) + if sys.hexversion < 0x3000000: + + __unicode__ = __str__ + + def __str__(self): + return _unicode_encode(self.__unicode__(), + encoding=_encodings['repo.content'], errors='strict') + class Manifest(object): parsers = (parseManifest2,) - def __init__(self, pkgdir, distdir, fetchlist_dict=None, + def __init__(self, pkgdir, distdir=None, fetchlist_dict=None, manifest1_compat=DeprecationWarning, from_scratch=False, thin=False, - allow_missing=False, allow_create=True, hashes=None): + allow_missing=False, allow_create=True, hashes=None, + find_invalid_path_char=None): """ Create new Manifest instance for package in pkgdir. Do not parse Manifest file if from_scratch == True (only for internal use) The fetchlist_dict parameter is required only for generation of @@ -124,6 +139,9 @@ class Manifest(object): "portage.manifest.Manifest constructor is deprecated.", DeprecationWarning, stacklevel=2) + if find_invalid_path_char is None: + find_invalid_path_char = _find_invalid_path_char + self._find_invalid_path_char = find_invalid_path_char self.pkgdir = _unicode_decode(pkgdir).rstrip(os.sep) + os.sep self.fhashdict = {} self.hashes = set() @@ -172,13 +190,12 @@ class Manifest(object): """Parse a manifest. If myhashdict is given then data will be added too it. Otherwise, a new dict will be created and returned.""" try: - fd = io.open(_unicode_encode(file_path, + with io.open(_unicode_encode(file_path, encoding=_encodings['fs'], errors='strict'), mode='r', - encoding=_encodings['repo.content'], errors='replace') - if myhashdict is None: - myhashdict = {} - self._parseDigests(fd, myhashdict=myhashdict, **kwargs) - fd.close() + encoding=_encodings['repo.content'], errors='replace') as f: + if myhashdict is None: + myhashdict = {} + self._parseDigests(f, myhashdict=myhashdict, **kwargs) return myhashdict except (OSError, IOError) as e: if e.errno == errno.ENOENT: @@ -197,9 +214,8 @@ class Manifest(object): """Parse manifest lines and return a list of manifest entries.""" for myline in mylines: myentry = None - mysplit = myline.split() for parser in self.parsers: - myentry = parser(mysplit) + myentry = parser(myline) if myentry is not None: yield myentry break # go to the next line @@ -254,9 +270,12 @@ class Manifest(object): (MANIFEST2_REQUIRED_HASH, t, f)) def write(self, sign=False, force=False): - """ Write Manifest instance to disk, optionally signing it """ + """ Write Manifest instance to disk, optionally signing it. Returns + True if the Manifest is actually written, and False if the write + is skipped due to existing Manifest being identical.""" + rval = False if not self.allow_create: - return + return rval self.checkIntegrity() try: myentries = list(self._createManifestEntries()) @@ -288,7 +307,8 @@ class Manifest(object): # thin manifests with no DIST entries, myentries is # non-empty for all currently known use cases. write_atomic(self.getFullname(), "".join("%s\n" % - str(myentry) for myentry in myentries)) + _unicode(myentry) for myentry in myentries)) + rval = True else: # With thin manifest, there's no need to have # a Manifest file if there are no DIST entries. @@ -297,6 +317,7 @@ class Manifest(object): except OSError as e: if e.errno != errno.ENOENT: raise + rval = True if sign: self.sign() @@ -304,6 +325,7 @@ class Manifest(object): if e.errno == errno.EACCES: raise PermissionDenied(str(e)) raise + return rval def sign(self): """ Sign the Manifest """ @@ -362,10 +384,11 @@ class Manifest(object): distfilehashes = self.fhashdict["DIST"] else: distfilehashes = {} - self.__init__(self.pkgdir, self.distdir, + self.__init__(self.pkgdir, distdir=self.distdir, fetchlist_dict=self.fetchlist_dict, from_scratch=True, thin=self.thin, allow_missing=self.allow_missing, - allow_create=self.allow_create, hashes=self.hashes) + allow_create=self.allow_create, hashes=self.hashes, + find_invalid_path_char=self._find_invalid_path_char) pn = os.path.basename(self.pkgdir.rstrip(os.path.sep)) cat = self._pkgdir_category() @@ -460,7 +483,8 @@ class Manifest(object): if pf is not None: mytype = "EBUILD" cpvlist.append(pf) - elif manifest2MiscfileFilter(f): + elif self._find_invalid_path_char(f) == -1 and \ + manifest2MiscfileFilter(f): mytype = "MISC" else: continue @@ -479,7 +503,8 @@ class Manifest(object): full_path = os.path.join(parentdir, f) recursive_files.append(full_path[cut_len:]) for f in recursive_files: - if not manifest2AuxfileFilter(f): + if self._find_invalid_path_char(f) != -1 or \ + not manifest2AuxfileFilter(f): continue self.fhashdict["AUX"][f] = perform_multiple_checksums( os.path.join(self.pkgdir, "files", f.lstrip(os.sep)), self.hashes) @@ -501,14 +526,17 @@ class Manifest(object): for t in MANIFEST2_IDENTIFIERS: self.checkTypeHashes(t, ignoreMissingFiles=ignoreMissingFiles) - def checkTypeHashes(self, idtype, ignoreMissingFiles=False): + def checkTypeHashes(self, idtype, ignoreMissingFiles=False, hash_filter=None): for f in self.fhashdict[idtype]: - self.checkFileHashes(idtype, f, ignoreMissing=ignoreMissingFiles) + self.checkFileHashes(idtype, f, ignoreMissing=ignoreMissingFiles, + hash_filter=hash_filter) - def checkFileHashes(self, ftype, fname, ignoreMissing=False): - myhashes = self.fhashdict[ftype][fname] + def checkFileHashes(self, ftype, fname, ignoreMissing=False, hash_filter=None): + digests = _filter_unaccelarated_hashes(self.fhashdict[ftype][fname]) + if hash_filter is not None: + digests = _apply_hash_filter(digests, hash_filter) try: - ok,reason = verify_all(self._getAbsname(ftype, fname), self.fhashdict[ftype][fname]) + ok, reason = verify_all(self._getAbsname(ftype, fname), digests) if not ok: raise DigestException(tuple([self._getAbsname(ftype, fname)]+list(reason))) return ok, reason diff --git a/portage_with_autodep/pym/portage/manifest.pyo b/portage_with_autodep/pym/portage/manifest.pyo Binary files differindex d482bbd..40d96e1 100644 --- a/portage_with_autodep/pym/portage/manifest.pyo +++ b/portage_with_autodep/pym/portage/manifest.pyo diff --git a/portage_with_autodep/pym/portage/news.py b/portage_with_autodep/pym/portage/news.py index bbd9325..408fb5c 100644 --- a/portage_with_autodep/pym/portage/news.py +++ b/portage_with_autodep/pym/portage/news.py @@ -1,8 +1,8 @@ # portage: news management code -# Copyright 2006-2011 Gentoo Foundation +# Copyright 2006-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import print_function +from __future__ import print_function, unicode_literals __all__ = ["NewsManager", "NewsItem", "DisplayRestriction", "DisplayProfileRestriction", "DisplayKeywordRestriction", @@ -13,6 +13,7 @@ import io import logging import os as _os import re +import portage from portage import OrderedDict from portage import os from portage import _encodings @@ -241,7 +242,8 @@ class NewsItem(object): for values in self.restrictions.values(): any_match = False for restriction in values: - if restriction.checkRestriction(**kwargs): + if restriction.checkRestriction( + **portage._native_kwargs(kwargs)): any_match = True if not any_match: all_match = False @@ -388,7 +390,7 @@ def count_unread_news(portdb, vardb, repos=None, update=True): # NOTE: The NewsManager typically handles permission errors by # returning silently, so PermissionDenied won't necessarily be # raised even if we do trigger a permission error above. - msg = _unicode_decode("Permission denied: '%s'\n") % (e,) + msg = "Permission denied: '%s'\n" % (e,) if msg in permission_msgs: pass else: diff --git a/portage_with_autodep/pym/portage/news.pyo b/portage_with_autodep/pym/portage/news.pyo Binary files differindex bbd247c..0a1d4ab 100644 --- a/portage_with_autodep/pym/portage/news.pyo +++ b/portage_with_autodep/pym/portage/news.pyo diff --git a/portage_with_autodep/pym/portage/output.py b/portage_with_autodep/pym/portage/output.py index 98bec81..fc1b042 100644 --- a/portage_with_autodep/pym/portage/output.py +++ b/portage_with_autodep/pym/portage/output.py @@ -1,4 +1,4 @@ -# Copyright 1998-2011 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __docformat__ = "epytext" @@ -7,6 +7,7 @@ import errno import io import formatter import re +import subprocess import sys import portage @@ -163,15 +164,12 @@ def _parse_color_map(config_root='/', onerror=None): token = token[1:-1] return token - f = None try: - f = io.open(_unicode_encode(myfile, + with io.open(_unicode_encode(myfile, encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['content'], errors='replace') - lineno = 0 - for line in f: - lineno += 1 - + mode='r', encoding=_encodings['content'], errors='replace') as f: + lines = f.readlines() + for lineno, line in enumerate(lines): commenter_pos = line.find("#") line = line[:commenter_pos].strip() @@ -229,9 +227,6 @@ def _parse_color_map(config_root='/', onerror=None): elif e.errno == errno.EACCES: raise PermissionDenied(myfile) raise - finally: - if f is not None: - f.close() def nc_len(mystr): tmp = re.sub(esc_seq + "^m]+m", "", mystr); @@ -244,7 +239,7 @@ _max_xtermTitle_len = 253 def xtermTitle(mystr, raw=False): global _disable_xtermTitle if _disable_xtermTitle is None: - _disable_xtermTitle = not (sys.stderr.isatty() and \ + _disable_xtermTitle = not (sys.__stderr__.isatty() and \ 'TERM' in os.environ and \ _legal_terms_re.match(os.environ['TERM']) is not None) @@ -277,15 +272,18 @@ def xtermTitleReset(): if dotitles and \ 'TERM' in os.environ and \ _legal_terms_re.match(os.environ['TERM']) is not None and \ - sys.stderr.isatty(): + sys.__stderr__.isatty(): from portage.process import find_binary, spawn shell = os.environ.get("SHELL") if not shell or not os.access(shell, os.EX_OK): shell = find_binary("sh") if shell: spawn([shell, "-c", prompt_command], env=os.environ, - fd_pipes={0:sys.stdin.fileno(),1:sys.stderr.fileno(), - 2:sys.stderr.fileno()}) + fd_pipes={ + 0: portage._get_stdin().fileno(), + 1: sys.__stderr__.fileno(), + 2: sys.__stderr__.fileno() + }) else: os.system(prompt_command) return @@ -425,28 +423,41 @@ class StyleWriter(formatter.DumbWriter): if self.style_listener: self.style_listener(styles) -def get_term_size(): +def get_term_size(fd=None): """ Get the number of lines and columns of the tty that is connected to - stdout. Returns a tuple of (lines, columns) or (0, 0) if an error + fd. Returns a tuple of (lines, columns) or (0, 0) if an error occurs. The curses module is used if available, otherwise the output of `stty size` is parsed. The lines and columns values are guaranteed to be greater than or equal to zero, since a negative COLUMNS variable is known to prevent some commands from working (see bug #394091). """ - if not sys.stdout.isatty(): + if fd is None: + fd = sys.stdout + if not hasattr(fd, 'isatty') or not fd.isatty(): return (0, 0) try: import curses try: - curses.setupterm() + curses.setupterm(term=os.environ.get("TERM", "unknown"), + fd=fd.fileno()) return curses.tigetnum('lines'), curses.tigetnum('cols') except curses.error: pass except ImportError: pass - st, out = portage.subprocess_getstatusoutput('stty size') - if st == os.EX_OK: + + try: + proc = subprocess.Popen(["stty", "size"], + stdout=subprocess.PIPE, stderr=fd) + except EnvironmentError as e: + if e.errno != errno.ENOENT: + raise + # stty command not found + return (0, 0) + + out = _unicode_decode(proc.communicate()[0]) + if proc.wait() == os.EX_OK: out = out.split() if len(out) == 2: try: @@ -631,11 +642,14 @@ class EOutput(object): class ProgressBar(object): """The interface is copied from the ProgressBar class from the EasyDialogs module (which is Mac only).""" - def __init__(self, title=None, maxval=0, label=None): - self._title = title + def __init__(self, title=None, maxval=0, label=None, max_desc_length=25): + self._title = title or "" self._maxval = maxval - self._label = maxval + self._label = label or "" self._curval = 0 + self._desc = "" + self._desc_max_length = max_desc_length + self._set_desc() @property def curval(self): @@ -659,10 +673,23 @@ class ProgressBar(object): def title(self, newstr): """Sets the text in the title bar of the progress dialog to newstr.""" self._title = newstr + self._set_desc() def label(self, newstr): """Sets the text in the progress box of the progress dialog to newstr.""" self._label = newstr + self._set_desc() + + def _set_desc(self): + self._desc = "%s%s" % ( + "%s: " % self._title if self._title else "", + "%s" % self._label if self._label else "" + ) + if len(self._desc) > self._desc_max_length: # truncate if too long + self._desc = "%s..." % self._desc[:self._desc_max_length - 3] + if len(self._desc): + self._desc = self._desc.ljust(self._desc_max_length) + def set(self, value, maxval=None): """ @@ -691,10 +718,10 @@ class ProgressBar(object): class TermProgressBar(ProgressBar): """A tty progress bar similar to wget's.""" - def __init__(self, **kwargs): + def __init__(self, fd=sys.stdout, **kwargs): ProgressBar.__init__(self, **kwargs) - lines, self.term_columns = get_term_size() - self.file = sys.stdout + lines, self.term_columns = get_term_size(fd) + self.file = fd self._min_columns = 11 self._max_columns = 80 # for indeterminate mode, ranges from 0.0 to 1.0 @@ -717,16 +744,18 @@ class TermProgressBar(ProgressBar): curval = self._curval maxval = self._maxval position = self._position - percentage_str_width = 4 + percentage_str_width = 5 square_brackets_width = 2 if cols < percentage_str_width: return "" - bar_space = cols - percentage_str_width - square_brackets_width + bar_space = cols - percentage_str_width - square_brackets_width - 1 + if self._desc: + bar_space -= self._desc_max_length if maxval == 0: max_bar_width = bar_space-3 - image = " " + _percent = "".ljust(percentage_str_width) if cols < min_columns: - return image + return "" if position <= 0.5: offset = 2 * position else: @@ -742,16 +771,16 @@ class TermProgressBar(ProgressBar): position = 0.5 self._position = position bar_width = int(offset * max_bar_width) - image = image + "[" + (bar_width * " ") + \ - "<=>" + ((max_bar_width - bar_width) * " ") + "]" + image = "%s%s%s" % (self._desc, _percent, + "[" + (bar_width * " ") + \ + "<=>" + ((max_bar_width - bar_width) * " ") + "]") return image else: percentage = int(100 * float(curval) / maxval) - if percentage == 100: - percentage_str_width += 1 - bar_space -= 1 max_bar_width = bar_space - 1 - image = ("%d%% " % percentage).rjust(percentage_str_width) + _percent = ("%d%% " % percentage).rjust(percentage_str_width) + image = "%s%s" % (self._desc, _percent) + if cols < min_columns: return image offset = float(curval) / maxval diff --git a/portage_with_autodep/pym/portage/output.pyo b/portage_with_autodep/pym/portage/output.pyo Binary files differindex 993a2de..2bb98b0 100644 --- a/portage_with_autodep/pym/portage/output.pyo +++ b/portage_with_autodep/pym/portage/output.pyo diff --git a/portage_with_autodep/pym/portage/package/__init__.pyo b/portage_with_autodep/pym/portage/package/__init__.pyo Binary files differindex 9d8f30c..a17fc87 100644 --- a/portage_with_autodep/pym/portage/package/__init__.pyo +++ b/portage_with_autodep/pym/portage/package/__init__.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/__init__.pyo b/portage_with_autodep/pym/portage/package/ebuild/__init__.pyo Binary files differindex 927b4bc..ff34626 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/__init__.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/__init__.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.py b/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.py index 0c613ce..af606f1 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.py @@ -11,7 +11,7 @@ from portage.dep import ExtendedAtomDict, _repo_separator, _slot_separator from portage.localization import _ from portage.package.ebuild._config.helper import ordered_by_atom_specificity from portage.util import grabdict_package, stack_lists, writemsg -from portage.versions import cpv_getkey, _pkg_str +from portage.versions import _pkg_str class KeywordsManager(object): """Manager class to handle keywords processing and validation""" @@ -77,7 +77,9 @@ class KeywordsManager(object): def getKeywords(self, cpv, slot, keywords, repo): - if not hasattr(cpv, 'slot'): + try: + cpv.slot + except AttributeError: pkg = _pkg_str(cpv, slot=slot, repo=repo) else: pkg = cpv @@ -91,6 +93,47 @@ class KeywordsManager(object): keywords.extend(pkg_keywords) return stack_lists(keywords, incremental=True) + def isStable(self, pkg, global_accept_keywords, backuped_accept_keywords): + mygroups = self.getKeywords(pkg, None, pkg._metadata["KEYWORDS"], None) + pgroups = global_accept_keywords.split() + + unmaskgroups = self.getPKeywords(pkg, None, None, + global_accept_keywords) + pgroups.extend(unmaskgroups) + + egroups = backuped_accept_keywords.split() + + if unmaskgroups or egroups: + pgroups = self._getEgroups(egroups, pgroups) + else: + pgroups = set(pgroups) + + if self._getMissingKeywords(pkg, pgroups, mygroups): + return False + + if pkg.cpv._settings.local_config: + # If replacing all keywords with unstable variants would mask the + # package, then it's considered stable. + unstable = [] + for kw in mygroups: + if kw[:1] != "~": + kw = "~" + kw + unstable.append(kw) + + return bool(self._getMissingKeywords(pkg, pgroups, set(unstable))) + else: + # For repoman, if the package has an effective stable keyword that + # intersects with the effective ACCEPT_KEYWORDS for the current + # profile, then consider it stable. + for kw in pgroups: + if kw[:1] != "~": + if kw in mygroups or '*' in mygroups: + return True + if kw == '*': + for x in mygroups: + if x[:1] != "~": + return True + return False def getMissingKeywords(self, cpv, @@ -237,7 +280,7 @@ class KeywordsManager(object): if not mygroups: # If KEYWORDS is empty then we still have to return something # in order to distinguish from the case of "none missing". - mygroups.append("**") + mygroups = ["**"] missing = mygroups return missing @@ -261,9 +304,11 @@ class KeywordsManager(object): """ pgroups = global_accept_keywords.split() - if not hasattr(cpv, 'slot'): + try: + cpv.slot + except AttributeError: cpv = _pkg_str(cpv, slot=slot, repo=repo) - cp = cpv_getkey(cpv) + cp = cpv.cp unmaskgroups = [] if self._p_accept_keywords: @@ -288,4 +333,3 @@ class KeywordsManager(object): for x in pkg_accept_keywords: unmaskgroups.extend(x) return unmaskgroups - diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.pyo Binary files differindex 15043f0..b922211 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/LicenseManager.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/LicenseManager.pyo Binary files differindex 4a38298..4ddda5e 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/LicenseManager.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/LicenseManager.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.py b/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.py index f7a1177..80b6a70 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.py @@ -1,6 +1,8 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ( 'LocationsManager', ) @@ -13,10 +15,12 @@ import portage from portage import os, eapi_is_supported, _encodings, _unicode_encode from portage.const import CUSTOM_PROFILE_PATH, GLOBAL_CONFIG_PATH, \ PROFILE_PATH, USER_CONFIG_PATH +from portage.eapi import eapi_allows_directories_on_profile_level_and_repository_level from portage.exception import DirectoryNotFound, ParseError from portage.localization import _ from portage.util import ensure_dirs, grabfile, \ normalize_path, shlex_split, writemsg +from portage.util._path import exists_raise_eaccess, isdir_raise_eaccess from portage.repository.config import parse_layout_conf, \ _portage1_profiles_allow_directories @@ -27,7 +31,7 @@ _PORTAGE1_DIRECTORIES = frozenset([ 'use.mask', 'use.force']) _profile_node = collections.namedtuple('_profile_node', - 'location portage1_directories') + 'location portage1_directories user_config') _allow_parent_colon = frozenset( ["portage-2"]) @@ -45,9 +49,13 @@ class LocationsManager(object): if self.eprefix is None: self.eprefix = portage.const.EPREFIX + elif self.eprefix: + self.eprefix = normalize_path(self.eprefix) + if self.eprefix == os.sep: + self.eprefix = "" if self.config_root is None: - self.config_root = self.eprefix + os.sep + self.config_root = portage.const.EPREFIX + os.sep self.config_root = normalize_path(os.path.abspath( self.config_root)).rstrip(os.path.sep) + os.path.sep @@ -72,14 +80,26 @@ class LocationsManager(object): known_repos = tuple(known_repos) if self.config_profile_path is None: + deprecated_profile_path = os.path.join( + self.config_root, 'etc', 'make.profile') self.config_profile_path = \ os.path.join(self.config_root, PROFILE_PATH) - if os.path.isdir(self.config_profile_path): + if isdir_raise_eaccess(self.config_profile_path): self.profile_path = self.config_profile_path + if isdir_raise_eaccess(deprecated_profile_path) and not \ + os.path.samefile(self.profile_path, + deprecated_profile_path): + # Don't warn if they refer to the same path, since + # that can be used for backward compatibility with + # old software. + writemsg("!!! %s\n" % + _("Found 2 make.profile dirs: " + "using '%s', ignoring '%s'") % + (self.profile_path, deprecated_profile_path), + noiselevel=-1) else: - self.config_profile_path = \ - os.path.join(self.abs_user_config, 'make.profile') - if os.path.isdir(self.config_profile_path): + self.config_profile_path = deprecated_profile_path + if isdir_raise_eaccess(self.config_profile_path): self.profile_path = self.config_profile_path else: self.profile_path = None @@ -99,9 +119,9 @@ class LocationsManager(object): self._addProfile(os.path.realpath(self.profile_path), repositories, known_repos) except ParseError as e: - writemsg(_("!!! Unable to parse profile: '%s'\n") % \ - self.profile_path, noiselevel=-1) - writemsg("!!! ParseError: %s\n" % str(e), noiselevel=-1) + if not portage._sync_disabled_warnings: + writemsg(_("!!! Unable to parse profile: '%s'\n") % self.profile_path, noiselevel=-1) + writemsg("!!! ParseError: %s\n" % str(e), noiselevel=-1) self.profiles = [] self.profiles_complex = [] @@ -111,14 +131,15 @@ class LocationsManager(object): if os.path.exists(custom_prof): self.user_profile_dir = custom_prof self.profiles.append(custom_prof) - self.profiles_complex.append(_profile_node(custom_prof, True)) + self.profiles_complex.append( + _profile_node(custom_prof, True, True)) del custom_prof self.profiles = tuple(self.profiles) self.profiles_complex = tuple(self.profiles_complex) def _check_var_directory(self, varname, var): - if not os.path.isdir(var): + if not isdir_raise_eaccess(var): writemsg(_("!!! Error: %s='%s' is not a directory. " "Please correct this.\n") % (varname, var), noiselevel=-1) @@ -130,33 +151,9 @@ class LocationsManager(object): allow_parent_colon = True repo_loc = None compat_mode = False - intersecting_repos = [x for x in known_repos if current_abs_path.startswith(x[0])] - if intersecting_repos: - # protect against nested repositories. Insane configuration, but the longest - # path will be the correct one. - repo_loc, layout_data = max(intersecting_repos, key=lambda x:len(x[0])) - allow_directories = any(x in _portage1_profiles_allow_directories - for x in layout_data['profile-formats']) - compat_mode = layout_data['profile-formats'] == ('portage-1-compat',) - allow_parent_colon = any(x in _allow_parent_colon - for x in layout_data['profile-formats']) - if compat_mode: - offenders = _PORTAGE1_DIRECTORIES.intersection(os.listdir(currentPath)) - offenders = sorted(x for x in offenders - if os.path.isdir(os.path.join(currentPath, x))) - if offenders: - warnings.warn(_("Profile '%(profile_path)s' in repository " - "'%(repo_name)s' is implicitly using 'portage-1' profile format, but " - "the repository profiles are not marked as that format. This will break " - "in the future. Please either convert the following paths " - "to files, or add\nprofile-formats = portage-1\nto the " - "repositories layout.conf. Files: '%(files)s'\n") - % dict(profile_path=currentPath, repo_name=repo_loc, - files=', '.join(offenders))) - - parentsFile = os.path.join(currentPath, "parent") eapi_file = os.path.join(currentPath, "eapi") + eapi = "0" f = None try: f = io.open(_unicode_encode(eapi_file, @@ -174,7 +171,38 @@ class LocationsManager(object): finally: if f is not None: f.close() - if os.path.exists(parentsFile): + + intersecting_repos = [x for x in known_repos if current_abs_path.startswith(x[0])] + if intersecting_repos: + # protect against nested repositories. Insane configuration, but the longest + # path will be the correct one. + repo_loc, layout_data = max(intersecting_repos, key=lambda x:len(x[0])) + allow_directories = eapi_allows_directories_on_profile_level_and_repository_level(eapi) or \ + any(x in _portage1_profiles_allow_directories for x in layout_data['profile-formats']) + compat_mode = not eapi_allows_directories_on_profile_level_and_repository_level(eapi) and \ + layout_data['profile-formats'] == ('portage-1-compat',) + allow_parent_colon = any(x in _allow_parent_colon + for x in layout_data['profile-formats']) + + if compat_mode: + offenders = _PORTAGE1_DIRECTORIES.intersection(os.listdir(currentPath)) + offenders = sorted(x for x in offenders + if os.path.isdir(os.path.join(currentPath, x))) + if offenders: + warnings.warn(_( + "\nThe selected profile is implicitly using the 'portage-1' format:\n" + "\tprofile = %(profile_path)s\n" + "But this repository is not using that format:\n" + "\trepo = %(repo_name)s\n" + "This will break in the future. Please convert these dirs to files:\n" + "\t%(files)s\n" + "Or, add this line to the repository's layout.conf:\n" + "\tprofile-formats = portage-1") + % dict(profile_path=currentPath, repo_name=repo_loc, + files='\n\t'.join(offenders))) + + parentsFile = os.path.join(currentPath, "parent") + if exists_raise_eaccess(parentsFile): parents = grabfile(parentsFile) if not parents: raise ParseError( @@ -196,7 +224,7 @@ class LocationsManager(object): # of the current repo, so realpath it. parentPath = os.path.realpath(parentPath) - if os.path.exists(parentPath): + if exists_raise_eaccess(parentPath): self._addProfile(parentPath, repositories, known_repos) else: raise ParseError( @@ -205,7 +233,7 @@ class LocationsManager(object): self.profiles.append(currentPath) self.profiles_complex.append( - _profile_node(currentPath, allow_directories)) + _profile_node(currentPath, allow_directories, False)) def _expand_parent_colon(self, parentsFile, parentPath, repo_loc, repositories): @@ -253,29 +281,10 @@ class LocationsManager(object): self.eroot = self.target_root.rstrip(os.sep) + self.eprefix + os.sep - # make.globals should not be relative to config_root - # because it only contains constants. However, if EPREFIX - # is set then there are two possible scenarios: - # 1) If $ROOT == "/" then make.globals should be - # relative to EPREFIX. - # 2) If $ROOT != "/" then the correct location of - # make.globals needs to be specified in the constructor - # parameters, since it's a property of the host system - # (and the current config represents the target system). self.global_config_path = GLOBAL_CONFIG_PATH - if self.eprefix: - if self.target_root == "/": - # case (1) above - self.global_config_path = os.path.join(self.eprefix, - GLOBAL_CONFIG_PATH.lstrip(os.sep)) - else: - # case (2) above - # For now, just assume make.globals is relative - # to EPREFIX. - # TODO: Pass in more info to the constructor, - # so we know the host system configuration. - self.global_config_path = os.path.join(self.eprefix, - GLOBAL_CONFIG_PATH.lstrip(os.sep)) + if portage.const.EPREFIX: + self.global_config_path = os.path.join(portage.const.EPREFIX, + GLOBAL_CONFIG_PATH.lstrip(os.sep)) def set_port_dirs(self, portdir, portdir_overlay): self.portdir = portdir @@ -287,7 +296,7 @@ class LocationsManager(object): for ov in shlex_split(self.portdir_overlay): ov = normalize_path(ov) profiles_dir = os.path.join(ov, "profiles") - if os.path.isdir(profiles_dir): + if isdir_raise_eaccess(profiles_dir): self.overlay_profiles.append(profiles_dir) self.profile_locations = [os.path.join(portdir, "profiles")] + self.overlay_profiles diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.pyo Binary files differindex c64d313..5ef52ec 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.py b/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.py index bce1152..aeb04d7 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ( @@ -8,11 +8,10 @@ __all__ = ( import warnings from portage import os -from portage.dep import ExtendedAtomDict, match_from_list, _repo_separator, _slot_separator +from portage.dep import ExtendedAtomDict, match_from_list from portage.localization import _ from portage.util import append_repo, grabfile_package, stack_lists, writemsg -from portage.versions import cpv_getkey -from _emerge.Package import Package +from portage.versions import _pkg_str class MaskManager(object): @@ -47,7 +46,7 @@ class MaskManager(object): "the repository profiles are not marked as that format. This will break " "in the future. Please either convert the following paths " "to files, or add\nprofile-formats = portage-1\nto the " - "repositories layout.conf.\n") + "repository's layout.conf.\n") % dict(repo_name=repo_config.name)) return pmask_cache[loc] @@ -185,12 +184,15 @@ class MaskManager(object): @return: A matching atom string or None if one is not found. """ - cp = cpv_getkey(cpv) - mask_atoms = self._pmaskdict.get(cp) + try: + cpv.slot + except AttributeError: + pkg = _pkg_str(cpv, slot=slot, repo=repo) + else: + pkg = cpv + + mask_atoms = self._pmaskdict.get(pkg.cp) if mask_atoms: - pkg = "".join((cpv, _slot_separator, slot)) - if repo and repo != Package.UNKNOWN_REPO: - pkg = "".join((pkg, _repo_separator, repo)) pkg_list = [pkg] for x in mask_atoms: if not match_from_list(x, pkg_list): @@ -219,8 +221,15 @@ class MaskManager(object): @return: A matching atom string or None if one is not found. """ - cp = cpv_getkey(cpv) - return self._getMaskAtom(cpv, slot, repo, self._punmaskdict.get(cp)) + try: + cpv.slot + except AttributeError: + pkg = _pkg_str(cpv, slot=slot, repo=repo) + else: + pkg = cpv + + return self._getMaskAtom(pkg, slot, repo, + self._punmaskdict.get(pkg.cp)) def getRawMaskAtom(self, cpv, slot, repo): diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.pyo Binary files differindex f48eb47..abc46de 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.py b/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.py index e1ec7f4..0d00810 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.py @@ -1,4 +1,4 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ( @@ -7,36 +7,49 @@ __all__ = ( from _emerge.Package import Package from portage import os -from portage.dep import dep_getrepo, dep_getslot, ExtendedAtomDict, remove_slot, _get_useflag_re +from portage.dep import Atom, dep_getrepo, dep_getslot, ExtendedAtomDict, remove_slot, _get_useflag_re, _repo_separator +from portage.eapi import eapi_has_use_aliases, eapi_supports_stable_use_forcing_and_masking +from portage.exception import InvalidAtom from portage.localization import _ -from portage.util import grabfile, grabdict_package, read_corresponding_eapi_file, stack_lists, writemsg -from portage.versions import cpv_getkey, _pkg_str +from portage.util import grabfile, grabdict, grabdict_package, read_corresponding_eapi_file, stack_lists, writemsg +from portage.versions import _pkg_str from portage.package.ebuild._config.helper import ordered_by_atom_specificity class UseManager(object): - def __init__(self, repositories, profiles, abs_user_config, user_config=True): + def __init__(self, repositories, profiles, abs_user_config, is_stable, + user_config=True): # file variable #-------------------------------- # repositories #-------------------------------- # use.mask _repo_usemask_dict + # use.stable.mask _repo_usestablemask_dict # use.force _repo_useforce_dict + # use.stable.force _repo_usestableforce_dict + # use.aliases _repo_usealiases_dict # package.use.mask _repo_pusemask_dict + # package.use.stable.mask _repo_pusestablemask_dict # package.use.force _repo_puseforce_dict + # package.use.stable.force _repo_pusestableforce_dict + # package.use.aliases _repo_pusealiases_dict #-------------------------------- # profiles #-------------------------------- # use.mask _usemask_list + # use.stable.mask _usestablemask_list # use.force _useforce_list + # use.stable.force _usestableforce_list # package.use.mask _pusemask_list + # package.use.stable.mask _pusestablemask_list # package.use _pkgprofileuse # package.use.force _puseforce_list + # package.use.stable.force _pusestableforce_list #-------------------------------- # user config #-------------------------------- - # package.use _pusedict + # package.use _pusedict # Dynamic variables tracked by the config class #-------------------------------- @@ -49,26 +62,61 @@ class UseManager(object): #-------------------------------- # puse + self._user_config = user_config + self._is_stable = is_stable self._repo_usemask_dict = self._parse_repository_files_to_dict_of_tuples("use.mask", repositories) + self._repo_usestablemask_dict = \ + self._parse_repository_files_to_dict_of_tuples("use.stable.mask", + repositories, eapi_filter=eapi_supports_stable_use_forcing_and_masking) self._repo_useforce_dict = self._parse_repository_files_to_dict_of_tuples("use.force", repositories) + self._repo_usestableforce_dict = \ + self._parse_repository_files_to_dict_of_tuples("use.stable.force", + repositories, eapi_filter=eapi_supports_stable_use_forcing_and_masking) self._repo_pusemask_dict = self._parse_repository_files_to_dict_of_dicts("package.use.mask", repositories) + self._repo_pusestablemask_dict = \ + self._parse_repository_files_to_dict_of_dicts("package.use.stable.mask", + repositories, eapi_filter=eapi_supports_stable_use_forcing_and_masking) self._repo_puseforce_dict = self._parse_repository_files_to_dict_of_dicts("package.use.force", repositories) + self._repo_pusestableforce_dict = \ + self._parse_repository_files_to_dict_of_dicts("package.use.stable.force", + repositories, eapi_filter=eapi_supports_stable_use_forcing_and_masking) self._repo_puse_dict = self._parse_repository_files_to_dict_of_dicts("package.use", repositories) self._usemask_list = self._parse_profile_files_to_tuple_of_tuples("use.mask", profiles) + self._usestablemask_list = \ + self._parse_profile_files_to_tuple_of_tuples("use.stable.mask", + profiles, eapi_filter=eapi_supports_stable_use_forcing_and_masking) self._useforce_list = self._parse_profile_files_to_tuple_of_tuples("use.force", profiles) + self._usestableforce_list = \ + self._parse_profile_files_to_tuple_of_tuples("use.stable.force", + profiles, eapi_filter=eapi_supports_stable_use_forcing_and_masking) self._pusemask_list = self._parse_profile_files_to_tuple_of_dicts("package.use.mask", profiles) + self._pusestablemask_list = \ + self._parse_profile_files_to_tuple_of_dicts("package.use.stable.mask", + profiles, eapi_filter=eapi_supports_stable_use_forcing_and_masking) self._pkgprofileuse = self._parse_profile_files_to_tuple_of_dicts("package.use", profiles, juststrings=True) self._puseforce_list = self._parse_profile_files_to_tuple_of_dicts("package.use.force", profiles) + self._pusestableforce_list = \ + self._parse_profile_files_to_tuple_of_dicts("package.use.stable.force", + profiles, eapi_filter=eapi_supports_stable_use_forcing_and_masking) self._pusedict = self._parse_user_files_to_extatomdict("package.use", abs_user_config, user_config) + self._repo_usealiases_dict = self._parse_repository_usealiases(repositories) + self._repo_pusealiases_dict = self._parse_repository_packageusealiases(repositories) + self.repositories = repositories - - def _parse_file_to_tuple(self, file_name, recursive=True): + + def _parse_file_to_tuple(self, file_name, recursive=True, eapi_filter=None): ret = [] lines = grabfile(file_name, recursive=recursive) eapi = read_corresponding_eapi_file(file_name) + if eapi_filter is not None and not eapi_filter(eapi): + if lines: + writemsg(_("--- EAPI '%s' does not support '%s': '%s'\n") % + (eapi, os.path.basename(file_name), file_name), + noiselevel=-1) + return () useflag_re = _get_useflag_re(eapi) for prefixed_useflag in lines: if prefixed_useflag[:1] == "-": @@ -82,11 +130,26 @@ class UseManager(object): ret.append(prefixed_useflag) return tuple(ret) - def _parse_file_to_dict(self, file_name, juststrings=False, recursive=True): + def _parse_file_to_dict(self, file_name, juststrings=False, recursive=True, + eapi_filter=None, user_config=False): ret = {} location_dict = {} - file_dict = grabdict_package(file_name, recursive=recursive, verify_eapi=True) - eapi = read_corresponding_eapi_file(file_name) + eapi = read_corresponding_eapi_file(file_name, default=None) + if eapi is None and not user_config: + eapi = "0" + if eapi is None: + ret = ExtendedAtomDict(dict) + else: + ret = {} + file_dict = grabdict_package(file_name, recursive=recursive, + allow_wildcard=(eapi is None), allow_repo=(eapi is None), + verify_eapi=(eapi is not None)) + if eapi is not None and eapi_filter is not None and not eapi_filter(eapi): + if file_dict: + writemsg(_("--- EAPI '%s' does not support '%s': '%s'\n") % + (eapi, os.path.basename(file_name), file_name), + noiselevel=-1) + return ret useflag_re = _get_useflag_re(eapi) for k, v in file_dict.items(): useflags = [] @@ -119,31 +182,116 @@ class UseManager(object): return ret - def _parse_repository_files_to_dict_of_tuples(self, file_name, repositories): + def _parse_repository_files_to_dict_of_tuples(self, file_name, repositories, eapi_filter=None): ret = {} for repo in repositories.repos_with_profiles(): - ret[repo.name] = self._parse_file_to_tuple(os.path.join(repo.location, "profiles", file_name)) + ret[repo.name] = self._parse_file_to_tuple(os.path.join(repo.location, "profiles", file_name), eapi_filter=eapi_filter) return ret - def _parse_repository_files_to_dict_of_dicts(self, file_name, repositories): + def _parse_repository_files_to_dict_of_dicts(self, file_name, repositories, eapi_filter=None): ret = {} for repo in repositories.repos_with_profiles(): - ret[repo.name] = self._parse_file_to_dict(os.path.join(repo.location, "profiles", file_name)) + ret[repo.name] = self._parse_file_to_dict(os.path.join(repo.location, "profiles", file_name), eapi_filter=eapi_filter) return ret - def _parse_profile_files_to_tuple_of_tuples(self, file_name, locations): + def _parse_profile_files_to_tuple_of_tuples(self, file_name, locations, + eapi_filter=None): return tuple(self._parse_file_to_tuple( os.path.join(profile.location, file_name), - recursive=profile.portage1_directories) + recursive=profile.portage1_directories, eapi_filter=eapi_filter) for profile in locations) - def _parse_profile_files_to_tuple_of_dicts(self, file_name, locations, juststrings=False): + def _parse_profile_files_to_tuple_of_dicts(self, file_name, locations, + juststrings=False, eapi_filter=None): return tuple(self._parse_file_to_dict( os.path.join(profile.location, file_name), juststrings, - recursive=profile.portage1_directories) + recursive=profile.portage1_directories, eapi_filter=eapi_filter, + user_config=profile.user_config) for profile in locations) - def getUseMask(self, pkg=None): + def _parse_repository_usealiases(self, repositories): + ret = {} + for repo in repositories.repos_with_profiles(): + file_name = os.path.join(repo.location, "profiles", "use.aliases") + eapi = read_corresponding_eapi_file(file_name) + useflag_re = _get_useflag_re(eapi) + raw_file_dict = grabdict(file_name, recursive=True) + file_dict = {} + for real_flag, aliases in raw_file_dict.items(): + if useflag_re.match(real_flag) is None: + writemsg(_("--- Invalid real USE flag in '%s': '%s'\n") % (file_name, real_flag), noiselevel=-1) + else: + for alias in aliases: + if useflag_re.match(alias) is None: + writemsg(_("--- Invalid USE flag alias for '%s' real USE flag in '%s': '%s'\n") % + (real_flag, file_name, alias), noiselevel=-1) + else: + if any(alias in v for k, v in file_dict.items() if k != real_flag): + writemsg(_("--- Duplicated USE flag alias in '%s': '%s'\n") % + (file_name, alias), noiselevel=-1) + else: + file_dict.setdefault(real_flag, []).append(alias) + ret[repo.name] = file_dict + return ret + + def _parse_repository_packageusealiases(self, repositories): + ret = {} + for repo in repositories.repos_with_profiles(): + file_name = os.path.join(repo.location, "profiles", "package.use.aliases") + eapi = read_corresponding_eapi_file(file_name) + useflag_re = _get_useflag_re(eapi) + lines = grabfile(file_name, recursive=True) + file_dict = {} + for line in lines: + elements = line.split() + atom = elements[0] + try: + atom = Atom(atom, eapi=eapi) + except InvalidAtom: + writemsg(_("--- Invalid atom in '%s': '%s'\n") % (file_name, atom)) + continue + if len(elements) == 1: + writemsg(_("--- Missing real USE flag for '%s' in '%s'\n") % (atom, file_name), noiselevel=-1) + continue + real_flag = elements[1] + if useflag_re.match(real_flag) is None: + writemsg(_("--- Invalid real USE flag for '%s' in '%s': '%s'\n") % (atom, file_name, real_flag), noiselevel=-1) + else: + for alias in elements[2:]: + if useflag_re.match(alias) is None: + writemsg(_("--- Invalid USE flag alias for '%s' real USE flag for '%s' in '%s': '%s'\n") % + (real_flag, atom, file_name, alias), noiselevel=-1) + else: + # Duplicated USE flag aliases in entries for different atoms + # matching the same package version are detected in getUseAliases(). + if any(alias in v for k, v in file_dict.get(atom.cp, {}).get(atom, {}).items() if k != real_flag): + writemsg(_("--- Duplicated USE flag alias for '%s' in '%s': '%s'\n") % + (atom, file_name, alias), noiselevel=-1) + else: + file_dict.setdefault(atom.cp, {}).setdefault(atom, {}).setdefault(real_flag, []).append(alias) + ret[repo.name] = file_dict + return ret + + def _isStable(self, pkg): + if self._user_config: + try: + return pkg.stable + except AttributeError: + # KEYWORDS is unavailable (prior to "depend" phase) + return False + + try: + pkg._metadata + except AttributeError: + # KEYWORDS is unavailable (prior to "depend" phase) + return False + + # Since repoman uses different config instances for + # different profiles, we have to be careful to do the + # stable check against the correct profile here. + return self._is_stable(pkg) + + def getUseMask(self, pkg=None, stable=None): if pkg is None: return frozenset(stack_lists( self._usemask_list, incremental=True)) @@ -155,7 +303,12 @@ class UseManager(object): repo = dep_getrepo(pkg) pkg = _pkg_str(remove_slot(pkg), slot=slot, repo=repo) cp = pkg.cp + + if stable is None: + stable = self._isStable(pkg) + usemask = [] + if hasattr(pkg, "repo") and pkg.repo != Package.UNKNOWN_REPO: repos = [] try: @@ -166,30 +319,56 @@ class UseManager(object): repos.append(pkg.repo) for repo in repos: usemask.append(self._repo_usemask_dict.get(repo, {})) + if stable: + usemask.append(self._repo_usestablemask_dict.get(repo, {})) cpdict = self._repo_pusemask_dict.get(repo, {}).get(cp) if cpdict: pkg_usemask = ordered_by_atom_specificity(cpdict, pkg) if pkg_usemask: usemask.extend(pkg_usemask) + if stable: + cpdict = self._repo_pusestablemask_dict.get(repo, {}).get(cp) + if cpdict: + pkg_usemask = ordered_by_atom_specificity(cpdict, pkg) + if pkg_usemask: + usemask.extend(pkg_usemask) + for i, pusemask_dict in enumerate(self._pusemask_list): if self._usemask_list[i]: usemask.append(self._usemask_list[i]) + if stable and self._usestablemask_list[i]: + usemask.append(self._usestablemask_list[i]) cpdict = pusemask_dict.get(cp) if cpdict: pkg_usemask = ordered_by_atom_specificity(cpdict, pkg) if pkg_usemask: usemask.extend(pkg_usemask) + if stable: + cpdict = self._pusestablemask_list[i].get(cp) + if cpdict: + pkg_usemask = ordered_by_atom_specificity(cpdict, pkg) + if pkg_usemask: + usemask.extend(pkg_usemask) + return frozenset(stack_lists(usemask, incremental=True)) - def getUseForce(self, pkg=None): + def getUseForce(self, pkg=None, stable=None): if pkg is None: return frozenset(stack_lists( self._useforce_list, incremental=True)) cp = getattr(pkg, "cp", None) if cp is None: - cp = cpv_getkey(remove_slot(pkg)) + slot = dep_getslot(pkg) + repo = dep_getrepo(pkg) + pkg = _pkg_str(remove_slot(pkg), slot=slot, repo=repo) + cp = pkg.cp + + if stable is None: + stable = self._isStable(pkg) + useforce = [] + if hasattr(pkg, "repo") and pkg.repo != Package.UNKNOWN_REPO: repos = [] try: @@ -200,25 +379,90 @@ class UseManager(object): repos.append(pkg.repo) for repo in repos: useforce.append(self._repo_useforce_dict.get(repo, {})) + if stable: + useforce.append(self._repo_usestableforce_dict.get(repo, {})) cpdict = self._repo_puseforce_dict.get(repo, {}).get(cp) if cpdict: pkg_useforce = ordered_by_atom_specificity(cpdict, pkg) if pkg_useforce: useforce.extend(pkg_useforce) + if stable: + cpdict = self._repo_pusestableforce_dict.get(repo, {}).get(cp) + if cpdict: + pkg_useforce = ordered_by_atom_specificity(cpdict, pkg) + if pkg_useforce: + useforce.extend(pkg_useforce) + for i, puseforce_dict in enumerate(self._puseforce_list): if self._useforce_list[i]: useforce.append(self._useforce_list[i]) + if stable and self._usestableforce_list[i]: + useforce.append(self._usestableforce_list[i]) cpdict = puseforce_dict.get(cp) if cpdict: pkg_useforce = ordered_by_atom_specificity(cpdict, pkg) if pkg_useforce: useforce.extend(pkg_useforce) + if stable: + cpdict = self._pusestableforce_list[i].get(cp) + if cpdict: + pkg_useforce = ordered_by_atom_specificity(cpdict, pkg) + if pkg_useforce: + useforce.extend(pkg_useforce) + return frozenset(stack_lists(useforce, incremental=True)) + def getUseAliases(self, pkg): + if hasattr(pkg, "eapi") and not eapi_has_use_aliases(pkg.eapi): + return {} + + cp = getattr(pkg, "cp", None) + if cp is None: + slot = dep_getslot(pkg) + repo = dep_getrepo(pkg) + pkg = _pkg_str(remove_slot(pkg), slot=slot, repo=repo) + cp = pkg.cp + + usealiases = {} + + if hasattr(pkg, "repo") and pkg.repo != Package.UNKNOWN_REPO: + repos = [] + try: + repos.extend(repo.name for repo in + self.repositories[pkg.repo].masters) + except KeyError: + pass + repos.append(pkg.repo) + for repo in repos: + usealiases_dict = self._repo_usealiases_dict.get(repo, {}) + for real_flag, aliases in usealiases_dict.items(): + for alias in aliases: + if any(alias in v for k, v in usealiases.items() if k != real_flag): + writemsg(_("--- Duplicated USE flag alias for '%s%s%s': '%s'\n") % + (pkg.cpv, _repo_separator, pkg.repo, alias), noiselevel=-1) + else: + usealiases.setdefault(real_flag, []).append(alias) + cp_usealiases_dict = self._repo_pusealiases_dict.get(repo, {}).get(cp) + if cp_usealiases_dict: + usealiases_dict_list = ordered_by_atom_specificity(cp_usealiases_dict, pkg) + for usealiases_dict in usealiases_dict_list: + for real_flag, aliases in usealiases_dict.items(): + for alias in aliases: + if any(alias in v for k, v in usealiases.items() if k != real_flag): + writemsg(_("--- Duplicated USE flag alias for '%s%s%s': '%s'\n") % + (pkg.cpv, _repo_separator, pkg.repo, alias), noiselevel=-1) + else: + usealiases.setdefault(real_flag, []).append(alias) + + return usealiases + def getPUSE(self, pkg): cp = getattr(pkg, "cp", None) if cp is None: - cp = cpv_getkey(remove_slot(pkg)) + slot = dep_getslot(pkg) + repo = dep_getrepo(pkg) + pkg = _pkg_str(remove_slot(pkg), slot=slot, repo=repo) + cp = pkg.cp ret = "" cpdict = self._pusedict.get(cp) if cpdict: diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.pyo Binary files differindex 2c9a609..16fa78e 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/VirtualsManager.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/VirtualsManager.pyo Binary files differindex b2ebd21..88010cc 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/VirtualsManager.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/VirtualsManager.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/__init__.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/__init__.pyo Binary files differindex b03cc29..f3c8238 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/__init__.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/__init__.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/env_var_validation.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/env_var_validation.pyo Binary files differindex aeee789..d78d7d2 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/env_var_validation.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/env_var_validation.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/features_set.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/features_set.pyo Binary files differindex 9854444..ef59bc0 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/features_set.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/features_set.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/helper.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/helper.pyo Binary files differindex f2b9261..d2a012f 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/helper.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/helper.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.py b/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.py index 1a75de9..f241e65 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.py @@ -1,6 +1,8 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ( 'case_insensitive_vars', 'default_globals', 'env_blacklist', \ 'environ_filter', 'environ_whitelist', 'environ_whitelist_re', @@ -13,14 +15,17 @@ import re # configuration files. env_blacklist = frozenset(( "A", "AA", "CATEGORY", "DEPEND", "DESCRIPTION", "EAPI", - "EBUILD_FORCE_TEST", "EBUILD_PHASE", "EBUILD_SKIP_MANIFEST", + "EBUILD_FORCE_TEST", "EBUILD_PHASE", + "EBUILD_PHASE_FUNC", "EBUILD_SKIP_MANIFEST", "ED", "EMERGE_FROM", "EPREFIX", "EROOT", - "GREP_OPTIONS", "HOMEPAGE", "INHERITED", "IUSE", + "GREP_OPTIONS", "HDEPEND", "HOMEPAGE", + "INHERITED", "IUSE", "IUSE_EFFECTIVE", "KEYWORDS", "LICENSE", "MERGE_TYPE", "PDEPEND", "PF", "PKGUSE", "PORTAGE_BACKGROUND", - "PORTAGE_BACKGROUND_UNMERGE", "PORTAGE_BUILDIR_LOCKED", - "PORTAGE_BUILT_USE", "PORTAGE_CONFIGROOT", "PORTAGE_IUSE", - "PORTAGE_NONFATAL", "PORTAGE_REPO_NAME", + "PORTAGE_BACKGROUND_UNMERGE", "PORTAGE_BUILDDIR_LOCKED", + "PORTAGE_BUILT_USE", "PORTAGE_CONFIGROOT", + "PORTAGE_INTERNAL_CALLER", "PORTAGE_IUSE", + "PORTAGE_NONFATAL", "PORTAGE_PIPE_FD", "PORTAGE_REPO_NAME", "PORTAGE_USE", "PROPERTIES", "PROVIDE", "RDEPEND", "REPOSITORY", "RESTRICT", "ROOT", "SLOT", "SRC_URI" )) @@ -39,7 +44,7 @@ environ_whitelist += [ "ACCEPT_LICENSE", "BASH_ENV", "BUILD_PREFIX", "COLUMNS", "D", "DISTDIR", "DOC_SYMLINKS_DIR", "EAPI", "EBUILD", "EBUILD_FORCE_TEST", - "EBUILD_PHASE", "ECLASSDIR", "ECLASS_DEPTH", "ED", + "EBUILD_PHASE", "EBUILD_PHASE_FUNC", "ECLASSDIR", "ECLASS_DEPTH", "ED", "EMERGE_FROM", "EPREFIX", "EROOT", "FEATURES", "FILESDIR", "HOME", "MERGE_TYPE", "NOCOLOR", "PATH", "PKGDIR", @@ -49,19 +54,26 @@ environ_whitelist += [ "PORTAGE_BINPKG_FILE", "PORTAGE_BINPKG_TAR_OPTS", "PORTAGE_BINPKG_TMPFILE", "PORTAGE_BIN_PATH", - "PORTAGE_BUILDDIR", "PORTAGE_BUNZIP2_COMMAND", "PORTAGE_BZIP2_COMMAND", - "PORTAGE_COLORMAP", "PORTAGE_COMPRESS_EXCLUDE_SUFFIXES", + "PORTAGE_BUILDDIR", "PORTAGE_BUILD_GROUP", "PORTAGE_BUILD_USER", + "PORTAGE_BUNZIP2_COMMAND", "PORTAGE_BZIP2_COMMAND", + "PORTAGE_COLORMAP", "PORTAGE_COMPRESS", + "PORTAGE_COMPRESS_EXCLUDE_SUFFIXES", "PORTAGE_CONFIGROOT", "PORTAGE_DEBUG", "PORTAGE_DEPCACHEDIR", + "PORTAGE_DOHTML_UNWARNED_SKIPPED_EXTENSIONS", + "PORTAGE_DOHTML_UNWARNED_SKIPPED_FILES", + "PORTAGE_DOHTML_WARN_ON_SKIPPED_FILES", "PORTAGE_EBUILD_EXIT_FILE", "PORTAGE_FEATURES", "PORTAGE_GID", "PORTAGE_GRPNAME", + "PORTAGE_INTERNAL_CALLER", "PORTAGE_INST_GID", "PORTAGE_INST_UID", - "PORTAGE_IPC_DAEMON", "PORTAGE_IUSE", - "PORTAGE_LOG_FILE", "PORTAGE_OVERRIDE_EPREFIX", - "PORTAGE_PYM_PATH", "PORTAGE_PYTHON", "PORTAGE_QUIET", - "PORTAGE_REPO_NAME", "PORTAGE_RESTRICT", + "PORTAGE_IPC_DAEMON", "PORTAGE_IUSE", "PORTAGE_ECLASS_LOCATIONS", + "PORTAGE_LOG_FILE", "PORTAGE_OVERRIDE_EPREFIX", "PORTAGE_PIPE_FD", + "PORTAGE_PYM_PATH", "PORTAGE_PYTHON", + "PORTAGE_PYTHONPATH", "PORTAGE_QUIET", + "PORTAGE_REPO_NAME", "PORTAGE_REPOSITORIES", "PORTAGE_RESTRICT", "PORTAGE_SIGPIPE_STATUS", "PORTAGE_TMPDIR", "PORTAGE_UPDATE_ENV", "PORTAGE_USERNAME", - "PORTAGE_VERBOSE", "PORTAGE_WORKDIR_MODE", + "PORTAGE_VERBOSE", "PORTAGE_WORKDIR_MODE", "PORTAGE_XATTR_EXCLUDE", "PORTDIR", "PORTDIR_OVERLAY", "PREROOTPATH", "PROFILE_PATHS", "REPLACING_VERSIONS", "REPLACED_BY_VERSION", "ROOT", "ROOTPATH", "T", "TMP", "TMPDIR", @@ -102,7 +114,7 @@ environ_whitelist += [ environ_whitelist += [ "CVS_RSH", "ECHANGELOG_USER", "GPG_AGENT_INFO", "LOG_SOCKET", - "SSH_AGENT_PID", "SSH_AUTH_SOCK" + "SSH_AGENT_PID", "SSH_AUTH_SOCK", "STY", "WINDOW", "XAUTHORITY", ] @@ -133,9 +145,11 @@ environ_filter += [ # portage config variables and variables set directly by portage environ_filter += [ - "ACCEPT_CHOSTS", "ACCEPT_KEYWORDS", "ACCEPT_PROPERTIES", "AUTOCLEAN", + "ACCEPT_CHOSTS", "ACCEPT_KEYWORDS", "ACCEPT_PROPERTIES", + "ACCEPT_RESTRICT", "AUTOCLEAN", "CLEAN_DELAY", "COLLISION_IGNORE", "CONFIG_PROTECT", "CONFIG_PROTECT_MASK", + "DCO_SIGNED_OFF_BY", "EGENCACHE_DEFAULT_OPTS", "EMERGE_DEFAULT_OPTS", "EMERGE_LOG_DIR", "EMERGE_WARNING_DELAY", @@ -144,8 +158,9 @@ environ_filter += [ "FETCHCOMMAND_RSYNC", "FETCHCOMMAND_SFTP", "GENTOO_MIRRORS", "NOCONFMEM", "O", "PORTAGE_BACKGROUND", "PORTAGE_BACKGROUND_UNMERGE", - "PORTAGE_BINHOST", - "PORTAGE_BINHOST_CHUNKSIZE", "PORTAGE_BUILDIR_LOCKED", + "PORTAGE_BINHOST", "PORTAGE_BINPKG_FORMAT", + "PORTAGE_BUILDDIR_LOCKED", + "PORTAGE_CHECKSUM_FILTER", "PORTAGE_ELOG_CLASSES", "PORTAGE_ELOG_MAILFROM", "PORTAGE_ELOG_MAILSUBJECT", "PORTAGE_ELOG_MAILURI", "PORTAGE_ELOG_SYSTEM", @@ -157,13 +172,20 @@ environ_filter += [ "PORTAGE_REPO_DUPLICATE_WARN", "PORTAGE_RO_DISTDIRS", "PORTAGE_RSYNC_EXTRA_OPTS", "PORTAGE_RSYNC_OPTS", - "PORTAGE_RSYNC_RETRIES", "PORTAGE_SYNC_STALE", - "PORTAGE_USE", "PORT_LOGDIR", "PORT_LOGDIR_CLEAN", + "PORTAGE_RSYNC_RETRIES", "PORTAGE_SSH_OPTS", "PORTAGE_SYNC_STALE", + "PORTAGE_USE", + "PORT_LOGDIR", "PORT_LOGDIR_CLEAN", "QUICKPKG_DEFAULT_OPTS", "REPOMAN_DEFAULT_OPTS", "RESUMECOMMAND", "RESUMECOMMAND_FTP", "RESUMECOMMAND_HTTP", "RESUMECOMMAND_HTTPS", "RESUMECOMMAND_RSYNC", "RESUMECOMMAND_SFTP", - "SYNC", "USE_EXPAND_HIDDEN", "USE_ORDER", + "UNINSTALL_IGNORE", "USE_EXPAND_HIDDEN", "USE_ORDER", + "__PORTAGE_HELPER" +] + +# No longer supported variables +environ_filter += [ + "SYNC" ] environ_filter = frozenset(environ_filter) diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.pyo Binary files differindex 06ea37e..92c5d32 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_eapi_invalid.py b/portage_with_autodep/pym/portage/package/ebuild/_eapi_invalid.py deleted file mode 100644 index d23677d..0000000 --- a/portage_with_autodep/pym/portage/package/ebuild/_eapi_invalid.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2012 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -import textwrap - -import portage -from portage.dep import _repo_separator -from portage.elog import elog_process -from portage.elog.messages import eerror - -def eapi_invalid(self, cpv, repo_name, settings, - eapi_var, eapi_parsed, eapi_lineno): - - msg = [] - msg.extend(textwrap.wrap(("EAPI assignment in ebuild '%s%s%s' does not" - " conform with PMS section 7.3.1 (see bug #402167):") % - (cpv, _repo_separator, repo_name), 70)) - - if not eapi_parsed: - # None means the assignment was not found, while an - # empty string indicates an (invalid) empty assingment. - msg.append( - "\tvalid EAPI assignment must" - " occur on or before line: %s" % - eapi_lineno) - else: - msg.append(("\tbash returned EAPI '%s' which does not match " - "assignment on line: %s") % - (eapi_var, eapi_lineno)) - - if 'parse-eapi-ebuild-head' in settings.features: - msg.extend(textwrap.wrap(("NOTE: This error will soon" - " become unconditionally fatal in a future version of Portage," - " but at this time, it can by made non-fatal by setting" - " FEATURES=-parse-eapi-ebuild-head in" - " make.conf."), 70)) - else: - msg.extend(textwrap.wrap(("NOTE: This error will soon" - " become unconditionally fatal in a future version of Portage." - " At the earliest opportunity, please enable" - " FEATURES=parse-eapi-ebuild-head in make.conf in order to" - " make this error fatal."), 70)) - - if portage.data.secpass >= 2: - # TODO: improve elog permission error handling (bug #416231) - for line in msg: - eerror(line, phase="other", key=cpv) - elog_process(cpv, settings, - phasefilter=("other",)) - - else: - out = portage.output.EOutput() - for line in msg: - out.eerror(line) diff --git a/portage_with_autodep/pym/portage/package/ebuild/_eapi_invalid.pyo b/portage_with_autodep/pym/portage/package/ebuild/_eapi_invalid.pyo Binary files differdeleted file mode 100644 index 0181c03..0000000 --- a/portage_with_autodep/pym/portage/package/ebuild/_eapi_invalid.pyo +++ /dev/null diff --git a/portage_with_autodep/pym/portage/package/ebuild/_ipc/ExitCommand.pyo b/portage_with_autodep/pym/portage/package/ebuild/_ipc/ExitCommand.pyo Binary files differindex 315cb0f..744bb83 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_ipc/ExitCommand.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_ipc/ExitCommand.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_ipc/IpcCommand.pyo b/portage_with_autodep/pym/portage/package/ebuild/_ipc/IpcCommand.pyo Binary files differindex 9f75518..ae66c3e 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_ipc/IpcCommand.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_ipc/IpcCommand.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.py b/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.py index 7bbb0e8..351c956 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.py @@ -1,12 +1,13 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + import io import portage from portage import os -from portage import _unicode_decode -from portage.dep import Atom +from portage.dep import Atom, _repo_name_re from portage.eapi import eapi_has_repo_deps from portage.elog import messages as elog_messages from portage.exception import InvalidAtom @@ -20,6 +21,12 @@ class QueryCommand(IpcCommand): _db = None + @classmethod + def get_db(cls): + if cls._db is not None: + return cls._db + return portage.db + def __init__(self, settings, phase): IpcCommand.__init__(self) self.settings = settings @@ -30,41 +37,46 @@ class QueryCommand(IpcCommand): @return: tuple of (stdout, stderr, returncode) """ - cmd, root, atom_str = argv + # Python 3: + # cmd, root, *args = argv + cmd = argv[0] + root = argv[1] + args = argv[2:] + + warnings = [] + warnings_str = '' + db = self.get_db() eapi = self.settings.get('EAPI') - allow_repo = eapi_has_repo_deps(eapi) - try: - atom = Atom(atom_str, allow_repo=allow_repo) - except InvalidAtom: - return ('', 'invalid atom: %s\n' % atom_str, 2) - warnings = [] - try: - atom = Atom(atom_str, allow_repo=allow_repo, eapi=eapi) - except InvalidAtom as e: - warnings.append(_unicode_decode("QA Notice: %s: %s") % (cmd, e)) + root = normalize_path(root).rstrip(os.path.sep) + os.path.sep + if root not in db: + return ('', '%s: Invalid ROOT: %s\n' % (cmd, root), 3) - use = self.settings.get('PORTAGE_BUILT_USE') - if use is None: - use = self.settings['PORTAGE_USE'] + portdb = db[root]["porttree"].dbapi + vardb = db[root]["vartree"].dbapi - use = frozenset(use.split()) - atom = atom.evaluate_conditionals(use) + if cmd in ('best_version', 'has_version'): + allow_repo = eapi_has_repo_deps(eapi) + try: + atom = Atom(args[0], allow_repo=allow_repo) + except InvalidAtom: + return ('', '%s: Invalid atom: %s\n' % (cmd, args[0]), 2) - db = self._db - if db is None: - db = portage.db + try: + atom = Atom(args[0], allow_repo=allow_repo, eapi=eapi) + except InvalidAtom as e: + warnings.append("QA Notice: %s: %s" % (cmd, e)) - warnings_str = '' - if warnings: - warnings_str = self._elog('eqawarn', warnings) + use = self.settings.get('PORTAGE_BUILT_USE') + if use is None: + use = self.settings['PORTAGE_USE'] - root = normalize_path(root).rstrip(os.path.sep) + os.path.sep - if root not in db: - return ('', 'invalid ROOT: %s\n' % root, 2) + use = frozenset(use.split()) + atom = atom.evaluate_conditionals(use) - vardb = db[root]["vartree"].dbapi + if warnings: + warnings_str = self._elog('eqawarn', warnings) if cmd == 'has_version': if vardb.match(atom): @@ -75,8 +87,35 @@ class QueryCommand(IpcCommand): elif cmd == 'best_version': m = best(vardb.match(atom)) return ('%s\n' % m, warnings_str, 0) + elif cmd in ('master_repositories', 'repository_path', 'available_eclasses', 'eclass_path', 'license_path'): + repo = _repo_name_re.match(args[0]) + if repo is None: + return ('', '%s: Invalid repository: %s\n' % (cmd, args[0]), 2) + try: + repo = portdb.repositories[args[0]] + except KeyError: + return ('', warnings_str, 1) + + if cmd == 'master_repositories': + return ('%s\n' % ' '.join(x.name for x in repo.masters), warnings_str, 0) + elif cmd == 'repository_path': + return ('%s\n' % repo.location, warnings_str, 0) + elif cmd == 'available_eclasses': + return ('%s\n' % ' '.join(sorted(repo.eclass_db.eclasses)), warnings_str, 0) + elif cmd == 'eclass_path': + try: + eclass = repo.eclass_db.eclasses[args[1]] + except KeyError: + return ('', warnings_str, 1) + return ('%s\n' % eclass.location, warnings_str, 0) + elif cmd == 'license_path': + paths = reversed([os.path.join(x.location, 'licenses', args[1]) for x in list(repo.masters) + [repo]]) + for path in paths: + if os.path.exists(path): + return ('%s\n' % path, warnings_str, 0) + return ('', warnings_str, 1) else: - return ('', 'invalid command: %s\n' % cmd, 2) + return ('', 'Invalid command: %s\n' % cmd, 3) def _elog(self, elog_funcname, lines): """ diff --git a/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.pyo b/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.pyo Binary files differindex 0e9ee96..3087272 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_ipc/__init__.pyo b/portage_with_autodep/pym/portage/package/ebuild/_ipc/__init__.pyo Binary files differindex d9f8d25..270e321 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_ipc/__init__.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_ipc/__init__.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.py b/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.py index 94f8c79..0fc53c8 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.py @@ -1,8 +1,9 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import tempfile +import portage from portage import os from portage import shutil from portage.const import EBUILD_PHASES @@ -10,10 +11,12 @@ from portage.elog import elog_process from portage.package.ebuild.config import config from portage.package.ebuild.doebuild import doebuild_environment from portage.package.ebuild.prepare_build_dirs import prepare_build_dirs +from portage.util._async.SchedulerInterface import SchedulerInterface +from portage.util._eventloop.EventLoop import EventLoop +from portage.util._eventloop.global_event_loop import global_event_loop from _emerge.EbuildPhase import EbuildPhase -from _emerge.PollScheduler import PollScheduler -def spawn_nofetch(portdb, ebuild_path, settings=None): +def spawn_nofetch(portdb, ebuild_path, settings=None, fd_pipes=None): """ This spawns pkg_nofetch if appropriate. The settings parameter is useful only if setcpv has already been called in order @@ -47,7 +50,7 @@ def spawn_nofetch(portdb, ebuild_path, settings=None): settings = config(clone=settings) if 'PORTAGE_PARALLEL_FETCHONLY' in settings: - return + return os.EX_OK # We must create our private PORTAGE_TMPDIR before calling # doebuild_environment(), since lots of variables such @@ -59,7 +62,7 @@ def spawn_nofetch(portdb, ebuild_path, settings=None): settings['PORTAGE_TMPDIR'] = private_tmpdir settings.backup_changes('PORTAGE_TMPDIR') # private temp dir was just created, so it's not locked yet - settings.pop('PORTAGE_BUILDIR_LOCKED', None) + settings.pop('PORTAGE_BUILDDIR_LOCKED', None) try: doebuild_environment(ebuild_path, 'nofetch', @@ -73,14 +76,18 @@ def spawn_nofetch(portdb, ebuild_path, settings=None): if 'fetch' not in restrict and \ 'nofetch' not in defined_phases: - return + return os.EX_OK prepare_build_dirs(settings=settings) ebuild_phase = EbuildPhase(background=False, - phase='nofetch', scheduler=PollScheduler().sched_iface, - settings=settings) + phase='nofetch', + scheduler=SchedulerInterface(portage._internal_caller and + global_event_loop() or EventLoop(main=False)), + fd_pipes=fd_pipes, settings=settings) ebuild_phase.start() ebuild_phase.wait() elog_process(settings.mycpv, settings) finally: shutil.rmtree(private_tmpdir) + + return ebuild_phase.returncode diff --git a/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.pyo b/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.pyo Binary files differindex ac449ea..d4c597c 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/config.py b/portage_with_autodep/pym/portage/package/ebuild/config.py index 97cbd99..92e6c3f 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/config.py +++ b/portage_with_autodep/pym/portage/package/ebuild/config.py @@ -1,6 +1,8 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = [ 'autouse', 'best_from_dict', 'check_config_instance', 'config', ] @@ -19,6 +21,8 @@ from _emerge.Package import Package import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.data:portage_gid', + 'portage.dbapi.vartree:vartree', + 'portage.package.ebuild.doebuild:_phase_func_map', ) from portage import bsd_chflags, \ load_mod, os, selinux, _unicode_decode @@ -29,10 +33,9 @@ from portage.const import CACHE_PATH, \ USER_VIRTUALS_FILE from portage.dbapi import dbapi from portage.dbapi.porttree import portdbapi -from portage.dbapi.vartree import vartree from portage.dep import Atom, isvalidatom, match_from_list, use_reduce, _repo_separator, _slot_separator from portage.eapi import eapi_exports_AA, eapi_exports_merge_type, \ - eapi_supports_prefix, eapi_exports_replace_vars + eapi_supports_prefix, eapi_exports_replace_vars, _get_eapi_attrs from portage.env.loaders import KeyValuePairFileLoader from portage.exception import InvalidDependString, PortageException from portage.localization import _ @@ -42,7 +45,8 @@ from portage.repository.config import load_repository_config from portage.util import ensure_dirs, getconfig, grabdict, \ grabdict_package, grabfile, grabfile_package, LazyItemsDict, \ normalize_path, shlex_split, stack_dictlist, stack_dicts, stack_lists, \ - writemsg, writemsg_level + writemsg, writemsg_level, _eapi_cache +from portage.util._path import exists_raise_eaccess, isdir_raise_eaccess from portage.versions import catpkgsplit, catsplit, cpv_getkey, _pkg_str from portage.package.ebuild._config import special_env_vars @@ -55,10 +59,29 @@ from portage.package.ebuild._config.LocationsManager import LocationsManager from portage.package.ebuild._config.MaskManager import MaskManager from portage.package.ebuild._config.VirtualsManager import VirtualsManager from portage.package.ebuild._config.helper import ordered_by_atom_specificity, prune_incremental +from portage.package.ebuild._config.unpack_dependencies import load_unpack_dependencies_configuration if sys.hexversion >= 0x3000000: basestring = str +_feature_flags_cache = {} + +def _get_feature_flags(eapi_attrs): + cache_key = (eapi_attrs.feature_flag_test, eapi_attrs.feature_flag_targetroot) + flags = _feature_flags_cache.get(cache_key) + if flags is not None: + return flags + + flags = [] + if eapi_attrs.feature_flag_test: + flags.append("test") + if eapi_attrs.feature_flag_targetroot: + flags.append("targetroot") + + flags = frozenset(flags) + _feature_flags_cache[cache_key] = flags + return flags + def autouse(myvartree, use_cache=1, mysettings=None): warnings.warn("portage.autouse() is deprecated", DeprecationWarning, stacklevel=2) @@ -123,9 +146,9 @@ class config(object): """ _constant_keys = frozenset(['PORTAGE_BIN_PATH', 'PORTAGE_GID', - 'PORTAGE_PYM_PATH']) + 'PORTAGE_PYM_PATH', 'PORTAGE_PYTHONPATH']) - _setcpv_aux_keys = ('DEFINED_PHASES', 'DEPEND', 'EAPI', + _setcpv_aux_keys = ('DEFINED_PHASES', 'DEPEND', 'EAPI', 'HDEPEND', 'INHERITED', 'IUSE', 'REQUIRED_USE', 'KEYWORDS', 'LICENSE', 'PDEPEND', 'PROPERTIES', 'PROVIDE', 'RDEPEND', 'SLOT', 'repository', 'RESTRICT', 'LICENSE',) @@ -146,7 +169,7 @@ class config(object): def __init__(self, clone=None, mycpv=None, config_profile_path=None, config_incrementals=None, config_root=None, target_root=None, eprefix=None, local_config=True, env=None, - _unmatched_removal=False): + _unmatched_removal=False, repositories=None): """ @param clone: If provided, init will use deepcopy to copy by value the instance. @type clone: Instance of config class. @@ -160,7 +183,8 @@ class config(object): @type config_incrementals: List @param config_root: path to read local config from (defaults to "/", see PORTAGE_CONFIGROOT) @type config_root: String - @param target_root: __init__ override of $ROOT env variable. + @param target_root: the target root, which typically corresponds to the + value of the $ROOT env variable (default is /) @type target_root: String @param eprefix: set the EPREFIX variable (default is portage.const.EPREFIX) @type eprefix: String @@ -173,14 +197,21 @@ class config(object): @param _unmatched_removal: Enabled by repoman when the --unmatched-removal option is given. @type _unmatched_removal: Boolean + @param repositories: Configuration of repositories. + Defaults to portage.repository.config.load_repository_config(). + @type repositories: Instance of portage.repository.config.RepoConfigLoader class. """ + # This is important when config is reloaded after emerge --sync. + _eapi_cache.clear() + # When initializing the global portage.settings instance, avoid # raising exceptions whenever possible since exceptions thrown # from 'import portage' or 'import portage.exceptions' statements # can practically render the api unusable for api consumers. tolerant = hasattr(portage, '_initializing_globals') self._tolerant = tolerant + self._unmatched_removal = _unmatched_removal self.locked = 0 self.mycpv = None @@ -191,8 +222,10 @@ class config(object): self.uvlist = [] self._accept_chost_re = None self._accept_properties = None + self._accept_restrict = None self._features_overrides = [] self._make_defaults = None + self._parent_stable = None # _unknown_features records unknown features that # have triggered warning messages, and ensures that @@ -205,6 +238,7 @@ class config(object): # For immutable attributes, use shallow copy for # speed and memory conservation. self._tolerant = clone._tolerant + self._unmatched_removal = clone._unmatched_removal self.categories = clone.categories self.depcachedir = clone.depcachedir self.incrementals = clone.incrementals @@ -213,6 +247,8 @@ class config(object): self.profiles = clone.profiles self.packages = clone.packages self.repositories = clone.repositories + self.unpack_dependencies = clone.unpack_dependencies + self._iuse_effective = clone._iuse_effective self._iuse_implicit_match = clone._iuse_implicit_match self._non_user_variables = clone._non_user_variables self._env_d_blacklist = clone._env_d_blacklist @@ -227,9 +263,12 @@ class config(object): self._setcpv_args_hash = clone._setcpv_args_hash # immutable attributes (internal policy ensures lack of mutation) - self._keywords_manager = clone._keywords_manager + self._locations_manager = clone._locations_manager self._use_manager = clone._use_manager - self._mask_manager = clone._mask_manager + # force instantiation of lazy immutable objects when cloning, so + # that they're not instantiated more than once + self._keywords_manager_obj = clone._keywords_manager + self._mask_manager_obj = clone._mask_manager # shared mutable attributes self._unknown_features = clone._unknown_features @@ -266,30 +305,55 @@ class config(object): #all LicenseManager instances. self._license_manager = clone._license_manager - self._virtuals_manager = copy.deepcopy(clone._virtuals_manager) + # force instantiation of lazy objects when cloning, so + # that they're not instantiated more than once + self._virtuals_manager_obj = copy.deepcopy(clone._virtuals_manager) self._accept_properties = copy.deepcopy(clone._accept_properties) self._ppropertiesdict = copy.deepcopy(clone._ppropertiesdict) + self._accept_restrict = copy.deepcopy(clone._accept_restrict) + self._paccept_restrict = copy.deepcopy(clone._paccept_restrict) self._penvdict = copy.deepcopy(clone._penvdict) self._expand_map = copy.deepcopy(clone._expand_map) else: + # lazily instantiated objects + self._keywords_manager_obj = None + self._mask_manager_obj = None + self._virtuals_manager_obj = None + locations_manager = LocationsManager(config_root=config_root, config_profile_path=config_profile_path, eprefix=eprefix, local_config=local_config, target_root=target_root) + self._locations_manager = locations_manager eprefix = locations_manager.eprefix config_root = locations_manager.config_root abs_user_config = locations_manager.abs_user_config + make_conf_paths = [ + os.path.join(config_root, 'etc', 'make.conf'), + os.path.join(config_root, MAKE_CONF_FILE) + ] + try: + if os.path.samefile(*make_conf_paths): + make_conf_paths.pop() + except OSError: + pass - make_conf = getconfig( - os.path.join(config_root, MAKE_CONF_FILE), - tolerant=tolerant, allow_sourcing=True) or {} - - make_conf.update(getconfig( - os.path.join(abs_user_config, 'make.conf'), - tolerant=tolerant, allow_sourcing=True, - expand=make_conf) or {}) + make_conf_count = 0 + make_conf = {} + for x in make_conf_paths: + mygcfg = getconfig(x, + tolerant=tolerant, allow_sourcing=True, + expand=make_conf, recursive=True) + if mygcfg is not None: + make_conf.update(mygcfg) + make_conf_count += 1 + + if make_conf_count == 2: + writemsg("!!! %s\n" % + _("Found 2 make.conf files, using both '%s' and '%s'") % + tuple(make_conf_paths), noiselevel=-1) # Allow ROOT setting to come from make.conf if it's not overridden # by the constructor argument (from the calling environment). @@ -315,15 +379,30 @@ class config(object): # lead to unexpected results. env_d = getconfig(os.path.join(eroot, "etc", "profile.env"), - expand=False) or {} + tolerant=tolerant, expand=False) or {} expand_map = env_d.copy() self._expand_map = expand_map # Allow make.globals to set default paths relative to ${EPREFIX}. expand_map["EPREFIX"] = eprefix - make_globals = getconfig(os.path.join( - self.global_config_path, 'make.globals'), expand=expand_map) + make_globals_path = os.path.join( + self.global_config_path, 'make.globals') + old_make_globals = os.path.join(config_root, + 'etc', 'make.globals') + if os.path.isfile(old_make_globals) and \ + not os.path.samefile(make_globals_path, old_make_globals): + # Don't warn if they refer to the same path, since + # that can be used for backward compatibility with + # old software. + writemsg("!!! %s\n" % + _("Found obsolete make.globals file: " + "'%s', (using '%s' instead)") % + (old_make_globals, make_globals_path), + noiselevel=-1) + + make_globals = getconfig(make_globals_path, + tolerant=tolerant, expand=expand_map) if make_globals is None: make_globals = {} @@ -412,6 +491,7 @@ class config(object): known_repos = [] portdir = "" portdir_overlay = "" + portdir_sync = None for confs in [make_globals, make_conf, self.configdict["env"]]: v = confs.get("PORTDIR") if v is not None: @@ -421,12 +501,52 @@ class config(object): if v is not None: portdir_overlay = v known_repos.extend(shlex_split(v)) + v = confs.get("SYNC") + if v is not None: + portdir_sync = v + known_repos = frozenset(known_repos) self["PORTDIR"] = portdir self["PORTDIR_OVERLAY"] = portdir_overlay + if portdir_sync: + self["SYNC"] = portdir_sync self.lookuplist = [self.configdict["env"]] - self.repositories = load_repository_config(self) + if repositories is None: + self.repositories = load_repository_config(self) + else: + self.repositories = repositories + + self['PORTAGE_REPOSITORIES'] = self.repositories.config_string() + self.backup_changes('PORTAGE_REPOSITORIES') + + #filling PORTDIR and PORTDIR_OVERLAY variable for compatibility + main_repo = self.repositories.mainRepo() + if main_repo is not None: + self["PORTDIR"] = main_repo.user_location + self.backup_changes("PORTDIR") + expand_map["PORTDIR"] = self["PORTDIR"] + + # repoman controls PORTDIR_OVERLAY via the environment, so no + # special cases are needed here. + portdir_overlay = list(self.repositories.repoUserLocationList()) + if portdir_overlay and portdir_overlay[0] == self["PORTDIR"]: + portdir_overlay = portdir_overlay[1:] + + new_ov = [] + if portdir_overlay: + for ov in portdir_overlay: + ov = normalize_path(ov) + if isdir_raise_eaccess(ov) or portage._sync_disabled_warnings: + new_ov.append(portage._shell_quote(ov)) + else: + writemsg(_("!!! Invalid PORTDIR_OVERLAY" + " (not a dir): '%s'\n") % ov, noiselevel=-1) + self["PORTDIR_OVERLAY"] = " ".join(new_ov) + self.backup_changes("PORTDIR_OVERLAY") + expand_map["PORTDIR_OVERLAY"] = self["PORTDIR_OVERLAY"] + + locations_manager.set_port_dirs(self["PORTDIR"], self["PORTDIR_OVERLAY"]) locations_manager.load_profiles(self.repositories, known_repos) profiles_complex = locations_manager.profiles_complex @@ -446,11 +566,13 @@ class config(object): x = Atom(x.lstrip('*')) self.prevmaskdict.setdefault(x.cp, []).append(x) + self.unpack_dependencies = load_unpack_dependencies_configuration(self.repositories) mygcfg = {} - if self.profiles: - mygcfg_dlists = [getconfig(os.path.join(x, "make.defaults"), - expand=expand_map) for x in self.profiles] + if profiles_complex: + mygcfg_dlists = [getconfig(os.path.join(x.location, "make.defaults"), + tolerant=tolerant, expand=expand_map, recursive=x.portage1_directories) + for x in profiles_complex] self._make_defaults = mygcfg_dlists mygcfg = stack_dicts(mygcfg_dlists, incrementals=self.incrementals) @@ -459,15 +581,11 @@ class config(object): self.configlist.append(mygcfg) self.configdict["defaults"]=self.configlist[-1] - mygcfg = getconfig( - os.path.join(config_root, MAKE_CONF_FILE), - tolerant=tolerant, allow_sourcing=True, - expand=expand_map) or {} - - mygcfg.update(getconfig( - os.path.join(abs_user_config, 'make.conf'), - tolerant=tolerant, allow_sourcing=True, - expand=expand_map) or {}) + mygcfg = {} + for x in make_conf_paths: + mygcfg.update(getconfig(x, + tolerant=tolerant, allow_sourcing=True, + expand=expand_map, recursive=True) or {}) # Don't allow the user to override certain variables in make.conf profile_only_variables = self.configdict["defaults"].get( @@ -520,66 +638,34 @@ class config(object): self.backup_changes("PORTAGE_CONFIGROOT") self["ROOT"] = target_root self.backup_changes("ROOT") - - # The PORTAGE_OVERRIDE_EPREFIX variable propagates the EPREFIX - # of this config instance to any portage commands or API - # consumers running in subprocesses. self["EPREFIX"] = eprefix self.backup_changes("EPREFIX") - self["PORTAGE_OVERRIDE_EPREFIX"] = eprefix - self.backup_changes("PORTAGE_OVERRIDE_EPREFIX") self["EROOT"] = eroot self.backup_changes("EROOT") + # The prefix of the running portage instance is used in the + # ebuild environment to implement the --host-root option for + # best_version and has_version. + self["PORTAGE_OVERRIDE_EPREFIX"] = portage.const.EPREFIX + self.backup_changes("PORTAGE_OVERRIDE_EPREFIX") + self._ppropertiesdict = portage.dep.ExtendedAtomDict(dict) + self._paccept_restrict = portage.dep.ExtendedAtomDict(dict) self._penvdict = portage.dep.ExtendedAtomDict(dict) - #filling PORTDIR and PORTDIR_OVERLAY variable for compatibility - main_repo = self.repositories.mainRepo() - if main_repo is not None: - self["PORTDIR"] = main_repo.user_location - self.backup_changes("PORTDIR") - - # repoman controls PORTDIR_OVERLAY via the environment, so no - # special cases are needed here. - portdir_overlay = list(self.repositories.repoUserLocationList()) - if portdir_overlay and portdir_overlay[0] == self["PORTDIR"]: - portdir_overlay = portdir_overlay[1:] - - new_ov = [] - if portdir_overlay: - shell_quote_re = re.compile(r"[\s\\\"'$`]") - for ov in portdir_overlay: - ov = normalize_path(ov) - if os.path.isdir(ov): - if shell_quote_re.search(ov) is not None: - ov = portage._shell_quote(ov) - new_ov.append(ov) - else: - writemsg(_("!!! Invalid PORTDIR_OVERLAY" - " (not a dir): '%s'\n") % ov, noiselevel=-1) - - self["PORTDIR_OVERLAY"] = " ".join(new_ov) - self.backup_changes("PORTDIR_OVERLAY") - - locations_manager.set_port_dirs(self["PORTDIR"], self["PORTDIR_OVERLAY"]) - self._repo_make_defaults = {} for repo in self.repositories.repos_with_profiles(): d = getconfig(os.path.join(repo.location, "profiles", "make.defaults"), - expand=self.configdict["globals"].copy()) or {} + tolerant=tolerant, expand=self.configdict["globals"].copy(), recursive=repo.portage1_profiles) or {} if d: for k in chain(self._env_blacklist, profile_only_variables, self._global_only_vars): d.pop(k, None) self._repo_make_defaults[repo.name] = d - #Read package.keywords and package.accept_keywords. - self._keywords_manager = KeywordsManager(profiles_complex, abs_user_config, \ - local_config, global_accept_keywords=self.configdict["defaults"].get("ACCEPT_KEYWORDS", "")) - #Read all USE related files from profiles and optionally from user config. - self._use_manager = UseManager(self.repositories, profiles_complex, abs_user_config, user_config=local_config) + self._use_manager = UseManager(self.repositories, profiles_complex, + abs_user_config, self._isStable, user_config=local_config) #Initialize all USE related variables we track ourselves. self.usemask = self._use_manager.getUseMask() self.useforce = self._use_manager.getUseForce() @@ -595,13 +681,6 @@ class config(object): self._license_manager.extract_global_changes( \ self.configdict["conf"].get("ACCEPT_LICENSE", "")) - #Read package.mask and package.unmask from profiles and optionally from user config - self._mask_manager = MaskManager(self.repositories, profiles_complex, - abs_user_config, user_config=local_config, - strict_umatched_removal=_unmatched_removal) - - self._virtuals_manager = VirtualsManager(self.profiles) - if local_config: #package.properties propdict = grabdict_package(os.path.join( @@ -616,6 +695,20 @@ class config(object): for k, v in propdict.items(): self._ppropertiesdict.setdefault(k.cp, {})[k] = v + # package.accept_restrict + d = grabdict_package(os.path.join( + abs_user_config, "package.accept_restrict"), + recursive=True, allow_wildcard=True, + allow_repo=True, verify_eapi=False) + v = d.pop("*/*", None) + if v is not None: + if "ACCEPT_RESTRICT" in self.configdict["conf"]: + self.configdict["conf"]["ACCEPT_RESTRICT"] += " " + " ".join(v) + else: + self.configdict["conf"]["ACCEPT_RESTRICT"] = " ".join(v) + for k, v in d.items(): + self._paccept_restrict.setdefault(k.cp, {})[k] = v + #package.env penvdict = grabdict_package(os.path.join( abs_user_config, "package.env"), recursive=1, allow_wildcard=True, \ @@ -702,21 +795,9 @@ class config(object): self.backupenv["USE_ORDER"] = "env:pkg:conf:defaults:pkginternal:repo:env.d" self.depcachedir = DEPCACHE_PATH - if eprefix: - # See comments about make.globals and EPREFIX - # above. DEPCACHE_PATH is similar. - if target_root == "/": - # case (1) above - self.depcachedir = os.path.join(eprefix, - DEPCACHE_PATH.lstrip(os.sep)) - else: - # case (2) above - # For now, just assume DEPCACHE_PATH is relative - # to EPREFIX. - # TODO: Pass in more info to the constructor, - # so we know the host system configuration. - self.depcachedir = os.path.join(eprefix, - DEPCACHE_PATH.lstrip(os.sep)) + if portage.const.EPREFIX: + self.depcachedir = os.path.join(portage.const.EPREFIX, + DEPCACHE_PATH.lstrip(os.sep)) if self.get("PORTAGE_DEPCACHEDIR", None): self.depcachedir = self["PORTAGE_DEPCACHEDIR"] @@ -783,12 +864,17 @@ class config(object): self[var] = default_val self.backup_changes(var) + if portage._internal_caller: + self["PORTAGE_INTERNAL_CALLER"] = "1" + self.backup_changes("PORTAGE_INTERNAL_CALLER") + # initialize self.features self.regenerate() if bsd_chflags: self.features.add('chflags') + self._iuse_effective = self._calc_iuse_effective() self._iuse_implicit_match = _iuse_implicit_match_cache(self) self._validate_commands() @@ -798,11 +884,6 @@ class config(object): self[k] = self[k].lower() self.backup_changes(k) - if main_repo is not None and not main_repo.sync: - main_repo_sync = self.get("SYNC") - if main_repo_sync: - main_repo.sync = main_repo_sync - # The first constructed config object initializes these modules, # and subsequent calls to the _init() functions have no effect. portage.output._init(config_root=self['PORTAGE_CONFIGROOT']) @@ -882,6 +963,32 @@ class config(object): noiselevel=-1) @property + def _keywords_manager(self): + if self._keywords_manager_obj is None: + self._keywords_manager_obj = KeywordsManager( + self._locations_manager.profiles_complex, + self._locations_manager.abs_user_config, + self.local_config, + global_accept_keywords=self.configdict["defaults"].get("ACCEPT_KEYWORDS", "")) + return self._keywords_manager_obj + + @property + def _mask_manager(self): + if self._mask_manager_obj is None: + self._mask_manager_obj = MaskManager(self.repositories, + self._locations_manager.profiles_complex, + self._locations_manager.abs_user_config, + user_config=self.local_config, + strict_umatched_removal=self._unmatched_removal) + return self._mask_manager_obj + + @property + def _virtuals_manager(self): + if self._virtuals_manager_obj is None: + self._virtuals_manager_obj = VirtualsManager(self.profiles) + return self._virtuals_manager_obj + + @property def pkeywordsdict(self): result = self._keywords_manager.pkeywordsdict.copy() for k, v in result.items(): @@ -919,13 +1026,23 @@ class config(object): writemsg(_("!!! INVALID ACCEPT_KEYWORDS: %s\n") % str(group), noiselevel=-1) - profile_broken = not self.profile_path or \ - not os.path.exists(os.path.join(self.profile_path, "parent")) and \ - os.path.exists(os.path.join(self["PORTDIR"], "profiles")) + profile_broken = False + + if not self.profile_path: + profile_broken = True + else: + # If any one of these files exists, then + # the profile is considered valid. + for x in ("make.defaults", "parent", + "packages", "use.force", "use.mask"): + if exists_raise_eaccess(os.path.join(self.profile_path, x)): + break + else: + profile_broken = True - if profile_broken: + if profile_broken and not portage._sync_disabled_warnings: abs_profile_path = None - for x in (PROFILE_PATH, 'etc/portage/make.profile'): + for x in (PROFILE_PATH, 'etc/make.profile'): x = os.path.join(self["PORTAGE_CONFIGROOT"], x) try: os.lstat(x) @@ -1091,8 +1208,11 @@ class config(object): the previously calculated USE settings. """ - def __init__(self, use, usemask, iuse_implicit, + def __init__(self, settings, unfiltered_use, + use, usemask, iuse_implicit, use_expand_split, use_expand_dict): + self._settings = settings + self._unfiltered_use = unfiltered_use self._use = use self._usemask = usemask self._iuse_implicit = iuse_implicit @@ -1147,13 +1267,32 @@ class config(object): # Don't export empty USE_EXPAND vars unless the user config # exports them as empty. This is required for vars such as # LINGUAS, where unset and empty have different meanings. + # The special '*' token is understood by ebuild.sh, which + # will unset the variable so that things like LINGUAS work + # properly (see bug #459350). if has_wildcard: - # ebuild.sh will see this and unset the variable so - # that things like LINGUAS work properly value = '*' else: if has_iuse: - value = '' + already_set = False + # Skip the first 'env' configdict, in order to + # avoid infinite recursion here, since that dict's + # __getitem__ calls the current __getitem__. + for d in self._settings.lookuplist[1:]: + if key in d: + already_set = True + break + + if not already_set: + for x in self._unfiltered_use: + if x[:prefix_len] == prefix: + already_set = True + break + + if already_set: + value = '' + else: + value = '*' else: # It's not in IUSE, so just allow the variable content # to pass through if it is defined somewhere. This @@ -1189,7 +1328,7 @@ class config(object): if not isinstance(mycpv, basestring): pkg = mycpv mycpv = pkg.cpv - mydb = pkg.metadata + mydb = pkg._metadata explicit_iuse = pkg.iuse.all args_hash = (mycpv, id(pkg)) if pkg.built: @@ -1210,6 +1349,7 @@ class config(object): iuse = "" pkg_configdict = self.configdict["pkg"] previous_iuse = pkg_configdict.get("IUSE") + previous_iuse_effective = pkg_configdict.get("IUSE_EFFECTIVE") previous_features = pkg_configdict.get("FEATURES") aux_keys = self._setcpv_aux_keys @@ -1221,6 +1361,7 @@ class config(object): pkg_configdict["CATEGORY"] = cat pkg_configdict["PF"] = pf repository = None + eapi = None if mydb: if not hasattr(mydb, "aux_get"): for k in aux_keys: @@ -1247,14 +1388,16 @@ class config(object): # Empty USE means this dbapi instance does not contain # built packages. built_use = None + eapi = pkg_configdict['EAPI'] repository = pkg_configdict.pop("repository", None) if repository is not None: pkg_configdict["PORTAGE_REPO_NAME"] = repository - slot = pkg_configdict["SLOT"] iuse = pkg_configdict["IUSE"] if pkg is None: - cpv_slot = _pkg_str(self.mycpv, slot=slot, repo=repository) + self.mycpv = _pkg_str(self.mycpv, metadata=pkg_configdict, + settings=self) + cpv_slot = self.mycpv else: cpv_slot = pkg pkginternaluse = [] @@ -1264,6 +1407,9 @@ class config(object): elif x.startswith("-"): pkginternaluse.append(x) pkginternaluse = " ".join(pkginternaluse) + + eapi_attrs = _get_eapi_attrs(eapi) + if pkginternaluse != self.configdict["pkginternal"].get("USE", ""): self.configdict["pkginternal"]["USE"] = pkginternaluse has_changed = True @@ -1394,30 +1540,70 @@ class config(object): # If reset() has not been called, it's safe to return # early if IUSE has not changed. - if not has_changed and previous_iuse == iuse: + if not has_changed and previous_iuse == iuse and \ + (previous_iuse_effective is not None == eapi_attrs.iuse_effective): return # Filter out USE flags that aren't part of IUSE. This has to # be done for every setcpv() call since practically every # package has different IUSE. use = set(self["USE"].split()) + unfiltered_use = frozenset(use) if explicit_iuse is None: explicit_iuse = frozenset(x.lstrip("+-") for x in iuse.split()) - iuse_implicit_match = self._iuse_implicit_match - portage_iuse = self._get_implicit_iuse() - portage_iuse.update(explicit_iuse) + + if eapi_attrs.iuse_effective: + iuse_implicit_match = self._iuse_effective_match + portage_iuse = set(self._iuse_effective) + portage_iuse.update(explicit_iuse) + self.configdict["pkg"]["IUSE_EFFECTIVE"] = \ + " ".join(sorted(portage_iuse)) + else: + iuse_implicit_match = self._iuse_implicit_match + portage_iuse = self._get_implicit_iuse() + portage_iuse.update(explicit_iuse) # PORTAGE_IUSE is not always needed so it's lazily evaluated. self.configdict["env"].addLazySingleton( "PORTAGE_IUSE", _lazy_iuse_regex, portage_iuse) - ebuild_force_test = self.get("EBUILD_FORCE_TEST") == "1" + if pkg is None: + raw_restrict = pkg_configdict.get("RESTRICT") + else: + raw_restrict = pkg._raw_metadata["RESTRICT"] + + restrict_test = False + if raw_restrict: + try: + if built_use is not None: + restrict = use_reduce(raw_restrict, + uselist=built_use, flat=True) + else: + # Use matchnone=True to ignore USE conditional parts + # of RESTRICT, since we want to know whether to mask + # the "test" flag _before_ we know the USE values + # that would be needed to evaluate the USE + # conditionals (see bug #273272). + restrict = use_reduce(raw_restrict, + matchnone=True, flat=True) + except PortageException: + pass + else: + restrict_test = "test" in restrict + + ebuild_force_test = not restrict_test and \ + self.get("EBUILD_FORCE_TEST") == "1" + if ebuild_force_test and \ not hasattr(self, "_ebuild_force_test_msg_shown"): self._ebuild_force_test_msg_shown = True writemsg(_("Forcing test.\n"), noiselevel=-1) - if "test" in self.features: - if "test" in self.usemask and not ebuild_force_test: + + if "test" in explicit_iuse or iuse_implicit_match("test"): + if "test" not in self.features: + use.discard("test") + elif restrict_test or \ + ("test" in self.usemask and not ebuild_force_test): # "test" is in IUSE and USE=test is masked, so execution # of src_test() probably is not reliable. Therefore, # temporarily disable FEATURES=test just for this package. @@ -1430,6 +1616,13 @@ class config(object): self.usemask = \ frozenset(x for x in self.usemask if x != "test") + if eapi_attrs.feature_flag_targetroot and \ + ("targetroot" in explicit_iuse or iuse_implicit_match("targetroot")): + if self["ROOT"] != "/": + use.add("targetroot") + else: + use.discard("targetroot") + # Allow _* flags from USE_EXPAND wildcards to pass through here. use.difference_update([x for x in use \ if (x not in explicit_iuse and \ @@ -1440,7 +1633,8 @@ class config(object): # comparison instead of startswith(). use_expand_split = set(x.lower() for \ x in self.get('USE_EXPAND', '').split()) - lazy_use_expand = self._lazy_use_expand(use, self.usemask, + lazy_use_expand = self._lazy_use_expand( + self, unfiltered_use, use, self.usemask, portage_iuse, use_expand_split, self._use_expand_dict) use_expand_iuses = {} @@ -1470,6 +1664,14 @@ class config(object): self.configdict['env'].addLazySingleton(k, lazy_use_expand.__getitem__, k) + for k in self.get("USE_EXPAND_UNPREFIXED", "").split(): + var_split = self.get(k, '').split() + var_split = [ x for x in var_split if x in use ] + if var_split: + self.configlist[-1][k] = ' '.join(var_split) + elif k in self: + self.configlist[-1][k] = '' + # Filtered for the ebuild environment. Store this in a separate # attribute since we still want to be able to see global USE # settings for things like emerge --info. @@ -1477,6 +1679,10 @@ class config(object): self.configdict["env"]["PORTAGE_USE"] = \ " ".join(sorted(x for x in use if x[-2:] != '_*')) + # Clear the eapi cache here rather than in the constructor, since + # setcpv triggers lazy instantiation of things like _use_manager. + _eapi_cache.clear() + def _grab_pkg_env(self, penv, container, protected_keys=None): if protected_keys is None: protected_keys = () @@ -1510,9 +1716,42 @@ class config(object): else: container[k] = v + def _iuse_effective_match(self, flag): + return flag in self._iuse_effective + + def _calc_iuse_effective(self): + """ + Beginning with EAPI 5, IUSE_EFFECTIVE is defined by PMS. + """ + iuse_effective = [] + iuse_effective.extend(self.get("IUSE_IMPLICIT", "").split()) + + # USE_EXPAND_IMPLICIT should contain things like ARCH, ELIBC, + # KERNEL, and USERLAND. + use_expand_implicit = frozenset( + self.get("USE_EXPAND_IMPLICIT", "").split()) + + # USE_EXPAND_UNPREFIXED should contain at least ARCH, and + # USE_EXPAND_VALUES_ARCH should contain all valid ARCH flags. + for v in self.get("USE_EXPAND_UNPREFIXED", "").split(): + if v not in use_expand_implicit: + continue + iuse_effective.extend( + self.get("USE_EXPAND_VALUES_" + v, "").split()) + + use_expand = frozenset(self.get("USE_EXPAND", "").split()) + for v in use_expand_implicit: + if v not in use_expand: + continue + lower_v = v.lower() + for x in self.get("USE_EXPAND_VALUES_" + v, "").split(): + iuse_effective.append(lower_v + "_" + x) + + return frozenset(iuse_effective) + def _get_implicit_iuse(self): """ - Some flags are considered to + Prior to EAPI 5, these flags are considered to be implicit members of IUSE: * Flags derived from ARCH * Flags derived from USE_EXPAND_HIDDEN variables @@ -1549,11 +1788,11 @@ class config(object): return iuse_implicit - def _getUseMask(self, pkg): - return self._use_manager.getUseMask(pkg) + def _getUseMask(self, pkg, stable=None): + return self._use_manager.getUseMask(pkg, stable=stable) - def _getUseForce(self, pkg): - return self._use_manager.getUseForce(pkg) + def _getUseForce(self, pkg, stable=None): + return self._use_manager.getUseForce(pkg, stable=stable) def _getMaskAtom(self, cpv, metadata): """ @@ -1618,6 +1857,11 @@ class config(object): return x return None + def _isStable(self, pkg): + return self._keywords_manager.isStable(pkg, + self.get("ACCEPT_KEYWORDS", ""), + self.configdict["backupenv"].get("ACCEPT_KEYWORDS", "")) + def _getKeywords(self, cpv, metadata): return self._keywords_manager.getKeywords(cpv, metadata["SLOT"], \ metadata.get("KEYWORDS", ""), metadata.get("repository")) @@ -1706,9 +1950,10 @@ class config(object): @return: A list of properties that have not been accepted. """ accept_properties = self._accept_properties - if not hasattr(cpv, 'slot'): - cpv = _pkg_str(cpv, slot=metadata["SLOT"], - repo=metadata.get("repository")) + try: + cpv.slot + except AttributeError: + cpv = _pkg_str(cpv, metadata=metadata, settings=self) cp = cpv_getkey(cpv) cpdict = self._ppropertiesdict.get(cp) if cpdict: @@ -1720,7 +1965,6 @@ class config(object): properties_str = metadata.get("PROPERTIES", "") properties = set(use_reduce(properties_str, matchall=1, flat=True)) - properties.discard('||') acceptable_properties = set() for x in accept_properties: @@ -1738,40 +1982,58 @@ class config(object): else: use = [] - properties_struct = use_reduce(properties_str, uselist=use, opconvert=True) - return self._getMaskedProperties(properties_struct, acceptable_properties) - - def _getMaskedProperties(self, properties_struct, acceptable_properties): - if not properties_struct: - return [] - if properties_struct[0] == "||": - ret = [] - for element in properties_struct[1:]: - if isinstance(element, list): - if element: - tmp = self._getMaskedProperties( - element, acceptable_properties) - if not tmp: - return [] - ret.extend(tmp) - else: - if element in acceptable_properties: - return[] - ret.append(element) - # Return all masked properties, since we don't know which combination - # (if any) the user will decide to unmask - return ret - - ret = [] - for element in properties_struct: - if isinstance(element, list): - if element: - ret.extend(self._getMaskedProperties(element, - acceptable_properties)) + return [x for x in use_reduce(properties_str, uselist=use, flat=True) + if x not in acceptable_properties] + + def _getMissingRestrict(self, cpv, metadata): + """ + Take a RESTRICT string and return a list of any tokens the user + may need to accept for the given package. The returned list will not + contain any tokens that have already been accepted. This method + can throw an InvalidDependString exception. + + @param cpv: The package name (for package.accept_restrict support) + @type cpv: String + @param metadata: A dictionary of raw package metadata + @type metadata: dict + @rtype: List + @return: A list of tokens that have not been accepted. + """ + accept_restrict = self._accept_restrict + try: + cpv.slot + except AttributeError: + cpv = _pkg_str(cpv, metadata=metadata, settings=self) + cp = cpv_getkey(cpv) + cpdict = self._paccept_restrict.get(cp) + if cpdict: + paccept_restrict_list = ordered_by_atom_specificity(cpdict, cpv) + if paccept_restrict_list: + accept_restrict = list(self._accept_restrict) + for x in paccept_restrict_list: + accept_restrict.extend(x) + + restrict_str = metadata.get("RESTRICT", "") + all_restricts = set(use_reduce(restrict_str, matchall=1, flat=True)) + + acceptable_restricts = set() + for x in accept_restrict: + if x == '*': + acceptable_restricts.update(all_restricts) + elif x == '-*': + acceptable_restricts.clear() + elif x[:1] == '-': + acceptable_restricts.discard(x[1:]) else: - if element not in acceptable_properties: - ret.append(element) - return ret + acceptable_restricts.add(x) + + if "?" in restrict_str: + use = metadata["USE"].split() + else: + use = [] + + return [x for x in use_reduce(restrict_str, uselist=use, flat=True) + if x not in acceptable_restricts] def _accept_chost(self, cpv, metadata): """ @@ -1840,7 +2102,8 @@ class config(object): """Reload things like /etc/profile.env that can change during runtime.""" env_d_filename = os.path.join(self["EROOT"], "etc", "profile.env") self.configdict["env.d"].clear() - env_d = getconfig(env_d_filename, expand=False) + env_d = getconfig(env_d_filename, + tolerant=self._tolerant, expand=False) if env_d: # env_d will be None if profile.env doesn't exist. for k in self._env_d_blacklist: @@ -1909,6 +2172,18 @@ class config(object): # repoman will accept any property self._accept_properties = ('*',) + if self.local_config: + mysplit = [] + for curdb in mydbs: + mysplit.extend(curdb.get('ACCEPT_RESTRICT', '').split()) + mysplit = prune_incremental(mysplit) + self.configlist[-1]['ACCEPT_RESTRICT'] = ' '.join(mysplit) + if tuple(mysplit) != self._accept_restrict: + self._accept_restrict = tuple(mysplit) + else: + # repoman will accept any property + self._accept_restrict = ('*',) + increment_lists = {} for k in myincrementals: incremental_list = [] @@ -1963,6 +2238,8 @@ class config(object): if v is not None: use_expand_dict[k] = v + use_expand_unprefixed = self.get("USE_EXPAND_UNPREFIXED", "").split() + # In order to best accomodate the long-standing practice of # setting default USE_EXPAND variables in the profile's # make.defaults, we translate these variables into their @@ -1976,6 +2253,12 @@ class config(object): continue use = cfg.get("USE", "") expand_use = [] + + for k in use_expand_unprefixed: + v = cfg.get(k) + if v is not None: + expand_use.extend(v.split()) + for k in use_expand_dict: v = cfg.get(k) if v is None: @@ -2013,6 +2296,17 @@ class config(object): iuse = [x.lstrip("+-") for x in iuse.split()] myflags = set() for curdb in self.uvlist: + + for k in use_expand_unprefixed: + v = curdb.get(k) + if v is None: + continue + for x in v.split(): + if x[:1] == "-": + myflags.discard(x[1:]) + else: + myflags.add(x) + cur_use_expand = [x for x in use_expand if x in curdb] mysplit = curdb.get("USE", "").split() if not mysplit and not cur_use_expand: @@ -2129,6 +2423,14 @@ class config(object): elif k in self: self.configlist[-1][k] = '' + for k in use_expand_unprefixed: + var_split = self.get(k, '').split() + var_split = [ x for x in var_split if x in myflags ] + if var_split: + self.configlist[-1][k] = ' '.join(var_split) + elif k in self: + self.configlist[-1][k] = '' + @property def virts_p(self): warnings.warn("portage config.virts_p attribute " + \ @@ -2189,8 +2491,22 @@ class config(object): elif mykey == "PORTAGE_PYM_PATH": return portage._pym_path + elif mykey == "PORTAGE_PYTHONPATH": + value = [x for x in \ + self.backupenv.get("PYTHONPATH", "").split(":") if x] + need_pym_path = True + if value: + try: + need_pym_path = not os.path.samefile(value[0], + portage._pym_path) + except OSError: + pass + if need_pym_path: + value.insert(0, portage._pym_path) + return ":".join(value) + elif mykey == "PORTAGE_GID": - return _unicode_decode(str(portage_gid)) + return "%s" % portage_gid for d in self.lookuplist: try: @@ -2277,6 +2593,7 @@ class config(object): environ_filter = self._environ_filter eapi = self.get('EAPI') + eapi_attrs = _get_eapi_attrs(eapi) phase = self.get('EBUILD_PHASE') filter_calling_env = False if self.mycpv is not None and \ @@ -2358,14 +2675,20 @@ class config(object): not eapi_exports_replace_vars(eapi): mydict.pop("REPLACED_BY_VERSION", None) + if phase is not None and eapi_attrs.exports_EBUILD_PHASE_FUNC: + phase_func = _phase_func_map.get(phase) + if phase_func is not None: + mydict["EBUILD_PHASE_FUNC"] = phase_func + return mydict def thirdpartymirrors(self): if getattr(self, "_thirdpartymirrors", None) is None: - profileroots = [os.path.join(self["PORTDIR"], "profiles")] - for x in shlex_split(self.get("PORTDIR_OVERLAY", "")): - profileroots.insert(0, os.path.join(x, "profiles")) - thirdparty_lists = [grabdict(os.path.join(x, "thirdpartymirrors")) for x in profileroots] + thirdparty_lists = [] + for repo_name in reversed(self.repositories.prepos_order): + thirdparty_lists.append(grabdict(os.path.join( + self.repositories[repo_name].location, + "profiles", "thirdpartymirrors"))) self._thirdpartymirrors = stack_dictlist(thirdparty_lists, incremental=True) return self._thirdpartymirrors diff --git a/portage_with_autodep/pym/portage/package/ebuild/config.pyo b/portage_with_autodep/pym/portage/package/ebuild/config.pyo Binary files differindex 742ee2b..f4123ed 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/config.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/config.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/deprecated_profile_check.py b/portage_with_autodep/pym/portage/package/ebuild/deprecated_profile_check.py index 3fab4da..fdb19b4 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/deprecated_profile_check.py +++ b/portage_with_autodep/pym/portage/package/ebuild/deprecated_profile_check.py @@ -1,10 +1,11 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ['deprecated_profile_check'] import io +import portage from portage import os, _encodings, _unicode_encode from portage.const import DEPRECATED_PROFILE_FILE from portage.localization import _ @@ -12,16 +13,32 @@ from portage.output import colorize from portage.util import writemsg def deprecated_profile_check(settings=None): - config_root = "/" + config_root = None + eprefix = None + deprecated_profile_file = None if settings is not None: config_root = settings["PORTAGE_CONFIGROOT"] - deprecated_profile_file = os.path.join(config_root, - DEPRECATED_PROFILE_FILE) - if not os.access(deprecated_profile_file, os.R_OK): - return False - dcontent = io.open(_unicode_encode(deprecated_profile_file, + eprefix = settings["EPREFIX"] + for x in reversed(settings.profiles): + deprecated_profile_file = os.path.join(x, "deprecated") + if os.access(deprecated_profile_file, os.R_OK): + break + else: + deprecated_profile_file = None + + if deprecated_profile_file is None: + deprecated_profile_file = os.path.join(config_root or "/", + DEPRECATED_PROFILE_FILE) + if not os.access(deprecated_profile_file, os.R_OK): + deprecated_profile_file = os.path.join(config_root or "/", + 'etc', 'make.profile', 'deprecated') + if not os.access(deprecated_profile_file, os.R_OK): + return + + with io.open(_unicode_encode(deprecated_profile_file, encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['content'], errors='replace').readlines() + mode='r', encoding=_encodings['content'], errors='replace') as f: + dcontent = f.readlines() writemsg(colorize("BAD", _("\n!!! Your current profile is " "deprecated and not supported anymore.")) + "\n", noiselevel=-1) writemsg(colorize("BAD", _("!!! Use eselect profile to update your " @@ -30,13 +47,37 @@ def deprecated_profile_check(settings=None): writemsg(colorize("BAD", _("!!! Please refer to the " "Gentoo Upgrading Guide.")) + "\n", noiselevel=-1) return True - newprofile = dcontent[0] + newprofile = dcontent[0].rstrip("\n") writemsg(colorize("BAD", _("!!! Please upgrade to the " - "following profile if possible:")) + "\n", noiselevel=-1) - writemsg(8*" " + colorize("GOOD", newprofile) + "\n", noiselevel=-1) + "following profile if possible:")) + "\n\n", noiselevel=-1) + writemsg(8*" " + colorize("GOOD", newprofile) + "\n\n", noiselevel=-1) if len(dcontent) > 1: writemsg(_("To upgrade do the following steps:\n"), noiselevel=-1) for myline in dcontent[1:]: writemsg(myline, noiselevel=-1) writemsg("\n\n", noiselevel=-1) + else: + writemsg(_("You may use the following command to upgrade:\n\n"), noiselevel=-1) + writemsg(8*" " + colorize("INFORM", 'eselect profile set ' + + newprofile) + "\n\n", noiselevel=-1) + + if settings is not None: + main_repo_loc = settings.repositories.mainRepoLocation() + new_profile_path = os.path.join(main_repo_loc, + "profiles", newprofile.rstrip("\n")) + + if os.path.isdir(new_profile_path): + new_config = portage.config(config_root=config_root, + config_profile_path=new_profile_path, + eprefix=eprefix) + + if not new_config.profiles: + writemsg("\n %s %s\n" % (colorize("WARN", "*"), + _("You must update portage before you " + "can migrate to the above profile.")), noiselevel=-1) + writemsg(" %s %s\n\n" % (colorize("WARN", "*"), + _("In order to update portage, " + "run 'emerge --oneshot portage'.")), + noiselevel=-1) + return True diff --git a/portage_with_autodep/pym/portage/package/ebuild/deprecated_profile_check.pyo b/portage_with_autodep/pym/portage/package/ebuild/deprecated_profile_check.pyo Binary files differindex 2b9362b..df43aab 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/deprecated_profile_check.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/deprecated_profile_check.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/digestcheck.py b/portage_with_autodep/pym/portage/package/ebuild/digestcheck.py index 8705639..e207ba8 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/digestcheck.py +++ b/portage_with_autodep/pym/portage/package/ebuild/digestcheck.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ['digestcheck'] @@ -6,6 +6,7 @@ __all__ = ['digestcheck'] import warnings from portage import os, _encodings, _unicode_decode +from portage.checksum import _hash_filter from portage.exception import DigestException, FileNotFound from portage.localization import _ from portage.output import EOutput @@ -28,6 +29,9 @@ def digestcheck(myfiles, mysettings, strict=False, justmanifest=None, mf=None): if mysettings.get("EBUILD_SKIP_MANIFEST") == "1": return 1 pkgdir = mysettings["O"] + hash_filter = _hash_filter(mysettings.get("PORTAGE_CHECKSUM_FILTER", "")) + if hash_filter.transparent: + hash_filter = None if mf is None: mf = mysettings.repositories.get_repo_for_location( os.path.dirname(os.path.dirname(pkgdir))) @@ -38,15 +42,16 @@ def digestcheck(myfiles, mysettings, strict=False, justmanifest=None, mf=None): if not mf.thin and strict and "PORTAGE_PARALLEL_FETCHONLY" not in mysettings: if mf.fhashdict.get("EBUILD"): eout.ebegin(_("checking ebuild checksums ;-)")) - mf.checkTypeHashes("EBUILD") + mf.checkTypeHashes("EBUILD", hash_filter=hash_filter) eout.eend(0) if mf.fhashdict.get("AUX"): eout.ebegin(_("checking auxfile checksums ;-)")) - mf.checkTypeHashes("AUX") + mf.checkTypeHashes("AUX", hash_filter=hash_filter) eout.eend(0) if mf.fhashdict.get("MISC"): eout.ebegin(_("checking miscfile checksums ;-)")) - mf.checkTypeHashes("MISC", ignoreMissingFiles=True) + mf.checkTypeHashes("MISC", ignoreMissingFiles=True, + hash_filter=hash_filter) eout.eend(0) for f in myfiles: eout.ebegin(_("checking %s ;-)") % f) @@ -58,7 +63,7 @@ def digestcheck(myfiles, mysettings, strict=False, justmanifest=None, mf=None): writemsg(_("\n!!! Missing digest for '%s'\n") % (f,), noiselevel=-1) return 0 - mf.checkFileHashes(ftype, f) + mf.checkFileHashes(ftype, f, hash_filter=hash_filter) eout.eend(0) except FileNotFound as e: eout.eend(1) diff --git a/portage_with_autodep/pym/portage/package/ebuild/digestcheck.pyo b/portage_with_autodep/pym/portage/package/ebuild/digestcheck.pyo Binary files differindex 66987a2..c6a8d4e 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/digestcheck.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/digestcheck.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/digestgen.py b/portage_with_autodep/pym/portage/package/ebuild/digestgen.py index 6ad3397..95d02db 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/digestgen.py +++ b/portage_with_autodep/pym/portage/package/ebuild/digestgen.py @@ -112,67 +112,64 @@ def digestgen(myarchives=None, mysettings=None, myportdb=None): missing_files.append(myfile) continue - if missing_files: - for myfile in missing_files: - uris = set() - all_restrict = set() - for cpv in distfiles_map[myfile]: - uris.update(myportdb.getFetchMap( - cpv, mytree=mytree)[myfile]) - restrict = myportdb.aux_get(cpv, ['RESTRICT'], - mytree=mytree)[0] - # Here we ignore conditional parts of RESTRICT since - # they don't apply unconditionally. Assume such - # conditionals only apply on the client side where - # digestgen() does not need to be called. - all_restrict.update(use_reduce(restrict, - flat=True, matchnone=True)) - - # fetch() uses CATEGORY and PF to display a message - # when fetch restriction is triggered. - cat, pf = catsplit(cpv) - mysettings["CATEGORY"] = cat - mysettings["PF"] = pf - - # fetch() uses PORTAGE_RESTRICT to control fetch - # restriction, which is only applied to files that - # are not fetchable via a mirror:// URI. - mysettings["PORTAGE_RESTRICT"] = " ".join(all_restrict) - - try: - st = os.stat(os.path.join( - mysettings["DISTDIR"],myfile)) - except OSError: - st = None - - if not fetch({myfile : uris}, mysettings): - myebuild = os.path.join(mysettings["O"], - catsplit(cpv)[1] + ".ebuild") - spawn_nofetch(myportdb, myebuild) - writemsg(_("!!! Fetch failed for %s, can't update " - "Manifest\n") % myfile, noiselevel=-1) - if myfile in dist_hashes and \ - st is not None and st.st_size > 0: - # stat result is obtained before calling fetch(), - # since fetch may rename the existing file if the - # digest does not match. - writemsg(_("!!! If you would like to " - "forcefully replace the existing " - "Manifest entry\n!!! for %s, use " - "the following command:\n") % myfile + \ - "!!! " + colorize("INFORM", - "ebuild --force %s manifest" % \ - os.path.basename(myebuild)) + "\n", - noiselevel=-1) - return 0 + for myfile in missing_files: + uris = set() + all_restrict = set() + for cpv in distfiles_map[myfile]: + uris.update(myportdb.getFetchMap( + cpv, mytree=mytree)[myfile]) + restrict = myportdb.aux_get(cpv, ['RESTRICT'], mytree=mytree)[0] + # Here we ignore conditional parts of RESTRICT since + # they don't apply unconditionally. Assume such + # conditionals only apply on the client side where + # digestgen() does not need to be called. + all_restrict.update(use_reduce(restrict, + flat=True, matchnone=True)) + + # fetch() uses CATEGORY and PF to display a message + # when fetch restriction is triggered. + cat, pf = catsplit(cpv) + mysettings["CATEGORY"] = cat + mysettings["PF"] = pf + + # fetch() uses PORTAGE_RESTRICT to control fetch + # restriction, which is only applied to files that + # are not fetchable via a mirror:// URI. + mysettings["PORTAGE_RESTRICT"] = " ".join(all_restrict) + + try: + st = os.stat(os.path.join(mysettings["DISTDIR"], myfile)) + except OSError: + st = None + + if not fetch({myfile : uris}, mysettings): + myebuild = os.path.join(mysettings["O"], + catsplit(cpv)[1] + ".ebuild") + spawn_nofetch(myportdb, myebuild) + writemsg(_("!!! Fetch failed for %s, can't update Manifest\n") + % myfile, noiselevel=-1) + if myfile in dist_hashes and \ + st is not None and st.st_size > 0: + # stat result is obtained before calling fetch(), + # since fetch may rename the existing file if the + # digest does not match. + cmd = colorize("INFORM", "ebuild --force %s manifest" % + os.path.basename(myebuild)) + writemsg((_( + "!!! If you would like to forcefully replace the existing Manifest entry\n" + "!!! for %s, use the following command:\n") % myfile) + + "!!! %s\n" % cmd, + noiselevel=-1) + return 0 + writemsg_stdout(_(">>> Creating Manifest for %s\n") % mysettings["O"]) try: mf.create(assumeDistHashesSometimes=True, assumeDistHashesAlways=( "assume-digests" in mysettings.features)) except FileNotFound as e: - writemsg(_("!!! File %s doesn't exist, can't update " - "Manifest\n") % e, noiselevel=-1) + writemsg(_("!!! File %s doesn't exist, can't update Manifest\n") + % e, noiselevel=-1) return 0 except PortagePackageException as e: writemsg(("!!! %s\n") % (e,), noiselevel=-1) diff --git a/portage_with_autodep/pym/portage/package/ebuild/digestgen.pyo b/portage_with_autodep/pym/portage/package/ebuild/digestgen.pyo Binary files differindex 66876ec..a4e9b62 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/digestgen.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/digestgen.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/doebuild.py b/portage_with_autodep/pym/portage/package/ebuild/doebuild.py index 610172f..5351c52 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/doebuild.py +++ b/portage_with_autodep/pym/portage/package/ebuild/doebuild.py @@ -1,14 +1,19 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ['doebuild', 'doebuild_environment', 'spawn', 'spawnebuild'] +import grp import gzip import errno import io from itertools import chain import logging import os as _os +import platform +import pwd import re import signal import stat @@ -25,7 +30,13 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.package.ebuild.digestcheck:digestcheck', 'portage.package.ebuild.digestgen:digestgen', 'portage.package.ebuild.fetch:fetch', + 'portage.package.ebuild._ipc.QueryCommand:QueryCommand', + 'portage.dep._slot_operator:evaluate_slot_operator_equal_deps', 'portage.package.ebuild._spawn_nofetch:spawn_nofetch', + 'portage.util._desktop_entry:validate_desktop_entry', + 'portage.util._async.SchedulerInterface:SchedulerInterface', + 'portage.util._eventloop.EventLoop:EventLoop', + 'portage.util._eventloop.global_event_loop:global_event_loop', 'portage.util.ExtractKernelVersion:ExtractKernelVersion' ) @@ -43,7 +54,7 @@ from portage.dep import Atom, check_required_use, \ from portage.eapi import eapi_exports_KV, eapi_exports_merge_type, \ eapi_exports_replace_vars, eapi_exports_REPOSITORY, \ eapi_has_required_use, eapi_has_src_prepare_and_src_configure, \ - eapi_has_pkg_pretend + eapi_has_pkg_pretend, _get_eapi_attrs from portage.elog import elog_process, _preload_elog_modules from portage.elog.messages import eerror, eqawarn from portage.exception import DigestException, FileNotFound, \ @@ -55,14 +66,13 @@ from portage.package.ebuild.prepare_build_dirs import prepare_build_dirs from portage.util import apply_recursive_permissions, \ apply_secpass_permissions, noiselimit, normalize_path, \ writemsg, writemsg_stdout, write_atomic -from portage.util.lafilefixer import rewrite_lafile +from portage.util.lafilefixer import rewrite_lafile from portage.versions import _pkgsplit from _emerge.BinpkgEnvExtractor import BinpkgEnvExtractor from _emerge.EbuildBuildDir import EbuildBuildDir from _emerge.EbuildPhase import EbuildPhase from _emerge.EbuildSpawnProcess import EbuildSpawnProcess from _emerge.Package import Package -from _emerge.PollScheduler import PollScheduler from _emerge.RootConfig import RootConfig _unsandboxed_phases = frozenset([ @@ -72,6 +82,40 @@ _unsandboxed_phases = frozenset([ "prerm", "setup" ]) +# phases in which IPC with host is allowed +_ipc_phases = frozenset([ + "setup", "pretend", + "preinst", "postinst", "prerm", "postrm", +]) + +# phases in which networking access is allowed +_networked_phases = frozenset([ + # for VCS fetching + "unpack", + # + for network-bound IPC +] + list(_ipc_phases)) + +_phase_func_map = { + "config": "pkg_config", + "setup": "pkg_setup", + "nofetch": "pkg_nofetch", + "unpack": "src_unpack", + "prepare": "src_prepare", + "configure": "src_configure", + "compile": "src_compile", + "test": "src_test", + "install": "src_install", + "preinst": "pkg_preinst", + "postinst": "pkg_postinst", + "prerm": "pkg_prerm", + "postrm": "pkg_postrm", + "info": "pkg_info", + "pretend": "pkg_pretend", +} + +_vdb_use_conditional_keys = Package._dep_keys + \ + ('LICENSE', 'PROPERTIES', 'PROVIDE', 'RESTRICT',) + def _doebuild_spawn(phase, settings, actionmap=None, **kwargs): """ All proper ebuild phases which execute ebuild.sh are spawned @@ -81,8 +125,18 @@ def _doebuild_spawn(phase, settings, actionmap=None, **kwargs): if phase in _unsandboxed_phases: kwargs['free'] = True + kwargs['ipc'] = 'ipc-sandbox' not in settings.features or \ + phase in _ipc_phases + kwargs['networked'] = 'network-sandbox' not in settings.features or \ + phase in _networked_phases + if phase == 'depend': kwargs['droppriv'] = 'userpriv' in settings.features + # It's not necessary to close_fds for this phase, since + # it should not spawn any daemons, and close_fds is + # best avoided since it can interact badly with some + # garbage collectors (see _setup_pipes docstring). + kwargs['close_fds'] = False if actionmap is not None and phase in actionmap: kwargs.update(actionmap[phase]["args"]) @@ -100,17 +154,24 @@ def _doebuild_spawn(phase, settings, actionmap=None, **kwargs): settings['EBUILD_PHASE'] = phase try: - return spawn(cmd, settings, **kwargs) + return spawn(cmd, settings, **portage._native_kwargs(kwargs)) finally: settings.pop('EBUILD_PHASE', None) -def _spawn_phase(phase, settings, actionmap=None, **kwargs): - if kwargs.get('returnpid'): - return _doebuild_spawn(phase, settings, actionmap=actionmap, **kwargs) +def _spawn_phase(phase, settings, actionmap=None, returnpid=False, + logfile=None, **kwargs): + + if returnpid: + return _doebuild_spawn(phase, settings, actionmap=actionmap, + returnpid=returnpid, logfile=logfile, **kwargs) + # The logfile argument is unused here, since EbuildPhase uses + # the PORTAGE_LOG_FILE variable if set. ebuild_phase = EbuildPhase(actionmap=actionmap, background=False, - phase=phase, scheduler=PollScheduler().sched_iface, - settings=settings) + phase=phase, scheduler=SchedulerInterface(portage._internal_caller and + global_event_loop() or EventLoop(main=False)), + settings=settings, **kwargs) + ebuild_phase.start() ebuild_phase.wait() return ebuild_phase.returncode @@ -123,19 +184,28 @@ def _doebuild_path(settings, eapi=None): # Note: PORTAGE_BIN_PATH may differ from the global constant # when portage is reinstalling itself. portage_bin_path = settings["PORTAGE_BIN_PATH"] - eprefix = settings["EPREFIX"] + eprefix = portage.const.EPREFIX prerootpath = [x for x in settings.get("PREROOTPATH", "").split(":") if x] rootpath = [x for x in settings.get("ROOTPATH", "").split(":") if x] + overrides = [x for x in settings.get( + "__PORTAGE_TEST_PATH_OVERRIDE", "").split(":") if x] prefixes = [] if eprefix: prefixes.append(eprefix) prefixes.append("/") - path = [] + path = overrides + + if "xattr" in settings.features: + path.append(os.path.join(portage_bin_path, "ebuild-helpers", "xattr")) - if eapi not in (None, "0", "1", "2", "3"): - path.append(os.path.join(portage_bin_path, "ebuild-helpers", "4")) + if eprefix and uid != 0 and "fakeroot" not in settings.features: + path.append(os.path.join(portage_bin_path, + "ebuild-helpers", "unprivileged")) + + if settings.get("USERLAND", "GNU") != "GNU": + path.append(os.path.join(portage_bin_path, "ebuild-helpers", "bsd")) path.append(os.path.join(portage_bin_path, "ebuild-helpers")) path.extend(prerootpath) @@ -254,10 +324,11 @@ def doebuild_environment(myebuild, mydo, myroot=None, settings=None, if hasattr(mydbapi, 'repositories'): repo = mydbapi.repositories.get_repo_for_location(mytree) mysettings['PORTDIR'] = repo.eclass_db.porttrees[0] - mysettings['PORTDIR_OVERLAY'] = ' '.join(repo.eclass_db.porttrees[1:]) + mysettings['PORTAGE_ECLASS_LOCATIONS'] = repo.eclass_db.eclass_locations_string mysettings.configdict["pkg"]["PORTAGE_REPO_NAME"] = repo.name mysettings["PORTDIR"] = os.path.realpath(mysettings["PORTDIR"]) + mysettings.pop("PORTDIR_OVERLAY", None) mysettings["DISTDIR"] = os.path.realpath(mysettings["DISTDIR"]) mysettings["RPMDIR"] = os.path.realpath(mysettings["RPMDIR"]) @@ -414,14 +485,14 @@ _doebuild_commands_without_builddir = ( 'fetch', 'fetchall', 'help', 'manifest' ) -def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, - fetchonly=0, cleanup=0, dbkey=None, use_cache=1, fetchall=0, tree=None, +def doebuild(myebuild, mydo, _unused=DeprecationWarning, settings=None, debug=0, listonly=0, + fetchonly=0, cleanup=0, dbkey=DeprecationWarning, use_cache=1, fetchall=0, tree=None, mydbapi=None, vartree=None, prev_mtimes=None, fd_pipes=None, returnpid=False): """ Wrapper function that invokes specific ebuild phases through the spawning of ebuild.sh - + @param myebuild: name of the ebuild to invoke the phase on (CPV) @type myebuild: String @param mydo: Phase to run @@ -464,13 +535,13 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, @return: 1. 0 for success 2. 1 for error - + Most errors have an accompanying error message. - + listonly and fetchonly are only really necessary for operations involving 'fetch' prev_mtimes are only necessary for merge operations. Other variables may not be strictly required, many have defaults that are set inside of doebuild. - + """ if settings is None: @@ -478,17 +549,22 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, mysettings = settings myroot = settings['EROOT'] - if _unused is not None and _unused != mysettings['EROOT']: + if _unused is not DeprecationWarning: warnings.warn("The third parameter of the " - "portage.doebuild() is now unused. Use " - "settings['ROOT'] instead.", + "portage.doebuild() is deprecated. Instead " + "settings['EROOT'] is used.", + DeprecationWarning, stacklevel=2) + + if dbkey is not DeprecationWarning: + warnings.warn("portage.doebuild() called " + "with deprecated dbkey argument.", DeprecationWarning, stacklevel=2) if not tree: writemsg("Warning: tree not specified to doebuild\n") tree = "porttree" - - # chunked out deps for each phase, so that ebuild binary can use it + + # chunked out deps for each phase, so that ebuild binary can use it # to collapse targets down. actionmap_deps={ "pretend" : [], @@ -503,7 +579,7 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, "package":["install"], "merge" :["install"], } - + if mydbapi is None: mydbapi = portage.db[myroot][tree].dbapi @@ -518,7 +594,7 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, "fetch", "fetchall", "digest", "unpack", "prepare", "configure", "compile", "test", "install", "rpm", "qmerge", "merge", - "package","unmerge", "manifest"] + "package", "unmerge", "manifest", "nofetch"] if mydo not in validcommands: validcommands.sort() @@ -532,8 +608,11 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, return 1 if returnpid and mydo != 'depend': - warnings.warn("portage.doebuild() called " + \ - "with returnpid parameter enabled. This usage will " + \ + # This case is not supported, since it bypasses the EbuildPhase class + # which implements important functionality (including post phase hooks + # and IPC for things like best/has_version and die). + warnings.warn("portage.doebuild() called " + "with returnpid parameter enabled. This usage will " "not be supported in the future.", DeprecationWarning, stacklevel=2) @@ -541,9 +620,6 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, fetchall = 1 mydo = "fetch" - parallel_fetchonly = mydo in ("fetch", "fetchall") and \ - "PORTAGE_PARALLEL_FETCHONLY" in mysettings - if mydo not in clean_phases and not os.path.exists(myebuild): writemsg("!!! doebuild: %s not found for %s\n" % (myebuild, mydo), noiselevel=-1) @@ -650,7 +726,7 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, # we can temporarily override PORTAGE_TMPDIR with a random temp dir # so that there's no need for locking and it can be used even if the # user isn't in the portage group. - if mydo in ("info",): + if not returnpid and mydo in ("info",): tmpdir = tempfile.mkdtemp() tmpdir_orig = mysettings["PORTAGE_TMPDIR"] mysettings["PORTAGE_TMPDIR"] = tmpdir @@ -661,9 +737,10 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, if mydo in clean_phases: builddir_lock = None if not returnpid and \ - 'PORTAGE_BUILDIR_LOCKED' not in mysettings: + 'PORTAGE_BUILDDIR_LOCKED' not in mysettings: builddir_lock = EbuildBuildDir( - scheduler=PollScheduler().sched_iface, + scheduler=(portage._internal_caller and + global_event_loop() or EventLoop(main=False)), settings=mysettings) builddir_lock.lock() try: @@ -679,42 +756,7 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, if returnpid: return _spawn_phase(mydo, mysettings, fd_pipes=fd_pipes, returnpid=returnpid) - elif isinstance(dbkey, dict): - warnings.warn("portage.doebuild() called " + \ - "with dict dbkey argument. This usage will " + \ - "not be supported in the future.", - DeprecationWarning, stacklevel=2) - mysettings["dbkey"] = "" - pr, pw = os.pipe() - fd_pipes = { - 0:sys.stdin.fileno(), - 1:sys.stdout.fileno(), - 2:sys.stderr.fileno(), - 9:pw} - mypids = _spawn_phase(mydo, mysettings, returnpid=True, - fd_pipes=fd_pipes) - os.close(pw) # belongs exclusively to the child process now - f = os.fdopen(pr, 'rb', 0) - for k, v in zip(auxdbkeys, - (_unicode_decode(line).rstrip('\n') for line in f)): - dbkey[k] = v - f.close() - retval = os.waitpid(mypids[0], 0)[1] - portage.process.spawned_pids.remove(mypids[0]) - # If it got a signal, return the signal that was sent, but - # shift in order to distinguish it from a return value. (just - # like portage.process.spawn() would do). - if retval & 0xff: - retval = (retval & 0xff) << 8 - else: - # Otherwise, return its exit code. - retval = retval >> 8 - if retval == os.EX_OK and len(dbkey) != len(auxdbkeys): - # Don't trust bash's returncode if the - # number of lines is incorrect. - retval = 1 - return retval - elif dbkey: + elif dbkey and dbkey is not DeprecationWarning: mysettings["dbkey"] = dbkey else: mysettings["dbkey"] = \ @@ -723,14 +765,25 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, return _spawn_phase(mydo, mysettings, fd_pipes=fd_pipes, returnpid=returnpid) - # Validate dependency metadata here to ensure that ebuilds with invalid - # data are never installed via the ebuild command. Don't bother when - # returnpid == True since there's no need to do this every time emerge - # executes a phase. + elif mydo == "nofetch": + + if returnpid: + writemsg("!!! doebuild: %s\n" % + _("returnpid is not supported for phase '%s'\n" % mydo), + noiselevel=-1) + + return spawn_nofetch(mydbapi, myebuild, settings=mysettings, + fd_pipes=fd_pipes) + if tree == "porttree": - rval = _validate_deps(mysettings, myroot, mydo, mydbapi) - if rval != os.EX_OK: - return rval + + if not returnpid: + # Validate dependency metadata here to ensure that ebuilds with + # invalid data are never installed via the ebuild command. Skip + # this when returnpid is True (assume the caller handled it). + rval = _validate_deps(mysettings, myroot, mydo, mydbapi) + if rval != os.EX_OK: + return rval else: # FEATURES=noauto only makes sense for porttree, and we don't want @@ -739,20 +792,25 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, if "noauto" in mysettings.features: mysettings.features.discard("noauto") - # The info phase is special because it uses mkdtemp so and - # user (not necessarily in the portage group) can run it. - if mydo not in ('info',) and \ + # If we are not using a private temp dir, then check access + # to the global temp dir. + if tmpdir is None and \ mydo not in _doebuild_commands_without_builddir: rval = _check_temp_dir(mysettings) if rval != os.EX_OK: return rval if mydo == "unmerge": + if returnpid: + writemsg("!!! doebuild: %s\n" % + _("returnpid is not supported for phase '%s'\n" % mydo), + noiselevel=-1) return unmerge(mysettings["CATEGORY"], mysettings["PF"], myroot, mysettings, vartree=vartree) phases_to_run = set() - if "noauto" in mysettings.features or \ + if returnpid or \ + "noauto" in mysettings.features or \ mydo not in actionmap_deps: phases_to_run.add(mydo) else: @@ -803,9 +861,10 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, if newstuff: if builddir_lock is None and \ - 'PORTAGE_BUILDIR_LOCKED' not in mysettings: + 'PORTAGE_BUILDDIR_LOCKED' not in mysettings: builddir_lock = EbuildBuildDir( - scheduler=PollScheduler().sched_iface, + scheduler=(portage._internal_caller and + global_event_loop() or EventLoop(main=False)), settings=mysettings) builddir_lock.lock() try: @@ -823,12 +882,12 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, # in order to satisfy the sane $PWD requirement (from bug #239560) # when pkg_nofetch is spawned. have_build_dirs = False - if not parallel_fetchonly and \ - mydo not in ('digest', 'fetch', 'help', 'manifest'): + if mydo not in ('digest', 'fetch', 'help', 'manifest'): if not returnpid and \ - 'PORTAGE_BUILDIR_LOCKED' not in mysettings: + 'PORTAGE_BUILDDIR_LOCKED' not in mysettings: builddir_lock = EbuildBuildDir( - scheduler=PollScheduler().sched_iface, + scheduler=(portage._internal_caller and + global_event_loop() or EventLoop(main=False)), settings=mysettings) builddir_lock.lock() mystatus = prepare_build_dirs(myroot, mysettings, cleanup) @@ -871,9 +930,8 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, else: vardb = vartree.dbapi cpv = mysettings.mycpv - cp = portage.versions.cpv_getkey(cpv) - slot = mysettings["SLOT"] - cpv_slot = cp + ":" + slot + cpv_slot = "%s%s%s" % \ + (cpv.cp, portage.dep._slot_separator, cpv.slot) mysettings["REPLACING_VERSIONS"] = " ".join( set(portage.versions.cpv_getversion(match) \ for match in vardb.match(cpv_slot) + \ @@ -883,8 +941,16 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, # the sandbox -- and stop now. if mydo in ("config", "help", "info", "postinst", "preinst", "pretend", "postrm", "prerm"): - return _spawn_phase(mydo, mysettings, - fd_pipes=fd_pipes, logfile=logfile, returnpid=returnpid) + if mydo in ("preinst", "postinst"): + env_file = os.path.join(os.path.dirname(mysettings["EBUILD"]), + "environment.bz2") + if os.path.isfile(env_file): + mysettings["PORTAGE_UPDATE_ENV"] = env_file + try: + return _spawn_phase(mydo, mysettings, + fd_pipes=fd_pipes, logfile=logfile, returnpid=returnpid) + finally: + mysettings.pop("PORTAGE_UPDATE_ENV", None) mycpv = "/".join((mysettings["CATEGORY"], mysettings["PF"])) @@ -925,7 +991,8 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, if not fetch(fetchme, mysettings, listonly=listonly, fetchonly=fetchonly, allow_missing_digests=True, digests=dist_digests): - spawn_nofetch(mydbapi, myebuild, settings=mysettings) + spawn_nofetch(mydbapi, myebuild, settings=mysettings, + fd_pipes=fd_pipes) if listonly: # The convention for listonly mode is to report # success in any case, even though fetch() may @@ -957,11 +1024,7 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, mf = None _doebuild_manifest_cache = None return not digestgen(mysettings=mysettings, myportdb=mydbapi) - elif mydo != 'fetch' and \ - "digest" in mysettings.features: - # Don't do this when called by emerge or when called just - # for fetch (especially parallel-fetch) since it's not needed - # and it can interfere with parallel tasks. + elif "digest" in mysettings.features: mf = None _doebuild_manifest_cache = None digestgen(mysettings=mysettings, myportdb=mydbapi) @@ -970,14 +1033,17 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, if mydo in ("digest", "manifest"): return 1 + if mydo == "fetch": + # Return after digestgen for FEATURES=digest support. + # Return before digestcheck, since fetch() already + # checked any relevant digests. + return 0 + # See above comment about fetching only when needed if tree == 'porttree' and \ not digestcheck(checkme, mysettings, "strict" in features, mf=mf): return 1 - if mydo == "fetch": - return 0 - # remove PORTAGE_ACTUAL_DISTDIR once cvs/svn is supported via SRC_URI if tree == 'porttree' and \ ((mydo != "setup" and "noauto" not in features) \ @@ -993,7 +1059,9 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, if len(actionmap_deps.get(x, [])): actionmap[x]["dep"] = ' '.join(actionmap_deps[x]) - if mydo in actionmap: + regular_actionmap_phase = mydo in actionmap + + if regular_actionmap_phase: bintree = None if mydo == "package": # Make sure the package directory exists before executing @@ -1017,6 +1085,9 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, actionmap, mysettings, debug, logfile=logfile, fd_pipes=fd_pipes, returnpid=returnpid) + if returnpid and isinstance(retval, list): + return retval + if retval == os.EX_OK: if mydo == "package" and bintree is not None: bintree.inject(mysettings.mycpv, @@ -1028,7 +1099,15 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, except OSError: pass - elif mydo=="qmerge": + elif returnpid: + writemsg("!!! doebuild: %s\n" % + _("returnpid is not supported for phase '%s'\n" % mydo), + noiselevel=-1) + + if regular_actionmap_phase: + # handled above + pass + elif mydo == "qmerge": # check to ensure install was run. this *only* pops up when users # forget it and are using ebuild if not os.path.exists( @@ -1045,7 +1124,8 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, mysettings["CATEGORY"], mysettings["PF"], mysettings["D"], os.path.join(mysettings["PORTAGE_BUILDDIR"], "build-info"), myroot, mysettings, myebuild=mysettings["EBUILD"], mytree=tree, - mydbapi=mydbapi, vartree=vartree, prev_mtimes=prev_mtimes) + mydbapi=mydbapi, vartree=vartree, prev_mtimes=prev_mtimes, + fd_pipes=fd_pipes) elif mydo=="merge": retval = spawnebuild("install", actionmap, mysettings, debug, alwaysdep=1, logfile=logfile, fd_pipes=fd_pipes, @@ -1061,7 +1141,9 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, mysettings["D"], os.path.join(mysettings["PORTAGE_BUILDDIR"], "build-info"), myroot, mysettings, myebuild=mysettings["EBUILD"], mytree=tree, mydbapi=mydbapi, - vartree=vartree, prev_mtimes=prev_mtimes) + vartree=vartree, prev_mtimes=prev_mtimes, + fd_pipes=fd_pipes) + else: writemsg_stdout(_("!!! Unknown mydo: %s\n") % mydo, noiselevel=-1) return 1 @@ -1161,7 +1243,9 @@ def _prepare_env_file(settings): """ env_extractor = BinpkgEnvExtractor(background=False, - scheduler=PollScheduler().sched_iface, settings=settings) + scheduler=(portage._internal_caller and + global_event_loop() or EventLoop(main=False)), + settings=settings) if env_extractor.dest_env_exists(): # There are lots of possible states when doebuild() @@ -1243,7 +1327,7 @@ def _spawn_actionmap(settings): misc_sh_binary = os.path.join(portage_bin_path, os.path.basename(MISC_SH_BINARY)) ebuild_sh = _shell_quote(ebuild_sh_binary) + " %s" - misc_sh = _shell_quote(misc_sh_binary) + " dyn_%s" + misc_sh = _shell_quote(misc_sh_binary) + " __dyn_%s" # args are for the to spawn function actionmap = { @@ -1299,10 +1383,10 @@ def _validate_deps(mysettings, myroot, mydo, mydbapi): if not pkg.built and \ mydo not in ("digest", "help", "manifest") and \ - pkg.metadata["REQUIRED_USE"] and \ - eapi_has_required_use(pkg.metadata["EAPI"]): - result = check_required_use(pkg.metadata["REQUIRED_USE"], - pkg.use.enabled, pkg.iuse.is_valid_flag) + pkg._metadata["REQUIRED_USE"] and \ + eapi_has_required_use(pkg.eapi): + result = check_required_use(pkg._metadata["REQUIRED_USE"], + pkg.use.enabled, pkg.iuse.is_valid_flag, eapi=pkg.eapi) if not result: reduced_noise = result.tounicode() writemsg("\n %s\n" % _("The following REQUIRED_USE flag" + \ @@ -1310,7 +1394,7 @@ def _validate_deps(mysettings, myroot, mydo, mydbapi): writemsg(" %s\n" % reduced_noise, noiselevel=-1) normalized_required_use = \ - " ".join(pkg.metadata["REQUIRED_USE"].split()) + " ".join(pkg._metadata["REQUIRED_USE"].split()) if reduced_noise != normalized_required_use: writemsg("\n %s\n" % _("The above constraints " + \ "are a subset of the following complete expression:"), @@ -1325,7 +1409,8 @@ def _validate_deps(mysettings, myroot, mydo, mydbapi): # XXX This would be to replace getstatusoutput completely. # XXX Issue: cannot block execution. Deadlock condition. -def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakeroot=0, **keywords): +def spawn(mystring, mysettings, debug=False, free=False, droppriv=False, + sesandbox=False, fakeroot=False, networked=True, ipc=True, **keywords): """ Spawn a subprocess with extra portage-specific options. Optiosn include: @@ -1355,6 +1440,10 @@ def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakero @type sesandbox: Boolean @param fakeroot: Run this command with faked root privileges @type fakeroot: Boolean + @param networked: Run this command with networking access enabled + @type networked: Boolean + @param ipc: Run this command with host IPC access enabled + @type ipc: Boolean @param keywords: Extra options encoded as a dict, to be passed to spawn @type keywords: Dictionary @rtype: Integer @@ -1367,29 +1456,90 @@ def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakero fd_pipes = keywords.get("fd_pipes") if fd_pipes is None: fd_pipes = { - 0:sys.stdin.fileno(), - 1:sys.stdout.fileno(), - 2:sys.stderr.fileno(), + 0:portage._get_stdin().fileno(), + 1:sys.__stdout__.fileno(), + 2:sys.__stderr__.fileno(), } # In some cases the above print statements don't flush stdout, so # it needs to be flushed before allowing a child process to use it # so that output always shows in the correct order. - stdout_filenos = (sys.stdout.fileno(), sys.stderr.fileno()) + stdout_filenos = (sys.__stdout__.fileno(), sys.__stderr__.fileno()) for fd in fd_pipes.values(): if fd in stdout_filenos: - sys.stdout.flush() - sys.stderr.flush() + sys.__stdout__.flush() + sys.__stderr__.flush() break features = mysettings.features + + # Use Linux namespaces if available + if uid == 0 and platform.system() == 'Linux': + keywords['unshare_net'] = not networked + keywords['unshare_ipc'] = not ipc + # TODO: Enable fakeroot to be used together with droppriv. The # fake ownership/permissions will have to be converted to real # permissions in the merge phase. fakeroot = fakeroot and uid != 0 and portage.process.fakeroot_capable - if droppriv and uid == 0 and portage_gid and portage_uid and \ - hasattr(os, "setgroups"): - keywords.update({"uid":portage_uid,"gid":portage_gid, - "groups":userpriv_groups,"umask":0o02}) + portage_build_uid = os.getuid() + portage_build_gid = os.getgid() + if uid == 0 and portage_uid and portage_gid and hasattr(os, "setgroups"): + if droppriv: + keywords.update({ + "uid": portage_uid, + "gid": portage_gid, + "groups": userpriv_groups, + "umask": 0o02 + }) + + # Adjust pty ownership so that subprocesses + # can directly access /dev/fd/{1,2}. + stdout_fd = fd_pipes.get(1) + if stdout_fd is not None: + try: + subprocess_tty = _os.ttyname(stdout_fd) + except OSError: + pass + else: + try: + parent_tty = _os.ttyname(sys.__stdout__.fileno()) + except OSError: + parent_tty = None + + if subprocess_tty != parent_tty: + _os.chown(subprocess_tty, + int(portage_uid), int(portage_gid)) + + if "userpriv" in features and "userpriv" not in mysettings["PORTAGE_RESTRICT"].split() and secpass >= 2: + # Since Python 3.4, getpwuid and getgrgid + # require int type (no proxies). + portage_build_uid = int(portage_uid) + portage_build_gid = int(portage_gid) + + if "PORTAGE_BUILD_USER" not in mysettings: + user = None + try: + user = pwd.getpwuid(portage_build_uid).pw_name + except KeyError: + if portage_build_uid == 0: + user = "root" + elif portage_build_uid == portage_uid: + user = portage.data._portage_username + if user is not None: + mysettings["PORTAGE_BUILD_USER"] = user + + if "PORTAGE_BUILD_GROUP" not in mysettings: + group = None + try: + group = grp.getgrgid(portage_build_gid).gr_name + except KeyError: + if portage_build_gid == 0: + group = "root" + elif portage_build_gid == portage_gid: + group = portage.data._portage_grpname + if group is not None: + mysettings["PORTAGE_BUILD_GROUP"] = group + if not free: free=((droppriv and "usersandbox" not in features) or \ (not droppriv and "sandbox" not in features and \ @@ -1423,12 +1573,15 @@ def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakero mysettings["PORTAGE_SANDBOX_T"]) if keywords.get("returnpid"): - return spawn_func(mystring, env=mysettings.environ(), **keywords) + return spawn_func(mystring, env=mysettings.environ(), + **portage._native_kwargs(keywords)) proc = EbuildSpawnProcess( background=False, args=mystring, - scheduler=PollScheduler().sched_iface, spawn_func=spawn_func, - settings=mysettings, **keywords) + scheduler=SchedulerInterface(portage._internal_caller and + global_event_loop() or EventLoop(main=False)), + spawn_func=spawn_func, + settings=mysettings, **portage._native_kwargs(keywords)) proc.start() proc.wait() @@ -1440,8 +1593,8 @@ def spawnebuild(mydo, actionmap, mysettings, debug, alwaysdep=0, logfile=None, fd_pipes=None, returnpid=False): if returnpid: - warnings.warn("portage.spawnebuild() called " + \ - "with returnpid parameter enabled. This usage will " + \ + warnings.warn("portage.spawnebuild() called " + "with returnpid parameter enabled. This usage will " "not be supported in the future.", DeprecationWarning, stacklevel=2) @@ -1534,7 +1687,52 @@ def _check_build_log(mysettings, out=None): configure_opts_warn = [] configure_opts_warn_re = re.compile( - r'^configure: WARNING: [Uu]nrecognized options: ') + r'^configure: WARNING: [Uu]nrecognized options: (.*)') + + qa_configure_opts = "" + try: + with io.open(_unicode_encode(os.path.join( + mysettings["PORTAGE_BUILDDIR"], + "build-info", "QA_CONFIGURE_OPTIONS"), + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') as qa_configure_opts_f: + qa_configure_opts = qa_configure_opts_f.read() + except IOError as e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + raise + + qa_configure_opts = qa_configure_opts.split() + if qa_configure_opts: + if len(qa_configure_opts) > 1: + qa_configure_opts = "|".join("(%s)" % x for x in qa_configure_opts) + qa_configure_opts = "^(%s)$" % qa_configure_opts + else: + qa_configure_opts = "^%s$" % qa_configure_opts[0] + qa_configure_opts = re.compile(qa_configure_opts) + + qa_am_maintainer_mode = [] + try: + with io.open(_unicode_encode(os.path.join( + mysettings["PORTAGE_BUILDDIR"], + "build-info", "QA_AM_MAINTAINER_MODE"), + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') as qa_am_maintainer_mode_f: + qa_am_maintainer_mode = [x for x in + qa_am_maintainer_mode_f.read().splitlines() if x] + except IOError as e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + raise + + if qa_am_maintainer_mode: + if len(qa_am_maintainer_mode) > 1: + qa_am_maintainer_mode = \ + "|".join("(%s)" % x for x in qa_am_maintainer_mode) + qa_am_maintainer_mode = "^(%s)$" % qa_am_maintainer_mode + else: + qa_am_maintainer_mode = "^%s$" % qa_am_maintainer_mode[0] + qa_am_maintainer_mode = re.compile(qa_am_maintainer_mode) # Exclude output from dev-libs/yaz-3.0.47 which looks like this: # @@ -1556,7 +1754,9 @@ def _check_build_log(mysettings, out=None): for line in f: line = _unicode_decode(line) if am_maintainer_mode_re.search(line) is not None and \ - am_maintainer_mode_exclude_re.search(line) is None: + am_maintainer_mode_exclude_re.search(line) is None and \ + (not qa_am_maintainer_mode or + qa_am_maintainer_mode.search(line) is None): am_maintainer_mode.append(line.rstrip("\n")) if bash_command_not_found_re.match(line) is not None and \ @@ -1566,8 +1766,11 @@ def _check_build_log(mysettings, out=None): if helper_missing_file_re.match(line) is not None: helper_missing_file.append(line.rstrip("\n")) - if configure_opts_warn_re.match(line) is not None: - configure_opts_warn.append(line.rstrip("\n")) + m = configure_opts_warn_re.match(line) + if m is not None: + for x in m.group(1).split(", "): + if not qa_configure_opts or qa_configure_opts.match(x) is None: + configure_opts_warn.append(x) if make_jobserver_re.match(line) is not None: make_jobserver.append(line.rstrip("\n")) @@ -1616,7 +1819,7 @@ def _check_build_log(mysettings, out=None): if configure_opts_warn: msg = [_("QA Notice: Unrecognized configure options:")] msg.append("") - msg.extend("\t" + line for line in configure_opts_warn) + msg.extend("\t%s" % x for x in configure_opts_warn) _eqawarn(msg) if make_jobserver: @@ -1629,7 +1832,7 @@ def _check_build_log(mysettings, out=None): if f_real is not None: f_real.close() -def _post_src_install_chost_fix(settings): +def _post_src_install_write_metadata(settings): """ It's possible that the ebuild has changed the CHOST variable, so revert it to the initial @@ -1637,10 +1840,16 @@ def _post_src_install_chost_fix(settings): due to local environment settings like in bug #386829. """ + eapi_attrs = _get_eapi_attrs(settings.configdict['pkg']['EAPI']) + build_info_dir = os.path.join(settings['PORTAGE_BUILDDIR'], 'build-info') - for k in ('IUSE',): - v = settings.get(k) + metadata_keys = ['IUSE'] + if eapi_attrs.iuse_effective: + metadata_keys.append('IUSE_EFFECTIVE') + + for k in metadata_keys: + v = settings.configdict['pkg'].get(k) if v is not None: write_atomic(os.path.join(build_info_dir, k), v + '\n') @@ -1652,9 +1861,59 @@ def _post_src_install_chost_fix(settings): if v is not None: write_atomic(os.path.join(build_info_dir, k), v + '\n') -_vdb_use_conditional_keys = ('DEPEND', 'LICENSE', 'PDEPEND', - 'PROPERTIES', 'PROVIDE', 'RDEPEND', 'RESTRICT',) -_vdb_use_conditional_atoms = frozenset(['DEPEND', 'PDEPEND', 'RDEPEND']) + with io.open(_unicode_encode(os.path.join(build_info_dir, + 'BUILD_TIME'), encoding=_encodings['fs'], errors='strict'), + mode='w', encoding=_encodings['repo.content'], + errors='strict') as f: + f.write("%.0f\n" % (time.time(),)) + + use = frozenset(settings['PORTAGE_USE'].split()) + for k in _vdb_use_conditional_keys: + v = settings.configdict['pkg'].get(k) + filename = os.path.join(build_info_dir, k) + if v is None: + try: + os.unlink(filename) + except OSError: + pass + continue + + if k.endswith('DEPEND'): + if eapi_attrs.slot_operator: + continue + token_class = Atom + else: + token_class = None + + v = use_reduce(v, uselist=use, token_class=token_class) + v = paren_enclose(v) + if not v: + try: + os.unlink(filename) + except OSError: + pass + continue + with io.open(_unicode_encode(os.path.join(build_info_dir, + k), encoding=_encodings['fs'], errors='strict'), + mode='w', encoding=_encodings['repo.content'], + errors='strict') as f: + f.write('%s\n' % v) + + if eapi_attrs.slot_operator: + deps = evaluate_slot_operator_equal_deps(settings, use, QueryCommand.get_db()) + for k, v in deps.items(): + filename = os.path.join(build_info_dir, k) + if not v: + try: + os.unlink(filename) + except OSError: + pass + continue + with io.open(_unicode_encode(os.path.join(build_info_dir, + k), encoding=_encodings['fs'], errors='strict'), + mode='w', encoding=_encodings['repo.content'], + errors='strict') as f: + f.write('%s\n' % v) def _preinst_bsdflags(mysettings): if bsd_chflags: @@ -1696,6 +1955,33 @@ def _post_src_install_uid_fix(mysettings, out): destdir = mysettings["D"] ed_len = len(mysettings["ED"]) unicode_errors = [] + desktop_file_validate = \ + portage.process.find_binary("desktop-file-validate") is not None + xdg_dirs = mysettings.get('XDG_DATA_DIRS', '/usr/share').split(':') + xdg_dirs = tuple(os.path.join(i, "applications") + os.sep + for i in xdg_dirs if i) + + qa_desktop_file = "" + try: + with io.open(_unicode_encode(os.path.join( + mysettings["PORTAGE_BUILDDIR"], + "build-info", "QA_DESKTOP_FILE"), + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') as f: + qa_desktop_file = f.read() + except IOError as e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + raise + + qa_desktop_file = qa_desktop_file.split() + if qa_desktop_file: + if len(qa_desktop_file) > 1: + qa_desktop_file = "|".join("(%s)" % x for x in qa_desktop_file) + qa_desktop_file = "^(%s)$" % qa_desktop_file + else: + qa_desktop_file = "^%s$" % qa_desktop_file[0] + qa_desktop_file = re.compile(qa_desktop_file) while True: @@ -1704,6 +1990,7 @@ def _post_src_install_uid_fix(mysettings, out): counted_inodes = set() fixlafiles_announced = False fixlafiles = "fixlafiles" in mysettings.features + desktopfile_errors = [] for parent, dirs, files in os.walk(destdir): try: @@ -1743,6 +2030,16 @@ def _post_src_install_uid_fix(mysettings, out): else: fpath = os.path.join(parent, fname) + fpath_relative = fpath[ed_len - 1:] + if desktop_file_validate and fname.endswith(".desktop") and \ + os.path.isfile(fpath) and \ + fpath_relative.startswith(xdg_dirs) and \ + not (qa_desktop_file and qa_desktop_file.match(fpath_relative.strip(os.sep)) is not None): + + desktop_validate = validate_desktop_entry(fpath) + if desktop_validate: + desktopfile_errors.extend(desktop_validate) + if fixlafiles and \ fname.endswith(".la") and os.path.isfile(fpath): f = open(_unicode_encode(fpath, @@ -1809,6 +2106,11 @@ def _post_src_install_uid_fix(mysettings, out): if not unicode_error: break + if desktopfile_errors: + for l in _merge_desktopfile_error(desktopfile_errors): + l = l.replace(mysettings["ED"], '/') + eqawarn(l, phase='install', key=mysettings.mycpv, out=out) + if unicode_errors: for l in _merge_unicode_error(unicode_errors): eqawarn(l, phase='install', key=mysettings.mycpv, out=out) @@ -1820,47 +2122,9 @@ def _post_src_install_uid_fix(mysettings, out): 'SIZE'), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['repo.content'], errors='strict') - f.write(_unicode_decode(str(size) + '\n')) + f.write('%d\n' % size) f.close() - f = io.open(_unicode_encode(os.path.join(build_info_dir, - 'BUILD_TIME'), encoding=_encodings['fs'], errors='strict'), - mode='w', encoding=_encodings['repo.content'], - errors='strict') - f.write(_unicode_decode("%.0f\n" % (time.time(),))) - f.close() - - use = frozenset(mysettings['PORTAGE_USE'].split()) - for k in _vdb_use_conditional_keys: - v = mysettings.configdict['pkg'].get(k) - filename = os.path.join(build_info_dir, k) - if v is None: - try: - os.unlink(filename) - except OSError: - pass - continue - - if k.endswith('DEPEND'): - token_class = Atom - else: - token_class = None - - v = use_reduce(v, uselist=use, token_class=token_class) - v = paren_enclose(v) - if not v: - try: - os.unlink(filename) - except OSError: - pass - continue - f = io.open(_unicode_encode(os.path.join(build_info_dir, - k), encoding=_encodings['fs'], errors='strict'), - mode='w', encoding=_encodings['repo.content'], - errors='strict') - f.write(_unicode_decode(v + '\n')) - f.close() - _reapply_bsdflags_to_image(mysettings) def _reapply_bsdflags_to_image(mysettings): @@ -2009,6 +2273,20 @@ def _post_src_install_soname_symlinks(mysettings, out): for line in qa_msg: eqawarn(line, key=mysettings.mycpv, out=out) +def _merge_desktopfile_error(errors): + lines = [] + + msg = _("QA Notice: This package installs one or more .desktop files " + "that do not pass validation.") + lines.extend(wrap(msg, 72)) + + lines.append("") + errors.sort() + lines.extend("\t" + x for x in errors) + lines.append("") + + return lines + def _merge_unicode_error(errors): lines = [] @@ -2065,11 +2343,6 @@ def _handle_self_update(settings, vardb): if settings["ROOT"] == "/" and \ portage.dep.match_from_list( portage.const.PORTAGE_PACKAGE_ATOM, [cpv]): - inherited = frozenset(settings.get('INHERITED', '').split()) - if not vardb.cpv_exists(cpv) or \ - '9999' in cpv or \ - 'git' in inherited or \ - 'git-2' in inherited: - _prepare_self_update(settings) - return True + _prepare_self_update(settings) + return True return False diff --git a/portage_with_autodep/pym/portage/package/ebuild/doebuild.pyo b/portage_with_autodep/pym/portage/package/ebuild/doebuild.pyo Binary files differindex a6ebb1d..846d99a 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/doebuild.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/doebuild.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/fetch.py b/portage_with_autodep/pym/portage/package/ebuild/fetch.py index b795b28..5316f03 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/fetch.py +++ b/portage_with_autodep/pym/portage/package/ebuild/fetch.py @@ -1,4 +1,4 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -14,6 +14,10 @@ import stat import sys import tempfile +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse import portage portage.proxy.lazyimport.lazyimport(globals(), @@ -25,7 +29,8 @@ portage.proxy.lazyimport.lazyimport(globals(), from portage import OrderedDict, os, selinux, shutil, _encodings, \ _shell_quote, _unicode_encode -from portage.checksum import hashfunc_map, perform_md5, verify_all +from portage.checksum import (hashfunc_map, perform_md5, verify_all, + _filter_unaccelarated_hashes, _hash_filter, _apply_hash_filter) from portage.const import BASH_BINARY, CUSTOM_MIRRORS_FILE, \ GLOBAL_CONFIG_PATH from portage.data import portage_gid, portage_uid, secpass, userpriv_groups @@ -63,9 +68,9 @@ def _spawn_fetch(settings, args, **kwargs): if "fd_pipes" not in kwargs: kwargs["fd_pipes"] = { - 0 : sys.stdin.fileno(), - 1 : sys.stdout.fileno(), - 2 : sys.stdout.fileno(), + 0 : portage._get_stdin().fileno(), + 1 : sys.__stdout__.fileno(), + 2 : sys.__stdout__.fileno(), } if "userfetch" in settings.features and \ @@ -184,7 +189,7 @@ def _check_digests(filename, digests, show_errors=1): return False return True -def _check_distfile(filename, digests, eout, show_errors=1): +def _check_distfile(filename, digests, eout, show_errors=1, hash_filter=None): """ @return a tuple of (match, stat_obj) where match is True if filename matches all given digests (if any) and stat_obj is a stat result, or @@ -210,6 +215,9 @@ def _check_distfile(filename, digests, eout, show_errors=1): # Zero-byte distfiles are always invalid. return (False, st) else: + digests = _filter_unaccelarated_hashes(digests) + if hash_filter is not None: + digests = _apply_hash_filter(digests, hash_filter) if _check_digests(filename, digests, show_errors=show_errors): eout.ebegin("%s %s ;-)" % (os.path.basename(filename), " ".join(sorted(digests)))) @@ -339,7 +347,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, _("!!! For fetching to a read-only filesystem, " "locking should be turned off.\n")), noiselevel=-1) writemsg(_("!!! This can be done by adding -distlocks to " - "FEATURES in /etc/make.conf\n"), noiselevel=-1) + "FEATURES in /etc/portage/make.conf\n"), noiselevel=-1) # use_locks = 0 # local mirrors are always added @@ -353,6 +361,9 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, if try_mirrors: mymirrors += [x.rstrip("/") for x in mysettings["GENTOO_MIRRORS"].split() if x] + hash_filter = _hash_filter(mysettings.get("PORTAGE_CHECKSUM_FILTER", "")) + if hash_filter.transparent: + hash_filter = None skip_manifest = mysettings.get("EBUILD_SKIP_MANIFEST") == "1" if skip_manifest: allow_missing_digests = True @@ -395,12 +406,16 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, for myfile, uri_set in myuris.items(): for myuri in uri_set: file_uri_tuples.append((myfile, myuri)) + if not uri_set: + file_uri_tuples.append((myfile, None)) else: for myuri in myuris: - file_uri_tuples.append((os.path.basename(myuri), myuri)) + if urlparse(myuri).scheme: + file_uri_tuples.append((os.path.basename(myuri), myuri)) + else: + file_uri_tuples.append((os.path.basename(myuri), None)) filedict = OrderedDict() - primaryuri_indexes={} primaryuri_dict = {} thirdpartymirror_uris = {} for myfile, myuri in file_uri_tuples: @@ -408,6 +423,8 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, filedict[myfile]=[] for y in range(0,len(locations)): filedict[myfile].append(locations[y]+"/distfiles/"+myfile) + if myuri is None: + continue if myuri[:9]=="mirror://": eidx = myuri.find("/", 9) if eidx != -1: @@ -422,10 +439,9 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, # now try the official mirrors if mirrorname in thirdpartymirrors: - random.shuffle(thirdpartymirrors[mirrorname]) - uris = [locmirr.rstrip("/") + "/" + path \ for locmirr in thirdpartymirrors[mirrorname]] + random.shuffle(uris) filedict[myfile].extend(uris) thirdpartymirror_uris.setdefault(myfile, []).extend(uris) @@ -438,26 +454,30 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, if restrict_fetch or force_mirror: # Only fetch from specific mirrors is allowed. continue - if "primaryuri" in restrict: - # Use the source site first. - if myfile in primaryuri_indexes: - primaryuri_indexes[myfile] += 1 - else: - primaryuri_indexes[myfile] = 0 - filedict[myfile].insert(primaryuri_indexes[myfile], myuri) - else: - filedict[myfile].append(myuri) primaryuris = primaryuri_dict.get(myfile) if primaryuris is None: primaryuris = [] primaryuri_dict[myfile] = primaryuris primaryuris.append(myuri) + # Order primaryuri_dict values to match that in SRC_URI. + for uris in primaryuri_dict.values(): + uris.reverse() + # Prefer thirdpartymirrors over normal mirrors in cases when # the file does not yet exist on the normal mirrors. for myfile, uris in thirdpartymirror_uris.items(): primaryuri_dict.setdefault(myfile, []).extend(uris) + # Now merge primaryuri values into filedict (includes mirrors + # explicitly referenced in SRC_URI). + if "primaryuri" in restrict: + for myfile, uris in filedict.items(): + filedict[myfile] = primaryuri_dict.get(myfile, []) + uris + else: + for myfile in filedict: + filedict[myfile] += primaryuri_dict.get(myfile, []) + can_fetch=True if listonly: @@ -635,7 +655,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, eout = EOutput() eout.quiet = mysettings.get("PORTAGE_QUIET") == "1" match, mystat = _check_distfile( - myfile_path, pruned_digests, eout) + myfile_path, pruned_digests, eout, hash_filter=hash_filter) if match: # Skip permission adjustment for symlinks, since we don't # want to modify anything outside of the primary DISTDIR, @@ -707,7 +727,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, for x in ro_distdirs: filename = os.path.join(x, myfile) match, mystat = _check_distfile( - filename, pruned_digests, eout) + filename, pruned_digests, eout, hash_filter=hash_filter) if match: readonly_file = filename break @@ -732,7 +752,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, "remaining space.\n"), noiselevel=-1) if userfetch: writemsg(_("!!! You may set FEATURES=\"-userfetch\"" - " in /etc/make.conf in order to fetch with\n" + " in /etc/portage/make.conf in order to fetch with\n" "!!! superuser privileges.\n"), noiselevel=-1) if fsmirrors and not os.path.exists(myfile_path) and has_space: @@ -793,8 +813,10 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, eout.eend(0) continue else: - verified_ok, reason = verify_all( - myfile_path, mydigests[myfile]) + digests = _filter_unaccelarated_hashes(mydigests[myfile]) + if hash_filter is not None: + digests = _apply_hash_filter(digests, hash_filter) + verified_ok, reason = verify_all(myfile_path, digests) if not verified_ok: writemsg(_("!!! Previously fetched" " file: '%s'\n") % myfile, noiselevel=-1) @@ -816,7 +838,6 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, eout = EOutput() eout.quiet = \ mysettings.get("PORTAGE_QUIET", None) == "1" - digests = mydigests.get(myfile) if digests: digests = list(digests) digests.sort() @@ -844,8 +865,8 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, protocol = loc[0:loc.find("://")] global_config_path = GLOBAL_CONFIG_PATH - if mysettings['EPREFIX']: - global_config_path = os.path.join(mysettings['EPREFIX'], + if portage.const.EPREFIX: + global_config_path = os.path.join(portage.const.EPREFIX, GLOBAL_CONFIG_PATH.lstrip(os.sep)) missing_file_param = False @@ -954,11 +975,16 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, writemsg_stdout(_(">>> Downloading '%s'\n") % \ _hide_url_passwd(loc)) variables = { - "DISTDIR": mysettings["DISTDIR"], "URI": loc, "FILE": myfile } + for k in ("DISTDIR", "PORTAGE_SSH_OPTS"): + try: + variables[k] = mysettings[k] + except KeyError: + pass + myfetch = shlex_split(locfetch) myfetch = [varexpand(x, mydict=variables) for x in myfetch] myret = -1 @@ -1051,7 +1077,10 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, # file NOW, for those users who don't have a stable/continuous # net connection. This way we have a chance to try to download # from another mirror... - verified_ok,reason = verify_all(mysettings["DISTDIR"]+"/"+myfile, mydigests[myfile]) + digests = _filter_unaccelarated_hashes(mydigests[myfile]) + if hash_filter is not None: + digests = _apply_hash_filter(digests, hash_filter) + verified_ok, reason = verify_all(myfile_path, digests) if not verified_ok: writemsg(_("!!! Fetched file: %s VERIFY FAILED!\n") % myfile, noiselevel=-1) @@ -1085,7 +1114,6 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, else: eout = EOutput() eout.quiet = mysettings.get("PORTAGE_QUIET", None) == "1" - digests = mydigests.get(myfile) if digests: eout.ebegin("%s %s ;-)" % \ (myfile, " ".join(sorted(digests)))) diff --git a/portage_with_autodep/pym/portage/package/ebuild/fetch.pyo b/portage_with_autodep/pym/portage/package/ebuild/fetch.pyo Binary files differindex 3bd81df..6e2c242 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/fetch.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/fetch.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.py b/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.py index 8a88c2f..70a6bf2 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.py +++ b/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ['getmaskingreason'] @@ -6,13 +6,12 @@ __all__ = ['getmaskingreason'] import portage from portage import os from portage.const import USER_CONFIG_PATH -from portage.dep import Atom, match_from_list, _slot_separator, _repo_separator +from portage.dep import Atom, match_from_list from portage.exception import InvalidAtom from portage.localization import _ from portage.repository.config import _gen_valid_repo from portage.util import grablines, normalize_path -from portage.versions import catpkgsplit -from _emerge.Package import Package +from portage.versions import catpkgsplit, _pkg_str def getmaskingreason(mycpv, metadata=None, settings=None, portdb=None, return_location=False, myrepo=None): @@ -60,23 +59,20 @@ def getmaskingreason(mycpv, metadata=None, settings=None, # Sometimes we can't access SLOT or repository due to corruption. pkg = mycpv - if metadata is not None: - pkg = "".join((mycpv, _slot_separator, metadata["SLOT"])) - # At this point myrepo should be None, a valid name, or - # Package.UNKNOWN_REPO which we ignore. - if myrepo is not None and myrepo != Package.UNKNOWN_REPO: - pkg = "".join((pkg, _repo_separator, myrepo)) + try: + pkg.slot + except AttributeError: + pkg = _pkg_str(mycpv, metadata=metadata, repo=myrepo) + cpv_slot_list = [pkg] - mycp=mysplit[0]+"/"+mysplit[1] + mycp = pkg.cp - # XXX- This is a temporary duplicate of code from the config constructor. - locations = [os.path.join(settings["PORTDIR"], "profiles")] + locations = [] + if pkg.repo in settings.repositories: + for repo in settings.repositories[pkg.repo].masters + (settings.repositories[pkg.repo],): + locations.append(os.path.join(repo.location, "profiles")) locations.extend(settings.profiles) - for ov in settings["PORTDIR_OVERLAY"].split(): - profdir = os.path.join(normalize_path(ov), "profiles") - if os.path.isdir(profdir): - locations.append(profdir) locations.append(os.path.join(settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH)) locations.reverse() diff --git a/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.pyo b/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.pyo Binary files differindex 1614244..6c0073d 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.py b/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.py index 9bf605d..c8954aa 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.py +++ b/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.py @@ -1,12 +1,15 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ['getmaskingstatus'] import sys import portage from portage import eapi_is_supported, _eapi_is_deprecated +from portage.exception import InvalidDependString from portage.localization import _ from portage.package.ebuild.config import config from portage.versions import catpkgsplit, _pkg_str @@ -48,7 +51,7 @@ def _getmaskingstatus(mycpv, settings, portdb, myrepo=None): # emerge passed in a Package instance pkg = mycpv mycpv = pkg.cpv - metadata = pkg.metadata + metadata = pkg._metadata installed = pkg.installed if metadata is None: @@ -65,10 +68,11 @@ def _getmaskingstatus(mycpv, settings, portdb, myrepo=None): else: metadata["USE"] = "" - if not hasattr(mycpv, 'slot'): + try: + mycpv.slot + except AttributeError: try: - mycpv = _pkg_str(mycpv, slot=metadata['SLOT'], - repo=metadata.get('repository')) + mycpv = _pkg_str(mycpv, metadata=metadata, settings=settings) except portage.exception.InvalidData: raise ValueError(_("invalid CPV: %s") % mycpv) @@ -83,6 +87,7 @@ def _getmaskingstatus(mycpv, settings, portdb, myrepo=None): mygroups = settings._getKeywords(mycpv, metadata) licenses = metadata["LICENSE"] properties = metadata["PROPERTIES"] + restrict = metadata["RESTRICT"] if not eapi_is_supported(eapi): return [_MaskReason("EAPI", "EAPI %s" % eapi)] elif _eapi_is_deprecated(eapi) and not installed: @@ -122,6 +127,13 @@ def _getmaskingstatus(mycpv, settings, portdb, myrepo=None): if gp=="*": kmask=None break + elif gp == "~*": + for x in pgroups: + if x[:1] == "~": + kmask = None + break + if kmask is None: + break elif gp=="-"+myarch and myarch in pgroups: kmask="-"+myarch break @@ -161,6 +173,15 @@ def _getmaskingstatus(mycpv, settings, portdb, myrepo=None): except portage.exception.InvalidDependString as e: rValue.append(_MaskReason("invalid", "PROPERTIES: "+str(e))) + try: + missing_restricts = settings._getMissingRestrict(mycpv, metadata) + if missing_restricts: + msg = list(missing_restricts) + msg.append("in RESTRICT") + rValue.append(_MaskReason("RESTRICT", " ".join(msg))) + except InvalidDependString as e: + rValue.append(_MaskReason("invalid", "RESTRICT: %s" % (e,))) + # Only show KEYWORDS masks for installed packages # if they're not masked for any other reason. if kmask and (not installed or not rValue): diff --git a/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.pyo b/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.pyo Binary files differindex 9cf1d9d..27bb3c7 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.py b/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.py index b8fbdc5..6782160 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.py +++ b/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.py @@ -1,6 +1,8 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ['prepare_build_dirs'] import errno @@ -338,12 +340,12 @@ def _prepare_workdir(mysettings): try: _ensure_log_subdirs(logdir, log_subdir) except PortageException as e: - writemsg(_unicode_decode("!!! %s\n") % (e,), noiselevel=-1) + writemsg("!!! %s\n" % (e,), noiselevel=-1) if os.access(log_subdir, os.W_OK): logdir_subdir_ok = True else: - writemsg(_unicode_decode("!!! %s: %s\n") % + writemsg("!!! %s: %s\n" % (_("Permission Denied"), log_subdir), noiselevel=-1) tmpdir_log_path = os.path.join( diff --git a/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.pyo b/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.pyo Binary files differindex 2dcfaea..fd6d446 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.pyo +++ b/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.pyo diff --git a/portage_with_autodep/pym/portage/process.py b/portage_with_autodep/pym/portage/process.py index d7d1037..7334c00 100644 --- a/portage_with_autodep/pym/portage/process.py +++ b/portage_with_autodep/pym/portage/process.py @@ -1,25 +1,30 @@ # portage.py -- core Portage functionality -# Copyright 1998-2012 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import atexit import errno +import fcntl import platform import signal +import socket +import struct import sys import traceback +import os as _os from portage import os from portage import _encodings from portage import _unicode_encode import portage portage.proxy.lazyimport.lazyimport(globals(), - 'portage.util:dump_traceback', + 'portage.util:dump_traceback,writemsg', ) from portage.const import BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY, AUTODEP_LIBRARY from portage.exception import CommandNotFound +from portage.util._ctypes import find_library, LoadLibrary, ctypes try: import resource @@ -30,10 +35,32 @@ except ImportError: if sys.hexversion >= 0x3000000: basestring = str -if os.path.isdir("/proc/%i/fd" % os.getpid()): +# Support PEP 446 for Python >=3.4 +try: + _set_inheritable = _os.set_inheritable +except AttributeError: + _set_inheritable = None + +try: + _FD_CLOEXEC = fcntl.FD_CLOEXEC +except AttributeError: + _FD_CLOEXEC = None + +# Prefer /proc/self/fd if available (/dev/fd +# doesn't work on solaris, see bug #474536). +for _fd_dir in ("/proc/self/fd", "/dev/fd"): + if os.path.isdir(_fd_dir): + break + else: + _fd_dir = None + +# /dev/fd does not work on FreeBSD, see bug #478446 +if platform.system() in ('FreeBSD',) and _fd_dir == '/dev/fd': + _fd_dir = None + +if _fd_dir is not None: def get_open_fds(): - return (int(fd) for fd in os.listdir("/proc/%i/fd" % os.getpid()) \ - if fd.isdigit()) + return (int(fd) for fd in os.listdir(_fd_dir) if fd.isdigit()) if platform.python_implementation() == 'PyPy': # EAGAIN observed with PyPy 1.8. @@ -46,6 +73,13 @@ if os.path.isdir("/proc/%i/fd" % os.getpid()): raise return range(max_fd_limit) +elif os.path.isdir("/proc/%s/fd" % os.getpid()): + # In order for this function to work in forked subprocesses, + # os.getpid() must be called from inside the function. + def get_open_fds(): + return (int(fd) for fd in os.listdir("/proc/%s/fd" % os.getpid()) + if fd.isdigit()) + else: def get_open_fds(): return range(max_fd_limit) @@ -62,7 +96,7 @@ fakeroot_capable = (os.path.isfile(FAKEROOT_BINARY) and def spawn_bash(mycommand, debug=False, opt_name=None, **keywords): """ Spawns a bash shell running a specific commands - + @param mycommand: The command for bash to run @type mycommand: String @param debug: Turn bash debugging on (set -x) @@ -91,7 +125,7 @@ def spawn_autodep(mycommand, opt_name=None, **keywords): # Core part: tell the loader to preload logging library keywords["env"]["LD_PRELOAD"]=AUTODEP_LIBRARY - return spawn_bash(mycommand, opt_name=opt_name, **keywords) + return spawn_bash(mycommand, opt_name=opt_name, **keywords) def spawn_sandbox(mycommand, opt_name=None, **keywords): if not sandbox_capable: @@ -154,33 +188,31 @@ def run_exitfuncs(): atexit.register(run_exitfuncs) -# We need to make sure that any processes spawned are killed off when -# we exit. spawn() takes care of adding and removing pids to this list -# as it creates and cleans up processes. -spawned_pids = [] -def cleanup(): - while spawned_pids: - pid = spawned_pids.pop() +# It used to be necessary for API consumers to remove pids from spawned_pids, +# since otherwise it would accumulate a pids endlessly. Now, spawned_pids is +# just an empty dummy list, so for backward compatibility, ignore ValueError +# for removal on non-existent items. +class _dummy_list(list): + def remove(self, item): + # TODO: Trigger a DeprecationWarning here, after stable portage + # has dummy spawned_pids. try: - # With waitpid and WNOHANG, only check the - # first element of the tuple since the second - # element may vary (bug #337465). - if os.waitpid(pid, os.WNOHANG)[0] == 0: - os.kill(pid, signal.SIGTERM) - os.waitpid(pid, 0) - except OSError: - # This pid has been cleaned up outside - # of spawn(). + list.remove(self, item) + except ValueError: pass -atexit_register(cleanup) +spawned_pids = _dummy_list() + +def cleanup(): + pass def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, uid=None, gid=None, groups=None, umask=None, logfile=None, - path_lookup=True, pre_exec=None): + path_lookup=True, pre_exec=None, close_fds=True, unshare_net=False, + unshare_ipc=False, cgroup=None): """ Spawns a given command. - + @param mycommand: the command to execute @type mycommand: String or List (Popen style list) @param env: A dict of Key=Value pairs for env variables @@ -188,6 +220,7 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, @param opt_name: an optional name for the spawn'd process (defaults to the binary name) @type opt_name: String @param fd_pipes: A dict of mapping for pipes, { '0': stdin, '1': stdout } for example + (default is {0:stdin, 1:stdout, 2:stderr}) @type fd_pipes: Dictionary @param returnpid: Return the Process IDs for a successful spawn. NOTE: This requires the caller clean up all the PIDs, otherwise spawn will clean them. @@ -206,10 +239,19 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, @type path_lookup: Boolean @param pre_exec: A function to be called with no arguments just prior to the exec call. @type pre_exec: callable - + @param close_fds: If True, then close all file descriptors except those + referenced by fd_pipes (default is True). + @type close_fds: Boolean + @param unshare_net: If True, networking will be unshared from the spawned process + @type unshare_net: Boolean + @param unshare_ipc: If True, IPC will be unshared from the spawned process + @type unshare_ipc: Boolean + @param cgroup: CGroup path to bind the process to + @type cgroup: String + logfile requires stdout and stderr to be assigned to this process (ie not pointed somewhere else.) - + """ # mycommand is either a str or a list @@ -239,9 +281,9 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, # default to propagating our stdin, stdout and stderr. if fd_pipes is None: fd_pipes = { - 0:sys.stdin.fileno(), - 1:sys.stdout.fileno(), - 2:sys.stderr.fileno(), + 0:portage._get_stdin().fileno(), + 1:sys.__stdout__.fileno(), + 2:sys.__stderr__.fileno(), } # mypids will hold the pids of all processes created. @@ -269,21 +311,40 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, fd_pipes[1] = pw fd_pipes[2] = pw - pid = os.fork() + # This caches the libc library lookup in the current + # process, so that it's only done once rather than + # for each child process. + if unshare_net or unshare_ipc: + find_library("c") - if pid == 0: - try: - _exec(binary, mycommand, opt_name, fd_pipes, - env, gid, groups, uid, umask, pre_exec) - except SystemExit: - raise - except Exception as e: - # We need to catch _any_ exception so that it doesn't - # propagate out of this function and cause exiting - # with anything other than os._exit() - sys.stderr.write("%s:\n %s\n" % (e, " ".join(mycommand))) - traceback.print_exc() - sys.stderr.flush() + parent_pid = os.getpid() + pid = None + try: + pid = os.fork() + + if pid == 0: + try: + _exec(binary, mycommand, opt_name, fd_pipes, + env, gid, groups, uid, umask, pre_exec, close_fds, + unshare_net, unshare_ipc, cgroup) + except SystemExit: + raise + except Exception as e: + # We need to catch _any_ exception so that it doesn't + # propagate out of this function and cause exiting + # with anything other than os._exit() + writemsg("%s:\n %s\n" % (e, " ".join(mycommand)), + noiselevel=-1) + traceback.print_exc() + sys.stderr.flush() + + finally: + if pid == 0 or (pid is None and os.getpid() != parent_pid): + # Call os._exit() from a finally block in order + # to suppress any finally blocks from earlier + # in the call stack (see bug #345289). This + # finally block has to be setup before the fork + # in order to avoid a race condition. os._exit(1) if not isinstance(pid, int): @@ -291,7 +352,6 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, # Add the pid to our local and the global pid lists. mypids.append(pid) - spawned_pids.append(pid) # If we started a tee process the write side of the pipe is no # longer needed, so close it. @@ -314,10 +374,6 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, # and wait for it. retval = os.waitpid(pid, 0)[1] - # When it's done, we can remove it from the - # global pid list as well. - spawned_pids.remove(pid) - if retval: # If it failed, kill off anything else that # isn't dead yet. @@ -328,7 +384,6 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, if os.waitpid(pid, os.WNOHANG)[0] == 0: os.kill(pid, signal.SIGTERM) os.waitpid(pid, 0) - spawned_pids.remove(pid) # If it got a signal, return the signal that was sent. if (retval & 0xff): @@ -341,11 +396,11 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, return 0 def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, - pre_exec): + pre_exec, close_fds, unshare_net, unshare_ipc, cgroup): """ Execute a given binary with options - + @param binary: Name of program to execute @type binary: String @param mycommand: Options for program @@ -366,10 +421,16 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, @type umask: Integer @param pre_exec: A function to be called with no arguments just prior to the exec call. @type pre_exec: callable + @param unshare_net: If True, networking will be unshared from the spawned process + @type unshare_net: Boolean + @param unshare_ipc: If True, IPC will be unshared from the spawned process + @type unshare_ipc: Boolean + @param cgroup: CGroup path to bind the process to + @type cgroup: String @rtype: None @return: Never returns (calls os.execve) """ - + # If the process we're creating hasn't been given a name # assign it the name of the executable. if not opt_name: @@ -384,6 +445,10 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, myargs = [opt_name] myargs.extend(mycommand[1:]) + # Avoid a potential UnicodeEncodeError from os.execve(). + myargs = [_unicode_encode(x, encoding=_encodings['fs'], + errors='strict') for x in myargs] + # Use default signal handlers in order to avoid problems # killing subprocesses as reported in bug #353239. signal.signal(signal.SIGINT, signal.SIG_DFL) @@ -396,15 +461,63 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, # the parent process (see bug #289486). signal.signal(signal.SIGQUIT, signal.SIG_DFL) - _setup_pipes(fd_pipes) + _setup_pipes(fd_pipes, close_fds=close_fds, inheritable=True) + + # Add to cgroup + # it's better to do it from the child since we can guarantee + # it is done before we start forking children + if cgroup: + with open(os.path.join(cgroup, 'cgroup.procs'), 'a') as f: + f.write('%d\n' % os.getpid()) + + # Unshare (while still uid==0) + if unshare_net or unshare_ipc: + filename = find_library("c") + if filename is not None: + libc = LoadLibrary(filename) + if libc is not None: + CLONE_NEWIPC = 0x08000000 + CLONE_NEWNET = 0x40000000 + + flags = 0 + if unshare_net: + flags |= CLONE_NEWNET + if unshare_ipc: + flags |= CLONE_NEWIPC + + try: + if libc.unshare(flags) != 0: + writemsg("Unable to unshare: %s\n" % ( + errno.errorcode.get(ctypes.get_errno(), '?')), + noiselevel=-1) + else: + if unshare_net: + # 'up' the loopback + IFF_UP = 0x1 + ifreq = struct.pack('16sh', b'lo', IFF_UP) + SIOCSIFFLAGS = 0x8914 + + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, 0) + try: + fcntl.ioctl(sock, SIOCSIFFLAGS, ifreq) + except IOError as e: + writemsg("Unable to enable loopback interface: %s\n" % ( + errno.errorcode.get(e.errno, '?')), + noiselevel=-1) + sock.close() + except AttributeError: + # unshare() not supported by libc + pass # Set requested process permissions. if gid: - os.setgid(gid) + # Cast proxies to int, in case it matters. + os.setgid(int(gid)) if groups: os.setgroups(groups) if uid: - os.setuid(uid) + # Cast proxies to int, in case it matters. + os.setuid(int(uid)) if umask: os.umask(umask) if pre_exec: @@ -413,9 +526,16 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, # And switch to the new process. os.execve(binary, myargs, env) -def _setup_pipes(fd_pipes, close_fds=True): +def _setup_pipes(fd_pipes, close_fds=True, inheritable=None): """Setup pipes for a forked process. + Even when close_fds is False, file descriptors referenced as + values in fd_pipes are automatically closed if they do not also + occur as keys in fd_pipes. It is assumed that the caller will + explicitly add them to the fd_pipes keys if they are intended + to remain open. This allows for convenient elimination of + unnecessary duplicate file descriptors. + WARNING: When not followed by exec, the close_fds behavior can trigger interference from destructors that close file descriptors. This interference happens when the garbage @@ -426,22 +546,92 @@ def _setup_pipes(fd_pipes, close_fds=True): and also with CPython under some circumstances (as triggered by xmpppy in bug #374335). In order to close a safe subset of file descriptors, see portage.locks._close_fds(). + + NOTE: When not followed by exec, even when close_fds is False, + it's still possible for dup2() calls to cause interference in a + way that's similar to the way that close_fds interferes (since + dup2() has to close the target fd if it happens to be open). + It's possible to avoid such interference by using allocated + file descriptors as the keys in fd_pipes. For example: + + pr, pw = os.pipe() + fd_pipes[pw] = pw + + By using the allocated pw file descriptor as the key in fd_pipes, + it's not necessary for dup2() to close a file descriptor (it + actually does nothing in this case), which avoids possible + interference. """ - my_fds = {} + + reverse_map = {} # To protect from cases where direct assignment could - # clobber needed fds ({1:2, 2:1}) we first dupe the fds - # into unused fds. - for fd in fd_pipes: - my_fds[fd] = os.dup(fd_pipes[fd]) - # Then assign them to what they should be. - for fd in my_fds: - os.dup2(my_fds[fd], fd) + # clobber needed fds ({1:2, 2:1}) we create a reverse map + # in order to know when it's necessary to create temporary + # backup copies with os.dup(). + for newfd, oldfd in fd_pipes.items(): + newfds = reverse_map.get(oldfd) + if newfds is None: + newfds = [] + reverse_map[oldfd] = newfds + newfds.append(newfd) + + # Assign newfds via dup2(), making temporary backups when + # necessary, and closing oldfd if the caller has not + # explicitly requested for it to remain open by adding + # it to the keys of fd_pipes. + while reverse_map: + + oldfd, newfds = reverse_map.popitem() + old_fdflags = None + + for newfd in newfds: + if newfd in reverse_map: + # Make a temporary backup before re-assignment, assuming + # that backup_fd won't collide with a key in reverse_map + # (since all of the keys correspond to open file + # descriptors, and os.dup() only allocates a previously + # unused file discriptors). + backup_fd = os.dup(newfd) + reverse_map[backup_fd] = reverse_map.pop(newfd) + + if oldfd != newfd: + os.dup2(oldfd, newfd) + if _set_inheritable is not None: + # Don't do this unless _set_inheritable is available, + # since it's used below to ensure correct state, and + # otherwise /dev/null stdin fails to inherit (at least + # with Python versions from 3.1 to 3.3). + if old_fdflags is None: + old_fdflags = fcntl.fcntl(oldfd, fcntl.F_GETFD) + fcntl.fcntl(newfd, fcntl.F_SETFD, old_fdflags) + + if _set_inheritable is not None: + + inheritable_state = None + if not (old_fdflags is None or _FD_CLOEXEC is None): + inheritable_state = not bool(old_fdflags & _FD_CLOEXEC) + + if inheritable is not None: + if inheritable_state is not inheritable: + _set_inheritable(newfd, inheritable) + + elif newfd in (0, 1, 2): + if inheritable_state is not True: + _set_inheritable(newfd, True) + + if oldfd not in fd_pipes: + # If oldfd is not a key in fd_pipes, then it's safe + # to close now, since we've already made all of the + # requested duplicates. This also closes every + # backup_fd that may have been created on previous + # iterations of this loop. + os.close(oldfd) if close_fds: # Then close _all_ fds that haven't been explicitly # requested to be kept open. for fd in get_open_fds(): - if fd not in my_fds: + if fd not in fd_pipes: try: os.close(fd) except OSError: @@ -450,14 +640,22 @@ def _setup_pipes(fd_pipes, close_fds=True): def find_binary(binary): """ Given a binary name, find the binary in PATH - + @param binary: Name of the binary to find @type string @rtype: None or string @return: full path to binary or None if the binary could not be located. """ - for path in os.environ.get("PATH", "").split(":"): - filename = "%s/%s" % (path, binary) - if os.access(filename, os.X_OK) and os.path.isfile(filename): + paths = os.environ.get("PATH", "") + if sys.hexversion >= 0x3000000 and isinstance(binary, bytes): + # return bytes when input is bytes + paths = paths.encode(sys.getfilesystemencoding(), 'surrogateescape') + paths = paths.split(b':') + else: + paths = paths.split(':') + + for path in paths: + filename = _os.path.join(path, binary) + if _os.access(filename, os.X_OK) and _os.path.isfile(filename): return filename return None diff --git a/portage_with_autodep/pym/portage/process.pyo b/portage_with_autodep/pym/portage/process.pyo Binary files differindex c53af30..323d956 100644 --- a/portage_with_autodep/pym/portage/process.pyo +++ b/portage_with_autodep/pym/portage/process.pyo diff --git a/portage_with_autodep/pym/portage/proxy/__init__.pyo b/portage_with_autodep/pym/portage/proxy/__init__.pyo Binary files differindex b3d096b..69a2a54 100644 --- a/portage_with_autodep/pym/portage/proxy/__init__.pyo +++ b/portage_with_autodep/pym/portage/proxy/__init__.pyo diff --git a/portage_with_autodep/pym/portage/proxy/lazyimport.py b/portage_with_autodep/pym/portage/proxy/lazyimport.py index ad4a542..3057c05 100644 --- a/portage_with_autodep/pym/portage/proxy/lazyimport.py +++ b/portage_with_autodep/pym/portage/proxy/lazyimport.py @@ -1,4 +1,4 @@ -# Copyright 2009 Gentoo Foundation +# Copyright 2009-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ['lazyimport'] @@ -32,7 +32,7 @@ def _preload_portage_submodules(): while True: remaining = False for name in list(_module_proxies): - if name.startswith('portage.'): + if name.startswith('portage.') or name.startswith('_emerge.'): if name in imported: continue imported.add(name) diff --git a/portage_with_autodep/pym/portage/proxy/lazyimport.pyo b/portage_with_autodep/pym/portage/proxy/lazyimport.pyo Binary files differindex 9da8089..cf4e80f 100644 --- a/portage_with_autodep/pym/portage/proxy/lazyimport.pyo +++ b/portage_with_autodep/pym/portage/proxy/lazyimport.pyo diff --git a/portage_with_autodep/pym/portage/proxy/objectproxy.py b/portage_with_autodep/pym/portage/proxy/objectproxy.py index 92b36d1..a755774 100644 --- a/portage_with_autodep/pym/portage/proxy/objectproxy.py +++ b/portage_with_autodep/pym/portage/proxy/objectproxy.py @@ -1,4 +1,4 @@ -# Copyright 2008-2009 Gentoo Foundation +# Copyright 2008-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import sys @@ -30,6 +30,13 @@ class ObjectProxy(object): result = object.__getattribute__(self, '_get_target')() return result(*args, **kwargs) + def __enter__(self): + return object.__getattribute__(self, '_get_target')().__enter__() + + def __exit__(self, exc_type, exc_value, traceback): + return object.__getattribute__(self, '_get_target')().__exit__( + exc_type, exc_value, traceback) + def __setitem__(self, key, value): object.__getattribute__(self, '_get_target')()[key] = value diff --git a/portage_with_autodep/pym/portage/proxy/objectproxy.pyo b/portage_with_autodep/pym/portage/proxy/objectproxy.pyo Binary files differindex f0919ff..a7c100a 100644 --- a/portage_with_autodep/pym/portage/proxy/objectproxy.pyo +++ b/portage_with_autodep/pym/portage/proxy/objectproxy.pyo diff --git a/portage_with_autodep/pym/portage/repository/__init__.pyo b/portage_with_autodep/pym/portage/repository/__init__.pyo Binary files differindex ab526c0..d98d5dc 100644 --- a/portage_with_autodep/pym/portage/repository/__init__.pyo +++ b/portage_with_autodep/pym/portage/repository/__init__.pyo diff --git a/portage_with_autodep/pym/portage/repository/config.py b/portage_with_autodep/pym/portage/repository/config.py index 20f1919..0d6edf4 100644 --- a/portage_with_autodep/pym/portage/repository/config.py +++ b/portage_with_autodep/pym/portage/repository/config.py @@ -1,6 +1,8 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + import io import logging import warnings @@ -8,25 +10,34 @@ import sys import re try: - from configparser import ParsingError + from configparser import Error as ConfigParserError if sys.hexversion >= 0x3020000: from configparser import ConfigParser as SafeConfigParser else: from configparser import SafeConfigParser except ImportError: - from ConfigParser import SafeConfigParser, ParsingError + from ConfigParser import SafeConfigParser, Error as ConfigParserError +import portage from portage import eclass_cache, os from portage.const import (MANIFEST2_HASH_FUNCTIONS, MANIFEST2_REQUIRED_HASH, - REPO_NAME_LOC, USER_CONFIG_PATH) + PORTAGE_BASE_PATH, REPO_NAME_LOC, USER_CONFIG_PATH) +from portage.eapi import eapi_allows_directories_on_profile_level_and_repository_level from portage.env.loaders import KeyValuePairFileLoader from portage.util import (normalize_path, read_corresponding_eapi_file, shlex_split, - stack_lists, writemsg, writemsg_level) + stack_lists, writemsg, writemsg_level, _recursive_file_list) +from portage.util._path import exists_raise_eaccess, isdir_raise_eaccess from portage.localization import _ from portage import _unicode_decode from portage import _unicode_encode from portage import _encodings from portage import manifest +if sys.hexversion >= 0x3000000: + basestring = str + +# Characters prohibited by repoman's file.name check. +_invalid_path_char_re = re.compile(r'[^a-zA-Z0-9._\-+:/]') + _valid_profile_formats = frozenset( ['pms', 'portage-1', 'portage-2']) @@ -48,38 +59,76 @@ def _gen_valid_repo(name): name = None return name +def _find_invalid_path_char(path, pos=0, endpos=None): + """ + Returns the position of the first invalid character found in basename, + or -1 if no invalid characters are found. + """ + if endpos is None: + endpos = len(path) + + m = _invalid_path_char_re.search(path, pos=pos, endpos=endpos) + if m is not None: + return m.start() + + return -1 + class RepoConfig(object): """Stores config of one repository""" __slots__ = ('aliases', 'allow_missing_manifest', 'allow_provide_virtual', 'cache_formats', 'create_manifest', 'disable_manifest', 'eapi', - 'eclass_db', 'eclass_locations', 'eclass_overrides', 'format', 'location', + 'eclass_db', 'eclass_locations', 'eclass_overrides', + 'find_invalid_path_char', 'force', 'format', 'local_config', 'location', 'main_repo', 'manifest_hashes', 'masters', 'missing_repo_name', 'name', 'portage1_profiles', 'portage1_profiles_compat', 'priority', - 'profile_formats', 'sign_commit', 'sign_manifest', 'sync', - 'thin_manifest', 'update_changelog', 'user_location') + 'profile_formats', 'sign_commit', 'sign_manifest', 'sync_cvs_repo', + 'sync_type', 'sync_uri', 'thin_manifest', 'update_changelog', + 'user_location', '_eapis_banned', '_eapis_deprecated', '_masters_orig') - def __init__(self, name, repo_opts): + def __init__(self, name, repo_opts, local_config=True): """Build a RepoConfig with options in repo_opts Try to read repo_name in repository location, but if it is not found use variable name as repository name""" - aliases = repo_opts.get('aliases') - if aliases is not None: - aliases = tuple(aliases.split()) + + force = repo_opts.get('force') + if force is not None: + force = tuple(force.split()) + self.force = force + if force is None: + force = () + + self.local_config = local_config + + if local_config or 'aliases' in force: + aliases = repo_opts.get('aliases') + if aliases is not None: + aliases = tuple(aliases.split()) + else: + aliases = None + self.aliases = aliases - eclass_overrides = repo_opts.get('eclass-overrides') - if eclass_overrides is not None: - eclass_overrides = tuple(eclass_overrides.split()) + if local_config or 'eclass-overrides' in force: + eclass_overrides = repo_opts.get('eclass-overrides') + if eclass_overrides is not None: + eclass_overrides = tuple(eclass_overrides.split()) + else: + eclass_overrides = None + self.eclass_overrides = eclass_overrides # Eclass databases and locations are computed later. self.eclass_db = None self.eclass_locations = None - # Masters from repos.conf override layout.conf. - masters = repo_opts.get('masters') - if masters is not None: - masters = tuple(masters.split()) + if local_config or 'masters' in force: + # Masters from repos.conf override layout.conf. + masters = repo_opts.get('masters') + if masters is not None: + masters = tuple(masters.split()) + else: + masters = None + self.masters = masters #The main-repo key makes only sense for the 'DEFAULT' section. @@ -93,11 +142,22 @@ class RepoConfig(object): priority = None self.priority = priority - sync = repo_opts.get('sync') - if sync is not None: - sync = sync.strip() - self.sync = sync + sync_cvs_repo = repo_opts.get('sync-cvs-repo') + if sync_cvs_repo is not None: + sync_cvs_repo = sync_cvs_repo.strip() + self.sync_cvs_repo = sync_cvs_repo or None + + sync_type = repo_opts.get('sync-type') + if sync_type is not None: + sync_type = sync_type.strip() + self.sync_type = sync_type or None + sync_uri = repo_opts.get('sync-uri') + if sync_uri is not None: + sync_uri = sync_uri.strip() + self.sync_uri = sync_uri or None + + # Not implemented. format = repo_opts.get('format') if format is not None: format = format.strip() @@ -106,7 +166,7 @@ class RepoConfig(object): location = repo_opts.get('location') self.user_location = location if location is not None and location.strip(): - if os.path.isdir(location): + if os.path.isdir(location) or portage._sync_disabled_warnings: location = os.path.realpath(location) else: location = None @@ -114,14 +174,23 @@ class RepoConfig(object): eapi = None missing = True + self.name = name if self.location is not None: eapi = read_corresponding_eapi_file(os.path.join(self.location, REPO_NAME_LOC)) - name, missing = self._read_valid_repo_name(self.location) - elif name == "DEFAULT": + self.name, missing = self._read_valid_repo_name(self.location) + if missing: + # The name from repos.conf has to be used here for + # things like emerge-webrsync to work when the repo + # is empty (bug #484950). + if name is not None: + self.name = name + if portage._sync_disabled_warnings: + missing = False + + elif name == "DEFAULT": missing = False self.eapi = eapi - self.name = name self.missing_repo_name = missing # sign_commit is disabled by default, since it requires Git >=1.7.9, # and key_id configured by `git config user.signingkey key_id` @@ -137,18 +206,20 @@ class RepoConfig(object): self.cache_formats = None self.portage1_profiles = True self.portage1_profiles_compat = False + self.find_invalid_path_char = _find_invalid_path_char + self._masters_orig = None # Parse layout.conf. if self.location: - layout_filename = os.path.join(self.location, "metadata", "layout.conf") layout_data = parse_layout_conf(self.location, self.name)[0] + self._masters_orig = layout_data['masters'] # layout.conf masters may be overridden here if we have a masters # setting from the user's repos.conf if self.masters is None: self.masters = layout_data['masters'] - if layout_data['aliases']: + if (local_config or 'aliases' in force) and layout_data['aliases']: aliases = self.aliases if aliases is None: aliases = () @@ -156,6 +227,12 @@ class RepoConfig(object): # them the ability to do incremental overrides self.aliases = layout_data['aliases'] + tuple(aliases) + if layout_data['repo-name']: + # allow layout.conf to override repository name + # useful when having two copies of the same repo enabled + # to avoid modifying profiles/repo_name in one of them + self.name = layout_data['repo-name'] + for value in ('allow-missing-manifest', 'allow-provide-virtual', 'cache-formats', 'create-manifest', 'disable-manifest', 'manifest-hashes', @@ -163,9 +240,19 @@ class RepoConfig(object): 'sign-commit', 'sign-manifest', 'thin-manifest', 'update-changelog'): setattr(self, value.lower().replace("-", "_"), layout_data[value]) - self.portage1_profiles = any(x in _portage1_profiles_allow_directories - for x in layout_data['profile-formats']) - self.portage1_profiles_compat = layout_data['profile-formats'] == ('portage-1-compat',) + self.portage1_profiles = eapi_allows_directories_on_profile_level_and_repository_level(eapi) or \ + any(x in _portage1_profiles_allow_directories for x in layout_data['profile-formats']) + self.portage1_profiles_compat = not eapi_allows_directories_on_profile_level_and_repository_level(eapi) and \ + layout_data['profile-formats'] == ('portage-1-compat',) + + self._eapis_banned = frozenset(layout_data['eapis-banned']) + self._eapis_deprecated = frozenset(layout_data['eapis-deprecated']) + + def eapi_is_banned(self, eapi): + return eapi in self._eapis_banned + + def eapi_is_deprecated(self, eapi): + return eapi in self._eapis_deprecated def iter_pregenerated_caches(self, auxdbkeys, readonly=True, force=False): """ @@ -178,7 +265,11 @@ class RepoConfig(object): if not formats: if not force: return - formats = ('pms',) + # The default egencache format was 'pms' prior to portage-2.1.11.32 + # (portage versions prior to portage-2.1.11.14 will NOT + # recognize md5-dict format unless it is explicitly listed in + # layout.conf). + formats = ('md5-dict',) for fmt in formats: name = None @@ -209,7 +300,8 @@ class RepoConfig(object): kwds['hashes'] = self.manifest_hashes if self.disable_manifest: kwds['from_scratch'] = True - return manifest.Manifest(*args, **kwds) + kwds['find_invalid_path_char'] = self.find_invalid_path_char + return manifest.Manifest(*args, **portage._native_kwargs(kwds)) def update(self, new_repo): """Update repository with options in another RepoConfig""" @@ -272,8 +364,12 @@ class RepoConfig(object): repo_msg.append(indent + "format: " + self.format) if self.user_location: repo_msg.append(indent + "location: " + self.user_location) - if self.sync: - repo_msg.append(indent + "sync: " + self.sync) + if self.sync_cvs_repo: + repo_msg.append(indent + "sync-cvs-repo: " + self.sync_cvs_repo) + if self.sync_type: + repo_msg.append(indent + "sync-type: " + self.sync_type) + if self.sync_uri: + repo_msg.append(indent + "sync-uri: " + self.sync_uri) if self.masters: repo_msg.append(indent + "masters: " + " ".join(master.name for master in self.masters)) if self.priority is not None: @@ -281,19 +377,19 @@ class RepoConfig(object): if self.aliases: repo_msg.append(indent + "aliases: " + " ".join(self.aliases)) if self.eclass_overrides: - repo_msg.append(indent + "eclass_overrides: " + \ + repo_msg.append(indent + "eclass-overrides: " + \ " ".join(self.eclass_overrides)) repo_msg.append("") return "\n".join(repo_msg) def __repr__(self): - return "<portage.repository.config.RepoConfig(name='%s', location='%s')>" % (self.name, _unicode_decode(self.location)) + return "<portage.repository.config.RepoConfig(name=%r, location=%r)>" % (self.name, _unicode_decode(self.location)) def __str__(self): d = {} for k in self.__slots__: d[k] = getattr(self, k, None) - return _unicode_decode("%s") % (d,) + return "%s" % (d,) if sys.hexversion < 0x3000000: @@ -306,11 +402,14 @@ class RepoConfigLoader(object): """Loads and store config of several repositories, loaded from PORTDIR_OVERLAY or repos.conf""" @staticmethod - def _add_repositories(portdir, portdir_overlay, prepos, ignored_map, ignored_location_map): + def _add_repositories(portdir, portdir_overlay, prepos, + ignored_map, ignored_location_map, local_config, default_portdir): """Add overlays in PORTDIR_OVERLAY as repositories""" overlays = [] + portdir_orig = None if portdir: portdir = normalize_path(portdir) + portdir_orig = portdir overlays.append(portdir) try: port_ov = [normalize_path(i) for i in shlex_split(portdir_overlay)] @@ -344,43 +443,57 @@ class RepoConfigLoader(object): #overlay priority is negative because we want them to be looked before any other repo base_priority = 0 for ov in overlays: - if os.path.isdir(ov): + # Ignore missing directory for 'gentoo' so that + # first sync with emerge-webrsync is possible. + if isdir_raise_eaccess(ov) or \ + (base_priority == 0 and ov is portdir): repo_opts = default_repo_opts.copy() repo_opts['location'] = ov - repo = RepoConfig(None, repo_opts) + repo = RepoConfig(None, repo_opts, local_config=local_config) # repos_conf_opts contains options from repos.conf repos_conf_opts = repos_conf.get(repo.name) if repos_conf_opts is not None: # Selectively copy only the attributes which # repos.conf is allowed to override. - for k in ('aliases', 'eclass_overrides', 'masters', 'priority'): + for k in ('aliases', 'eclass_overrides', 'force', 'masters', + 'priority', 'sync_cvs_repo', 'sync_type', 'sync_uri'): v = getattr(repos_conf_opts, k, None) if v is not None: setattr(repo, k, v) if repo.name in prepos: + # Silently ignore when PORTDIR overrides the location + # setting from the default repos.conf (bug #478544). old_location = prepos[repo.name].location - if old_location is not None and old_location != repo.location: + if old_location is not None and \ + old_location != repo.location and \ + not (base_priority == 0 and + old_location == default_portdir): ignored_map.setdefault(repo.name, []).append(old_location) ignored_location_map[old_location] = repo.name if old_location == portdir: portdir = repo.user_location - if ov == portdir and portdir not in port_ov: - repo.priority = -1000 - elif repo.priority is None: - repo.priority = base_priority - base_priority += 1 + if repo.priority is None: + if base_priority == 0 and ov == portdir_orig: + # If it's the original PORTDIR setting and it's not + # in PORTDIR_OVERLAY, then it will be assigned a + # special priority setting later. + pass + else: + repo.priority = base_priority + base_priority += 1 prepos[repo.name] = repo else: - writemsg(_("!!! Invalid PORTDIR_OVERLAY" - " (not a dir): '%s'\n") % ov, noiselevel=-1) + + if not portage._sync_disabled_warnings: + writemsg(_("!!! Invalid PORTDIR_OVERLAY (not a dir): '%s'\n") % ov, noiselevel=-1) return portdir @staticmethod - def _parse(paths, prepos, ignored_map, ignored_location_map): + def _parse(paths, prepos, ignored_map, ignored_location_map, local_config, portdir): """Parse files in paths to load config""" parser = SafeConfigParser() @@ -388,49 +501,78 @@ class RepoConfigLoader(object): try: # Python >=3.2 read_file = parser.read_file + source_kwarg = 'source' except AttributeError: read_file = parser.readfp + source_kwarg = 'filename' + recursive_paths = [] for p in paths: - f = None - try: - f = io.open(_unicode_encode(p, - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content'], - errors='replace') - except EnvironmentError: - pass + if isinstance(p, basestring): + recursive_paths.extend(_recursive_file_list(p)) else: + recursive_paths.append(p) + + for p in recursive_paths: + if isinstance(p, basestring): + f = None try: - read_file(f) - except ParsingError as e: - writemsg(_unicode_decode( - _("!!! Error while reading repo config file: %s\n") - ) % e, noiselevel=-1) - finally: - if f is not None: - f.close() - - prepos['DEFAULT'] = RepoConfig("DEFAULT", parser.defaults()) + f = io.open(_unicode_encode(p, + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') + except EnvironmentError: + pass + else: + # The 'source' keyword argument is needed since otherwise + # ConfigParser in Python <3.3.3 may throw a TypeError + # because it assumes that f.name is a native string rather + # than binary when constructing error messages. + kwargs = {source_kwarg: p} + read_file(f, **portage._native_kwargs(kwargs)) + finally: + if f is not None: + f.close() + elif isinstance(p, io.StringIO): + kwargs = {source_kwarg: "<io.StringIO>"} + read_file(p, **portage._native_kwargs(kwargs)) + else: + raise TypeError("Unsupported type %r of element %r of 'paths' argument" % (type(p), p)) + + prepos['DEFAULT'] = RepoConfig("DEFAULT", + parser.defaults(), local_config=local_config) + for sname in parser.sections(): optdict = {} for oname in parser.options(sname): optdict[oname] = parser.get(sname, oname) - repo = RepoConfig(sname, optdict) - if repo.location and not os.path.exists(repo.location): - writemsg(_("!!! Invalid repos.conf entry '%s'" - " (not a dir): '%s'\n") % (sname, repo.location), noiselevel=-1) + repo = RepoConfig(sname, optdict, local_config=local_config) + + if repo.sync_type is not None and repo.sync_uri is None: + writemsg_level("!!! %s\n" % _("Repository '%s' has sync-type attribute, but is missing sync-uri attribute") % + sname, level=logging.ERROR, noiselevel=-1) continue - if repo.name in prepos: - old_location = prepos[repo.name].location - if old_location is not None and repo.location is not None and old_location != repo.location: - ignored_map.setdefault(repo.name, []).append(old_location) - ignored_location_map[old_location] = repo.name - prepos[repo.name].update(repo) - else: - prepos[repo.name] = repo + if repo.sync_uri is not None and repo.sync_type is None: + writemsg_level("!!! %s\n" % _("Repository '%s' has sync-uri attribute, but is missing sync-type attribute") % + sname, level=logging.ERROR, noiselevel=-1) + continue + + if repo.sync_type not in (None, "cvs", "git", "rsync"): + writemsg_level("!!! %s\n" % _("Repository '%s' has sync-type attribute set to unsupported value: '%s'") % + (sname, repo.sync_type), level=logging.ERROR, noiselevel=-1) + continue + + if repo.sync_type == "cvs" and repo.sync_cvs_repo is None: + writemsg_level("!!! %s\n" % _("Repository '%s' has sync-type=cvs, but is missing sync-cvs-repo attribute") % + sname, level=logging.ERROR, noiselevel=-1) + continue + + # For backward compatibility with locations set via PORTDIR and + # PORTDIR_OVERLAY, delay validation of the location and repo.name + # until after PORTDIR and PORTDIR_OVERLAY have been processed. + prepos[sname] = repo def __init__(self, paths, settings): """Load config from files in paths""" @@ -441,15 +583,42 @@ class RepoConfigLoader(object): ignored_map = {} ignored_location_map = {} - portdir = settings.get('PORTDIR', '') - portdir_overlay = settings.get('PORTDIR_OVERLAY', '') + if "PORTAGE_REPOSITORIES" in settings: + portdir = "" + portdir_overlay = "" + portdir_sync = "" + else: + portdir = settings.get("PORTDIR", "") + portdir_overlay = settings.get("PORTDIR_OVERLAY", "") + portdir_sync = settings.get("SYNC", "") - self._parse(paths, prepos, ignored_map, ignored_location_map) + try: + self._parse(paths, prepos, ignored_map, + ignored_location_map, settings.local_config, + portdir) + except ConfigParserError as e: + writemsg( + _("!!! Error while reading repo config file: %s\n") % e, + noiselevel=-1) + # The configparser state is unreliable (prone to quirky + # exceptions) after it has thrown an error, so use empty + # config and try to fall back to PORTDIR{,_OVERLAY}. + prepos.clear() + prepos['DEFAULT'] = RepoConfig('DEFAULT', + {}, local_config=settings.local_config) + location_map.clear() + treemap.clear() + ignored_map.clear() + ignored_location_map.clear() + + default_portdir = os.path.join(os.sep, + settings['EPREFIX'].lstrip(os.sep), 'usr', 'portage') # If PORTDIR_OVERLAY contains a repo with the same repo_name as # PORTDIR, then PORTDIR is overridden. portdir = self._add_repositories(portdir, portdir_overlay, prepos, - ignored_map, ignored_location_map) + ignored_map, ignored_location_map, settings.local_config, + default_portdir) if portdir and portdir.strip(): portdir = os.path.realpath(portdir) @@ -460,9 +629,51 @@ class RepoConfigLoader(object): for repo in prepos.values() if repo.location is not None and repo.missing_repo_name) - #Take aliases into account. - new_prepos = {} - for repo_name, repo in prepos.items(): + # Do this before expanding aliases, so that location_map and + # treemap consistently map unaliased names whenever available. + for repo_name, repo in list(prepos.items()): + if repo.location is None: + if repo_name != 'DEFAULT': + # Skip this warning for repoman (bug #474578). + if settings.local_config and paths: + writemsg_level("!!! %s\n" % _("Section '%s' in repos.conf is missing location attribute") % + repo.name, level=logging.ERROR, noiselevel=-1) + del prepos[repo_name] + continue + else: + if not portage._sync_disabled_warnings: + if not isdir_raise_eaccess(repo.location): + writemsg_level("!!! %s\n" % _("Section '%s' in repos.conf has location attribute set " + "to nonexistent directory: '%s'") % + (repo_name, repo.location), level=logging.ERROR, noiselevel=-1) + + # Ignore missing directory for 'gentoo' so that + # first sync with emerge-webrsync is possible. + if repo.name != 'gentoo': + del prepos[repo_name] + continue + + # After removing support for PORTDIR_OVERLAY, the following check can be: + # if repo.missing_repo_name: + if repo.missing_repo_name and repo.name != repo_name: + writemsg_level("!!! %s\n" % _("Section '%s' in repos.conf refers to repository " + "without repository name set in '%s'") % + (repo_name, os.path.join(repo.location, REPO_NAME_LOC)), level=logging.ERROR, noiselevel=-1) + del prepos[repo_name] + continue + + if repo.name != repo_name: + writemsg_level("!!! %s\n" % _("Section '%s' in repos.conf has name different " + "from repository name '%s' set inside repository") % + (repo_name, repo.name), level=logging.ERROR, noiselevel=-1) + del prepos[repo_name] + continue + + location_map[repo.location] = repo_name + treemap[repo_name] = repo.location + + # Add alias mappings, but never replace unaliased mappings. + for repo_name, repo in list(prepos.items()): names = set() names.add(repo_name) if repo.aliases: @@ -470,36 +681,55 @@ class RepoConfigLoader(object): names.update(aliases) for name in names: - if name in new_prepos: + if name in prepos and prepos[name].location is not None: + if name == repo_name: + # unaliased names already handled earlier + continue writemsg_level(_("!!! Repository name or alias '%s', " + \ "defined for repository '%s', overrides " + \ "existing alias or repository.\n") % (name, repo_name), level=logging.WARNING, noiselevel=-1) - new_prepos[name] = repo - prepos = new_prepos + # Never replace an unaliased mapping with + # an aliased mapping. + continue + prepos[name] = repo + if repo.location is not None: + if repo.location not in location_map: + # Never replace an unaliased mapping with + # an aliased mapping. + location_map[repo.location] = name + treemap[name] = repo.location + + main_repo = prepos['DEFAULT'].main_repo + if main_repo is None or main_repo not in prepos: + #setting main_repo if it was not set in repos.conf + main_repo = location_map.get(portdir) + if main_repo is not None: + prepos['DEFAULT'].main_repo = main_repo + else: + prepos['DEFAULT'].main_repo = None + if portdir and not portage._sync_disabled_warnings: + writemsg(_("!!! main-repo not set in DEFAULT and PORTDIR is empty.\n"), noiselevel=-1) - for (name, r) in prepos.items(): - if r.location is not None: - location_map[r.location] = name - treemap[name] = r.location + if main_repo is not None and prepos[main_repo].priority is None: + # This happens if main-repo has been set in repos.conf. + prepos[main_repo].priority = -1000 - # filter duplicates from aliases, by only including - # items where repo.name == key + # Backward compatible SYNC support for mirrorselect. + if portdir_sync and main_repo is not None: + if portdir_sync.startswith("rsync://"): + prepos[main_repo].sync_uri = portdir_sync + prepos[main_repo].sync_type = "rsync" - prepos_order = sorted(prepos.items(), key=lambda r:r[1].priority or 0) + # Include repo.name in sort key, for predictable sorting + # even when priorities are equal. + prepos_order = sorted(prepos.items(), + key=lambda r:(r[1].priority or 0, r[1].name)) + # filter duplicates from aliases, by only including + # items where repo.name == key prepos_order = [repo.name for (key, repo) in prepos_order - if repo.name == key and repo.location is not None] - - if prepos['DEFAULT'].main_repo is None or \ - prepos['DEFAULT'].main_repo not in prepos: - #setting main_repo if it was not set in repos.conf - if portdir in location_map: - prepos['DEFAULT'].main_repo = location_map[portdir] - elif portdir in ignored_location_map: - prepos['DEFAULT'].main_repo = ignored_location_map[portdir] - else: - prepos['DEFAULT'].main_repo = None - writemsg(_("!!! main-repo not set in DEFAULT and PORTDIR is empty. \n"), noiselevel=-1) + if repo.name == key and key != 'DEFAULT' and + repo.location is not None] self.prepos = prepos self.prepos_order = prepos_order @@ -578,6 +808,17 @@ class RepoConfigLoader(object): eclass_db.append(tree_db) repo.eclass_db = eclass_db + for repo_name, repo in prepos.items(): + if repo_name == "DEFAULT": + continue + + if repo._masters_orig is None and self.mainRepo() and \ + repo.name != self.mainRepo().name and not portage._sync_disabled_warnings: + writemsg_level("!!! %s\n" % _("Repository '%s' is missing masters attribute in '%s'") % + (repo.name, os.path.join(repo.location, "metadata", "layout.conf")) + + "!!! %s\n" % _("Set 'masters = %s' in this file for future compatibility") % + self.mainRepo().name, level=logging.WARNING, noiselevel=-1) + self._prepos_changed = True self._repo_location_list = [] @@ -613,10 +854,10 @@ class RepoConfigLoader(object): def mainRepo(self): """Returns the main repo""" - maid_repo = self.prepos['DEFAULT'].main_repo - if maid_repo is None: + main_repo = self.prepos['DEFAULT'].main_repo + if main_repo is None: return None - return self.prepos[maid_repo] + return self.prepos[main_repo] def _check_locations(self): """Check if repositories location are correct and show a warning message if not""" @@ -625,7 +866,7 @@ class RepoConfigLoader(object): if r.location is None: writemsg(_("!!! Location not set for repository %s\n") % name, noiselevel=-1) else: - if not os.path.isdir(r.location): + if not isdir_raise_eaccess(r.location) and not portage._sync_disabled_warnings: self.prepos_order.remove(name) writemsg(_("!!! Invalid Repository Location" " (not a dir): '%s'\n") % r.location, noiselevel=-1) @@ -650,19 +891,66 @@ class RepoConfigLoader(object): def get_repo_for_location(self, location): return self.prepos[self.get_name_for_location(location)] + def __setitem__(self, repo_name, repo): + # self.prepos[repo_name] = repo + raise NotImplementedError + def __getitem__(self, repo_name): return self.prepos[repo_name] + def __delitem__(self, repo_name): + if repo_name == self.prepos['DEFAULT'].main_repo: + self.prepos['DEFAULT'].main_repo = None + location = self.prepos[repo_name].location + del self.prepos[repo_name] + if repo_name in self.prepos_order: + self.prepos_order.remove(repo_name) + for k, v in self.location_map.copy().items(): + if v == repo_name: + del self.location_map[k] + if repo_name in self.treemap: + del self.treemap[repo_name] + self._repo_location_list = tuple(x for x in self._repo_location_list if x != location) + def __iter__(self): for repo_name in self.prepos_order: yield self.prepos[repo_name] -def load_repository_config(settings): - #~ repoconfigpaths = [os.path.join(settings.global_config_path, "repos.conf")] + def __contains__(self, repo_name): + return repo_name in self.prepos + + def config_string(self): + str_or_int_keys = ("format", "location", "main_repo", "priority", "sync_cvs_repo", "sync_type", "sync_uri") + str_tuple_keys = ("aliases", "eclass_overrides", "force") + repo_config_tuple_keys = ("masters",) + keys = str_or_int_keys + str_tuple_keys + repo_config_tuple_keys + config_string = "" + for repo_name, repo in sorted(self.prepos.items()): + config_string += "\n[%s]\n" % repo_name + for key in sorted(keys): + if key == "main_repo" and repo_name != "DEFAULT": + continue + if getattr(repo, key) is not None: + if key in str_or_int_keys: + config_string += "%s = %s\n" % (key.replace("_", "-"), getattr(repo, key)) + elif key in str_tuple_keys: + config_string += "%s = %s\n" % (key.replace("_", "-"), " ".join(getattr(repo, key))) + elif key in repo_config_tuple_keys: + config_string += "%s = %s\n" % (key.replace("_", "-"), " ".join(x.name for x in getattr(repo, key))) + return config_string.lstrip("\n") + +def load_repository_config(settings, extra_files=None): repoconfigpaths = [] - if settings.local_config: - repoconfigpaths.append(os.path.join(settings["PORTAGE_CONFIGROOT"], - USER_CONFIG_PATH, "repos.conf")) + if "PORTAGE_REPOSITORIES" in settings: + repoconfigpaths.append(io.StringIO(settings["PORTAGE_REPOSITORIES"])) + else: + if portage._working_copy: + repoconfigpaths.append(os.path.join(PORTAGE_BASE_PATH, "cnf", "repos.conf")) + else: + repoconfigpaths.append(os.path.join(settings.global_config_path, "repos.conf")) + repoconfigpaths.append(os.path.join(settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH, "repos.conf")) + if extra_files: + repoconfigpaths.extend(extra_files) return RepoConfigLoader(repoconfigpaths, settings) def _get_repo_name(repo_location, cached=None): @@ -696,6 +984,9 @@ def parse_layout_conf(repo_location, repo_name=None): data['allow-provide-virtual'] = \ layout_data.get('allow-provide-virtuals', 'false').lower() == 'true' + data['eapis-banned'] = tuple(layout_data.get('eapis-banned', '').split()) + data['eapis-deprecated'] = tuple(layout_data.get('eapis-deprecated', '').split()) + data['sign-commit'] = layout_data.get('sign-commits', 'false').lower() \ == 'true' @@ -705,6 +996,8 @@ def parse_layout_conf(repo_location, repo_name=None): data['thin-manifest'] = layout_data.get('thin-manifests', 'false').lower() \ == 'true' + data['repo-name'] = _gen_valid_repo(layout_data.get('repo-name', '')) + manifest_policy = layout_data.get('use-manifests', 'strict').lower() data['allow-missing-manifest'] = manifest_policy != 'strict' data['create-manifest'] = manifest_policy != 'false' @@ -712,10 +1005,19 @@ def parse_layout_conf(repo_location, repo_name=None): # for compatibility w/ PMS, fallback to pms; but also check if the # cache exists or not. - cache_formats = layout_data.get('cache-formats', 'pms').lower().split() - if 'pms' in cache_formats and not os.path.isdir( - os.path.join(repo_location, 'metadata', 'cache')): - cache_formats.remove('pms') + cache_formats = layout_data.get('cache-formats', '').lower().split() + if not cache_formats: + # Auto-detect cache formats, and prefer md5-cache if available. + # This behavior was deployed in portage-2.1.11.14, so that the + # default egencache format could eventually be changed to md5-dict + # in portage-2.1.11.32. WARNING: Versions prior to portage-2.1.11.14 + # will NOT recognize md5-dict format unless it is explicitly + # listed in layout.conf. + cache_formats = [] + if os.path.isdir(os.path.join(repo_location, 'metadata', 'md5-cache')): + cache_formats.append('md5-dict') + if os.path.isdir(os.path.join(repo_location, 'metadata', 'cache')): + cache_formats.append('pms') data['cache-formats'] = tuple(cache_formats) manifest_hashes = layout_data.get('manifest-hashes') @@ -754,7 +1056,7 @@ def parse_layout_conf(repo_location, repo_name=None): raw_formats = layout_data.get('profile-formats') if raw_formats is None: - if eapi in ('4-python',): + if eapi_allows_directories_on_profile_level_and_repository_level(eapi): raw_formats = ('portage-1',) else: raw_formats = ('portage-1-compat',) diff --git a/portage_with_autodep/pym/portage/repository/config.pyo b/portage_with_autodep/pym/portage/repository/config.pyo Binary files differindex f9ee26d..b289c2c 100644 --- a/portage_with_autodep/pym/portage/repository/config.pyo +++ b/portage_with_autodep/pym/portage/repository/config.pyo diff --git a/portage_with_autodep/pym/portage/tests/__init__.py b/portage_with_autodep/pym/portage/tests/__init__.py index 492ece4..84e732a 100644 --- a/portage_with_autodep/pym/portage/tests/__init__.py +++ b/portage_with_autodep/pym/portage/tests/__init__.py @@ -1,5 +1,5 @@ # tests/__init__.py -- Portage Unit Test functionality -# Copyright 2006-2011 Gentoo Foundation +# Copyright 2006-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -7,26 +7,40 @@ from __future__ import print_function import sys import time import unittest -from optparse import OptionParser, OptionValueError try: from unittest.runner import _TextTestResult # new in python-2.7 except ImportError: from unittest import _TextTestResult +try: + # They added the skip framework to python-2.7. + # Drop this once we drop python-2.6 support. + unittest_skip_shims = False + import unittest.SkipTest as SkipTest # new in python-2.7 +except ImportError: + unittest_skip_shims = True + +import portage from portage import os from portage import _encodings from portage import _unicode_decode +from portage.util._argparse import ArgumentParser def main(): suite = unittest.TestSuite() basedir = os.path.dirname(os.path.realpath(__file__)) usage = "usage: %s [options] [tests to run]" % os.path.basename(sys.argv[0]) - parser = OptionParser(usage=usage) - parser.add_option("-l", "--list", help="list all tests", + parser = ArgumentParser(usage=usage) + parser.add_argument("-l", "--list", help="list all tests", action="store_true", dest="list_tests") - (options, args) = parser.parse_args(args=sys.argv) + options, args = parser.parse_known_args(args=sys.argv) + + if (os.environ.get('NOCOLOR') in ('yes', 'true') or + os.environ.get('TERM') == 'dumb' or + not sys.stdout.isatty()): + portage.output.nocolor() if options.list_tests: testdir = os.path.dirname(sys.argv[0]) @@ -70,15 +84,12 @@ def getTestFromCommandLine(args, base_path): def getTestDirs(base_path): TEST_FILE = b'__test__' - svn_dirname = b'.svn' testDirs = [] # the os.walk help mentions relative paths as being quirky # I was tired of adding dirs to the list, so now we add __test__ # to each dir we want tested. for root, dirs, files in os.walk(base_path): - if svn_dirname in dirs: - dirs.remove(svn_dirname) try: root = _unicode_decode(root, encoding=_encodings['fs'], errors='strict') @@ -93,7 +104,7 @@ def getTestDirs(base_path): def getTestNames(path): files = os.listdir(path) - files = [ f[:-3] for f in files if f.startswith("test") and f.endswith(".py") ] + files = [f[:-3] for f in files if f.startswith("test") and f.endswith(".py")] files.sort() return files @@ -134,14 +145,14 @@ class TextTestResult(_TextTestResult): self.portage_skipped = [] def addTodo(self, test, info): - self.todoed.append((test,info)) + self.todoed.append((test, info)) if self.showAll: self.stream.writeln("TODO") elif self.dots: self.stream.write(".") def addPortageSkip(self, test, info): - self.portage_skipped.append((test,info)) + self.portage_skipped.append((test, info)) if self.showAll: self.stream.writeln("SKIP") elif self.dots: @@ -185,10 +196,14 @@ class TestCase(unittest.TestCase): except: result.addError(self, sys.exc_info()) return + ok = False try: testMethod() ok = True + except SkipTest as e: + result.addPortageSkip(self, "%s: SKIP: %s" % + (testMethod, str(e))) except self.failureException: if self.portage_skip is not None: if self.portage_skip is True: @@ -197,13 +212,14 @@ class TestCase(unittest.TestCase): result.addPortageSkip(self, "%s: SKIP: %s" % (testMethod, self.portage_skip)) elif self.todo: - result.addTodo(self,"%s: TODO" % testMethod) + result.addTodo(self, "%s: TODO" % testMethod) else: result.addFailure(self, sys.exc_info()) except (KeyboardInterrupt, SystemExit): raise except: result.addError(self, sys.exc_info()) + try: self.tearDown() except SystemExit: @@ -213,7 +229,8 @@ class TestCase(unittest.TestCase): except: result.addError(self, sys.exc_info()) ok = False - if ok: result.addSuccess(self) + if ok: + result.addSuccess(self) finally: result.stopTest(self) @@ -230,10 +247,48 @@ class TestCase(unittest.TestCase): except excClass: return else: - if hasattr(excClass,'__name__'): excName = excClass.__name__ + if hasattr(excClass, '__name__'): excName = excClass.__name__ else: excName = str(excClass) raise self.failureException("%s not raised: %s" % (excName, msg)) + def assertExists(self, path): + """Make sure |path| exists""" + if not os.path.exists(path): + msg = ['path is missing: %s' % (path,)] + while path != '/': + path = os.path.dirname(path) + if not path: + # If we're given something like "foo", abort once we get to "". + break + result = os.path.exists(path) + msg.append('\tos.path.exists(%s): %s' % (path, result)) + if result: + msg.append('\tcontents: %r' % os.listdir(path)) + break + raise self.failureException('\n'.join(msg)) + + def assertNotExists(self, path): + """Make sure |path| does not exist""" + if os.path.exists(path): + raise self.failureException('path exists when it should not: %s' % path) + +if unittest_skip_shims: + # Shim code for <python-2.7. + class SkipTest(Exception): + """unittest.SkipTest shim for <python-2.7""" + + def skipTest(self, reason): + raise SkipTest(reason) + setattr(TestCase, 'skipTest', skipTest) + + def assertIn(self, member, container, msg=None): + self.assertTrue(member in container, msg=msg) + setattr(TestCase, 'assertIn', assertIn) + + def assertNotIn(self, member, container, msg=None): + self.assertFalse(member in container, msg=msg) + setattr(TestCase, 'assertNotIn', assertNotIn) + class TextTestRunner(unittest.TextTestRunner): """ We subclass unittest.TextTestRunner to output SKIP for tests that fail but are skippable @@ -271,8 +326,8 @@ class TextTestRunner(unittest.TextTestRunner): self.stream.writeln("OK") return result -test_cps = ['sys-apps/portage','virtual/portage'] -test_versions = ['1.0', '1.0-r1','2.3_p4','1.0_alpha57'] -test_slots = [ None, '1','gentoo-sources-2.6.17','spankywashere'] -test_usedeps = ['foo','-bar', ('foo','bar'), - ('foo','-bar'), ('foo?', '!bar?') ] +test_cps = ['sys-apps/portage', 'virtual/portage'] +test_versions = ['1.0', '1.0-r1', '2.3_p4', '1.0_alpha57'] +test_slots = [None, '1', 'gentoo-sources-2.6.17', 'spankywashere'] +test_usedeps = ['foo', '-bar', ('foo', 'bar'), + ('foo', '-bar'), ('foo?', '!bar?')] diff --git a/portage_with_autodep/pym/portage/tests/__init__.pyo b/portage_with_autodep/pym/portage/tests/__init__.pyo Binary files differindex 0e961b8..aa0215b 100644 --- a/portage_with_autodep/pym/portage/tests/__init__.pyo +++ b/portage_with_autodep/pym/portage/tests/__init__.pyo diff --git a/portage_with_autodep/pym/portage/tests/lint/__init__.pyo b/portage_with_autodep/pym/portage/tests/lint/__init__.pyo Binary files differindex a1241e5..2a1215a 100644 --- a/portage_with_autodep/pym/portage/tests/lint/__init__.pyo +++ b/portage_with_autodep/pym/portage/tests/lint/__init__.pyo diff --git a/portage_with_autodep/pym/portage/tests/lint/test_bash_syntax.py b/portage_with_autodep/pym/portage/tests/lint/test_bash_syntax.py index aef8d74..fdbb6fe 100644 --- a/portage_with_autodep/pym/portage/tests/lint/test_bash_syntax.py +++ b/portage_with_autodep/pym/portage/tests/lint/test_bash_syntax.py @@ -1,20 +1,26 @@ -# Copyright 2010 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from itertools import chain import stat +import subprocess +import sys -from portage.const import BASH_BINARY, PORTAGE_BIN_PATH +from portage.const import BASH_BINARY, PORTAGE_BASE_PATH, PORTAGE_BIN_PATH from portage.tests import TestCase from portage import os -from portage import subprocess_getstatusoutput from portage import _encodings -from portage import _shell_quote from portage import _unicode_decode, _unicode_encode class BashSyntaxTestCase(TestCase): def testBashSyntax(self): - for parent, dirs, files in os.walk(PORTAGE_BIN_PATH): + locations = [PORTAGE_BIN_PATH] + misc_dir = os.path.join(PORTAGE_BASE_PATH, "misc") + if os.path.isdir(misc_dir): + locations.append(misc_dir) + for parent, dirs, files in \ + chain.from_iterable(os.walk(x) for x in locations): parent = _unicode_decode(parent, encoding=_encodings['fs'], errors='strict') for x in files: @@ -36,7 +42,13 @@ class BashSyntaxTestCase(TestCase): f.close() if line[:2] == '#!' and \ 'bash' in line: - cmd = "%s -n %s" % (_shell_quote(BASH_BINARY), _shell_quote(x)) - status, output = subprocess_getstatusoutput(cmd) + cmd = [BASH_BINARY, "-n", x] + cmd = [_unicode_encode(x, + encoding=_encodings['fs'], errors='strict') for x in cmd] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + output = _unicode_decode(proc.communicate()[0], + encoding=_encodings['fs']) + status = proc.wait() self.assertEqual(os.WIFEXITED(status) and \ os.WEXITSTATUS(status) == os.EX_OK, True, msg=output) diff --git a/portage_with_autodep/pym/portage/tests/lint/test_bash_syntax.pyo b/portage_with_autodep/pym/portage/tests/lint/test_bash_syntax.pyo Binary files differindex a7ddc80..b066527 100644 --- a/portage_with_autodep/pym/portage/tests/lint/test_bash_syntax.pyo +++ b/portage_with_autodep/pym/portage/tests/lint/test_bash_syntax.pyo diff --git a/portage_with_autodep/pym/portage/tests/lint/test_compile_modules.py b/portage_with_autodep/pym/portage/tests/lint/test_compile_modules.py index f90a666..1d44e68 100644 --- a/portage_with_autodep/pym/portage/tests/lint/test_compile_modules.py +++ b/portage_with_autodep/pym/portage/tests/lint/test_compile_modules.py @@ -1,4 +1,4 @@ -# Copyright 2009-2010 Gentoo Foundation +# Copyright 2009-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import itertools @@ -10,8 +10,6 @@ from portage import os from portage import _encodings from portage import _unicode_decode, _unicode_encode -import py_compile - class CompileModulesTestCase(TestCase): def testCompileModules(self): @@ -34,13 +32,13 @@ class CompileModulesTestCase(TestCase): do_compile = True else: # Check for python shebang - f = open(_unicode_encode(x, - encoding=_encodings['fs'], errors='strict'), 'rb') - line = _unicode_decode(f.readline(), - encoding=_encodings['content'], errors='replace') - f.close() - if line[:2] == '#!' and \ - 'python' in line: + with open(_unicode_encode(x, + encoding=_encodings['fs'], errors='strict'), 'rb') as f: + line = _unicode_decode(f.readline(), + encoding=_encodings['content'], errors='replace') + if line[:2] == '#!' and 'python' in line: do_compile = True if do_compile: - py_compile.compile(x, cfile='/dev/null', doraise=True) + with open(_unicode_encode(x, + encoding=_encodings['fs'], errors='strict'), 'rb') as f: + compile(f.read(), x, 'exec') diff --git a/portage_with_autodep/pym/portage/tests/lint/test_compile_modules.pyo b/portage_with_autodep/pym/portage/tests/lint/test_compile_modules.pyo Binary files differindex 7b1460d..8e32e1f 100644 --- a/portage_with_autodep/pym/portage/tests/lint/test_compile_modules.pyo +++ b/portage_with_autodep/pym/portage/tests/lint/test_compile_modules.pyo diff --git a/portage_with_autodep/pym/portage/tests/lint/test_import_modules.py b/portage_with_autodep/pym/portage/tests/lint/test_import_modules.py index 8d257c5..34261f4 100644 --- a/portage_with_autodep/pym/portage/tests/lint/test_import_modules.py +++ b/portage_with_autodep/pym/portage/tests/lint/test_import_modules.py @@ -1,4 +1,4 @@ -# Copyright 2011 Gentoo Foundation +# Copyright 2011-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from portage.const import PORTAGE_PYM_PATH diff --git a/portage_with_autodep/pym/portage/tests/lint/test_import_modules.pyo b/portage_with_autodep/pym/portage/tests/lint/test_import_modules.pyo Binary files differindex b3a1d61..b566748 100644 --- a/portage_with_autodep/pym/portage/tests/lint/test_import_modules.pyo +++ b/portage_with_autodep/pym/portage/tests/lint/test_import_modules.pyo diff --git a/portage_with_autodep/pym/portage/tests/runTests b/portage_with_autodep/pym/portage/tests/runTests index 4c10087..60bcf31 100755 --- a/portage_with_autodep/pym/portage/tests/runTests +++ b/portage_with_autodep/pym/portage/tests/runTests @@ -1,18 +1,25 @@ #!/usr/bin/python -Wd # runTests.py -- Portage Unit Test Functionality -# Copyright 2006 Gentoo Foundation +# Copyright 2006-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import os, sys import os.path as osp import grp +import platform import pwd import signal def debug_signal(signum, frame): import pdb pdb.set_trace() -signal.signal(signal.SIGUSR1, debug_signal) + +if platform.python_implementation() == 'Jython': + debug_signum = signal.SIGUSR2 # bug #424259 +else: + debug_signum = signal.SIGUSR1 + +signal.signal(debug_signum, debug_signal) # Pretend that the current user's uid/gid are the 'portage' uid/gid, # so things go smoothly regardless of the current user and global @@ -22,23 +29,33 @@ os.environ["PORTAGE_GRPNAME"] = grp.getgrgid(os.getgid()).gr_name # Insert our parent dir so we can do shiny import "tests" # This line courtesy of Marienz and Pkgcore ;) -sys.path.insert(0, osp.dirname(osp.dirname(osp.dirname(osp.abspath(__file__))))) +sys.path.insert(0, osp.dirname(osp.dirname(osp.dirname(osp.realpath(__file__))))) import portage +portage._internal_caller = True # Ensure that we don't instantiate portage.settings, so that tests should # work the same regardless of global configuration file state/existence. portage._disable_legacy_globals() +if os.environ.get('NOCOLOR') in ('yes', 'true'): + portage.output.nocolor() + import portage.tests as tests from portage.const import PORTAGE_BIN_PATH path = os.environ.get("PATH", "").split(":") path = [x for x in path if x] -if not path or not os.path.samefile(path[0], PORTAGE_BIN_PATH): + +insert_bin_path = True +try: + insert_bin_path = not path or \ + not os.path.samefile(path[0], PORTAGE_BIN_PATH) +except OSError: + pass + +if insert_bin_path: path.insert(0, PORTAGE_BIN_PATH) os.environ["PATH"] = ":".join(path) -del path - if __name__ == "__main__": sys.exit(tests.main()) diff --git a/portage_with_autodep/pym/portage/update.py b/portage_with_autodep/pym/portage/update.py index 34e4663..92aba9a 100644 --- a/portage_with_autodep/pym/portage/update.py +++ b/portage_with_autodep/pym/portage/update.py @@ -1,11 +1,14 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + import errno import io import re import stat import sys +import warnings from portage import os from portage import _encodings @@ -13,50 +16,107 @@ from portage import _unicode_decode from portage import _unicode_encode import portage portage.proxy.lazyimport.lazyimport(globals(), - 'portage.dep:Atom,dep_getkey,isvalidatom,' + \ - 'remove_slot', + 'portage.dep:Atom,dep_getkey,isvalidatom,match_from_list', 'portage.util:ConfigProtect,new_protect_filename,' + \ 'normalize_path,write_atomic,writemsg', - 'portage.util.listdir:_ignorecvs_dirs', - 'portage.versions:ververify' + 'portage.versions:_get_slot_re', ) -from portage.const import USER_CONFIG_PATH +from portage.const import USER_CONFIG_PATH, VCS_DIRS +from portage.eapi import _get_eapi_attrs from portage.exception import DirectoryNotFound, InvalidAtom, PortageException from portage.localization import _ if sys.hexversion >= 0x3000000: long = int + _unicode = str +else: + _unicode = unicode ignored_dbentries = ("CONTENTS", "environment.bz2") -def update_dbentry(update_cmd, mycontent): +def update_dbentry(update_cmd, mycontent, eapi=None, parent=None): + + if parent is not None: + eapi = parent.eapi + if update_cmd[0] == "move": - old_value = str(update_cmd[1]) - if old_value in mycontent: - new_value = str(update_cmd[2]) - old_value = re.escape(old_value); - mycontent = re.sub(old_value+"(:|$|\\s)", new_value+"\\1", mycontent) - def myreplace(matchobj): - # Strip slot and * operator if necessary - # so that ververify works. - ver = remove_slot(matchobj.group(2)) - ver = ver.rstrip("*") - if ververify(ver): - return "%s-%s" % (new_value, matchobj.group(2)) - else: - return "".join(matchobj.groups()) - mycontent = re.sub("(%s-)(\\S*)" % old_value, myreplace, mycontent) + old_value = _unicode(update_cmd[1]) + new_value = _unicode(update_cmd[2]) + + # Use isvalidatom() to check if this move is valid for the + # EAPI (characters allowed in package names may vary). + if old_value in mycontent and isvalidatom(new_value, eapi=eapi): + # this split preserves existing whitespace + split_content = re.split(r'(\s+)', mycontent) + modified = False + for i, token in enumerate(split_content): + if old_value not in token: + continue + try: + atom = Atom(token, eapi=eapi) + except InvalidAtom: + continue + if atom.cp != old_value: + continue + + new_atom = Atom(token.replace(old_value, new_value, 1), + eapi=eapi) + + # Avoid creating self-blockers for bug #367215. + if new_atom.blocker and parent is not None and \ + parent.cp == new_atom.cp and \ + match_from_list(new_atom, [parent]): + continue + + split_content[i] = _unicode(new_atom) + modified = True + + if modified: + mycontent = "".join(split_content) + elif update_cmd[0] == "slotmove" and update_cmd[1].operator is None: - pkg, origslot, newslot = update_cmd[1:] - old_value = "%s:%s" % (pkg, origslot) - if old_value in mycontent: - old_value = re.escape(old_value) - new_value = "%s:%s" % (pkg, newslot) - mycontent = re.sub(old_value+"($|\\s)", new_value+"\\1", mycontent) + orig_atom, origslot, newslot = update_cmd[1:] + orig_cp = orig_atom.cp + + # We don't support versioned slotmove atoms here, since it can be + # difficult to determine if the version constraints really match + # the atoms that we're trying to update. + if orig_atom.version is None and orig_cp in mycontent: + # this split preserves existing whitespace + split_content = re.split(r'(\s+)', mycontent) + modified = False + for i, token in enumerate(split_content): + if orig_cp not in token: + continue + try: + atom = Atom(token, eapi=eapi) + except InvalidAtom: + continue + if atom.cp != orig_cp: + continue + if atom.slot is None or atom.slot != origslot: + continue + + slot_part = newslot + if atom.sub_slot is not None: + if atom.sub_slot == origslot: + sub_slot = newslot + else: + sub_slot = atom.sub_slot + slot_part += "/" + sub_slot + if atom.slot_operator is not None: + slot_part += atom.slot_operator + + split_content[i] = atom.with_slot(slot_part) + modified = True + + if modified: + mycontent = "".join(split_content) + return mycontent -def update_dbentries(update_iter, mydata): +def update_dbentries(update_iter, mydata, eapi=None, parent=None): """Performs update commands and returns a dict containing only the updated items.""" updated_items = {} @@ -70,7 +130,8 @@ def update_dbentries(update_iter, mydata): is_encoded = mycontent is not orig_content orig_content = mycontent for update_cmd in update_iter: - mycontent = update_dbentry(update_cmd, mycontent) + mycontent = update_dbentry(update_cmd, mycontent, + eapi=eapi, parent=parent) if mycontent != orig_content: if is_encoded: mycontent = _unicode_encode(mycontent, @@ -79,10 +140,14 @@ def update_dbentries(update_iter, mydata): updated_items[k] = mycontent return updated_items -def fixdbentries(update_iter, dbdir): +def fixdbentries(update_iter, dbdir, eapi=None, parent=None): """Performs update commands which result in search and replace operations for each of the files in dbdir (excluding CONTENTS and environment.bz2). Returns True when actual modifications are necessary and False otherwise.""" + + warnings.warn("portage.update.fixdbentries() is deprecated", + DeprecationWarning, stacklevel=2) + mydata = {} for myfile in [f for f in os.listdir(dbdir) if f not in ignored_dbentries]: file_path = os.path.join(dbdir, myfile) @@ -91,7 +156,8 @@ def fixdbentries(update_iter, dbdir): mode='r', encoding=_encodings['repo.content'], errors='replace') as f: mydata[myfile] = f.read() - updated_items = update_dbentries(update_iter, mydata) + updated_items = update_dbentries(update_iter, mydata, + eapi=eapi, parent=parent) for myfile, mycontent in updated_items.items(): file_path = os.path.join(dbdir, myfile) write_atomic(file_path, mycontent, encoding=_encodings['repo.content']) @@ -143,6 +209,8 @@ def grab_updates(updpath, prev_mtimes=None): def parse_updates(mycontent): """Valid updates are returned as a list of split update commands.""" + eapi_attrs = _get_eapi_attrs(None) + slot_re = _get_slot_re(eapi_attrs) myupd = [] errors = [] mylines = mycontent.splitlines() @@ -194,12 +262,28 @@ def parse_updates(mycontent): errors.append(_("ERROR: Malformed update entry '%s'") % myline) continue + invalid_slot = False + for slot in (origslot, newslot): + m = slot_re.match(slot) + if m is None: + invalid_slot = True + break + if "/" in slot: + # EAPI 4-slot-abi style SLOT is currently not supported. + invalid_slot = True + break + + if invalid_slot: + errors.append(_("ERROR: Malformed update entry '%s'") % myline) + continue + # The list of valid updates is filtered by continue statements above. myupd.append(mysplit) return myupd, errors def update_config_files(config_root, protect, protect_mask, update_iter, match_callback = None): - """Perform global updates on /etc/portage/package.*. + """Perform global updates on /etc/portage/package.*, /etc/portage/profile/package.*, + /etc/portage/profile/packages and /etc/portage/sets. config_root - location of files to update protect - list of paths from CONFIG_PROTECT protect_mask - list of paths from CONFIG_PROTECT_MASK @@ -222,9 +306,15 @@ def update_config_files(config_root, protect, protect_mask, update_iter, match_c "package.accept_keywords", "package.env", "package.keywords", "package.license", "package.mask", "package.properties", - "package.unmask", "package.use" + "package.unmask", "package.use", "sets" ] - myxfiles += [os.path.join("profile", x) for x in myxfiles] + myxfiles += [os.path.join("profile", x) for x in ( + "packages", "package.accept_keywords", + "package.keywords", "package.mask", + "package.unmask", "package.use", + "package.use.force", "package.use.mask", + "package.use.stable.force", "package.use.stable.mask" + )] abs_user_config = os.path.join(config_root, USER_CONFIG_PATH) recursivefiles = [] for x in myxfiles: @@ -243,7 +333,7 @@ def update_config_files(config_root, protect, protect_mask, update_iter, match_c except UnicodeDecodeError: dirs.remove(y_enc) continue - if y.startswith(".") or y in _ignorecvs_dirs: + if y.startswith(".") or y in VCS_DIRS: dirs.remove(y_enc) for y in files: try: @@ -273,7 +363,6 @@ def update_config_files(config_root, protect, protect_mask, update_iter, match_c if f is not None: f.close() - # update /etc/portage/packages.* ignore_line_re = re.compile(r'^#|^\s*$') if repo_dict is None: update_items = [(None, update_iter)] @@ -293,6 +382,9 @@ def update_config_files(config_root, protect, protect_mask, update_iter, match_c if atom[:1] == "-": # package.mask supports incrementals atom = atom[1:] + if atom[:1] == "*": + # packages file supports "*"-prefixed atoms as indication of system packages. + atom = atom[1:] if not isvalidatom(atom): continue new_atom = update_dbentry(update_cmd, atom) diff --git a/portage_with_autodep/pym/portage/update.pyo b/portage_with_autodep/pym/portage/update.pyo Binary files differindex 9628ea9..2979af7 100644 --- a/portage_with_autodep/pym/portage/update.pyo +++ b/portage_with_autodep/pym/portage/update.pyo diff --git a/portage_with_autodep/pym/portage/util/ExtractKernelVersion.py b/portage_with_autodep/pym/portage/util/ExtractKernelVersion.py index 69bd58a..af4a4fe 100644 --- a/portage_with_autodep/pym/portage/util/ExtractKernelVersion.py +++ b/portage_with_autodep/pym/portage/util/ExtractKernelVersion.py @@ -61,18 +61,18 @@ def ExtractKernelVersion(base_dir): # Grab a list of files named localversion* and sort them localversions = os.listdir(base_dir) - for x in range(len(localversions)-1,-1,-1): + for x in range(len(localversions) - 1, -1, -1): if localversions[x][:12] != "localversion": del localversions[x] localversions.sort() # Append the contents of each to the version string, stripping ALL whitespace for lv in localversions: - version += "".join( " ".join( grabfile( base_dir+ "/" + lv ) ).split() ) + version += "".join(" ".join(grabfile(base_dir + "/" + lv)).split()) # Check the .config for a CONFIG_LOCALVERSION and append that too, also stripping whitespace kernelconfig = getconfig(base_dir+"/.config") if kernelconfig and "CONFIG_LOCALVERSION" in kernelconfig: version += "".join(kernelconfig["CONFIG_LOCALVERSION"].split()) - return (version,None) + return (version, None) diff --git a/portage_with_autodep/pym/portage/util/ExtractKernelVersion.pyo b/portage_with_autodep/pym/portage/util/ExtractKernelVersion.pyo Binary files differindex d0302fd..132515d 100644 --- a/portage_with_autodep/pym/portage/util/ExtractKernelVersion.pyo +++ b/portage_with_autodep/pym/portage/util/ExtractKernelVersion.pyo diff --git a/portage_with_autodep/pym/portage/util/SlotObject.py b/portage_with_autodep/pym/portage/util/SlotObject.py index a59dfc1..4bb6822 100644 --- a/portage_with_autodep/pym/portage/util/SlotObject.py +++ b/portage_with_autodep/pym/portage/util/SlotObject.py @@ -48,4 +48,3 @@ class SlotObject(object): setattr(obj, myattr, getattr(self, myattr)) return obj - diff --git a/portage_with_autodep/pym/portage/util/SlotObject.pyo b/portage_with_autodep/pym/portage/util/SlotObject.pyo Binary files differindex 11d0ec7..08133cb 100644 --- a/portage_with_autodep/pym/portage/util/SlotObject.pyo +++ b/portage_with_autodep/pym/portage/util/SlotObject.pyo diff --git a/portage_with_autodep/pym/portage/util/__init__.py b/portage_with_autodep/pym/portage/util/__init__.py index 2e0a32b..24553da 100644 --- a/portage_with_autodep/pym/portage/util/__init__.py +++ b/portage_with_autodep/pym/portage/util/__init__.py @@ -1,6 +1,8 @@ -# Copyright 2004-2012 Gentoo Foundation +# Copyright 2004-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ['apply_permissions', 'apply_recursive_permissions', 'apply_secpass_permissions', 'apply_stat_permissions', 'atomic_ofstream', 'cmp_sort_key', 'ConfigProtect', 'dump_traceback', 'ensure_dirs', @@ -31,21 +33,26 @@ import portage portage.proxy.lazyimport.lazyimport(globals(), 'pickle', 'portage.dep:Atom', - 'portage.util.listdir:_ignorecvs_dirs' + 'subprocess', ) from portage import os -from portage import subprocess_getstatusoutput from portage import _encodings from portage import _os_merge from portage import _unicode_encode from portage import _unicode_decode +from portage.const import VCS_DIRS from portage.exception import InvalidAtom, PortageException, FileNotFound, \ OperationNotPermitted, ParseError, PermissionDenied, ReadOnlyFileSystem from portage.localization import _ from portage.proxy.objectproxy import ObjectProxy from portage.cache.mappings import UserDict +if sys.hexversion >= 0x3000000: + _unicode = str +else: + _unicode = unicode + noiselimit = 0 def initialize_logger(level=logging.WARN): @@ -57,7 +64,7 @@ def initialize_logger(level=logging.WARN): """ logging.basicConfig(level=logging.WARN, format='[%(levelname)-4s] %(message)s') -def writemsg(mystr,noiselevel=0,fd=None): +def writemsg(mystr, noiselevel=0, fd=None): """Prints out warning and debug messages based on the noiselimit setting""" global noiselimit if fd is None: @@ -75,7 +82,7 @@ def writemsg(mystr,noiselevel=0,fd=None): fd.write(mystr) fd.flush() -def writemsg_stdout(mystr,noiselevel=0): +def writemsg_stdout(mystr, noiselevel=0): """Prints messages stdout based on the noiselimit setting""" writemsg(mystr, noiselevel=noiselevel, fd=sys.stdout) @@ -100,7 +107,7 @@ def writemsg_level(msg, level=0, noiselevel=0): writemsg(msg, noiselevel=noiselevel, fd=fd) def normalize_path(mypath): - """ + """ os.path.normpath("//foo") returns "//foo" instead of "/foo" We dislike this behavior so we create our own normpath func to fix it. @@ -120,8 +127,8 @@ def grabfile(myfilename, compat_level=0, recursive=0, remember_source_file=False """This function grabs the lines in a file, normalizes whitespace and returns lines in a list; if a line begins with a #, it is ignored, as are empty lines""" - mylines=grablines(myfilename, recursive, remember_source_file=True) - newlines=[] + mylines = grablines(myfilename, recursive, remember_source_file=True) + newlines = [] for x, source_file in mylines: #the split/join thing removes leading and trailing whitespace, and converts any whitespace in the line @@ -139,10 +146,10 @@ def grabfile(myfilename, compat_level=0, recursive=0, remember_source_file=False myline = " ".join(myline) if not myline: continue - if myline[0]=="#": + if myline[0] == "#": # Check if we have a compat-level string. BC-integration data. # '##COMPAT==>N<==' 'some string attached to it' - mylinetest = myline.split("<==",1) + mylinetest = myline.split("<==", 1) if len(mylinetest) == 2: myline_potential = mylinetest[1] mylinetest = mylinetest[0].split("##COMPAT==>") @@ -159,7 +166,7 @@ def grabfile(myfilename, compat_level=0, recursive=0, remember_source_file=False newlines.append(myline) return newlines -def map_dictlist_vals(func,myDict): +def map_dictlist_vals(func, myDict): """Performs a function on each value of each key in a dictlist. Returns a new dictlist.""" new_dl = {} @@ -173,7 +180,7 @@ def stack_dictlist(original_dicts, incremental=0, incrementals=[], ignore_none=0 Stacks an array of dict-types into one array. Optionally merging or overwriting matching key/value pairs for the dict[key]->list. Returns a single dict. Higher index in lists is preferenced. - + Example usage: >>> from portage.util import stack_dictlist >>> print stack_dictlist( [{'a':'b'},{'x':'y'}]) @@ -188,7 +195,7 @@ def stack_dictlist(original_dicts, incremental=0, incrementals=[], ignore_none=0 >>> { 'KEYWORDS':['alpha'] } >>> print stack_dictlist( [a,b], incrementals=['KEYWORDS']) >>> { 'KEYWORDS':['alpha'] } - + @param original_dicts a list of (dictionary objects or None) @type list @param incremental True or false depending on whether new keys should overwrite @@ -199,7 +206,7 @@ def stack_dictlist(original_dicts, incremental=0, incrementals=[], ignore_none=0 @type list @param ignore_none Appears to be ignored, but probably was used long long ago. @type boolean - + """ final_dict = {} for mydict in original_dicts: @@ -208,7 +215,7 @@ def stack_dictlist(original_dicts, incremental=0, incrementals=[], ignore_none=0 for y in mydict: if not y in final_dict: final_dict[y] = [] - + for thing in mydict[y]: if thing: if incremental or y in incrementals: @@ -245,12 +252,13 @@ def stack_dicts(dicts, incremental=0, incrementals=[], ignore_none=0): def append_repo(atom_list, repo_name, remember_source_file=False): """ Takes a list of valid atoms without repo spec and appends ::repo_name. + If an atom already has a repo part, then it is preserved (see bug #461948). """ if remember_source_file: - return [(Atom(atom + "::" + repo_name, allow_wildcard=True, allow_repo=True), source) \ + return [(atom.repo is not None and atom or atom.with_repo(repo_name), source) \ for atom, source in atom_list] else: - return [Atom(atom + "::" + repo_name, allow_wildcard=True, allow_repo=True) \ + return [atom.repo is not None and atom or atom.with_repo(repo_name) \ for atom in atom_list] def stack_lists(lists, incremental=1, remember_source_file=False, @@ -334,7 +342,7 @@ def stack_lists(lists, incremental=1, remember_source_file=False, def grabdict(myfilename, juststrings=0, empty=0, recursive=0, incremental=1): """ This function grabs the lines in a file, normalizes whitespace and returns lines in a dictionary - + @param myfilename: file to process @type myfilename: string (path) @param juststrings: only return strings @@ -350,9 +358,9 @@ def grabdict(myfilename, juststrings=0, empty=0, recursive=0, incremental=1): 1. Returns the lines in a file in a dictionary, for example: 'sys-apps/portage x86 amd64 ppc' would return - { "sys-apps/portage" : [ 'x86', 'amd64', 'ppc' ] + {"sys-apps/portage" : ['x86', 'amd64', 'ppc']} """ - newdict={} + newdict = {} for x in grablines(myfilename, recursive): #the split/join thing removes leading and trailing whitespace, and converts any whitespace in the line #into single spaces. @@ -379,52 +387,75 @@ def grabdict(myfilename, juststrings=0, empty=0, recursive=0, incremental=1): newdict[k] = " ".join(v) return newdict -def read_corresponding_eapi_file(filename): +_eapi_cache = {} + +def read_corresponding_eapi_file(filename, default="0"): """ Read the 'eapi' file from the directory 'filename' is in. Returns "0" if the file is not present or invalid. """ - default = "0" eapi_file = os.path.join(os.path.dirname(filename), "eapi") try: - f = io.open(_unicode_encode(eapi_file, + eapi = _eapi_cache[eapi_file] + except KeyError: + pass + else: + if eapi is None: + return default + return eapi + + eapi = None + try: + with io.open(_unicode_encode(eapi_file, encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content'], errors='replace') - lines = f.readlines() + mode='r', encoding=_encodings['repo.content'], errors='replace') as f: + lines = f.readlines() if len(lines) == 1: eapi = lines[0].rstrip("\n") else: writemsg(_("--- Invalid 'eapi' file (doesn't contain exactly one line): %s\n") % (eapi_file), noiselevel=-1) - eapi = default - f.close() except IOError: - eapi = default + pass + _eapi_cache[eapi_file] = eapi + if eapi is None: + return default return eapi def grabdict_package(myfilename, juststrings=0, recursive=0, allow_wildcard=False, allow_repo=False, verify_eapi=False, eapi=None): """ Does the same thing as grabdict except it validates keys with isvalidatom()""" - pkgs=grabdict(myfilename, juststrings, empty=1, recursive=recursive) - if not pkgs: - return pkgs - if verify_eapi and eapi is None: - eapi = read_corresponding_eapi_file(myfilename) - # We need to call keys() here in order to avoid the possibility of - # "RuntimeError: dictionary changed size during iteration" - # when an invalid atom is deleted. + if recursive: + file_list = _recursive_file_list(myfilename) + else: + file_list = [myfilename] + atoms = {} - for k, v in pkgs.items(): - try: - k = Atom(k, allow_wildcard=allow_wildcard, allow_repo=allow_repo, eapi=eapi) - except InvalidAtom as e: - writemsg(_("--- Invalid atom in %s: %s\n") % (myfilename, e), - noiselevel=-1) - else: - atoms[k] = v + for filename in file_list: + d = grabdict(filename, juststrings=False, + empty=True, recursive=False, incremental=True) + if not d: + continue + if verify_eapi and eapi is None: + eapi = read_corresponding_eapi_file(myfilename) + + for k, v in d.items(): + try: + k = Atom(k, allow_wildcard=allow_wildcard, + allow_repo=allow_repo, eapi=eapi) + except InvalidAtom as e: + writemsg(_("--- Invalid atom in %s: %s\n") % (filename, e), + noiselevel=-1) + else: + atoms.setdefault(k, []).extend(v) + + if juststrings: + for k, v in atoms.items(): + atoms[k] = " ".join(v) + return atoms def grabfile_package(myfilename, compatlevel=0, recursive=0, allow_wildcard=False, allow_repo=False, @@ -447,10 +478,10 @@ def grabfile_package(myfilename, compatlevel=0, recursive=0, allow_wildcard=Fals try: pkg = Atom(pkg, allow_wildcard=allow_wildcard, allow_repo=allow_repo, eapi=eapi) except InvalidAtom as e: - writemsg(_("--- Invalid atom in %s: %s\n") % (myfilename, e), + writemsg(_("--- Invalid atom in %s: %s\n") % (source_file, e), noiselevel=-1) else: - if pkg_orig == str(pkg): + if pkg_orig == _unicode(pkg): # normal atom, so return as Atom instance if remember_source_file: atoms.append((pkg, source_file)) @@ -464,34 +495,73 @@ def grabfile_package(myfilename, compatlevel=0, recursive=0, allow_wildcard=Fals atoms.append(pkg_orig) return atoms +def _recursive_basename_filter(f): + return not f.startswith(".") and not f.endswith("~") + +def _recursive_file_list(path): + # path may be a regular file or a directory + + def onerror(e): + if e.errno == PermissionDenied.errno: + raise PermissionDenied(path) + + stack = [os.path.split(path)] + + while stack: + parent, fname = stack.pop() + fullpath = os.path.join(parent, fname) + + try: + st = os.stat(fullpath) + except OSError as e: + onerror(e) + continue + + if stat.S_ISDIR(st.st_mode): + if fname in VCS_DIRS or not _recursive_basename_filter(fname): + continue + try: + children = os.listdir(fullpath) + except OSError as e: + onerror(e) + continue + + # Sort in reverse, since we pop from the end of the stack. + # Include regular files in the stack, so files are sorted + # together with directories. + children.sort(reverse=True) + stack.extend((fullpath, x) for x in children) + + elif stat.S_ISREG(st.st_mode): + if _recursive_basename_filter(fname): + yield fullpath + def grablines(myfilename, recursive=0, remember_source_file=False): - mylines=[] - if recursive and os.path.isdir(myfilename): - if os.path.basename(myfilename) in _ignorecvs_dirs: - return mylines - dirlist = os.listdir(myfilename) - dirlist.sort() - for f in dirlist: - if not f.startswith(".") and not f.endswith("~"): - mylines.extend(grablines( - os.path.join(myfilename, f), recursive, remember_source_file)) + mylines = [] + if recursive: + for f in _recursive_file_list(myfilename): + mylines.extend(grablines(f, recursive=False, + remember_source_file=remember_source_file)) + else: try: - myfile = io.open(_unicode_encode(myfilename, + with io.open(_unicode_encode(myfilename, encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['content'], errors='replace') - if remember_source_file: - mylines = [(line, myfilename) for line in myfile.readlines()] - else: - mylines = myfile.readlines() - myfile.close() + mode='r', encoding=_encodings['content'], errors='replace') as myfile: + if remember_source_file: + mylines = [(line, myfilename) for line in myfile.readlines()] + else: + mylines = myfile.readlines() except IOError as e: if e.errno == PermissionDenied.errno: raise PermissionDenied(myfilename) - pass + elif e.errno in (errno.ENOENT, errno.ESTALE): + pass + else: + raise return mylines -def writedict(mydict,myfilename,writekey=True): +def writedict(mydict, myfilename, writekey=True): """Writes out a dict to a file; writekey=0 mode doesn't write out the key and assumes all values are strings, not lists.""" lines = [] @@ -517,18 +587,39 @@ def shlex_split(s): rval = [_unicode_decode(x) for x in rval] return rval -class _tolerant_shlex(shlex.shlex): +class _getconfig_shlex(shlex.shlex): + + def __init__(self, portage_tolerant=False, **kwargs): + shlex.shlex.__init__(self, **kwargs) + self.__portage_tolerant = portage_tolerant + def sourcehook(self, newfile): try: return shlex.shlex.sourcehook(self, newfile) except EnvironmentError as e: - writemsg(_("!!! Parse error in '%s': source command failed: %s\n") % \ - (self.infile, str(e)), noiselevel=-1) + if e.errno == PermissionDenied.errno: + raise PermissionDenied(newfile) + if e.errno not in (errno.ENOENT, errno.ENOTDIR): + writemsg("open('%s', 'r'): %s\n" % (newfile, e), noiselevel=-1) + raise + + msg = self.error_leader() + if e.errno == errno.ENOTDIR: + msg += _("%s: Not a directory") % newfile + else: + msg += _("%s: No such file or directory") % newfile + + if self.__portage_tolerant: + writemsg("%s\n" % msg, noiselevel=-1) + else: + raise ParseError(msg) return (newfile, io.StringIO()) _invalid_var_name_re = re.compile(r'^\d|\W') -def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): +def getconfig(mycfg, tolerant=False, allow_sourcing=False, expand=True, + recursive=False): + if isinstance(expand, dict): # Some existing variable definitions have been # passed in, for use in substitutions. @@ -537,6 +628,21 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): else: expand_map = {} mykeys = {} + + if recursive: + # Emulate source commands so that syntax error messages + # can display real file names and line numbers. + if not expand: + expand_map = False + fname = None + for fname in _recursive_file_list(mycfg): + mykeys.update(getconfig(fname, tolerant=tolerant, + allow_sourcing=allow_sourcing, expand=expand_map, + recursive=False) or {}) + if fname is None: + return None + return mykeys + f = None try: # NOTE: shlex doesn't support unicode objects with Python 2 @@ -561,49 +667,53 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): if f is not None: f.close() + # Since this file has unicode_literals enabled, and Python 2's + # shlex implementation does not support unicode, the following code + # uses _native_string() to encode unicode literals when necessary. + # Workaround for avoiding a silent error in shlex that is # triggered by a source statement at the end of the file # without a trailing newline after the source statement. - if content and content[-1] != '\n': - content += '\n' + if content and content[-1] != portage._native_string('\n'): + content += portage._native_string('\n') # Warn about dos-style line endings since that prevents # people from being able to source them with bash. - if '\r' in content: + if portage._native_string('\r') in content: writemsg(("!!! " + _("Please use dos2unix to convert line endings " + \ "in config file: '%s'") + "\n") % mycfg, noiselevel=-1) lex = None try: - if tolerant: - shlex_class = _tolerant_shlex - else: - shlex_class = shlex.shlex # The default shlex.sourcehook() implementation # only joins relative paths when the infile # attribute is properly set. - lex = shlex_class(content, infile=mycfg, posix=True) - lex.wordchars = string.digits + string.ascii_letters + \ - "~!@#$%*_\:;?,./-+{}" - lex.quotes="\"'" + lex = _getconfig_shlex(instream=content, infile=mycfg, posix=True, + portage_tolerant=tolerant) + lex.wordchars = portage._native_string(string.digits + + string.ascii_letters + "~!@#$%*_\:;?,./-+{}") + lex.quotes = portage._native_string("\"'") if allow_sourcing: - lex.source="source" - while 1: - key=lex.get_token() + lex.source = portage._native_string("source") + + while True: + key = _unicode_decode(lex.get_token()) if key == "export": - key = lex.get_token() + key = _unicode_decode(lex.get_token()) if key is None: #normal end of file - break; - equ=lex.get_token() - if (equ==''): + break + + equ = _unicode_decode(lex.get_token()) + if not equ: msg = lex.error_leader() + _("Unexpected EOF") if not tolerant: raise ParseError(msg) else: writemsg("%s\n" % msg, noiselevel=-1) return mykeys - elif (equ!='='): + + elif equ != "=": msg = lex.error_leader() + \ _("Invalid token '%s' (not '=')") % (equ,) if not tolerant: @@ -611,7 +721,8 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): else: writemsg("%s\n" % msg, noiselevel=-1) return mykeys - val=lex.get_token() + + val = _unicode_decode(lex.get_token()) if val is None: msg = lex.error_leader() + \ _("Unexpected end of config file: variable '%s'") % (key,) @@ -620,8 +731,6 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): else: writemsg("%s\n" % msg, noiselevel=-1) return mykeys - key = _unicode_decode(key) - val = _unicode_decode(val) if _invalid_var_name_re.search(key) is not None: msg = lex.error_leader() + \ @@ -642,7 +751,7 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): except Exception as e: if isinstance(e, ParseError) or lex is None: raise - msg = _unicode_decode("%s%s") % (lex.error_leader(), e) + msg = "%s%s" % (lex.error_leader(), e) writemsg("%s\n" % msg, noiselevel=-1) raise @@ -660,10 +769,10 @@ def varexpand(mystring, mydict=None, error_leader=None): This code is used by the configfile code, as well as others (parser) This would be a good bunch of code to port to C. """ - numvars=0 - #in single, double quotes - insing=0 - indoub=0 + numvars = 0 + # in single, double quotes + insing = 0 + indoub = 0 pos = 0 length = len(mystring) newstring = [] @@ -675,7 +784,7 @@ def varexpand(mystring, mydict=None, error_leader=None): else: newstring.append("'") # Quote removal is handled by shlex. insing=not insing - pos=pos+1 + pos += 1 continue elif current == '"': if (insing): @@ -683,9 +792,9 @@ def varexpand(mystring, mydict=None, error_leader=None): else: newstring.append('"') # Quote removal is handled by shlex. indoub=not indoub - pos=pos+1 + pos += 1 continue - if (not insing): + if not insing: #expansion time if current == "\n": #convert newlines to spaces @@ -700,7 +809,7 @@ def varexpand(mystring, mydict=None, error_leader=None): # escaped newline characters. Note that we don't handle # escaped quotes here, since getconfig() uses shlex # to handle that earlier. - if (pos+1>=len(mystring)): + if pos + 1 >= len(mystring): newstring.append(current) break else: @@ -722,15 +831,15 @@ def varexpand(mystring, mydict=None, error_leader=None): newstring.append(mystring[pos - 2:pos]) continue elif current == "$": - pos=pos+1 - if mystring[pos]=="{": - pos=pos+1 - braced=True + pos += 1 + if mystring[pos] == "{": + pos += 1 + braced = True else: - braced=False - myvstart=pos + braced = False + myvstart = pos while mystring[pos] in _varexpand_word_chars: - if (pos+1)>=len(mystring): + if pos + 1 >= len(mystring): if braced: msg = _varexpand_unexpected_eof_msg if error_leader is not None: @@ -738,20 +847,20 @@ def varexpand(mystring, mydict=None, error_leader=None): writemsg(msg + "\n", noiselevel=-1) return "" else: - pos=pos+1 + pos += 1 break - pos=pos+1 - myvarname=mystring[myvstart:pos] + pos += 1 + myvarname = mystring[myvstart:pos] if braced: - if mystring[pos]!="}": + if mystring[pos] != "}": msg = _varexpand_unexpected_eof_msg if error_leader is not None: msg = error_leader() + msg writemsg(msg + "\n", noiselevel=-1) return "" else: - pos=pos+1 - if len(myvarname)==0: + pos += 1 + if len(myvarname) == 0: msg = "$" if braced: msg += "{}" @@ -760,7 +869,7 @@ def varexpand(mystring, mydict=None, error_leader=None): msg = error_leader() + msg writemsg(msg + "\n", noiselevel=-1) return "" - numvars=numvars+1 + numvars += 1 if myvarname in mydict: newstring.append(mydict[myvarname]) else: @@ -775,9 +884,9 @@ def varexpand(mystring, mydict=None, error_leader=None): # broken and removed, but can still be imported pickle_write = None -def pickle_read(filename,default=None,debug=0): +def pickle_read(filename, default=None, debug=0): if not os.access(filename, os.R_OK): - writemsg(_("pickle_read(): File not readable. '")+filename+"'\n",1) + writemsg(_("pickle_read(): File not readable. '") + filename + "'\n", 1) return default data = None try: @@ -786,12 +895,12 @@ def pickle_read(filename,default=None,debug=0): mypickle = pickle.Unpickler(myf) data = mypickle.load() myf.close() - del mypickle,myf - writemsg(_("pickle_read(): Loaded pickle. '")+filename+"'\n",1) + del mypickle, myf + writemsg(_("pickle_read(): Loaded pickle. '") + filename + "'\n", 1) except SystemExit as e: raise except Exception as e: - writemsg(_("!!! Failed to load pickle: ")+str(e)+"\n",1) + writemsg(_("!!! Failed to load pickle: ") + str(e) + "\n", 1) data = default return data @@ -819,6 +928,9 @@ class cmp_sort_key(object): list.sort(), making it easier to port code for python-3.0 compatibility. It works by generating key objects which use the given cmp function to implement their __lt__ method. + + Beginning with Python 2.7 and 3.2, equivalent functionality is provided + by functools.cmp_to_key(). """ __slots__ = ("_cmp_func",) @@ -911,6 +1023,10 @@ def apply_permissions(filename, uid=-1, gid=-1, mode=-1, mask=-1, modified = False + # Since Python 3.4, chown requires int type (no proxies). + uid = int(uid) + gid = int(gid) + if stat_cached is None: try: if follow_links: @@ -1130,7 +1246,7 @@ class atomic_ofstream(ObjectProxy): object.__setattr__(self, '_file', open_func(_unicode_encode(tmp_name, encoding=_encodings['fs'], errors='strict'), - mode=mode, **kargs)) + mode=mode, **portage._native_kwargs(kargs))) return except IOError as e: if canonical_path == filename: @@ -1212,7 +1328,7 @@ class atomic_ofstream(ObjectProxy): self.close() def __del__(self): - """If the user does not explicitely call close(), it is + """If the user does not explicitly call close(), it is assumed that an error has occurred, so we abort().""" try: f = object.__getattribute__(self, '_file') @@ -1391,9 +1507,9 @@ class LazyItemsDict(UserDict): lazy_item = self.lazy_items.get(k) if lazy_item is not None: if not lazy_item.singleton: - raise TypeError(_unicode_decode("LazyItemsDict " + \ + raise TypeError("LazyItemsDict " + \ "deepcopy is unsafe with lazy items that are " + \ - "not singletons: key=%s value=%s") % (k, lazy_item,)) + "not singletons: key=%s value=%s" % (k, lazy_item,)) UserDict.__setitem__(result, k_copy, deepcopy(self[k], memo)) return result @@ -1565,13 +1681,13 @@ def find_updated_config_files(target_root, config_protect): """ Return a tuple of configuration files that needs to be updated. The tuple contains lists organized like this: - [ protected_dir, file_list ] + [protected_dir, file_list] If the protected config isn't a protected_dir but a procted_file, list is: - [ protected_file, None ] + [protected_file, None] If no configuration files needs to be updated, None is returned """ - os = _os_merge + encoding = _encodings['fs'] if config_protect: # directories with some protect files in them @@ -1603,10 +1719,24 @@ def find_updated_config_files(target_root, config_protect): mycommand = "find '%s' -maxdepth 1 -name '._cfg????_%s'" % \ os.path.split(x.rstrip(os.path.sep)) mycommand += " ! -name '.*~' ! -iname '.*.bak' -print0" - a = subprocess_getstatusoutput(mycommand) - - if a[0] == 0: - files = a[1].split('\0') + cmd = shlex_split(mycommand) + + if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000: + # Python 3.1 _execvp throws TypeError for non-absolute executable + # path passed as bytes (see http://bugs.python.org/issue8513). + fullname = portage.process.find_binary(cmd[0]) + if fullname is None: + raise portage.exception.CommandNotFound(cmd[0]) + cmd[0] = fullname + + cmd = [_unicode_encode(arg, encoding=encoding, errors='strict') + for arg in cmd] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + output = _unicode_decode(proc.communicate()[0], encoding=encoding) + status = proc.wait() + if os.WIFEXITED(status) and os.WEXITSTATUS(status) == os.EX_OK: + files = output.split('\0') # split always produces an empty string as the last element if files and not files[-1]: del files[-1] diff --git a/portage_with_autodep/pym/portage/util/__init__.pyo b/portage_with_autodep/pym/portage/util/__init__.pyo Binary files differindex 941b286..9bab394 100644 --- a/portage_with_autodep/pym/portage/util/__init__.pyo +++ b/portage_with_autodep/pym/portage/util/__init__.pyo diff --git a/portage_with_autodep/pym/portage/util/_desktop_entry.py b/portage_with_autodep/pym/portage/util/_desktop_entry.py index 7901780..0b49547 100644 --- a/portage_with_autodep/pym/portage/util/_desktop_entry.py +++ b/portage_with_autodep/pym/portage/util/_desktop_entry.py @@ -1,7 +1,8 @@ -# Copyright 2012 Gentoo Foundation +# Copyright 2012-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import io +import re import subprocess import sys @@ -10,7 +11,9 @@ try: except ImportError: from ConfigParser import Error as ConfigParserError, RawConfigParser +import portage from portage import _encodings, _unicode_encode, _unicode_decode +from portage.util import writemsg def parse_desktop_entry(path): """ @@ -31,45 +34,71 @@ def parse_desktop_entry(path): encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace') as f: - read_file(f) + content = f.read() + + # In Python 3.2, read_file does not support bytes in file names + # (see bug #429544), so use StringIO to hide the file name. + read_file(io.StringIO(content)) return parser -_ignored_service_errors = ( - 'error: required key "Name" in group "Desktop Entry" is not present', - 'error: key "Actions" is present in group "Desktop Entry", but the type is "Service" while this key is only valid for type "Application"', - 'error: key "MimeType" is present in group "Desktop Entry", but the type is "Service" while this key is only valid for type "Application"', +_trivial_warnings = re.compile(r' looks redundant with value ') + +_ignored_errors = ( + # Ignore error for emacs.desktop: + # https://bugs.freedesktop.org/show_bug.cgi?id=35844#c6 + 'error: (will be fatal in the future): value "TextEditor" in key "Categories" in group "Desktop Entry" requires another category to be present among the following categories: Utility', + 'warning: key "Encoding" in group "Desktop Entry" is deprecated' +) + +_ShowIn_exemptions = ( + # See bug #480586. + 'contains an unregistered value "Pantheon"', ) def validate_desktop_entry(path): args = ["desktop-file-validate", path] - if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000: - # Python 3.1 does not support bytes in Popen args. - args = [_unicode_encode(x, errors='strict') for x in args] + + if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000: + # Python 3.1 _execvp throws TypeError for non-absolute executable + # path passed as bytes (see http://bugs.python.org/issue8513). + fullname = portage.process.find_binary(args[0]) + if fullname is None: + raise portage.exception.CommandNotFound(args[0]) + args[0] = fullname + + args = [_unicode_encode(x, errors='strict') for x in args] proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output_lines = _unicode_decode(proc.communicate()[0]).splitlines() proc.wait() if output_lines: - try: - desktop_entry = parse_desktop_entry(path) - except ConfigParserError: - pass - else: - if desktop_entry.has_section("Desktop Entry"): - try: - entry_type = desktop_entry.get("Desktop Entry", "Type") - except ConfigParserError: - pass - else: - if entry_type == "Service": - # Filter false errors for Type=Service (bug #414125). - filtered_output = [] - for line in output_lines: - if line[len(path)+2:] in _ignored_service_errors: - continue - filtered_output.append(line) - output_lines = filtered_output + filtered_output = [] + for line in output_lines: + msg = line[len(path)+2:] + # "hint:" output is new in desktop-file-utils-0.21 + if msg.startswith('hint: ') or msg in _ignored_errors: + continue + if 'for key "NotShowIn" in group "Desktop Entry"' in msg or \ + 'for key "OnlyShowIn" in group "Desktop Entry"' in msg: + exempt = False + for s in _ShowIn_exemptions: + if s in msg: + exempt = True + break + if exempt: + continue + filtered_output.append(line) + output_lines = filtered_output + + if output_lines: + output_lines = [line for line in output_lines + if _trivial_warnings.search(line) is None] return output_lines + +if __name__ == "__main__": + for arg in sys.argv[1:]: + for line in validate_desktop_entry(arg): + writemsg(line + "\n", noiselevel=-1) diff --git a/portage_with_autodep/pym/portage/util/_desktop_entry.pyo b/portage_with_autodep/pym/portage/util/_desktop_entry.pyo Binary files differindex 7dec17a..c338c3f 100644 --- a/portage_with_autodep/pym/portage/util/_desktop_entry.pyo +++ b/portage_with_autodep/pym/portage/util/_desktop_entry.pyo diff --git a/portage_with_autodep/pym/portage/util/_dyn_libs/LinkageMapELF.py b/portage_with_autodep/pym/portage/util/_dyn_libs/LinkageMapELF.py index e71ac73..3920f94 100644 --- a/portage_with_autodep/pym/portage/util/_dyn_libs/LinkageMapELF.py +++ b/portage_with_autodep/pym/portage/util/_dyn_libs/LinkageMapELF.py @@ -1,4 +1,4 @@ -# Copyright 1998-2011 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import errno @@ -26,7 +26,7 @@ class LinkageMapELF(object): _soname_map_class = slot_dict_class( ("consumers", "providers"), prefix="") - class _obj_properies_class(object): + class _obj_properties_class(object): __slots__ = ("arch", "needed", "runpaths", "soname", "alt_paths", "owner",) @@ -316,7 +316,7 @@ class LinkageMapELF(object): myprops = obj_properties.get(obj_key) if myprops is None: indexed = False - myprops = self._obj_properies_class( + myprops = self._obj_properties_class( arch, needed, path, soname, [], owner) obj_properties[obj_key] = myprops # All object paths are added into the obj_properties tuple. @@ -678,7 +678,7 @@ class LinkageMapELF(object): rValue[soname].add(provider) return rValue - def findConsumers(self, obj, exclude_providers=None): + def findConsumers(self, obj, exclude_providers=None, greedy=True): """ Find consumers of an object or object key. @@ -715,6 +715,9 @@ class LinkageMapELF(object): '/usr/lib/libssl.so.0.9.8'), and return True if the library is owned by a provider which is planned for removal. @type exclude_providers: collection + @param greedy: If True, then include consumers that are satisfied + by alternative providers, otherwise omit them. Default is True. + @type greedy: Boolean @rtype: set of strings (example: set(['/bin/foo', '/usr/bin/bar'])) @return: The return value is a soname -> set-of-library-paths, where set-of-library-paths satisfy soname. @@ -769,16 +772,19 @@ class LinkageMapELF(object): defpath_keys = set(self._path_key(x) for x in self._defpath) satisfied_consumer_keys = set() if soname_node is not None: - if exclude_providers is not None: + if exclude_providers is not None or not greedy: relevant_dir_keys = set() for provider_key in soname_node.providers: + if not greedy and provider_key == obj_key: + continue provider_objs = self._obj_properties[provider_key].alt_paths for p in provider_objs: provider_excluded = False - for excluded_provider_isowner in exclude_providers: - if excluded_provider_isowner(p): - provider_excluded = True - break + if exclude_providers is not None: + for excluded_provider_isowner in exclude_providers: + if excluded_provider_isowner(p): + provider_excluded = True + break if not provider_excluded: # This provider is not excluded. It will # satisfy a consumer of this soname if it diff --git a/portage_with_autodep/pym/portage/util/_dyn_libs/LinkageMapELF.pyo b/portage_with_autodep/pym/portage/util/_dyn_libs/LinkageMapELF.pyo Binary files differindex c1e5603..4907057 100644 --- a/portage_with_autodep/pym/portage/util/_dyn_libs/LinkageMapELF.pyo +++ b/portage_with_autodep/pym/portage/util/_dyn_libs/LinkageMapELF.pyo diff --git a/portage_with_autodep/pym/portage/util/_dyn_libs/PreservedLibsRegistry.pyo b/portage_with_autodep/pym/portage/util/_dyn_libs/PreservedLibsRegistry.pyo Binary files differindex 8cdd7cb..8d18cdc 100644 --- a/portage_with_autodep/pym/portage/util/_dyn_libs/PreservedLibsRegistry.pyo +++ b/portage_with_autodep/pym/portage/util/_dyn_libs/PreservedLibsRegistry.pyo diff --git a/portage_with_autodep/pym/portage/util/_dyn_libs/__init__.pyo b/portage_with_autodep/pym/portage/util/_dyn_libs/__init__.pyo Binary files differindex 960b66e..86d7f56 100644 --- a/portage_with_autodep/pym/portage/util/_dyn_libs/__init__.pyo +++ b/portage_with_autodep/pym/portage/util/_dyn_libs/__init__.pyo diff --git a/portage_with_autodep/pym/portage/util/_eventloop/EventLoop.py b/portage_with_autodep/pym/portage/util/_eventloop/EventLoop.py index bbbce52..9ffcc74 100644 --- a/portage_with_autodep/pym/portage/util/_eventloop/EventLoop.py +++ b/portage_with_autodep/pym/portage/util/_eventloop/EventLoop.py @@ -1,20 +1,37 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import errno -import fcntl import logging import os import select import signal +import sys import time +try: + import fcntl +except ImportError: + # http://bugs.jython.org/issue1074 + fcntl = None + +try: + import threading +except ImportError: + import dummy_threading as threading + from portage.util import writemsg_level from ..SlotObject import SlotObject from .PollConstants import PollConstants from .PollSelectAdapter import PollSelectAdapter class EventLoop(object): + """ + An event loop, intended to be compatible with the GLib event loop. + Call the iteration method in order to execute one iteration of the + loop. The idle_add and timeout_add methods serve as thread-safe + means to interact with the loop's thread. + """ supports_multiprocessing = True @@ -43,7 +60,9 @@ class EventLoop(object): that global_event_loop does not need constructor arguments) @type main: bool """ - self._use_signal = main + self._use_signal = main and fcntl is not None + self._thread_rlock = threading.RLock() + self._thread_condition = threading.Condition(self._thread_rlock) self._poll_event_queue = [] self._poll_event_handlers = {} self._poll_event_handler_ids = {} @@ -52,14 +71,48 @@ class EventLoop(object): self._idle_callbacks = {} self._timeout_handlers = {} self._timeout_interval = None - self._poll_obj = create_poll_instance() - self.IO_ERR = PollConstants.POLLERR - self.IO_HUP = PollConstants.POLLHUP - self.IO_IN = PollConstants.POLLIN - self.IO_NVAL = PollConstants.POLLNVAL - self.IO_OUT = PollConstants.POLLOUT - self.IO_PRI = PollConstants.POLLPRI + self._poll_obj = None + try: + select.epoll + except AttributeError: + pass + else: + try: + epoll_obj = select.epoll() + except IOError: + # This happens with Linux 2.4 kernels: + # IOError: [Errno 38] Function not implemented + pass + else: + + # FD_CLOEXEC is enabled by default in Python >=3.4. + if sys.hexversion < 0x3040000 and fcntl is not None: + try: + fcntl.FD_CLOEXEC + except AttributeError: + pass + else: + fcntl.fcntl(epoll_obj.fileno(), fcntl.F_SETFD, + fcntl.fcntl(epoll_obj.fileno(), + fcntl.F_GETFD) | fcntl.FD_CLOEXEC) + + self._poll_obj = _epoll_adapter(epoll_obj) + self.IO_ERR = select.EPOLLERR + self.IO_HUP = select.EPOLLHUP + self.IO_IN = select.EPOLLIN + self.IO_NVAL = 0 + self.IO_OUT = select.EPOLLOUT + self.IO_PRI = select.EPOLLPRI + + if self._poll_obj is None: + self._poll_obj = create_poll_instance() + self.IO_ERR = PollConstants.POLLERR + self.IO_HUP = PollConstants.POLLHUP + self.IO_IN = PollConstants.POLLIN + self.IO_NVAL = PollConstants.POLLNVAL + self.IO_OUT = PollConstants.POLLOUT + self.IO_PRI = PollConstants.POLLPRI self._child_handlers = {} self._sigchld_read = None @@ -67,6 +120,14 @@ class EventLoop(object): self._sigchld_src_id = None self._pid = os.getpid() + def _new_source_id(self): + """ + Generate a new source id. This method is thread-safe. + """ + with self._thread_rlock: + self._event_handler_id += 1 + return self._event_handler_id + def _poll(self, timeout=None): """ All poll() calls pass through here. The poll events @@ -85,9 +146,11 @@ class EventLoop(object): try: self._poll_event_queue.extend(self._poll_obj.poll(timeout)) break - except select.error as e: + except (IOError, select.error) as e: # Silently handle EINTR, which is normal when we have - # received a signal such as SIGINT. + # received a signal such as SIGINT (epoll objects may + # raise IOError rather than select.error, at least in + # Python 3.2). if not (e.args and e.args[0] == errno.EINTR): writemsg_level("\n!!! select error: %s\n" % (e,), level=logging.ERROR, noiselevel=-1) @@ -101,7 +164,19 @@ class EventLoop(object): def iteration(self, *args): """ - Like glib.MainContext.iteration(), runs a single iteration. + Like glib.MainContext.iteration(), runs a single iteration. In order + to avoid blocking forever when may_block is True (the default), + callers must be careful to ensure that at least one of the following + conditions is met: + 1) An event source or timeout is registered which is guaranteed + to trigger at least on event (a call to an idle function + only counts as an event if it returns a False value which + causes it to stop being called) + 2) Another thread is guaranteed to call one of the thread-safe + methods which notify iteration to stop waiting (such as + idle_add or timeout_add). + These rules ensure that iteration is able to block until an event + arrives, without doing any busy waiting that would waste CPU time. @type may_block: bool @param may_block: if True the call may block waiting for an event (default is True). @@ -120,23 +195,32 @@ class EventLoop(object): event_queue = self._poll_event_queue event_handlers = self._poll_event_handlers events_handled = 0 + timeouts_checked = False if not event_handlers: - if self._run_timeouts(): - events_handled += 1 - if not event_handlers: - if not events_handled and may_block and \ - self._timeout_interval is not None: + with self._thread_condition: + if self._run_timeouts(): + events_handled += 1 + timeouts_checked = True + if not event_handlers and not events_handled and may_block: # Block so that we don't waste cpu time by looping too # quickly. This makes EventLoop useful for code that needs # to wait for timeout callbacks regardless of whether or # not any IO handlers are currently registered. - try: - self._poll(timeout=self._timeout_interval) - except StopIteration: - pass + timeout = self._get_poll_timeout() + if timeout is None: + wait_timeout = None + else: + wait_timeout = float(timeout) / 1000 + # NOTE: In order to avoid a possible infinite wait when + # wait_timeout is None, the previous _run_timeouts() + # call must have returned False *with* _thread_condition + # acquired. Otherwise, we would risk going to sleep after + # our only notify event has already passed. + self._thread_condition.wait(wait_timeout) if self._run_timeouts(): events_handled += 1 + timeouts_checked = True # If any timeouts have executed, then return immediately, # in order to minimize latency in termination of iteration @@ -147,14 +231,18 @@ class EventLoop(object): if not event_queue: if may_block: - if self._child_handlers: - if self._timeout_interval is None: - timeout = self._sigchld_interval - else: - timeout = min(self._sigchld_interval, - self._timeout_interval) - else: - timeout = self._timeout_interval + timeout = self._get_poll_timeout() + + # Avoid blocking for IO if there are any timeout + # or idle callbacks available to process. + if timeout != 0 and not timeouts_checked: + if self._run_timeouts(): + events_handled += 1 + timeouts_checked = True + if events_handled: + # Minimize latency for loops controlled + # by timeout or idle callback events. + timeout = 0 else: timeout = 0 @@ -170,17 +258,37 @@ class EventLoop(object): while event_queue: events_handled += 1 f, event = event_queue.pop() - x = event_handlers[f] + try: + x = event_handlers[f] + except KeyError: + # This is known to be triggered by the epoll + # implementation in qemu-user-1.2.2, and appears + # to be harmless (see bug #451326). + continue if not x.callback(f, event, *x.args): self.source_remove(x.source_id) - # Run timeouts last, in order to minimize latency in - # termination of iteration loops that they may control. - if self._run_timeouts(): - events_handled += 1 + if not timeouts_checked: + if self._run_timeouts(): + events_handled += 1 + timeouts_checked = True return bool(events_handled) + def _get_poll_timeout(self): + + with self._thread_rlock: + if self._child_handlers: + if self._timeout_interval is None: + timeout = self._sigchld_interval + else: + timeout = min(self._sigchld_interval, + self._timeout_interval) + else: + timeout = self._timeout_interval + + return timeout + def child_watch_add(self, pid, callback, data=None): """ Like glib.child_watch_add(), sets callback to be called with the @@ -201,18 +309,29 @@ class EventLoop(object): @rtype: int @return: an integer ID """ - self._event_handler_id += 1 - source_id = self._event_handler_id + source_id = self._new_source_id() self._child_handlers[source_id] = self._child_callback_class( callback=callback, data=data, pid=pid, source_id=source_id) if self._use_signal: if self._sigchld_read is None: self._sigchld_read, self._sigchld_write = os.pipe() + fcntl.fcntl(self._sigchld_read, fcntl.F_SETFL, fcntl.fcntl(self._sigchld_read, fcntl.F_GETFL) | os.O_NONBLOCK) + # FD_CLOEXEC is enabled by default in Python >=3.4. + if sys.hexversion < 0x3040000: + try: + fcntl.FD_CLOEXEC + except AttributeError: + pass + else: + fcntl.fcntl(self._sigchld_read, fcntl.F_SETFD, + fcntl.fcntl(self._sigchld_read, + fcntl.F_GETFD) | fcntl.FD_CLOEXEC) + # The IO watch is dynamically registered and unregistered as # needed, since we don't want to consider it as a valid source # of events when there are no child listeners. It's important @@ -276,22 +395,25 @@ class EventLoop(object): """ Like glib.idle_add(), if callback returns False it is automatically removed from the list of event sources and will - not be called again. + not be called again. This method is thread-safe. @type callback: callable @param callback: a function to call @rtype: int @return: an integer ID """ - self._event_handler_id += 1 - source_id = self._event_handler_id - self._idle_callbacks[source_id] = self._idle_callback_class( - args=args, callback=callback, source_id=source_id) + with self._thread_condition: + source_id = self._new_source_id() + self._idle_callbacks[source_id] = self._idle_callback_class( + args=args, callback=callback, source_id=source_id) + self._thread_condition.notify() return source_id def _run_idle_callbacks(self): + # assumes caller has acquired self._thread_rlock if not self._idle_callbacks: - return + return False + state_change = 0 # Iterate of our local list, since self._idle_callbacks can be # modified during the exection of these callbacks. for x in list(self._idle_callbacks.values()): @@ -304,26 +426,32 @@ class EventLoop(object): x.calling = True try: if not x.callback(*x.args): + state_change += 1 self.source_remove(x.source_id) finally: x.calling = False + return bool(state_change) + def timeout_add(self, interval, function, *args): """ Like glib.timeout_add(), interval argument is the number of milliseconds between calls to your function, and your function should return False to stop being called, or True to continue being called. Any additional positional arguments given here - are passed to your function when it's called. + are passed to your function when it's called. This method is + thread-safe. """ - self._event_handler_id += 1 - source_id = self._event_handler_id - self._timeout_handlers[source_id] = \ - self._timeout_handler_class( - interval=interval, function=function, args=args, - source_id=source_id, timestamp=time.time()) - if self._timeout_interval is None or self._timeout_interval > interval: - self._timeout_interval = interval + with self._thread_condition: + source_id = self._new_source_id() + self._timeout_handlers[source_id] = \ + self._timeout_handler_class( + interval=interval, function=function, args=args, + source_id=source_id, timestamp=time.time()) + if self._timeout_interval is None or \ + self._timeout_interval > interval: + self._timeout_interval = interval + self._thread_condition.notify() return source_id def _run_timeouts(self): @@ -333,37 +461,40 @@ class EventLoop(object): if self._poll_child_processes(): calls += 1 - self._run_idle_callbacks() - - if not self._timeout_handlers: - return bool(calls) + with self._thread_rlock: - ready_timeouts = [] - current_time = time.time() - for x in self._timeout_handlers.values(): - elapsed_seconds = current_time - x.timestamp - # elapsed_seconds < 0 means the system clock has been adjusted - if elapsed_seconds < 0 or \ - (x.interval - 1000 * elapsed_seconds) <= 0: - ready_timeouts.append(x) + if self._run_idle_callbacks(): + calls += 1 - # Iterate of our local list, since self._timeout_handlers can be - # modified during the exection of these callbacks. - for x in ready_timeouts: - if x.source_id not in self._timeout_handlers: - # it got cancelled while executing another timeout - continue - if x.calling: - # don't call it recursively - continue - calls += 1 - x.calling = True - try: - x.timestamp = time.time() - if not x.function(*x.args): - self.source_remove(x.source_id) - finally: - x.calling = False + if not self._timeout_handlers: + return bool(calls) + + ready_timeouts = [] + current_time = time.time() + for x in self._timeout_handlers.values(): + elapsed_seconds = current_time - x.timestamp + # elapsed_seconds < 0 means the system clock has been adjusted + if elapsed_seconds < 0 or \ + (x.interval - 1000 * elapsed_seconds) <= 0: + ready_timeouts.append(x) + + # Iterate of our local list, since self._timeout_handlers can be + # modified during the exection of these callbacks. + for x in ready_timeouts: + if x.source_id not in self._timeout_handlers: + # it got cancelled while executing another timeout + continue + if x.calling: + # don't call it recursively + continue + calls += 1 + x.calling = True + try: + x.timestamp = time.time() + if not x.function(*x.args): + self.source_remove(x.source_id) + finally: + x.calling = False return bool(calls) @@ -385,8 +516,7 @@ class EventLoop(object): """ if f in self._poll_event_handlers: raise AssertionError("fd %d is already registered" % f) - self._event_handler_id += 1 - source_id = self._event_handler_id + source_id = self._new_source_id() self._poll_event_handler_ids[source_id] = f self._poll_event_handlers[f] = self._io_handler_class( args=args, callback=callback, f=f, source_id=source_id) @@ -406,18 +536,21 @@ class EventLoop(object): self.source_remove(self._sigchld_src_id) self._sigchld_src_id = None return True - idle_callback = self._idle_callbacks.pop(reg_id, None) - if idle_callback is not None: - return True - timeout_handler = self._timeout_handlers.pop(reg_id, None) - if timeout_handler is not None: - if timeout_handler.interval == self._timeout_interval: - if self._timeout_handlers: - self._timeout_interval = \ - min(x.interval for x in self._timeout_handlers.values()) - else: - self._timeout_interval = None - return True + + with self._thread_rlock: + idle_callback = self._idle_callbacks.pop(reg_id, None) + if idle_callback is not None: + return True + timeout_handler = self._timeout_handlers.pop(reg_id, None) + if timeout_handler is not None: + if timeout_handler.interval == self._timeout_interval: + if self._timeout_handlers: + self._timeout_interval = min(x.interval + for x in self._timeout_handlers.values()) + else: + self._timeout_interval = None + return True + f = self._poll_event_handler_ids.pop(reg_id, None) if f is None: return False @@ -467,7 +600,12 @@ def can_poll_device(): return _can_poll_device p = select.poll() - p.register(dev_null.fileno(), PollConstants.POLLIN) + try: + p.register(dev_null.fileno(), PollConstants.POLLIN) + except TypeError: + # Jython: Object 'org.python.core.io.FileIO@f8f175' is not watchable + _can_poll_device = False + return _can_poll_device invalid_request = False for f, event in p.poll(): @@ -488,3 +626,37 @@ def create_poll_instance(): if can_poll_device(): return select.poll() return PollSelectAdapter() + +class _epoll_adapter(object): + """ + Wraps a select.epoll instance in order to make it compatible + with select.poll instances. This is necessary since epoll instances + interpret timeout arguments differently. Note that the file descriptor + that is associated with an epoll instance will close automatically when + it is garbage collected, so it's not necessary to close it explicitly. + """ + __slots__ = ('_epoll_obj',) + + def __init__(self, epoll_obj): + self._epoll_obj = epoll_obj + + def register(self, fd, *args): + self._epoll_obj.register(fd, *args) + + def unregister(self, fd): + self._epoll_obj.unregister(fd) + + def poll(self, *args): + if len(args) > 1: + raise TypeError( + "poll expected at most 2 arguments, got " + \ + repr(1 + len(args))) + timeout = -1 + if args: + timeout = args[0] + if timeout is None or timeout < 0: + timeout = -1 + elif timeout != 0: + timeout = float(timeout) / 1000 + + return self._epoll_obj.poll(timeout) diff --git a/portage_with_autodep/pym/portage/util/_eventloop/EventLoop.pyo b/portage_with_autodep/pym/portage/util/_eventloop/EventLoop.pyo Binary files differindex 6ce2883..948018f 100644 --- a/portage_with_autodep/pym/portage/util/_eventloop/EventLoop.pyo +++ b/portage_with_autodep/pym/portage/util/_eventloop/EventLoop.pyo diff --git a/portage_with_autodep/pym/portage/util/_eventloop/GlibEventLoop.pyo b/portage_with_autodep/pym/portage/util/_eventloop/GlibEventLoop.pyo Binary files differindex d3453a4..d149863 100644 --- a/portage_with_autodep/pym/portage/util/_eventloop/GlibEventLoop.pyo +++ b/portage_with_autodep/pym/portage/util/_eventloop/GlibEventLoop.pyo diff --git a/portage_with_autodep/pym/portage/util/_eventloop/PollConstants.pyo b/portage_with_autodep/pym/portage/util/_eventloop/PollConstants.pyo Binary files differindex 6c7c953..876c5b1 100644 --- a/portage_with_autodep/pym/portage/util/_eventloop/PollConstants.pyo +++ b/portage_with_autodep/pym/portage/util/_eventloop/PollConstants.pyo diff --git a/portage_with_autodep/pym/portage/util/_eventloop/PollSelectAdapter.py b/portage_with_autodep/pym/portage/util/_eventloop/PollSelectAdapter.py index 17e63d9..244788c 100644 --- a/portage_with_autodep/pym/portage/util/_eventloop/PollSelectAdapter.py +++ b/portage_with_autodep/pym/portage/util/_eventloop/PollSelectAdapter.py @@ -64,7 +64,7 @@ class PollSelectAdapter(object): if timeout is not None and timeout < 0: timeout = None if timeout is not None: - select_args.append(timeout / 1000) + select_args.append(float(timeout) / 1000) select_events = select.select(*select_args) poll_events = [] diff --git a/portage_with_autodep/pym/portage/util/_eventloop/PollSelectAdapter.pyo b/portage_with_autodep/pym/portage/util/_eventloop/PollSelectAdapter.pyo Binary files differindex e9ecc51..eac94ca 100644 --- a/portage_with_autodep/pym/portage/util/_eventloop/PollSelectAdapter.pyo +++ b/portage_with_autodep/pym/portage/util/_eventloop/PollSelectAdapter.pyo diff --git a/portage_with_autodep/pym/portage/util/_eventloop/__init__.pyo b/portage_with_autodep/pym/portage/util/_eventloop/__init__.pyo Binary files differindex 69864a6..2b2f9a1 100644 --- a/portage_with_autodep/pym/portage/util/_eventloop/__init__.pyo +++ b/portage_with_autodep/pym/portage/util/_eventloop/__init__.pyo diff --git a/portage_with_autodep/pym/portage/util/_eventloop/global_event_loop.pyo b/portage_with_autodep/pym/portage/util/_eventloop/global_event_loop.pyo Binary files differindex 3d57192..7a7f087 100644 --- a/portage_with_autodep/pym/portage/util/_eventloop/global_event_loop.pyo +++ b/portage_with_autodep/pym/portage/util/_eventloop/global_event_loop.pyo diff --git a/portage_with_autodep/pym/portage/util/_pty.pyo b/portage_with_autodep/pym/portage/util/_pty.pyo Binary files differindex 70b5eb0..0fa01cf 100644 --- a/portage_with_autodep/pym/portage/util/_pty.pyo +++ b/portage_with_autodep/pym/portage/util/_pty.pyo diff --git a/portage_with_autodep/pym/portage/util/_urlopen.py b/portage_with_autodep/pym/portage/util/_urlopen.py index 307624b..15f041a 100644 --- a/portage_with_autodep/pym/portage/util/_urlopen.py +++ b/portage_with_autodep/pym/portage/util/_urlopen.py @@ -1,7 +1,11 @@ # Copyright 2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +import io import sys +from datetime import datetime +from time import mktime +from email.utils import formatdate, parsedate try: from urllib.request import urlopen as _urlopen @@ -14,29 +18,74 @@ except ImportError: import urllib2 as urllib_request from urllib import splituser as urllib_parse_splituser -def urlopen(url): - try: - return _urlopen(url) - except SystemExit: - raise - except Exception: - if sys.hexversion < 0x3000000: - raise - parse_result = urllib_parse.urlparse(url) - if parse_result.scheme not in ("http", "https") or \ - not parse_result.username: - raise - - return _new_urlopen(url) - -def _new_urlopen(url): - # This is experimental code for bug #413983. +if sys.hexversion >= 0x3000000: + long = int + +# to account for the difference between TIMESTAMP of the index' contents +# and the file-'mtime' +TIMESTAMP_TOLERANCE = 5 + +def urlopen(url, if_modified_since=None): parse_result = urllib_parse.urlparse(url) - netloc = urllib_parse_splituser(parse_result.netloc)[1] - url = urllib_parse.urlunparse((parse_result.scheme, netloc, parse_result.path, parse_result.params, parse_result.query, parse_result.fragment)) - password_manager = urllib_request.HTTPPasswordMgrWithDefaultRealm() - if parse_result.username is not None: - password_manager.add_password(None, url, parse_result.username, parse_result.password) - auth_handler = urllib_request.HTTPBasicAuthHandler(password_manager) - opener = urllib_request.build_opener(auth_handler) - return opener.open(url) + if parse_result.scheme not in ("http", "https"): + return _urlopen(url) + else: + netloc = urllib_parse_splituser(parse_result.netloc)[1] + url = urllib_parse.urlunparse((parse_result.scheme, netloc, parse_result.path, parse_result.params, parse_result.query, parse_result.fragment)) + password_manager = urllib_request.HTTPPasswordMgrWithDefaultRealm() + request = urllib_request.Request(url) + request.add_header('User-Agent', 'Gentoo Portage') + if if_modified_since: + request.add_header('If-Modified-Since', _timestamp_to_http(if_modified_since)) + if parse_result.username is not None: + password_manager.add_password(None, url, parse_result.username, parse_result.password) + auth_handler = CompressedResponseProcessor(password_manager) + opener = urllib_request.build_opener(auth_handler) + hdl = opener.open(request) + if hdl.headers.get('last-modified', ''): + try: + add_header = hdl.headers.add_header + except AttributeError: + # Python 2 + add_header = hdl.headers.addheader + add_header('timestamp', _http_to_timestamp(hdl.headers.get('last-modified'))) + return hdl + +def _timestamp_to_http(timestamp): + dt = datetime.fromtimestamp(float(long(timestamp)+TIMESTAMP_TOLERANCE)) + stamp = mktime(dt.timetuple()) + return formatdate(timeval=stamp, localtime=False, usegmt=True) + +def _http_to_timestamp(http_datetime_string): + tuple = parsedate(http_datetime_string) + timestamp = mktime(tuple) + return str(long(timestamp)) + +class CompressedResponseProcessor(urllib_request.HTTPBasicAuthHandler): + # Handler for compressed responses. + + def http_request(self, req): + req.add_header('Accept-Encoding', 'bzip2,gzip,deflate') + return req + https_request = http_request + + def http_response(self, req, response): + decompressed = None + if response.headers.get('content-encoding') == 'bzip2': + import bz2 + decompressed = io.BytesIO(bz2.decompress(response.read())) + elif response.headers.get('content-encoding') == 'gzip': + from gzip import GzipFile + decompressed = GzipFile(fileobj=io.BytesIO(response.read()), mode='r') + elif response.headers.get('content-encoding') == 'deflate': + import zlib + try: + decompressed = io.BytesIO(zlib.decompress(response.read())) + except zlib.error: # they ignored RFC1950 + decompressed = io.BytesIO(zlib.decompress(response.read(), -zlib.MAX_WBITS)) + if decompressed: + old_response = response + response = urllib_request.addinfourl(decompressed, old_response.headers, old_response.url, old_response.code) + response.msg = old_response.msg + return response + https_response = http_response diff --git a/portage_with_autodep/pym/portage/util/_urlopen.pyo b/portage_with_autodep/pym/portage/util/_urlopen.pyo Binary files differindex 9f51de8..d548069 100644 --- a/portage_with_autodep/pym/portage/util/_urlopen.pyo +++ b/portage_with_autodep/pym/portage/util/_urlopen.pyo diff --git a/portage_with_autodep/pym/portage/util/digraph.py b/portage_with_autodep/pym/portage/util/digraph.py index f3ae658..fc1fb86 100644 --- a/portage_with_autodep/pym/portage/util/digraph.py +++ b/portage_with_autodep/pym/portage/util/digraph.py @@ -1,12 +1,13 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ['digraph'] from collections import deque import sys -from portage import _unicode_decode from portage.util import writemsg class digraph(object): @@ -16,24 +17,24 @@ class digraph(object): def __init__(self): """Create an empty digraph""" - + # { node : ( { child : priority } , { parent : priority } ) } self.nodes = {} self.order = [] def add(self, node, parent, priority=0): """Adds the specified node with the specified parent. - + If the dep is a soft-dep and the node already has a hard relationship to the parent, the relationship is left as hard.""" - + if node not in self.nodes: self.nodes[node] = ({}, {}, node) self.order.append(node) - + if not parent: return - + if parent not in self.nodes: self.nodes[parent] = ({}, {}, parent) self.order.append(parent) @@ -50,15 +51,15 @@ class digraph(object): """Removes the specified node from the digraph, also removing and ties to other nodes in the digraph. Raises KeyError if the node doesn't exist.""" - + if node not in self.nodes: raise KeyError(node) - + for parent in self.nodes[node][1]: del self.nodes[parent][0][node] for child in self.nodes[node][0]: del self.nodes[child][1][node] - + del self.nodes[node] self.order.remove(node) @@ -157,10 +158,10 @@ class digraph(object): def leaf_nodes(self, ignore_priority=None): """Return all nodes that have no children - + If ignore_soft_deps is True, soft deps are not counted as children in calculations.""" - + leaf_nodes = [] if ignore_priority is None: for node in self.order: @@ -191,10 +192,10 @@ class digraph(object): def root_nodes(self, ignore_priority=None): """Return all nodes that have no parents. - + If ignore_soft_deps is True, soft deps are not counted as parents in calculations.""" - + root_nodes = [] if ignore_priority is None: for node in self.order: @@ -272,18 +273,17 @@ class digraph(object): def debug_print(self): def output(s): writemsg(s, noiselevel=-1) - # Use _unicode_decode() to force unicode format + # Use unicode_literals to force unicode format # strings for python-2.x safety, ensuring that # node.__unicode__() is used when necessary. for node in self.nodes: - output(_unicode_decode("%s ") % (node,)) + output("%s " % (node,)) if self.nodes[node][0]: output("depends on\n") else: output("(no children)\n") for child, priorities in self.nodes[node][0].items(): - output(_unicode_decode(" %s (%s)\n") % \ - (child, priorities[-1],)) + output(" %s (%s)\n" % (child, priorities[-1],)) def bfs(self, start, ignore_priority=None): if start not in self: diff --git a/portage_with_autodep/pym/portage/util/digraph.pyo b/portage_with_autodep/pym/portage/util/digraph.pyo Binary files differindex 8e503a6..fc00aa7 100644 --- a/portage_with_autodep/pym/portage/util/digraph.pyo +++ b/portage_with_autodep/pym/portage/util/digraph.pyo diff --git a/portage_with_autodep/pym/portage/util/env_update.py b/portage_with_autodep/pym/portage/util/env_update.py index ace4077..5fddaac 100644 --- a/portage_with_autodep/pym/portage/util/env_update.py +++ b/portage_with_autodep/pym/portage/util/env_update.py @@ -1,16 +1,17 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ['env_update'] import errno +import glob import io import stat import sys import time import portage -from portage import os, _encodings, _unicode_encode +from portage import os, _encodings, _unicode_decode, _unicode_encode from portage.checksum import prelink_capable from portage.data import ostype from portage.exception import ParseError @@ -88,7 +89,8 @@ def _env_update(makelinks, target_root, prev_mtimes, contents, env, eprefix = settings.get("EPREFIX", "") eprefix_lstrip = eprefix.lstrip(os.sep) - envd_dir = os.path.join(target_root, eprefix_lstrip, "etc", "env.d") + eroot = normalize_path(os.path.join(target_root, eprefix_lstrip)).rstrip(os.sep) + os.sep + envd_dir = os.path.join(eroot, "etc", "env.d") ensure_dirs(envd_dir, mode=0o755) fns = listdir(envd_dir, EmptyOnError=1) fns.sort() @@ -164,15 +166,14 @@ def _env_update(makelinks, target_root, prev_mtimes, contents, env, they won't be overwritten by this dict.update call.""" env.update(myconfig) - ldsoconf_path = os.path.join( - target_root, eprefix_lstrip, "etc", "ld.so.conf") + ldsoconf_path = os.path.join(eroot, "etc", "ld.so.conf") try: myld = io.open(_unicode_encode(ldsoconf_path, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['content'], errors='replace') - myldlines=myld.readlines() + myldlines = myld.readlines() myld.close() - oldld=[] + oldld = [] for x in myldlines: #each line has at least one char (a newline) if x[:1] == "#": @@ -193,20 +194,34 @@ def _env_update(makelinks, target_root, prev_mtimes, contents, env, myfd.write(x + "\n") myfd.close() + potential_lib_dirs = set() + for lib_dir_glob in ('usr/lib*', 'lib*'): + x = os.path.join(eroot, lib_dir_glob) + for y in glob.glob(_unicode_encode(x, + encoding=_encodings['fs'], errors='strict')): + try: + y = _unicode_decode(y, + encoding=_encodings['fs'], errors='strict') + except UnicodeDecodeError: + continue + if os.path.basename(y) != 'libexec': + potential_lib_dirs.add(y[len(eroot):]) + # Update prelink.conf if we are prelink-enabled if prelink_capable: - newprelink = atomic_ofstream(os.path.join( - target_root, eprefix_lstrip, "etc", "prelink.conf")) + prelink_d = os.path.join(eroot, 'etc', 'prelink.conf.d') + ensure_dirs(prelink_d) + newprelink = atomic_ofstream(os.path.join(prelink_d, 'portage.conf')) newprelink.write("# prelink.conf autogenerated by env-update; make all changes to\n") newprelink.write("# contents of /etc/env.d directory\n") - for x in ["/bin","/sbin","/usr/bin","/usr/sbin","/lib","/usr/lib"]: - newprelink.write("-l %s\n" % (x,)); - prelink_paths = [] - prelink_paths += specials.get("LDPATH", []) - prelink_paths += specials.get("PATH", []) - prelink_paths += specials.get("PRELINK_PATH", []) - prelink_path_mask = specials.get("PRELINK_PATH_MASK", []) + for x in sorted(potential_lib_dirs) + ['bin', 'sbin']: + newprelink.write('-l /%s\n' % (x,)); + prelink_paths = set() + prelink_paths |= set(specials.get('LDPATH', [])) + prelink_paths |= set(specials.get('PATH', [])) + prelink_paths |= set(specials.get('PRELINK_PATH', [])) + prelink_path_mask = specials.get('PRELINK_PATH_MASK', []) for x in prelink_paths: if not x: continue @@ -227,12 +242,26 @@ def _env_update(makelinks, target_root, prev_mtimes, contents, env, newprelink.write("-b %s\n" % (x,)) newprelink.close() + # Migration code path. If /etc/prelink.conf was generated by us, then + # point it to the new stuff until the prelink package re-installs. + prelink_conf = os.path.join(eroot, 'etc', 'prelink.conf') + try: + with open(_unicode_encode(prelink_conf, + encoding=_encodings['fs'], errors='strict'), 'rb') as f: + if f.readline() == b'# prelink.conf autogenerated by env-update; make all changes to\n': + f = atomic_ofstream(prelink_conf) + f.write('-c /etc/prelink.conf.d/*.conf\n') + f.close() + except IOError as e: + if e.errno != errno.ENOENT: + raise + current_time = long(time.time()) mtime_changed = False + lib_dirs = set() - for lib_dir in set(specials["LDPATH"] + \ - ['usr/lib','usr/lib64','usr/lib32','lib','lib64','lib32']): - x = os.path.join(target_root, eprefix_lstrip, lib_dir.lstrip(os.sep)) + for lib_dir in set(specials['LDPATH']) | potential_lib_dirs: + x = os.path.join(eroot, lib_dir.lstrip(os.sep)) try: newldpathtime = os.stat(x)[stat.ST_MTIME] lib_dirs.add(normalize_path(x)) @@ -292,7 +321,7 @@ def _env_update(makelinks, target_root, prev_mtimes, contents, env, writemsg_level(_(">>> Regenerating %setc/ld.so.cache...\n") % \ (target_root,)) os.system("cd / ; %s -X -r '%s'" % (ldconfig, target_root)) - elif ostype in ("FreeBSD","DragonFly"): + elif ostype in ("FreeBSD", "DragonFly"): writemsg_level(_(">>> Regenerating %svar/run/ld-elf.so.hints...\n") % \ target_root) os.system(("cd / ; %s -elf -i " + \ @@ -308,11 +337,10 @@ def _env_update(makelinks, target_root, prev_mtimes, contents, env, cenvnotice += "# GO INTO /etc/csh.cshrc NOT /etc/csh.env\n\n" #create /etc/profile.env for bash support - outfile = atomic_ofstream(os.path.join( - target_root, eprefix_lstrip, "etc", "profile.env")) + outfile = atomic_ofstream(os.path.join(eroot, "etc", "profile.env")) outfile.write(penvnotice) - env_keys = [ x for x in env if x != "LDPATH" ] + env_keys = [x for x in env if x != "LDPATH"] env_keys.sort() for k in env_keys: v = env[k] @@ -323,8 +351,7 @@ def _env_update(makelinks, target_root, prev_mtimes, contents, env, outfile.close() #create /etc/csh.env for (t)csh support - outfile = atomic_ofstream(os.path.join( - target_root, eprefix_lstrip, "etc", "csh.env")) + outfile = atomic_ofstream(os.path.join(eroot, "etc", "csh.env")) outfile.write(cenvnotice) for x in env_keys: outfile.write("setenv %s '%s'\n" % (x, env[x])) diff --git a/portage_with_autodep/pym/portage/util/env_update.pyo b/portage_with_autodep/pym/portage/util/env_update.pyo Binary files differindex ee3b187..2398289 100644 --- a/portage_with_autodep/pym/portage/util/env_update.pyo +++ b/portage_with_autodep/pym/portage/util/env_update.pyo diff --git a/portage_with_autodep/pym/portage/util/lafilefixer.py b/portage_with_autodep/pym/portage/util/lafilefixer.py index 54ff20d..2562d9a 100644 --- a/portage_with_autodep/pym/portage/util/lafilefixer.py +++ b/portage_with_autodep/pym/portage/util/lafilefixer.py @@ -11,7 +11,7 @@ from portage.exception import InvalidData # This an re-implementaion of dev-util/lafilefixer-0.5. # rewrite_lafile() takes the contents of an lafile as a string # It then parses the dependency_libs and inherited_linker_flags -# entries. +# entries. # We insist on dependency_libs being present. inherited_linker_flags # is optional. # There are strict rules about the syntax imposed by libtool's libltdl. @@ -21,7 +21,7 @@ from portage.exception import InvalidData # lafilefixer does). # What it does: # * Replaces all .la files with absolut paths in dependency_libs with -# corresponding -l* and -L* entries +# corresponding -l* and -L* entries # (/usr/lib64/libfoo.la -> -L/usr/lib64 -lfoo) # * Moves various flags (see flag_re below) to inherited_linker_flags, # if such an entry was present. @@ -36,7 +36,7 @@ from portage.exception import InvalidData dep_libs_re = re.compile(b"dependency_libs='(?P<value>[^']*)'$") inh_link_flags_re = re.compile(b"inherited_linker_flags='(?P<value>[^']*)'$") -#regexes for replacing stuff in -L entries. +#regexes for replacing stuff in -L entries. #replace 'X11R6/lib' and 'local/lib' with 'lib', no idea what's this about. X11_local_sub = re.compile(b"X11R6/lib|local/lib") #get rid of the '..' @@ -129,11 +129,11 @@ def rewrite_lafile(contents): #This allows us to place all -L entries at the beginning #of 'dependency_libs'. ladir = dep_libs_entry - + ladir = X11_local_sub.sub(b"lib", ladir) ladir = pkgconfig_sub1.sub(b"usr", ladir) ladir = pkgconfig_sub2.sub(b"\g<usrlib>", ladir) - + if ladir not in libladir: libladir.append(ladir) diff --git a/portage_with_autodep/pym/portage/util/lafilefixer.pyo b/portage_with_autodep/pym/portage/util/lafilefixer.pyo Binary files differindex a6e06ab..5378d82 100644 --- a/portage_with_autodep/pym/portage/util/lafilefixer.pyo +++ b/portage_with_autodep/pym/portage/util/lafilefixer.pyo diff --git a/portage_with_autodep/pym/portage/util/listdir.py b/portage_with_autodep/pym/portage/util/listdir.py index c2628cb..2012e14 100644 --- a/portage_with_autodep/pym/portage/util/listdir.py +++ b/portage_with_autodep/pym/portage/util/listdir.py @@ -1,36 +1,33 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ['cacheddir', 'listdir'] import errno import stat -import time +import sys + +if sys.hexversion < 0x3000000: + from itertools import izip as zip from portage import os +from portage.const import VCS_DIRS from portage.exception import DirectoryNotFound, PermissionDenied, PortageException -from portage.util import normalize_path, writemsg - -_ignorecvs_dirs = ('CVS', 'RCS', 'SCCS', '.svn', '.git') +from portage.util import normalize_path + +# The global dircache is no longer supported, since it could +# be a memory leak for API consumers. Any cacheddir callers +# should use higher-level caches instead, when necessary. +# TODO: Remove dircache variable after stable portage does +# not use is (keep it for now, in case API consumers clear +# it manually). dircache = {} -cacheHit = 0 -cacheMiss = 0 -cacheStale = 0 def cacheddir(my_original_path, ignorecvs, ignorelist, EmptyOnError, followSymlinks=True): - global cacheHit,cacheMiss,cacheStale mypath = normalize_path(my_original_path) - if mypath in dircache: - cacheHit += 1 - cached_mtime, list, ftype = dircache[mypath] - else: - cacheMiss += 1 - cached_mtime, list, ftype = -1, [], [] try: pathstat = os.stat(mypath) - if stat.S_ISDIR(pathstat[stat.ST_MODE]): - mtime = pathstat.st_mtime - else: + if not stat.S_ISDIR(pathstat.st_mode): raise DirectoryNotFound(mypath) except EnvironmentError as e: if e.errno == PermissionDenied.errno: @@ -39,19 +36,16 @@ def cacheddir(my_original_path, ignorecvs, ignorelist, EmptyOnError, followSymli return [], [] except PortageException: return [], [] - # Python retuns mtime in seconds, so if it was changed in the last few seconds, it could be invalid - if mtime != cached_mtime or time.time() - mtime < 4: - if mypath in dircache: - cacheStale += 1 + else: try: - list = os.listdir(mypath) + fpaths = os.listdir(mypath) except EnvironmentError as e: if e.errno != errno.EACCES: raise del e raise PermissionDenied(mypath) ftype = [] - for x in list: + for x in fpaths: try: if followSymlinks: pathstat = os.stat(mypath+"/"+x) @@ -68,23 +62,22 @@ def cacheddir(my_original_path, ignorecvs, ignorelist, EmptyOnError, followSymli ftype.append(3) except (IOError, OSError): ftype.append(3) - dircache[mypath] = mtime, list, ftype - - ret_list = [] - ret_ftype = [] - for x in range(0, len(list)): - if list[x] in ignorelist: - pass - elif ignorecvs: - if list[x][:2] != ".#" and \ - not (ftype[x] == 1 and list[x] in _ignorecvs_dirs): - ret_list.append(list[x]) - ret_ftype.append(ftype[x]) - else: - ret_list.append(list[x]) - ret_ftype.append(ftype[x]) - - writemsg("cacheddirStats: H:%d/M:%d/S:%d\n" % (cacheHit, cacheMiss, cacheStale),10) + + if ignorelist or ignorecvs: + ret_list = [] + ret_ftype = [] + for file_path, file_type in zip(fpaths, ftype): + if file_path in ignorelist: + pass + elif ignorecvs: + if file_path[:2] != ".#" and \ + not (file_type == 1 and file_path in VCS_DIRS): + ret_list.append(file_path) + ret_ftype.append(file_type) + else: + ret_list = fpaths + ret_ftype = ftype + return ret_list, ret_ftype def listdir(mypath, recursive=False, filesonly=False, ignorecvs=False, ignorelist=[], followSymlinks=True, @@ -98,7 +91,7 @@ def listdir(mypath, recursive=False, filesonly=False, ignorecvs=False, ignorelis @type recursive: Boolean @param filesonly; Only return files, not more directories @type filesonly: Boolean - @param ignorecvs: Ignore CVS directories ('CVS','SCCS','.svn','.git') + @param ignorecvs: Ignore VCS directories @type ignorecvs: Boolean @param ignorelist: List of filenames/directories to exclude @type ignorelist: List @@ -112,40 +105,35 @@ def listdir(mypath, recursive=False, filesonly=False, ignorecvs=False, ignorelis @return: A list of files and directories (or just files or just directories) or an empty list. """ - list, ftype = cacheddir(mypath, ignorecvs, ignorelist, EmptyOnError, followSymlinks) + fpaths, ftype = cacheddir(mypath, ignorecvs, ignorelist, EmptyOnError, followSymlinks) - if list is None: - list=[] + if fpaths is None: + fpaths = [] if ftype is None: - ftype=[] + ftype = [] if not (filesonly or dirsonly or recursive): - return list + return fpaths if recursive: - x=0 - while x<len(ftype): - if ftype[x] == 1: - l,f = cacheddir(mypath+"/"+list[x], ignorecvs, ignorelist, EmptyOnError, - followSymlinks) - - l=l[:] - for y in range(0,len(l)): - l[y]=list[x]+"/"+l[y] - list=list+l - ftype=ftype+f - x+=1 + stack = list(zip(fpaths, ftype)) + fpaths = [] + ftype = [] + while stack: + file_path, file_type = stack.pop() + fpaths.append(file_path) + ftype.append(file_type) + if file_type == 1: + subdir_list, subdir_types = cacheddir( + os.path.join(mypath, file_path), ignorecvs, + ignorelist, EmptyOnError, followSymlinks) + stack.extend((os.path.join(file_path, x), x_type) + for x, x_type in zip(subdir_list, subdir_types)) + if filesonly: - rlist=[] - for x in range(0,len(ftype)): - if ftype[x]==0: - rlist=rlist+[list[x]] + fpaths = [x for x, x_type in zip(fpaths, ftype) if x_type == 0] + elif dirsonly: - rlist = [] - for x in range(0, len(ftype)): - if ftype[x] == 1: - rlist = rlist + [list[x]] - else: - rlist=list + fpaths = [x for x, x_type in zip(fpaths, ftype) if x_type == 1] - return rlist + return fpaths diff --git a/portage_with_autodep/pym/portage/util/listdir.pyo b/portage_with_autodep/pym/portage/util/listdir.pyo Binary files differindex 0f02d6d..e4d40be 100644 --- a/portage_with_autodep/pym/portage/util/listdir.pyo +++ b/portage_with_autodep/pym/portage/util/listdir.pyo diff --git a/portage_with_autodep/pym/portage/util/movefile.py b/portage_with_autodep/pym/portage/util/movefile.py index 10577b5..452e77f 100644 --- a/portage_with_autodep/pym/portage/util/movefile.py +++ b/portage_with_autodep/pym/portage/util/movefile.py @@ -1,18 +1,22 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = ['movefile'] import errno +import fnmatch import os as _os import shutil as _shutil import stat +import sys import subprocess import textwrap import portage from portage import bsd_chflags, _encodings, _os_overrides, _selinux, \ - _unicode_decode, _unicode_encode, _unicode_func_wrapper,\ + _unicode_decode, _unicode_encode, _unicode_func_wrapper, \ _unicode_module_wrapper from portage.const import MOVE_BINARY from portage.exception import OperationNotSupported @@ -24,43 +28,113 @@ def _apply_stat(src_stat, dest): _os.chown(dest, src_stat.st_uid, src_stat.st_gid) _os.chmod(dest, stat.S_IMODE(src_stat.st_mode)) +_xattr_excluder_cache = {} + +def _get_xattr_excluder(pattern): + + try: + value = _xattr_excluder_cache[pattern] + except KeyError: + value = _xattr_excluder(pattern) + _xattr_excluder_cache[pattern] = value + + return value + +class _xattr_excluder(object): + + __slots__ = ('_pattern_split',) + + def __init__(self, pattern): + + if pattern is None: + self._pattern_split = None + else: + pattern = pattern.split() + if not pattern: + self._pattern_split = None + else: + pattern.sort() + self._pattern_split = tuple(pattern) + + def __call__(self, attr): + + if self._pattern_split is None: + return False + + match = fnmatch.fnmatch + for x in self._pattern_split: + if match(attr, x): + return True + + return False + if hasattr(_os, "getxattr"): # Python >=3.3 and GNU/Linux - def _copyxattr(src, dest): - for attr in _os.listxattr(src): + def _copyxattr(src, dest, exclude=None): + + try: + attrs = _os.listxattr(src) + except OSError as e: + if e.errno != OperationNotSupported.errno: + raise + attrs = () + if attrs: + if exclude is not None and isinstance(attrs[0], bytes): + exclude = exclude.encode(_encodings['fs']) + exclude = _get_xattr_excluder(exclude) + + for attr in attrs: + if exclude(attr): + continue try: _os.setxattr(dest, attr, _os.getxattr(src, attr)) raise_exception = False except OSError: raise_exception = True if raise_exception: - raise OperationNotSupported("Filesystem containing file '%s' does not support extended attributes" % dest) + raise OperationNotSupported(_("Filesystem containing file '%s' " + "does not support extended attribute '%s'") % + (_unicode_decode(dest), _unicode_decode(attr))) else: try: import xattr except ImportError: xattr = None if xattr is not None: - def _copyxattr(src, dest): - for attr in xattr.list(src): + def _copyxattr(src, dest, exclude=None): + + try: + attrs = xattr.list(src) + except IOError as e: + if e.errno != OperationNotSupported.errno: + raise + attrs = () + + if attrs: + if exclude is not None and isinstance(attrs[0], bytes): + exclude = exclude.encode(_encodings['fs']) + exclude = _get_xattr_excluder(exclude) + + for attr in attrs: + if exclude(attr): + continue try: xattr.set(dest, attr, xattr.get(src, attr)) raise_exception = False except IOError: raise_exception = True if raise_exception: - raise OperationNotSupported("Filesystem containing file '%s' does not support extended attributes" % dest) + raise OperationNotSupported(_("Filesystem containing file '%s' " + "does not support extended attribute '%s'") % + (_unicode_decode(dest), _unicode_decode(attr))) else: - _devnull = open("/dev/null", "wb") try: - subprocess.call(["getfattr", "--version"], stdout=_devnull) - subprocess.call(["setfattr", "--version"], stdout=_devnull) - _has_getfattr_and_setfattr = True + with open(_os.devnull, 'wb') as f: + subprocess.call(["getfattr", "--version"], stdout=f) + subprocess.call(["setfattr", "--version"], stdout=f) except OSError: - _has_getfattr_and_setfattr = False - _devnull.close() - if _has_getfattr_and_setfattr: - def _copyxattr(src, dest): + def _copyxattr(src, dest, exclude=None): + # TODO: implement exclude getfattr_process = subprocess.Popen(["getfattr", "-d", "--absolute-names", src], stdout=subprocess.PIPE) getfattr_process.wait() extended_attributes = getfattr_process.stdout.readlines() @@ -72,14 +146,15 @@ else: if setfattr_process.returncode != 0: raise OperationNotSupported("Filesystem containing file '%s' does not support extended attributes" % dest) else: - def _copyxattr(src, dest): + def _copyxattr(src, dest, exclude=None): pass def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, hardlink_candidates=None, encoding=_encodings['fs']): """moves a file from src to dest, preserving all permissions and attributes; mtime will - be preserved even when moving across filesystems. Returns true on success and false on - failure. Move is atomic.""" + be preserved even when moving across filesystems. Returns mtime as integer on success + and None on failure. mtime is expressed in seconds in Python <3.3 and nanoseconds in + Python >=3.3. Move is atomic.""" if mysettings is None: mysettings = portage.settings @@ -102,22 +177,22 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, try: if not sstat: - sstat=os.lstat(src) + sstat = os.lstat(src) except SystemExit as e: raise except Exception as e: writemsg("!!! %s\n" % _("Stating source file failed... movefile()"), noiselevel=-1) - writemsg(_unicode_decode("!!! %s\n") % (e,), noiselevel=-1) + writemsg("!!! %s\n" % (e,), noiselevel=-1) return None - destexists=1 + destexists = 1 try: - dstat=os.lstat(dest) + dstat = os.lstat(dest) except (OSError, IOError): - dstat=os.lstat(os.path.dirname(dest)) - destexists=0 + dstat = os.lstat(os.path.dirname(dest)) + destexists = 0 if bsd_chflags: if destexists and dstat.st_flags != 0: @@ -132,7 +207,7 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, if stat.S_ISLNK(dstat[stat.ST_MODE]): try: os.unlink(dest) - destexists=0 + destexists = 0 except SystemExit as e: raise except Exception as e: @@ -140,7 +215,7 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, if stat.S_ISLNK(sstat[stat.ST_MODE]): try: - target=os.readlink(src) + target = os.readlink(src) if mysettings and "D" in mysettings and \ target.startswith(mysettings["D"]): target = target[len(mysettings["D"])-1:] @@ -159,17 +234,32 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, if e.errno not in (errno.ENOENT, errno.EEXIST) or \ target != os.readlink(dest): raise - lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID]) - # utime() only works on the target of a symlink, so it's not - # possible to perserve mtime on symlinks. - return os.lstat(dest)[stat.ST_MTIME] + lchown(dest, sstat[stat.ST_UID], sstat[stat.ST_GID]) + + try: + _os.unlink(src_bytes) + except OSError: + pass + + if sys.hexversion >= 0x3030000: + try: + os.utime(dest, ns=(sstat.st_mtime_ns, sstat.st_mtime_ns), follow_symlinks=False) + except NotImplementedError: + # utimensat() and lutimes() missing in libc. + return os.stat(dest, follow_symlinks=False).st_mtime_ns + else: + return sstat.st_mtime_ns + else: + # utime() in Python <3.3 only works on the target of a symlink, so it's not + # possible to preserve mtime on symlinks. + return os.lstat(dest)[stat.ST_MTIME] except SystemExit as e: raise except Exception as e: writemsg("!!! %s\n" % _("failed to properly create symlink:"), noiselevel=-1) writemsg("!!! %s -> %s\n" % (dest, target), noiselevel=-1) - writemsg(_unicode_decode("!!! %s\n") % (e,), noiselevel=-1) + writemsg("!!! %s\n" % (e,), noiselevel=-1) return None hardlinked = False @@ -204,9 +294,13 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, writemsg("!!! %s\n" % (e,), noiselevel=-1) return None hardlinked = True + try: + _os.unlink(src_bytes) + except OSError: + pass break - renamefailed=1 + renamefailed = 1 if hardlinked: renamefailed = False if not hardlinked and (selinux_enabled or sstat.st_dev == dstat.st_dev): @@ -214,14 +308,14 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, if selinux_enabled: selinux.rename(src, dest) else: - os.rename(src,dest) - renamefailed=0 + os.rename(src, dest) + renamefailed = 0 except OSError as e: if e.errno != errno.EXDEV: # Some random error. writemsg("!!! %s\n" % _("Failed to move %(src)s to %(dest)s") % {"src": src, "dest": dest}, noiselevel=-1) - writemsg(_unicode_decode("!!! %s\n") % (e,), noiselevel=-1) + writemsg("!!! %s\n" % (e,), noiselevel=-1) return None # Invalid cross-device-link 'bind' mounted or actually Cross-Device if renamefailed: @@ -233,7 +327,8 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, _copyfile(src_bytes, dest_tmp_bytes) if xattr_enabled: try: - _copyxattr(src_bytes, dest_tmp_bytes) + _copyxattr(src_bytes, dest_tmp_bytes, + exclude=mysettings.get("PORTAGE_XATTR_EXCLUDE", "security.* system.nfs4_acl")) except SystemExit: raise except: @@ -252,7 +347,7 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, except Exception as e: writemsg("!!! %s\n" % _('copy %(src)s -> %(dest)s failed.') % {"src": src, "dest": dest}, noiselevel=-1) - writemsg(_unicode_decode("!!! %s\n") % (e,), noiselevel=-1) + writemsg("!!! %s\n" % (e,), noiselevel=-1) return None else: #we don't yet handle special, so we need to fall back to /bin/mv @@ -265,35 +360,54 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, writemsg("!!! %s\n" % a, noiselevel=-1) return None # failure - # Always use stat_obj[stat.ST_MTIME] for the integral timestamp which - # is returned, since the stat_obj.st_mtime float attribute rounds *up* + # In Python <3.3 always use stat_obj[stat.ST_MTIME] for the integral timestamp + # which is returned, since the stat_obj.st_mtime float attribute rounds *up* # if the nanosecond part of the timestamp is 999999881 ns or greater. try: if hardlinked: - newmtime = os.stat(dest)[stat.ST_MTIME] + if sys.hexversion >= 0x3030000: + newmtime = os.stat(dest).st_mtime_ns + else: + newmtime = os.stat(dest)[stat.ST_MTIME] else: # Note: It is not possible to preserve nanosecond precision # (supported in POSIX.1-2008 via utimensat) with the IEEE 754 # double precision float which only has a 53 bit significand. if newmtime is not None: - os.utime(dest, (newmtime, newmtime)) + if sys.hexversion >= 0x3030000: + os.utime(dest, ns=(newmtime, newmtime)) + else: + os.utime(dest, (newmtime, newmtime)) else: - newmtime = sstat[stat.ST_MTIME] + if sys.hexversion >= 0x3030000: + newmtime = sstat.st_mtime_ns + else: + newmtime = sstat[stat.ST_MTIME] if renamefailed: - # If rename succeeded then timestamps are automatically - # preserved with complete precision because the source - # and destination inode are the same. Otherwise, round - # down to the nearest whole second since python's float - # st_mtime cannot be used to preserve the st_mtim.tv_nsec - # field with complete precision. Note that we have to use - # stat_obj[stat.ST_MTIME] here because the float - # stat_obj.st_mtime rounds *up* sometimes. - os.utime(dest, (newmtime, newmtime)) + if sys.hexversion >= 0x3030000: + # If rename succeeded then timestamps are automatically + # preserved with complete precision because the source + # and destination inodes are the same. Otherwise, manually + # update timestamps with nanosecond precision. + os.utime(dest, ns=(newmtime, newmtime)) + else: + # If rename succeeded then timestamps are automatically + # preserved with complete precision because the source + # and destination inodes are the same. Otherwise, round + # down to the nearest whole second since python's float + # st_mtime cannot be used to preserve the st_mtim.tv_nsec + # field with complete precision. Note that we have to use + # stat_obj[stat.ST_MTIME] here because the float + # stat_obj.st_mtime rounds *up* sometimes. + os.utime(dest, (newmtime, newmtime)) except OSError: # The utime can fail here with EPERM even though the move succeeded. # Instead of failing, use stat to return the mtime if possible. try: - newmtime = os.stat(dest)[stat.ST_MTIME] + if sys.hexversion >= 0x3030000: + newmtime = os.stat(dest).st_mtime_ns + else: + newmtime = os.stat(dest)[stat.ST_MTIME] except OSError as e: writemsg(_("!!! Failed to stat in movefile()\n"), noiselevel=-1) writemsg("!!! %s\n" % dest, noiselevel=-1) diff --git a/portage_with_autodep/pym/portage/util/movefile.pyo b/portage_with_autodep/pym/portage/util/movefile.pyo Binary files differindex 1228ee7..690b9b6 100644 --- a/portage_with_autodep/pym/portage/util/movefile.pyo +++ b/portage_with_autodep/pym/portage/util/movefile.pyo diff --git a/portage_with_autodep/pym/portage/util/mtimedb.pyo b/portage_with_autodep/pym/portage/util/mtimedb.pyo Binary files differindex fda479a..6fe9b37 100644 --- a/portage_with_autodep/pym/portage/util/mtimedb.pyo +++ b/portage_with_autodep/pym/portage/util/mtimedb.pyo diff --git a/portage_with_autodep/pym/portage/util/whirlpool.py b/portage_with_autodep/pym/portage/util/whirlpool.py index c696f6f..170ae73 100644 --- a/portage_with_autodep/pym/portage/util/whirlpool.py +++ b/portage_with_autodep/pym/portage/util/whirlpool.py @@ -639,6 +639,8 @@ def WhirlpoolInit(ctx): return def WhirlpoolAdd(source, sourceBits, ctx): + if not isinstance(source, bytes): + raise TypeError("Expected %s, got %s" % (bytes, type(source))) if sys.hexversion < 0x3000000: source = [ord(s)&0xff for s in source] diff --git a/portage_with_autodep/pym/portage/util/whirlpool.pyo b/portage_with_autodep/pym/portage/util/whirlpool.pyo Binary files differindex 4bcf49a..98e2815 100644 --- a/portage_with_autodep/pym/portage/util/whirlpool.pyo +++ b/portage_with_autodep/pym/portage/util/whirlpool.pyo diff --git a/portage_with_autodep/pym/portage/versions.py b/portage_with_autodep/pym/portage/versions.py index db14e99..3bfc388 100644 --- a/portage_with_autodep/pym/portage/versions.py +++ b/portage_with_autodep/pym/portage/versions.py @@ -1,7 +1,9 @@ # versions.py -- core Portage functionality -# Copyright 1998-2012 Gentoo Foundation +# Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + __all__ = [ 'best', 'catpkgsplit', 'catsplit', 'cpv_getkey', 'cpv_getversion', 'cpv_sort_key', 'pkgcmp', 'pkgsplit', @@ -23,7 +25,7 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.util:cmp_sort_key', ) from portage import _unicode_decode -from portage.eapi import eapi_allows_dots_in_PN +from portage.eapi import _get_eapi_attrs from portage.exception import InvalidData from portage.localization import _ @@ -31,6 +33,10 @@ _unknown_repo = "__unknown__" # \w is [a-zA-Z0-9_] +# PMS 3.1.3: A slot name may contain any of the characters [A-Za-z0-9+_.-]. +# It must not begin with a hyphen or a dot. +_slot = r'([\w+][\w+.-]*)' + # 2.1.1 A category name may contain any of the characters [A-Za-z0-9+_.-]. # It must not begin with a hyphen or a dot. _cat = r'[\w+][\w+.-]*' @@ -65,6 +71,42 @@ suffix_regexp = re.compile("^(alpha|beta|rc|pre|p)(\\d*)$") suffix_value = {"pre": -2, "p": 0, "alpha": -4, "beta": -3, "rc": -1} endversion_keys = ["pre", "p", "alpha", "beta", "rc"] +_slot_re_cache = {} + +def _get_slot_re(eapi_attrs): + cache_key = eapi_attrs.slot_operator + slot_re = _slot_re_cache.get(cache_key) + if slot_re is not None: + return slot_re + + if eapi_attrs.slot_operator: + slot_re = _slot + r'(/' + _slot + r')?' + else: + slot_re = _slot + + slot_re = re.compile('^' + slot_re + '$', re.VERBOSE | re.UNICODE) + + _slot_re_cache[cache_key] = slot_re + return slot_re + +_pv_re_cache = {} + +def _get_pv_re(eapi_attrs): + cache_key = eapi_attrs.dots_in_PN + pv_re = _pv_re_cache.get(cache_key) + if pv_re is not None: + return pv_re + + if eapi_attrs.dots_in_PN: + pv_re = _pv['dots_allowed_in_PN'] + else: + pv_re = _pv['dots_disallowed_in_PN'] + + pv_re = re.compile(r'^' + pv_re + r'$', re.VERBOSE | re.UNICODE) + + _pv_re_cache[cache_key] = pv_re + return pv_re + def ververify(myver, silent=1): if ver_regexp.match(myver): return 1 @@ -251,17 +293,6 @@ def pkgcmp(pkg1, pkg2): return None return vercmp("-".join(pkg1[1:]), "-".join(pkg2[1:])) -_pv_re = { - "dots_disallowed_in_PN": re.compile('^' + _pv['dots_disallowed_in_PN'] + '$', re.VERBOSE), - "dots_allowed_in_PN": re.compile('^' + _pv['dots_allowed_in_PN'] + '$', re.VERBOSE), -} - -def _get_pv_re(eapi): - if eapi is None or eapi_allows_dots_in_PN(eapi): - return _pv_re["dots_allowed_in_PN"] - else: - return _pv_re["dots_disallowed_in_PN"] - def _pkgsplit(mypkg, eapi=None): """ @param mypkg: pv @@ -269,7 +300,7 @@ def _pkgsplit(mypkg, eapi=None): 1. None if input is invalid. 2. (pn, ver, rev) if input is pv """ - m = _get_pv_re(eapi).match(mypkg) + m = _get_pv_re(_get_eapi_attrs(eapi)).match(mypkg) if m is None: return None @@ -284,7 +315,7 @@ def _pkgsplit(mypkg, eapi=None): return (m.group('pn'), m.group('ver'), rev) -_cat_re = re.compile('^%s$' % _cat) +_cat_re = re.compile('^%s$' % _cat, re.UNICODE) _missing_cat = 'null' def catpkgsplit(mydata, silent=1, eapi=None): @@ -329,14 +360,25 @@ class _pkg_str(_unicode): manually convert them to a plain unicode object first. """ - def __new__(cls, cpv, slot=None, repo=None, eapi=None): + def __new__(cls, cpv, metadata=None, settings=None, eapi=None, + repo=None, slot=None): return _unicode.__new__(cls, cpv) - def __init__(self, cpv, slot=None, repo=None, eapi=None): + def __init__(self, cpv, metadata=None, settings=None, eapi=None, + repo=None, slot=None): if not isinstance(cpv, _unicode): # Avoid TypeError from _unicode.__init__ with PyPy. cpv = _unicode_decode(cpv) _unicode.__init__(cpv) + if metadata is not None: + self.__dict__['_metadata'] = metadata + slot = metadata.get('SLOT', slot) + repo = metadata.get('repository', repo) + eapi = metadata.get('EAPI', eapi) + if settings is not None: + self.__dict__['_settings'] = settings + if eapi is not None: + self.__dict__['eapi'] = eapi self.__dict__['cpv_split'] = catpkgsplit(cpv, eapi=eapi) if self.cpv_split is None: raise InvalidData(cpv) @@ -348,7 +390,25 @@ class _pkg_str(_unicode): # for match_from_list introspection self.__dict__['cpv'] = self if slot is not None: - self.__dict__['slot'] = slot + eapi_attrs = _get_eapi_attrs(eapi) + slot_match = _get_slot_re(eapi_attrs).match(slot) + if slot_match is None: + # Avoid an InvalidAtom exception when creating SLOT atoms + self.__dict__['slot'] = '0' + self.__dict__['sub_slot'] = '0' + self.__dict__['slot_invalid'] = slot + else: + if eapi_attrs.slot_operator: + slot_split = slot.split("/") + self.__dict__['slot'] = slot_split[0] + if len(slot_split) > 1: + self.__dict__['sub_slot'] = slot_split[1] + else: + self.__dict__['sub_slot'] = slot_split[0] + else: + self.__dict__['slot'] = slot + self.__dict__['sub_slot'] = slot + if repo is not None: repo = _gen_valid_repo(repo) if not repo: @@ -359,6 +419,25 @@ class _pkg_str(_unicode): raise AttributeError("_pkg_str instances are immutable", self.__class__, name, value) + @property + def stable(self): + try: + return self._stable + except AttributeError: + try: + metadata = self._metadata + settings = self._settings + except AttributeError: + raise AttributeError('stable') + if not settings.local_config: + # Since repoman uses different config instances for + # different profiles, our local instance does not + # refer to the correct profile. + raise AssertionError('invalid context') + stable = settings._isStable(self) + self.__dict__['_stable'] = stable + return stable + def pkgsplit(mypkg, silent=1, eapi=None): """ @param mypkg: either a pv or cpv diff --git a/portage_with_autodep/pym/portage/versions.pyo b/portage_with_autodep/pym/portage/versions.pyo Binary files differindex eae2743..d59719e 100644 --- a/portage_with_autodep/pym/portage/versions.pyo +++ b/portage_with_autodep/pym/portage/versions.pyo diff --git a/portage_with_autodep/pym/portage/xml/__init__.pyo b/portage_with_autodep/pym/portage/xml/__init__.pyo Binary files differindex 15f1b77..3c787a2 100644 --- a/portage_with_autodep/pym/portage/xml/__init__.pyo +++ b/portage_with_autodep/pym/portage/xml/__init__.pyo diff --git a/portage_with_autodep/pym/portage/xml/metadata.py b/portage_with_autodep/pym/portage/xml/metadata.py index 25f801a..fcd9dc0 100644 --- a/portage_with_autodep/pym/portage/xml/metadata.py +++ b/portage_with_autodep/pym/portage/xml/metadata.py @@ -1,4 +1,4 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 """Provides an easy-to-use python interface to Gentoo's metadata.xml file. @@ -28,6 +28,8 @@ 'Thomas Mills Hinkle' """ +from __future__ import unicode_literals + __all__ = ('MetaDataXML',) import sys @@ -42,18 +44,23 @@ if sys.hexversion < 0x2070000 or \ else: try: import xml.etree.cElementTree as etree - except (ImportError, SystemError): + except (SystemExit, KeyboardInterrupt): + raise + except (ImportError, SystemError, RuntimeError, Exception): + # broken or missing xml support + # http://bugs.python.org/issue14988 import xml.etree.ElementTree as etree try: from xml.parsers.expat import ExpatError -except (ImportError, SystemError): +except (SystemExit, KeyboardInterrupt): + raise +except (ImportError, SystemError, RuntimeError, Exception): ExpatError = SyntaxError import re import xml.etree.ElementTree -import portage -from portage import os, _unicode_decode +from portage import _encodings, _unicode_encode from portage.util import unique_everseen class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder): @@ -197,12 +204,13 @@ class MetaDataXML(object): self._xml_tree = None try: - self._xml_tree = etree.parse(metadata_xml_path, + self._xml_tree = etree.parse(_unicode_encode(metadata_xml_path, + encoding=_encodings['fs'], errors='strict'), parser=etree.XMLParser(target=_MetadataTreeBuilder())) except ImportError: pass except ExpatError as e: - raise SyntaxError(_unicode_decode("%s") % (e,)) + raise SyntaxError("%s" % (e,)) if isinstance(herds, etree.ElementTree): herds_etree = herds @@ -235,7 +243,8 @@ class MetaDataXML(object): if self._herdstree is None: try: - self._herdstree = etree.parse(self._herds_path, + self._herdstree = etree.parse(_unicode_encode(self._herds_path, + encoding=_encodings['fs'], errors='strict'), parser=etree.XMLParser(target=_MetadataTreeBuilder())) except (ImportError, IOError, SyntaxError): return None diff --git a/portage_with_autodep/pym/portage/xml/metadata.pyo b/portage_with_autodep/pym/portage/xml/metadata.pyo Binary files differindex 0103456..4e8910e 100644 --- a/portage_with_autodep/pym/portage/xml/metadata.pyo +++ b/portage_with_autodep/pym/portage/xml/metadata.pyo diff --git a/portage_with_autodep/pym/portage/xpak.py b/portage_with_autodep/pym/portage/xpak.py index 3262326..73f84ab 100644 --- a/portage_with_autodep/pym/portage/xpak.py +++ b/portage_with_autodep/pym/portage/xpak.py @@ -324,7 +324,7 @@ class tbz2(object): """ self.scan() # Don't care about condition... We'll rewrite the data anyway. - if break_hardlinks and self.filestat.st_nlink > 1: + if break_hardlinks and self.filestat and self.filestat.st_nlink > 1: tmp_fname = "%s.%d" % (self.file, os.getpid()) shutil.copyfile(self.file, tmp_fname) try: diff --git a/portage_with_autodep/pym/portage/xpak.pyo b/portage_with_autodep/pym/portage/xpak.pyo Binary files differindex 361a709..eec8ebc 100644 --- a/portage_with_autodep/pym/portage/xpak.pyo +++ b/portage_with_autodep/pym/portage/xpak.pyo |